Add latest changes from gitlab-org/gitlab@master

4 files changed, 115 insertions, 80 deletions

diff --git a/scripts/lib/glfm/constants.rb b/scripts/lib/glfm/constants.rb
index d020d2fec5c..c4feef8464e 100644
--- a/scripts/lib/glfm/constants.rb
+++ b/scripts/lib/glfm/constants.rb
@@ -15,24 +15,25 @@ module Glfm
 
     # GitLab Flavored Markdown specification files
     specification_input_glfm_path = specification_path.join('input/gitlab_flavored_markdown')
-    GLFM_INTRO_MD_PATH = specification_input_glfm_path.join('glfm_intro.md')
-    GLFM_OFFICIAL_SPECIFICATION_EXAMPLES_MD_PATH =
-      specification_input_glfm_path.join('glfm_official_specification_examples.md')
-    GLFM_INTERNAL_EXTENSION_EXAMPLES_MD_PATH = specification_input_glfm_path.join('glfm_internal_extension_examples.md')
+    GLFM_OFFICIAL_SPECIFICATION_MD_PATH =
+      specification_input_glfm_path.join('glfm_official_specification.md')
+    GLFM_INTERNAL_EXTENSIONS_MD_PATH = specification_input_glfm_path.join('glfm_internal_extensions.md')
     GLFM_EXAMPLE_STATUS_YML_PATH = specification_input_glfm_path.join('glfm_example_status.yml')
     GLFM_EXAMPLE_METADATA_YML_PATH =
       specification_input_glfm_path.join('glfm_example_metadata.yml')
     GLFM_EXAMPLE_NORMALIZATIONS_YML_PATH = specification_input_glfm_path.join('glfm_example_normalizations.yml')
-    GLFM_SPEC_OUTPUT_PATH = specification_path.join('output')
-    GLFM_SPEC_TXT_PATH = GLFM_SPEC_OUTPUT_PATH.join('spec.txt')
-    GLFM_SPEC_HTML_PATH = GLFM_SPEC_OUTPUT_PATH.join('spec.html')
+    GLFM_OUTPUT_SPEC_PATH = specification_path.join('output_spec')
+    GLFM_SPEC_TXT_PATH = GLFM_OUTPUT_SPEC_PATH.join('spec.txt')
+    GLFM_SPEC_HTML_PATH = GLFM_OUTPUT_SPEC_PATH.join('spec.html')
 
     # Example Snapshot (ES) files
-    EXAMPLE_SNAPSHOTS_PATH = File.expand_path("../../../glfm_specification/example_snapshots", __dir__)
-    ES_EXAMPLES_INDEX_YML_PATH = File.join(EXAMPLE_SNAPSHOTS_PATH, 'examples_index.yml')
-    ES_MARKDOWN_YML_PATH = File.join(EXAMPLE_SNAPSHOTS_PATH, 'markdown.yml')
-    ES_HTML_YML_PATH = File.join(EXAMPLE_SNAPSHOTS_PATH, 'html.yml')
-    ES_PROSEMIRROR_JSON_YML_PATH = File.join(EXAMPLE_SNAPSHOTS_PATH, 'prosemirror_json.yml')
+    ES_OUTPUT_EXAMPLE_SNAPSHOTS_PATH = specification_path.join('output_example_snapshots')
+    ES_SNAPSHOT_SPEC_MD_PATH = ES_OUTPUT_EXAMPLE_SNAPSHOTS_PATH.join('snapshot_spec.md')
+    ES_SNAPSHOT_SPEC_HTML_PATH = ES_OUTPUT_EXAMPLE_SNAPSHOTS_PATH.join('snapshot_spec.html')
+    ES_EXAMPLES_INDEX_YML_PATH = ES_OUTPUT_EXAMPLE_SNAPSHOTS_PATH.join('examples_index.yml')
+    ES_MARKDOWN_YML_PATH = ES_OUTPUT_EXAMPLE_SNAPSHOTS_PATH.join('markdown.yml')
+    ES_HTML_YML_PATH = ES_OUTPUT_EXAMPLE_SNAPSHOTS_PATH.join('html.yml')
+    ES_PROSEMIRROR_JSON_YML_PATH = ES_OUTPUT_EXAMPLE_SNAPSHOTS_PATH.join('prosemirror_json.yml')
 
     # Other constants used for processing files
     GLFM_SPEC_TXT_HEADER = <<~MARKDOWN
@@ -41,8 +42,9 @@ module Glfm
       version: alpha
       ...
     MARKDOWN
-    INTRODUCTION_HEADER_LINE_TEXT = /\A# Introduction\Z/.freeze
-    END_TESTS_COMMENT_LINE_TEXT = /\A<!-- END TESTS -->\Z/.freeze
+    INTRODUCTION_HEADER_LINE_TEXT = '# Introduction'
+    BEGIN_TESTS_COMMENT_LINE_TEXT = '<!-- BEGIN TESTS -->'
+    END_TESTS_COMMENT_LINE_TEXT = '<!-- END TESTS -->'
     MARKDOWN_TEMPFILE_BASENAME = %w[MARKDOWN_TEMPFILE_ .yml].freeze
     METADATA_TEMPFILE_BASENAME = %w[METADATA_TEMPFILE_ .yml].freeze
     STATIC_HTML_TEMPFILE_BASENAME = %w[STATIC_HTML_TEMPFILE_ .yml].freeze
diff --git a/scripts/lib/glfm/update_example_snapshots.rb b/scripts/lib/glfm/update_example_snapshots.rb
index 9075260e748..8f817d0173e 100644
--- a/scripts/lib/glfm/update_example_snapshots.rb
+++ b/scripts/lib/glfm/update_example_snapshots.rb
@@ -30,12 +30,12 @@ module Glfm
     def process(skip_static_and_wysiwyg: false)
       output('Updating example snapshots...')
 
-      output("Reading #{GLFM_SPEC_TXT_PATH}...")
-      glfm_spec_txt_lines = File.open(GLFM_SPEC_TXT_PATH).readlines
+      output("Reading #{ES_SNAPSHOT_SPEC_MD_PATH}...")
+      es_snapshot_spec_md_lines = File.open(ES_SNAPSHOT_SPEC_MD_PATH).readlines
 
-      # Parse all the examples from `spec.txt`, using a Ruby port of the Python `get_tests`
+      # Parse all the examples from `snapshot_spec.md`, using a Ruby port of the Python `get_tests`
       # function the from original CommonMark/GFM `spec_test.py` script.
-      all_examples = parse_examples(glfm_spec_txt_lines)
+      all_examples = parse_examples(es_snapshot_spec_md_lines)
 
       add_example_names(all_examples)
 
@@ -55,7 +55,7 @@ module Glfm
       #    in the H1 header count. So, even though due to the concatenation it appears before the
       #    GitLab examples sections, it doesn't result in their header counts being off by +1.
       # 5. If an example contains the 'disabled' string extension, it is skipped (and will thus
-      #    result in a skip in the `spec_txt_example_position`). This behavior is taken from the
+      #    result in a skip in the `spec_example_position`). This behavior is taken from the
       #    GFM `spec_test.py` script (but it's NOT in the original CommonMark `spec_test.py`).
       # 6. If a section contains ONLY disabled examples, the section numbering will still be
       #    incremented to match the rendered HTML specification section numbering.
@@ -202,7 +202,7 @@ module Glfm
       ) do |example, hash|
         name = example.fetch(:name).to_sym
         hash[name] = {
-          'spec_txt_example_position' => example.fetch(:example),
+          'spec_example_position' => example.fetch(:example),
           'source_specification' => source_specification_for_extensions(example.fetch(:extensions))
         }
       end
diff --git a/scripts/lib/glfm/update_specification.rb b/scripts/lib/glfm/update_specification.rb
index c7264547e44..521c0919f64 100644
--- a/scripts/lib/glfm/update_specification.rb
+++ b/scripts/lib/glfm/update_specification.rb
@@ -23,17 +23,42 @@ module Glfm
     def process(skip_spec_html_generation: false)
       output('Updating specification...')
 
+      # read and optionally update `input/github_flavored_markdown/ghfm_spec_v_x.yy.md`
       ghfm_spec_lines = load_ghfm_spec
-      glfm_spec_txt_string = build_glfm_spec_txt(ghfm_spec_lines)
+
+      # create `output_spec/spec.txt`
+      glfm_spec_txt_header_lines = GLFM_SPEC_TXT_HEADER.split("\n").map { |line| "#{line}\n" }
+      official_spec_lines = readlines_from_path!(GLFM_OFFICIAL_SPECIFICATION_MD_PATH)
+
+      glfm_spec_txt_string = (glfm_spec_txt_header_lines + official_spec_lines).join('')
       write_glfm_spec_txt(glfm_spec_txt_string)
 
+      # create `output_example_snapshots/snapshot_spec.md`
+      ghfm_spec_example_lines = extract_ghfm_spec_example_lines(ghfm_spec_lines)
+      official_spec_example_lines =
+        extract_glfm_spec_example_lines(official_spec_lines, GLFM_OFFICIAL_SPECIFICATION_MD_PATH)
+      internal_extension_lines = readlines_from_path!(GLFM_INTERNAL_EXTENSIONS_MD_PATH)
+      internal_extension_example_lines =
+        extract_glfm_spec_example_lines(internal_extension_lines, GLFM_INTERNAL_EXTENSIONS_MD_PATH)
+      snapshot_spec_md_string = (
+        glfm_spec_txt_header_lines +
+          ghfm_spec_example_lines +
+          official_spec_example_lines +
+          ["\n"] +
+          internal_extension_example_lines
+      ).join('')
+      write_snapshot_spec_md(snapshot_spec_md_string)
+
       if skip_spec_html_generation
-        output("Skipping GLFM spec.html generation...")
+        output("Skipping GLFM spec.html and snapshot_spec.html generation...")
         return
       end
 
-      glfm_spec_html_string = generate_glfm_spec_html(glfm_spec_txt_string)
-      write_glfm_spec_html(glfm_spec_html_string)
+      # create `output_spec/spec.html` and `output_snapshot_examples/snapshot_spec.html`
+      spec_html_string, snapshot_spec_html_string =
+        generate_spec_html_files(glfm_spec_txt_string, snapshot_spec_md_string)
+      write_spec_html(spec_html_string)
+      write_snapshot_spec_html(snapshot_spec_html_string)
     end
 
     private
@@ -68,9 +93,7 @@ module Glfm
       ghfm_spec_txt_uri_parsed = URI.parse(GHFM_SPEC_TXT_URI)
       ghfm_spec_txt_uri_io = ghfm_spec_txt_uri_parsed.open
 
-      # Read IO stream into an array of lines for easy processing later
-      ghfm_spec_lines = ghfm_spec_txt_uri_io.readlines
-      raise "Unable to read lines from #{GHFM_SPEC_TXT_URI}" if ghfm_spec_lines.empty?
+      ghfm_spec_lines = readlines_from_io!(ghfm_spec_txt_uri_io, GHFM_SPEC_TXT_URI)
 
       # Make sure the GHFM spec version has not changed
       validate_expected_spec_version!(ghfm_spec_lines[2])
@@ -95,59 +118,42 @@ module Glfm
           "Expected 'version: #{GHFM_SPEC_VERSION}', got '#{version_line}'"
     end
 
-    def build_glfm_spec_txt(ghfm_spec_txt_lines)
-      glfm_spec_txt_lines = ghfm_spec_txt_lines.dup
-      replace_header(glfm_spec_txt_lines)
-      replace_intro_section(glfm_spec_txt_lines)
-      insert_examples(glfm_spec_txt_lines)
-      glfm_spec_txt_lines.join('')
-    end
-
-    def replace_header(spec_txt_lines)
-      spec_txt_lines[0, spec_txt_lines.index("...\n") + 1] = GLFM_SPEC_TXT_HEADER
-    end
-
-    def replace_intro_section(spec_txt_lines)
-      glfm_intro_md_lines = File.open(GLFM_INTRO_MD_PATH).readlines
-      raise "Unable to read lines from #{GLFM_INTRO_MD_PATH}" if glfm_intro_md_lines.empty?
-
-      ghfm_intro_header_begin_index = spec_txt_lines.index do |line|
-        line =~ INTRODUCTION_HEADER_LINE_TEXT
+    def extract_ghfm_spec_example_lines(spec_lines)
+      # In the GHFM spec.txt format, all we have to identify the headers containing examples
+      # is the presence of a single initial H1 named "Introduction" before the first
+      # header containing examples, and the <!-- END TESTS --> comment after the last header
+      # containing examples.
+      path = GHFM_SPEC_MD_PATH
+      first_examples_header_index = spec_lines.index do |line|
+        line.start_with?('# ') && !line.start_with?(INTRODUCTION_HEADER_LINE_TEXT)
       end
-      raise "Unable to locate introduction header line in #{GHFM_SPEC_MD_PATH}" if ghfm_intro_header_begin_index.nil?
+      raise "Unable to find first examples header in #{path}" unless first_examples_header_index
 
-      # Find the index of the next header after the introduction header, starting from the index
-      # of the introduction header this is the length of the intro section
-      ghfm_intro_section_length = spec_txt_lines[ghfm_intro_header_begin_index + 1..].index do |line|
-        line.start_with?('# ')
+      end_tests_comment_index = spec_lines.index do |line|
+        line.start_with?(END_TESTS_COMMENT_LINE_TEXT)
       end
+      raise "Unable to locate 'END TESTS' comment line in #{path}" if end_tests_comment_index.nil?
 
-      # Replace the intro section with the GitLab flavored Markdown intro section
-      spec_txt_lines[ghfm_intro_header_begin_index, ghfm_intro_section_length] = glfm_intro_md_lines
+      spec_lines[first_examples_header_index..(end_tests_comment_index - 1)]
     end
 
-    def insert_examples(spec_txt_lines)
-      official_spec_lines = File.open(GLFM_OFFICIAL_SPECIFICATION_EXAMPLES_MD_PATH).readlines
-      raise "Unable to read lines from #{GLFM_OFFICIAL_SPECIFICATION_EXAMPLES_MD_PATH}" if official_spec_lines.empty?
-
-      internal_extension_lines = File.open(GLFM_INTERNAL_EXTENSION_EXAMPLES_MD_PATH).readlines
-      raise "Unable to read lines from #{GLFM_INTERNAL_EXTENSION_EXAMPLES_MD_PATH}" if internal_extension_lines.empty?
+    def extract_glfm_spec_example_lines(spec_lines, path)
+      # In the GLFM input markdown files (unlike the GLFM spec.txt format), we have control over
+      # the contents, so we can use explicit <!-- BEGIN TESTS --> and <!-- END TESTS -->
+      # is the presence of a single initial H1 named "Introduction" before the first
+      # header containing examples, and the <!-- END TESTS --> comment after the last header
+      # containing examples.
+      begin_tests_comment_line_index = spec_lines.index do |line|
+        line.start_with?(BEGIN_TESTS_COMMENT_LINE_TEXT)
+      end
+      raise "Unable to locate 'BEGIN TESTS' comment line in #{path}" unless begin_tests_comment_line_index
 
-      ghfm_end_tests_comment_index = spec_txt_lines.index do |line|
-        line =~ END_TESTS_COMMENT_LINE_TEXT
+      end_tests_comment_index = spec_lines.index do |line|
+        line.start_with?(END_TESTS_COMMENT_LINE_TEXT)
       end
-      raise "Unable to locate 'END TESTS' comment line in #{GHFM_SPEC_MD_PATH}" if ghfm_end_tests_comment_index.nil?
-
-      # Insert the GLFM examples before the 'END TESTS' comment line
-      spec_txt_lines[ghfm_end_tests_comment_index - 1] = [
-        "\n",
-        official_spec_lines,
-        "\n",
-        internal_extension_lines,
-        "\n"
-      ].flatten
-
-      spec_txt_lines
+      raise "Unable to locate 'END TESTS' comment line in #{path}" if end_tests_comment_index.nil?
+
+      spec_lines[(begin_tests_comment_line_index + 1)..(end_tests_comment_index - 1)]
     end
 
     def write_glfm_spec_txt(glfm_spec_txt_string)
@@ -156,13 +162,21 @@ module Glfm
       write_file(GLFM_SPEC_TXT_PATH, glfm_spec_txt_string)
     end
 
-    def generate_glfm_spec_html(glfm_spec_txt_string)
-      output("Generating spec.html from spec.txt markdown...")
+    def write_snapshot_spec_md(snapshot_spec_md_string)
+      output("Writing #{ES_SNAPSHOT_SPEC_MD_PATH}...")
+      FileUtils.mkdir_p(Pathname.new(ES_SNAPSHOT_SPEC_MD_PATH).dirname)
+      write_file(ES_SNAPSHOT_SPEC_MD_PATH, snapshot_spec_md_string)
+    end
+
+    def generate_spec_html_files(spec_txt_string, snapshot_spec_md_string)
+      output("Generating spec.html and snapshot_spec.html from spec.txt and snapshot_spec.md markdown...")
 
       input_markdown_yml_string = <<~MARKDOWN
         ---
         spec_txt: |
-        #{glfm_spec_txt_string.gsub(/^/, '  ')}
+        #{spec_txt_string.gsub(/^/, '  ')}
+        snapshot_spec_md: |
+        #{snapshot_spec_md_string.gsub(/^/, '  ')}
       MARKDOWN
 
       # NOTE: We must copy the input YAML file used by the `render_static_html.rb`
@@ -190,14 +204,33 @@ module Glfm
       cmd = %(bin/rspec #{__dir__}/render_static_html.rb)
       run_external_cmd(cmd)
 
-      output("Reading generated spec.html from tempfile #{static_html_tempfile_path}...")
-      YAML.safe_load(File.open(static_html_tempfile_path), symbolize_names: true).fetch(:spec_txt)
+      output("Reading generated html from tempfile #{static_html_tempfile_path}...")
+      rendered_html_hash = YAML.safe_load(File.open(static_html_tempfile_path), symbolize_names: true)
+      [rendered_html_hash.fetch(:spec_txt), rendered_html_hash.fetch(:snapshot_spec_md)]
     end
 
-    def write_glfm_spec_html(glfm_spec_html_string)
+    def write_spec_html(spec_html_string)
       output("Writing #{GLFM_SPEC_TXT_PATH}...")
       FileUtils.mkdir_p(Pathname.new(GLFM_SPEC_HTML_PATH).dirname)
-      write_file(GLFM_SPEC_HTML_PATH, "#{glfm_spec_html_string}\n")
+      write_file(GLFM_SPEC_HTML_PATH, "#{spec_html_string}\n")
+    end
+
+    def write_snapshot_spec_html(snapshot_spec_html_string)
+      output("Writing #{ES_SNAPSHOT_SPEC_HTML_PATH}...")
+      FileUtils.mkdir_p(Pathname.new(ES_SNAPSHOT_SPEC_HTML_PATH).dirname)
+      write_file(ES_SNAPSHOT_SPEC_HTML_PATH, "#{snapshot_spec_html_string}\n")
+    end
+
+    def readlines_from_path!(path)
+      io = File.open(path)
+      readlines_from_io!(io, path)
+    end
+
+    def readlines_from_io!(io, uri_or_path)
+      lines = io.readlines
+      raise "Unable to read lines from #{uri_or_path}" if lines.empty?
+
+      lines
     end
   end
 end
diff --git a/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb b/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb
index 0b824fc589d..a928c78bfc6 100644
--- a/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb
+++ b/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb
@@ -12,7 +12,7 @@ module Glfm
     include Shared
 
     def process
-      verify_cmd = "git status --porcelain #{GLFM_SPEC_OUTPUT_PATH} #{EXAMPLE_SNAPSHOTS_PATH}"
+      verify_cmd = "git status --porcelain #{GLFM_OUTPUT_SPEC_PATH} #{ES_OUTPUT_EXAMPLE_SNAPSHOTS_PATH}"
       verify_cmd_output = run_external_cmd(verify_cmd)
       unless verify_cmd_output.empty?
         msg = "ERROR: Cannot run `#{__FILE__}` because `#{verify_cmd}` shows the following uncommitted changes:\n" \