diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-10-27 03:10:35 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-10-27 03:10:35 +0300 |
commit | 135dd0646bd1402985d82d99eb1e667a0f695e8b (patch) | |
tree | 7e062800b550e3ab36b8d8f9b1270c563aee6f2d /scripts/lib | |
parent | f825fd1d881ce077ad868a70fd8d7db6a49e4700 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'scripts/lib')
-rw-r--r-- | scripts/lib/glfm/constants.rb | 30 | ||||
-rw-r--r-- | scripts/lib/glfm/update_example_snapshots.rb | 12 | ||||
-rw-r--r-- | scripts/lib/glfm/update_specification.rb | 151 | ||||
-rw-r--r-- | scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb | 2 |
4 files changed, 115 insertions, 80 deletions
diff --git a/scripts/lib/glfm/constants.rb b/scripts/lib/glfm/constants.rb index d020d2fec5c..c4feef8464e 100644 --- a/scripts/lib/glfm/constants.rb +++ b/scripts/lib/glfm/constants.rb @@ -15,24 +15,25 @@ module Glfm # GitLab Flavored Markdown specification files specification_input_glfm_path = specification_path.join('input/gitlab_flavored_markdown') - GLFM_INTRO_MD_PATH = specification_input_glfm_path.join('glfm_intro.md') - GLFM_OFFICIAL_SPECIFICATION_EXAMPLES_MD_PATH = - specification_input_glfm_path.join('glfm_official_specification_examples.md') - GLFM_INTERNAL_EXTENSION_EXAMPLES_MD_PATH = specification_input_glfm_path.join('glfm_internal_extension_examples.md') + GLFM_OFFICIAL_SPECIFICATION_MD_PATH = + specification_input_glfm_path.join('glfm_official_specification.md') + GLFM_INTERNAL_EXTENSIONS_MD_PATH = specification_input_glfm_path.join('glfm_internal_extensions.md') GLFM_EXAMPLE_STATUS_YML_PATH = specification_input_glfm_path.join('glfm_example_status.yml') GLFM_EXAMPLE_METADATA_YML_PATH = specification_input_glfm_path.join('glfm_example_metadata.yml') GLFM_EXAMPLE_NORMALIZATIONS_YML_PATH = specification_input_glfm_path.join('glfm_example_normalizations.yml') - GLFM_SPEC_OUTPUT_PATH = specification_path.join('output') - GLFM_SPEC_TXT_PATH = GLFM_SPEC_OUTPUT_PATH.join('spec.txt') - GLFM_SPEC_HTML_PATH = GLFM_SPEC_OUTPUT_PATH.join('spec.html') + GLFM_OUTPUT_SPEC_PATH = specification_path.join('output_spec') + GLFM_SPEC_TXT_PATH = GLFM_OUTPUT_SPEC_PATH.join('spec.txt') + GLFM_SPEC_HTML_PATH = GLFM_OUTPUT_SPEC_PATH.join('spec.html') # Example Snapshot (ES) files - EXAMPLE_SNAPSHOTS_PATH = File.expand_path("../../../glfm_specification/example_snapshots", __dir__) - ES_EXAMPLES_INDEX_YML_PATH = File.join(EXAMPLE_SNAPSHOTS_PATH, 'examples_index.yml') - ES_MARKDOWN_YML_PATH = File.join(EXAMPLE_SNAPSHOTS_PATH, 'markdown.yml') - ES_HTML_YML_PATH = File.join(EXAMPLE_SNAPSHOTS_PATH, 'html.yml') - ES_PROSEMIRROR_JSON_YML_PATH = File.join(EXAMPLE_SNAPSHOTS_PATH, 'prosemirror_json.yml') + ES_OUTPUT_EXAMPLE_SNAPSHOTS_PATH = specification_path.join('output_example_snapshots') + ES_SNAPSHOT_SPEC_MD_PATH = ES_OUTPUT_EXAMPLE_SNAPSHOTS_PATH.join('snapshot_spec.md') + ES_SNAPSHOT_SPEC_HTML_PATH = ES_OUTPUT_EXAMPLE_SNAPSHOTS_PATH.join('snapshot_spec.html') + ES_EXAMPLES_INDEX_YML_PATH = ES_OUTPUT_EXAMPLE_SNAPSHOTS_PATH.join('examples_index.yml') + ES_MARKDOWN_YML_PATH = ES_OUTPUT_EXAMPLE_SNAPSHOTS_PATH.join('markdown.yml') + ES_HTML_YML_PATH = ES_OUTPUT_EXAMPLE_SNAPSHOTS_PATH.join('html.yml') + ES_PROSEMIRROR_JSON_YML_PATH = ES_OUTPUT_EXAMPLE_SNAPSHOTS_PATH.join('prosemirror_json.yml') # Other constants used for processing files GLFM_SPEC_TXT_HEADER = <<~MARKDOWN @@ -41,8 +42,9 @@ module Glfm version: alpha ... MARKDOWN - INTRODUCTION_HEADER_LINE_TEXT = /\A# Introduction\Z/.freeze - END_TESTS_COMMENT_LINE_TEXT = /\A<!-- END TESTS -->\Z/.freeze + INTRODUCTION_HEADER_LINE_TEXT = '# Introduction' + BEGIN_TESTS_COMMENT_LINE_TEXT = '<!-- BEGIN TESTS -->' + END_TESTS_COMMENT_LINE_TEXT = '<!-- END TESTS -->' MARKDOWN_TEMPFILE_BASENAME = %w[MARKDOWN_TEMPFILE_ .yml].freeze METADATA_TEMPFILE_BASENAME = %w[METADATA_TEMPFILE_ .yml].freeze STATIC_HTML_TEMPFILE_BASENAME = %w[STATIC_HTML_TEMPFILE_ .yml].freeze diff --git a/scripts/lib/glfm/update_example_snapshots.rb b/scripts/lib/glfm/update_example_snapshots.rb index 9075260e748..8f817d0173e 100644 --- a/scripts/lib/glfm/update_example_snapshots.rb +++ b/scripts/lib/glfm/update_example_snapshots.rb @@ -30,12 +30,12 @@ module Glfm def process(skip_static_and_wysiwyg: false) output('Updating example snapshots...') - output("Reading #{GLFM_SPEC_TXT_PATH}...") - glfm_spec_txt_lines = File.open(GLFM_SPEC_TXT_PATH).readlines + output("Reading #{ES_SNAPSHOT_SPEC_MD_PATH}...") + es_snapshot_spec_md_lines = File.open(ES_SNAPSHOT_SPEC_MD_PATH).readlines - # Parse all the examples from `spec.txt`, using a Ruby port of the Python `get_tests` + # Parse all the examples from `snapshot_spec.md`, using a Ruby port of the Python `get_tests` # function the from original CommonMark/GFM `spec_test.py` script. - all_examples = parse_examples(glfm_spec_txt_lines) + all_examples = parse_examples(es_snapshot_spec_md_lines) add_example_names(all_examples) @@ -55,7 +55,7 @@ module Glfm # in the H1 header count. So, even though due to the concatenation it appears before the # GitLab examples sections, it doesn't result in their header counts being off by +1. # 5. If an example contains the 'disabled' string extension, it is skipped (and will thus - # result in a skip in the `spec_txt_example_position`). This behavior is taken from the + # result in a skip in the `spec_example_position`). This behavior is taken from the # GFM `spec_test.py` script (but it's NOT in the original CommonMark `spec_test.py`). # 6. If a section contains ONLY disabled examples, the section numbering will still be # incremented to match the rendered HTML specification section numbering. @@ -202,7 +202,7 @@ module Glfm ) do |example, hash| name = example.fetch(:name).to_sym hash[name] = { - 'spec_txt_example_position' => example.fetch(:example), + 'spec_example_position' => example.fetch(:example), 'source_specification' => source_specification_for_extensions(example.fetch(:extensions)) } end diff --git a/scripts/lib/glfm/update_specification.rb b/scripts/lib/glfm/update_specification.rb index c7264547e44..521c0919f64 100644 --- a/scripts/lib/glfm/update_specification.rb +++ b/scripts/lib/glfm/update_specification.rb @@ -23,17 +23,42 @@ module Glfm def process(skip_spec_html_generation: false) output('Updating specification...') + # read and optionally update `input/github_flavored_markdown/ghfm_spec_v_x.yy.md` ghfm_spec_lines = load_ghfm_spec - glfm_spec_txt_string = build_glfm_spec_txt(ghfm_spec_lines) + + # create `output_spec/spec.txt` + glfm_spec_txt_header_lines = GLFM_SPEC_TXT_HEADER.split("\n").map { |line| "#{line}\n" } + official_spec_lines = readlines_from_path!(GLFM_OFFICIAL_SPECIFICATION_MD_PATH) + + glfm_spec_txt_string = (glfm_spec_txt_header_lines + official_spec_lines).join('') write_glfm_spec_txt(glfm_spec_txt_string) + # create `output_example_snapshots/snapshot_spec.md` + ghfm_spec_example_lines = extract_ghfm_spec_example_lines(ghfm_spec_lines) + official_spec_example_lines = + extract_glfm_spec_example_lines(official_spec_lines, GLFM_OFFICIAL_SPECIFICATION_MD_PATH) + internal_extension_lines = readlines_from_path!(GLFM_INTERNAL_EXTENSIONS_MD_PATH) + internal_extension_example_lines = + extract_glfm_spec_example_lines(internal_extension_lines, GLFM_INTERNAL_EXTENSIONS_MD_PATH) + snapshot_spec_md_string = ( + glfm_spec_txt_header_lines + + ghfm_spec_example_lines + + official_spec_example_lines + + ["\n"] + + internal_extension_example_lines + ).join('') + write_snapshot_spec_md(snapshot_spec_md_string) + if skip_spec_html_generation - output("Skipping GLFM spec.html generation...") + output("Skipping GLFM spec.html and snapshot_spec.html generation...") return end - glfm_spec_html_string = generate_glfm_spec_html(glfm_spec_txt_string) - write_glfm_spec_html(glfm_spec_html_string) + # create `output_spec/spec.html` and `output_snapshot_examples/snapshot_spec.html` + spec_html_string, snapshot_spec_html_string = + generate_spec_html_files(glfm_spec_txt_string, snapshot_spec_md_string) + write_spec_html(spec_html_string) + write_snapshot_spec_html(snapshot_spec_html_string) end private @@ -68,9 +93,7 @@ module Glfm ghfm_spec_txt_uri_parsed = URI.parse(GHFM_SPEC_TXT_URI) ghfm_spec_txt_uri_io = ghfm_spec_txt_uri_parsed.open - # Read IO stream into an array of lines for easy processing later - ghfm_spec_lines = ghfm_spec_txt_uri_io.readlines - raise "Unable to read lines from #{GHFM_SPEC_TXT_URI}" if ghfm_spec_lines.empty? + ghfm_spec_lines = readlines_from_io!(ghfm_spec_txt_uri_io, GHFM_SPEC_TXT_URI) # Make sure the GHFM spec version has not changed validate_expected_spec_version!(ghfm_spec_lines[2]) @@ -95,59 +118,42 @@ module Glfm "Expected 'version: #{GHFM_SPEC_VERSION}', got '#{version_line}'" end - def build_glfm_spec_txt(ghfm_spec_txt_lines) - glfm_spec_txt_lines = ghfm_spec_txt_lines.dup - replace_header(glfm_spec_txt_lines) - replace_intro_section(glfm_spec_txt_lines) - insert_examples(glfm_spec_txt_lines) - glfm_spec_txt_lines.join('') - end - - def replace_header(spec_txt_lines) - spec_txt_lines[0, spec_txt_lines.index("...\n") + 1] = GLFM_SPEC_TXT_HEADER - end - - def replace_intro_section(spec_txt_lines) - glfm_intro_md_lines = File.open(GLFM_INTRO_MD_PATH).readlines - raise "Unable to read lines from #{GLFM_INTRO_MD_PATH}" if glfm_intro_md_lines.empty? - - ghfm_intro_header_begin_index = spec_txt_lines.index do |line| - line =~ INTRODUCTION_HEADER_LINE_TEXT + def extract_ghfm_spec_example_lines(spec_lines) + # In the GHFM spec.txt format, all we have to identify the headers containing examples + # is the presence of a single initial H1 named "Introduction" before the first + # header containing examples, and the <!-- END TESTS --> comment after the last header + # containing examples. + path = GHFM_SPEC_MD_PATH + first_examples_header_index = spec_lines.index do |line| + line.start_with?('# ') && !line.start_with?(INTRODUCTION_HEADER_LINE_TEXT) end - raise "Unable to locate introduction header line in #{GHFM_SPEC_MD_PATH}" if ghfm_intro_header_begin_index.nil? + raise "Unable to find first examples header in #{path}" unless first_examples_header_index - # Find the index of the next header after the introduction header, starting from the index - # of the introduction header this is the length of the intro section - ghfm_intro_section_length = spec_txt_lines[ghfm_intro_header_begin_index + 1..].index do |line| - line.start_with?('# ') + end_tests_comment_index = spec_lines.index do |line| + line.start_with?(END_TESTS_COMMENT_LINE_TEXT) end + raise "Unable to locate 'END TESTS' comment line in #{path}" if end_tests_comment_index.nil? - # Replace the intro section with the GitLab flavored Markdown intro section - spec_txt_lines[ghfm_intro_header_begin_index, ghfm_intro_section_length] = glfm_intro_md_lines + spec_lines[first_examples_header_index..(end_tests_comment_index - 1)] end - def insert_examples(spec_txt_lines) - official_spec_lines = File.open(GLFM_OFFICIAL_SPECIFICATION_EXAMPLES_MD_PATH).readlines - raise "Unable to read lines from #{GLFM_OFFICIAL_SPECIFICATION_EXAMPLES_MD_PATH}" if official_spec_lines.empty? - - internal_extension_lines = File.open(GLFM_INTERNAL_EXTENSION_EXAMPLES_MD_PATH).readlines - raise "Unable to read lines from #{GLFM_INTERNAL_EXTENSION_EXAMPLES_MD_PATH}" if internal_extension_lines.empty? + def extract_glfm_spec_example_lines(spec_lines, path) + # In the GLFM input markdown files (unlike the GLFM spec.txt format), we have control over + # the contents, so we can use explicit <!-- BEGIN TESTS --> and <!-- END TESTS --> + # is the presence of a single initial H1 named "Introduction" before the first + # header containing examples, and the <!-- END TESTS --> comment after the last header + # containing examples. + begin_tests_comment_line_index = spec_lines.index do |line| + line.start_with?(BEGIN_TESTS_COMMENT_LINE_TEXT) + end + raise "Unable to locate 'BEGIN TESTS' comment line in #{path}" unless begin_tests_comment_line_index - ghfm_end_tests_comment_index = spec_txt_lines.index do |line| - line =~ END_TESTS_COMMENT_LINE_TEXT + end_tests_comment_index = spec_lines.index do |line| + line.start_with?(END_TESTS_COMMENT_LINE_TEXT) end - raise "Unable to locate 'END TESTS' comment line in #{GHFM_SPEC_MD_PATH}" if ghfm_end_tests_comment_index.nil? - - # Insert the GLFM examples before the 'END TESTS' comment line - spec_txt_lines[ghfm_end_tests_comment_index - 1] = [ - "\n", - official_spec_lines, - "\n", - internal_extension_lines, - "\n" - ].flatten - - spec_txt_lines + raise "Unable to locate 'END TESTS' comment line in #{path}" if end_tests_comment_index.nil? + + spec_lines[(begin_tests_comment_line_index + 1)..(end_tests_comment_index - 1)] end def write_glfm_spec_txt(glfm_spec_txt_string) @@ -156,13 +162,21 @@ module Glfm write_file(GLFM_SPEC_TXT_PATH, glfm_spec_txt_string) end - def generate_glfm_spec_html(glfm_spec_txt_string) - output("Generating spec.html from spec.txt markdown...") + def write_snapshot_spec_md(snapshot_spec_md_string) + output("Writing #{ES_SNAPSHOT_SPEC_MD_PATH}...") + FileUtils.mkdir_p(Pathname.new(ES_SNAPSHOT_SPEC_MD_PATH).dirname) + write_file(ES_SNAPSHOT_SPEC_MD_PATH, snapshot_spec_md_string) + end + + def generate_spec_html_files(spec_txt_string, snapshot_spec_md_string) + output("Generating spec.html and snapshot_spec.html from spec.txt and snapshot_spec.md markdown...") input_markdown_yml_string = <<~MARKDOWN --- spec_txt: | - #{glfm_spec_txt_string.gsub(/^/, ' ')} + #{spec_txt_string.gsub(/^/, ' ')} + snapshot_spec_md: | + #{snapshot_spec_md_string.gsub(/^/, ' ')} MARKDOWN # NOTE: We must copy the input YAML file used by the `render_static_html.rb` @@ -190,14 +204,33 @@ module Glfm cmd = %(bin/rspec #{__dir__}/render_static_html.rb) run_external_cmd(cmd) - output("Reading generated spec.html from tempfile #{static_html_tempfile_path}...") - YAML.safe_load(File.open(static_html_tempfile_path), symbolize_names: true).fetch(:spec_txt) + output("Reading generated html from tempfile #{static_html_tempfile_path}...") + rendered_html_hash = YAML.safe_load(File.open(static_html_tempfile_path), symbolize_names: true) + [rendered_html_hash.fetch(:spec_txt), rendered_html_hash.fetch(:snapshot_spec_md)] end - def write_glfm_spec_html(glfm_spec_html_string) + def write_spec_html(spec_html_string) output("Writing #{GLFM_SPEC_TXT_PATH}...") FileUtils.mkdir_p(Pathname.new(GLFM_SPEC_HTML_PATH).dirname) - write_file(GLFM_SPEC_HTML_PATH, "#{glfm_spec_html_string}\n") + write_file(GLFM_SPEC_HTML_PATH, "#{spec_html_string}\n") + end + + def write_snapshot_spec_html(snapshot_spec_html_string) + output("Writing #{ES_SNAPSHOT_SPEC_HTML_PATH}...") + FileUtils.mkdir_p(Pathname.new(ES_SNAPSHOT_SPEC_HTML_PATH).dirname) + write_file(ES_SNAPSHOT_SPEC_HTML_PATH, "#{snapshot_spec_html_string}\n") + end + + def readlines_from_path!(path) + io = File.open(path) + readlines_from_io!(io, path) + end + + def readlines_from_io!(io, uri_or_path) + lines = io.readlines + raise "Unable to read lines from #{uri_or_path}" if lines.empty? + + lines end end end diff --git a/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb b/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb index 0b824fc589d..a928c78bfc6 100644 --- a/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb +++ b/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb @@ -12,7 +12,7 @@ module Glfm include Shared def process - verify_cmd = "git status --porcelain #{GLFM_SPEC_OUTPUT_PATH} #{EXAMPLE_SNAPSHOTS_PATH}" + verify_cmd = "git status --porcelain #{GLFM_OUTPUT_SPEC_PATH} #{ES_OUTPUT_EXAMPLE_SNAPSHOTS_PATH}" verify_cmd_output = run_external_cmd(verify_cmd) unless verify_cmd_output.empty? msg = "ERROR: Cannot run `#{__FILE__}` because `#{verify_cmd}` shows the following uncommitted changes:\n" \ |