diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-05-17 03:08:52 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-05-17 03:08:52 +0300 |
commit | 159a7788ca18140da04e24c45ab557da99864789 (patch) | |
tree | 929bc82d0fbdd8fdcd6b522bc1e1f9bcb71db3a8 /scripts/lib | |
parent | cffe2c2c348d86d67298fa6516d49c36d696ab2d (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'scripts/lib')
-rw-r--r-- | scripts/lib/glfm/constants.rb | 13 | ||||
-rw-r--r-- | scripts/lib/glfm/parse_examples.rb | 169 | ||||
-rw-r--r-- | scripts/lib/glfm/render_static_html.rb | 50 | ||||
-rw-r--r-- | scripts/lib/glfm/render_wysiwyg_html_and_json.js | 152 | ||||
-rw-r--r-- | scripts/lib/glfm/shared.rb | 15 | ||||
-rw-r--r-- | scripts/lib/glfm/update_example_snapshots.rb | 245 |
6 files changed, 644 insertions, 0 deletions
diff --git a/scripts/lib/glfm/constants.rb b/scripts/lib/glfm/constants.rb index 70f38e45cf5..e5917fc5cdb 100644 --- a/scripts/lib/glfm/constants.rb +++ b/scripts/lib/glfm/constants.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'pathname' + module Glfm module Constants # Root dir containing all specification files @@ -15,8 +17,16 @@ module Glfm specification_input_glfm_path = specification_path.join('input/gitlab_flavored_markdown') GLFM_INTRO_TXT_PATH = specification_input_glfm_path.join('glfm_intro.txt') GLFM_EXAMPLES_TXT_PATH = specification_input_glfm_path.join('glfm_canonical_examples.txt') + GLFM_EXAMPLE_STATUS_YML_PATH = specification_input_glfm_path.join('glfm_example_status.yml') GLFM_SPEC_TXT_PATH = specification_path.join('output/spec.txt') + # Example Snapshot (ES) files + es_fixtures_path = File.expand_path("../../../spec/fixtures/glfm/example_snapshots", __dir__) + ES_EXAMPLES_INDEX_YML_PATH = File.join(es_fixtures_path, 'examples_index.yml') + ES_MARKDOWN_YML_PATH = File.join(es_fixtures_path, 'markdown.yml') + ES_HTML_YML_PATH = File.join(es_fixtures_path, 'html.yml') + ES_PROSEMIRROR_JSON_YML_PATH = File.join(es_fixtures_path, 'prosemirror_json.yml') + # Other constants used for processing files GLFM_SPEC_TXT_HEADER = <<~GLFM_SPEC_TXT_HEADER --- @@ -26,5 +36,8 @@ module Glfm GLFM_SPEC_TXT_HEADER INTRODUCTION_HEADER_LINE_TEXT = /\A# Introduction\Z/.freeze END_TESTS_COMMENT_LINE_TEXT = /\A<!-- END TESTS -->\Z/.freeze + MARKDOWN_TEMPFILE_BASENAME = %w[MARKDOWN_TEMPFILE_ .yml].freeze + STATIC_HTML_TEMPFILE_BASENAME = %w[STATIC_HTML_TEMPFILE_ .yml].freeze + WYSIWYG_HTML_AND_JSON_TEMPFILE_BASENAME = %w[WYSIWYG_HTML_AND_JSON_TEMPFILE_ .yml].freeze end end diff --git a/scripts/lib/glfm/parse_examples.rb b/scripts/lib/glfm/parse_examples.rb new file mode 100644 index 00000000000..1c6afb800c3 --- /dev/null +++ b/scripts/lib/glfm/parse_examples.rb @@ -0,0 +1,169 @@ +# frozen_string_literal: true + +# This module contains a Ruby port of Python logic from the `get_tests` method of the +# `spec_test.py` script (see copy of original code in a comment at the bottom of this file): +# https://github.com/github/cmark-gfm/blob/5dfedc7/test/spec_tests.py#L82 +# +# The logic and structure is as unchanged as possible from the original Python - no +# cleanup or refactoring was done. +# +# Changes from the original logic were made to follow Ruby/GitLab syntax, idioms, and linting rules. +# +# Additional logic was also added to: +# 1. Capture all nested headers, not just the most recent. +# 2. Raise an exception if an unexpected state is encountered. +# +# Comments indicate where changes or additions were made. +module Glfm + module ParseExamples + REGULAR_TEXT = 0 + MARKDOWN_EXAMPLE = 1 + HTML_OUTPUT = 2 + EXAMPLE_BACKTICKS_LENGTH = 32 + EXAMPLE_BACKTICKS_STRING = '`' * EXAMPLE_BACKTICKS_LENGTH + + # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity + def parse_examples(spec_txt_lines) + line_number = 0 + start_line = 0 + example_number = 0 + markdown_lines = [] + html_lines = [] + state = REGULAR_TEXT # 0 regular text, 1 markdown example, 2 html output + extensions = [] + headertext = '' # most recent header text + headers = [] # all nested headers since last H2 - new logic compared to original Python code + tests = [] + + h1_regex = /\A# / # new logic compared to original Python code + h2_regex = /\A## / # new logic compared to original Python code + header_regex = /\A#+ / # Added beginning of line anchor to original Python code + + spec_txt_lines.each do |line| + line_number += 1 + stripped_line = line.strip + if stripped_line.start_with?("#{EXAMPLE_BACKTICKS_STRING} example") + # If beginning line of an example block... + state = MARKDOWN_EXAMPLE + extensions = stripped_line[(EXAMPLE_BACKTICKS_LENGTH + " example".length)..].split + elsif stripped_line == EXAMPLE_BACKTICKS_STRING + # Else if end line of an example block... + state = REGULAR_TEXT + example_number += 1 + end_line = line_number + unless extensions.include?('disabled') + tests << + { + markdown: markdown_lines.join.tr('→', "\t"), + html: html_lines.join.tr('→', "\t"), + example: example_number, + start_line: start_line, + end_line: end_line, + section: headertext, + extensions: extensions, + headers: headers.dup # new logic compared to original Python code + } + start_line = 0 + markdown_lines = [] + html_lines = [] + end + elsif stripped_line == "." + # Else if the example divider line... + state = HTML_OUTPUT + # Else if we are currently in a markdown example... + elsif state == MARKDOWN_EXAMPLE + start_line = line_number - 1 if start_line == 0 + + markdown_lines.append(line) + elsif state == HTML_OUTPUT + # Else if we are currently in the html output... + html_lines.append(line) + elsif state == REGULAR_TEXT && line =~ header_regex + # Else if we are in regular text and it is a header line + # NOTE: This assumes examples are only nested up to 2 levels deep (H2) + + # Extract the header text from the line + headertext = line.gsub(header_regex, '').strip + + # reset the headers array if we found a new H1 + headers = [] if line =~ h1_regex # new logic compared to original Python code + + # pop the last entry from the headers array if we found a new H2 + headers.pop if headers.length == 2 && line =~ h2_regex # new logic compared to original Python code + + # push the new header text to the headers array + headers << headertext # New logic compared to original Python code + else + # Else if we are in regular text... + + # This else block is new logic compared to original Python code + + # Sanity check for state machine + raise 'Unexpected state encountered when parsing examples' unless state == REGULAR_TEXT + + # no-op - skips any other non-header regular text lines + end + end + # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity + + tests + end + end +end + +# Original `get_tests` method from spec_test.py: +# rubocop:disable Style/BlockComments +# rubocop:disable Style/AsciiComments +=begin +def get_tests(specfile): + line_number = 0 + start_line = 0 + end_line = 0 + example_number = 0 + markdown_lines = [] + html_lines = [] + state = 0 # 0 regular text, 1 markdown example, 2 html output + extensions = [] + headertext = '' + tests = [] + + header_re = re.compile('#+ ') + + with open(specfile, 'r', encoding='utf-8', newline='\n') as specf: + for line in specf: + line_number = line_number + 1 + l = line.strip() + if l.startswith("`" * 32 + " example"): + state = 1 + extensions = l[32 + len(" example"):].split() + elif l == "`" * 32: + state = 0 + example_number = example_number + 1 + end_line = line_number + if 'disabled' not in extensions: + tests.append({ + "markdown":''.join(markdown_lines).replace('→',"\t"), + "html":''.join(html_lines).replace('→',"\t"), + "example": example_number, + "start_line": start_line, + "end_line": end_line, + "section": headertext, + "extensions": extensions}) + start_line = 0 + markdown_lines = [] + html_lines = [] + elif l == ".": + state = 2 + elif state == 1: + if start_line == 0: + start_line = line_number - 1 + markdown_lines.append(line) + elif state == 2: + html_lines.append(line) + elif state == 0 and re.match(header_re, line): + headertext = header_re.sub('', line).strip() + return tests + +=end +# rubocop:enable Style/BlockComments +# rubocop:enable Style/AsciiComments diff --git a/scripts/lib/glfm/render_static_html.rb b/scripts/lib/glfm/render_static_html.rb new file mode 100644 index 00000000000..d533e586fe1 --- /dev/null +++ b/scripts/lib/glfm/render_static_html.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require_relative 'constants' +require_relative 'shared' + +# Purpose: +# - Reads a set of markdown examples from a hash which has been serialized to disk +# - Converts each example to static HTML using the `markdown` helper +# - Writes the HTML for each example to a hash which is serialized to disk +# +# It should be invoked via `rails runner` from the Rails root directory. +# It is intended to be invoked from the `update_example_snapshots.rb` script class. +module Glfm + class RenderStaticHtml + include Constants + include Shared + + def process + markdown_yml_path = ARGV[0] + markdown_hash = YAML.load_file(markdown_yml_path) + + # NOTE: We COULD parallelize this loop like the Javascript WYSIWYG example generation does, + # but it wouldn't save much time. Most of the time is spent loading the Rails environment + # via `rails runner`. In initial testing, this loop only took ~7 seconds while the entire + # script took ~20 seconds. Unfortunately, there's no easy way to execute + # `ApplicationController.helpers.markdown` without using `rails runner` + static_html_hash = markdown_hash.transform_values do |markdown| + ApplicationController.helpers.markdown(markdown) + end + + static_html_tempfile_path = Dir::Tmpname.create(STATIC_HTML_TEMPFILE_BASENAME) do |path| + tmpfile = File.open(path, 'w') + YAML.dump(static_html_hash, tmpfile) + tmpfile.close + end + + # Write the path to the output file to stdout + print static_html_tempfile_path + end + end +end + +# current_user must be in global scope for `markdown` helper to work. Currently it's not supported +# to pass it in the context. +def current_user + # TODO: This will likely need to be a more realistic user object for some of the GLFM examples + User.new +end + +Glfm::RenderStaticHtml.new.process diff --git a/scripts/lib/glfm/render_wysiwyg_html_and_json.js b/scripts/lib/glfm/render_wysiwyg_html_and_json.js new file mode 100644 index 00000000000..58b440d7ab2 --- /dev/null +++ b/scripts/lib/glfm/render_wysiwyg_html_and_json.js @@ -0,0 +1,152 @@ +import fs from 'fs'; +import { DOMSerializer } from 'prosemirror-model'; +import jsYaml from 'js-yaml'; +// TODO: DRY up duplication with spec/frontend/content_editor/services/markdown_serializer_spec.js +// See https://gitlab.com/groups/gitlab-org/-/epics/7719#plan +import Blockquote from '~/content_editor/extensions/blockquote'; +import Bold from '~/content_editor/extensions/bold'; +import BulletList from '~/content_editor/extensions/bullet_list'; +import Code from '~/content_editor/extensions/code'; +import CodeBlockHighlight from '~/content_editor/extensions/code_block_highlight'; +import DescriptionItem from '~/content_editor/extensions/description_item'; +import DescriptionList from '~/content_editor/extensions/description_list'; +import Details from '~/content_editor/extensions/details'; +import DetailsContent from '~/content_editor/extensions/details_content'; +import Division from '~/content_editor/extensions/division'; +import Emoji from '~/content_editor/extensions/emoji'; +import Figure from '~/content_editor/extensions/figure'; +import FigureCaption from '~/content_editor/extensions/figure_caption'; +import FootnoteDefinition from '~/content_editor/extensions/footnote_definition'; +import FootnoteReference from '~/content_editor/extensions/footnote_reference'; +import FootnotesSection from '~/content_editor/extensions/footnotes_section'; +import HardBreak from '~/content_editor/extensions/hard_break'; +import Heading from '~/content_editor/extensions/heading'; +import HorizontalRule from '~/content_editor/extensions/horizontal_rule'; +import Image from '~/content_editor/extensions/image'; +import InlineDiff from '~/content_editor/extensions/inline_diff'; +import Italic from '~/content_editor/extensions/italic'; +import Link from '~/content_editor/extensions/link'; +import ListItem from '~/content_editor/extensions/list_item'; +import OrderedList from '~/content_editor/extensions/ordered_list'; +import Strike from '~/content_editor/extensions/strike'; +import Table from '~/content_editor/extensions/table'; +import TableCell from '~/content_editor/extensions/table_cell'; +import TableHeader from '~/content_editor/extensions/table_header'; +import TableRow from '~/content_editor/extensions/table_row'; +import TaskItem from '~/content_editor/extensions/task_item'; +import TaskList from '~/content_editor/extensions/task_list'; +import createMarkdownDeserializer from '~/content_editor/services/remark_markdown_deserializer'; +import { createTestEditor } from 'jest/content_editor/test_utils'; +import { setTestTimeout } from 'jest/__helpers__/timeout'; + +const tiptapEditor = createTestEditor({ + extensions: [ + Blockquote, + Bold, + BulletList, + Code, + CodeBlockHighlight, + DescriptionItem, + DescriptionList, + Details, + DetailsContent, + Division, + Emoji, + FootnoteDefinition, + FootnoteReference, + FootnotesSection, + Figure, + FigureCaption, + HardBreak, + Heading, + HorizontalRule, + Image, + InlineDiff, + Italic, + Link, + ListItem, + OrderedList, + Strike, + Table, + TableCell, + TableHeader, + TableRow, + TaskItem, + TaskList, + ], +}); + +async function renderMarkdownToHTMLAndJSON(markdown, schema, deserializer) { + let prosemirrorDocument; + try { + const { document } = await deserializer.deserialize({ schema, content: markdown }); + prosemirrorDocument = document; + } catch (e) { + const errorMsg = `Error - check implementation:\n${e.message}`; + return { + html: errorMsg, + json: errorMsg, + }; + } + + const documentFragment = DOMSerializer.fromSchema(schema).serializeFragment( + prosemirrorDocument.content, + ); + const htmlString = documentFragment.firstChild.outerHTML; + + const json = prosemirrorDocument.toJSON(); + const jsonString = JSON.stringify(json, null, 2); + return { html: htmlString, json: jsonString }; +} + +function renderHtmlAndJsonForAllExamples(markdownExamples) { + const { schema } = tiptapEditor; + const deserializer = createMarkdownDeserializer(); + const exampleNames = Object.keys(markdownExamples); + + return exampleNames.reduce(async (promisedExamples, exampleName) => { + const markdown = markdownExamples[exampleName]; + const htmlAndJson = await renderMarkdownToHTMLAndJSON(markdown, schema, deserializer); + const examples = await promisedExamples; + examples[exampleName] = htmlAndJson; + return examples; + }, Promise.resolve({})); +} + +/* eslint-disable no-undef */ +jest.mock('~/emoji'); + +// The purpose of this file is to deserialize markdown examples +// to WYSIWYG HTML and to prosemirror documents in JSON form, using +// the logic implemented as part of the Content Editor. +// +// It reads an input YAML file containing all the markdown examples, +// and outputs a YAML files containing the rendered HTML and JSON +// corresponding each markdown example. +// +// The input and output file paths are provides as command line arguments. +// +// Although it is implemented as a Jest test, it is not a unit test. We use +// Jest because that is the simplest environment in which to execute the +// relevant Content Editor logic. +// +// +// This script should be invoked via jest with the a command similar to the following: +// yarn jest --testMatch '**/render_wysiwyg_html_and_json.js' ./scripts/lib/glfm/render_wysiwyg_html_and_json.js +it('serializes html to prosemirror json', async () => { + setTestTimeout(20000); + + const inputMarkdownTempfilePath = process.env.INPUT_MARKDOWN_YML_PATH; + expect(inputMarkdownTempfilePath).not.toBeUndefined(); + const outputWysiwygHtmlAndJsonTempfilePath = + process.env.OUTPUT_WYSIWYG_HTML_AND_JSON_TEMPFILE_PATH; + expect(outputWysiwygHtmlAndJsonTempfilePath).not.toBeUndefined(); + /* eslint-enable no-undef */ + + const markdownExamples = jsYaml.safeLoad(fs.readFileSync(inputMarkdownTempfilePath), {}); + + const htmlAndJsonExamples = await renderHtmlAndJsonForAllExamples(markdownExamples); + + const htmlAndJsonExamplesYamlString = jsYaml.safeDump(htmlAndJsonExamples, {}); + fs.writeFileSync(outputWysiwygHtmlAndJsonTempfilePath, htmlAndJsonExamplesYamlString); +}); diff --git a/scripts/lib/glfm/shared.rb b/scripts/lib/glfm/shared.rb index 92ca067f9f8..f11c66eb8be 100644 --- a/scripts/lib/glfm/shared.rb +++ b/scripts/lib/glfm/shared.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true require 'fileutils' +require 'open3' module Glfm module Shared @@ -24,5 +25,19 @@ module Glfm def output(string) puts string end + + def run_external_cmd(cmd) + # noinspection RubyMismatchedArgumentType + rails_root = File.expand_path('../../../', __dir__) + + # See https://stackoverflow.com/a/20001569/25192 + stdout_and_stderr_str, status = Open3.capture2e(cmd, chdir: rails_root) + + return stdout_and_stderr_str if status.success? + + warn("Error running command `#{cmd}`\n") + warn(stdout_and_stderr_str) + raise + end end end diff --git a/scripts/lib/glfm/update_example_snapshots.rb b/scripts/lib/glfm/update_example_snapshots.rb new file mode 100644 index 00000000000..42ba305565d --- /dev/null +++ b/scripts/lib/glfm/update_example_snapshots.rb @@ -0,0 +1,245 @@ +# frozen_string_literal: true +require 'fileutils' +require 'open-uri' +require 'yaml' +require 'psych' +require 'tempfile' +require 'open3' +require_relative 'constants' +require_relative 'shared' +require_relative 'parse_examples' + +# IMPORTANT NOTE: See https://docs.gitlab.com/ee/development/gitlab_flavored_markdown/specification_guide/ +# for details on the implementation and usage of this script. This developers guide +# contains diagrams and documentation of this script, +# including explanations and examples of all files it reads and writes. +module Glfm + class UpdateExampleSnapshots + include Constants + include Shared + include ParseExamples + + # skip_static_and_wysiwyg can be used to skip the backend/frontend html and prosemirror JSON + # generation which depends on external calls. This allows for faster processing in unit tests + # which do not require it. + def process(skip_static_and_wysiwyg: false) + output('Updating example snapshots...') + + output('(Skipping static HTML generation)') if skip_static_and_wysiwyg + + glfm_spec_txt_lines, _glfm_examples_status_lines = read_input_files + + # Parse all the examples from `spec.txt`, using a Ruby port of the Python `get_tests` + # function the from original CommonMark/GFM `spec_test.py` script. + all_examples = parse_examples(glfm_spec_txt_lines) + + add_example_names(all_examples) + write_snapshot_example_files(all_examples, skip_static_and_wysiwyg: skip_static_and_wysiwyg) + end + + private + + def read_input_files + [ + GLFM_SPEC_TXT_PATH, + GLFM_EXAMPLE_STATUS_YML_PATH + ].map do |path| + output("Reading #{path}...") + File.open(path).readlines + end + end + + def add_example_names(all_examples) + # NOTE: This method assumes: + # 1. Section 2 is the first section which contains examples + # 2. Examples are always nested exactly than 2 levels deep in an H2 + # 3. We assume that the Appendix doesn't ever contain any examples, so it doesn't show up + # in the H1 header count. So, even though due to the concatenation it appears before the + # GitLab examples sections, it doesn't result in their header counts being off by +1. + + h1_count = 1 # examples start in H1 section 2; section 1 is the overview with no examples. + h2_count = 0 + previous_h1 = '' + previous_h2 = '' + index_within_h2 = 0 + all_examples.each do |example| + headers = example[:headers] + + if headers[0] != previous_h1 + h1_count += 1 + h2_count = 0 + previous_h1 = headers[0] + end + + if headers[1] != previous_h2 + h2_count += 1 + previous_h2 = headers[1] + index_within_h2 = 0 + end + + index_within_h2 += 1 + + # convert headers array to lowercase string with underscores, and double underscores between headers + formatted_headers_text = headers.join('__').tr('-', '_').tr(' ', '_').downcase + + hierarchy_level = "#{h1_count.to_s.rjust(2, '0')}_#{h2_count.to_s.rjust(2, '0')}" + position_within_section = index_within_h2.to_s.rjust(2, '0') + name = "#{hierarchy_level}__#{formatted_headers_text}__#{position_within_section}" + converted_name = name.tr('(', '').tr(')', '') # remove any parens from the name + example[:name] = converted_name + end + end + + def write_snapshot_example_files(all_examples, skip_static_and_wysiwyg:) + write_examples_index_yml(all_examples) + + write_markdown_yml(all_examples) + + if skip_static_and_wysiwyg + output("Skipping static/WYSIWYG HTML and prosemirror JSON generation...") + return + end + + markdown_yml_tempfile_path = write_markdown_yml_tempfile + static_html_hash = generate_static_html(markdown_yml_tempfile_path) + wysiwyg_html_and_json_hash = generate_wysiwyg_html_and_json(markdown_yml_tempfile_path) + + write_html_yml(all_examples, static_html_hash, wysiwyg_html_and_json_hash) + + write_prosemirror_json_yml(all_examples, wysiwyg_html_and_json_hash) + end + + def write_examples_index_yml(all_examples) + generate_and_write_for_all_examples( + all_examples, ES_EXAMPLES_INDEX_YML_PATH, literal_scalars: false + ) do |example, hash| + hash[example.fetch(:name)] = { + 'spec_txt_example_position' => example.fetch(:example), + 'source_specification' => + if example[:extensions].empty? + 'commonmark' + elsif example[:extensions].include?('gitlab') + 'gitlab' + else + 'github' + end + } + end + end + + def write_markdown_yml(all_examples) + generate_and_write_for_all_examples(all_examples, ES_MARKDOWN_YML_PATH) do |example, hash| + hash[example.fetch(:name)] = example.fetch(:markdown) + end + end + + def write_markdown_yml_tempfile + # NOTE: We must copy the markdown YAML file to a separate temporary file for the + # `render_static_html.rb` script to read it, because the script is run in a + # separate process, and during unit testing we are unable to substitute the mock + # StringIO when reading the input file in the subprocess. + Dir::Tmpname.create(MARKDOWN_TEMPFILE_BASENAME) do |path| + io = File.open(ES_MARKDOWN_YML_PATH) + io.seek(0) # rewind the file. This is necessary when testing with a mock StringIO + contents = io.read + write_file(path, contents) + end + end + + def generate_static_html(markdown_yml_tempfile_path) + output("Generating static HTML from markdown examples...") + + # NOTE 1: We shell out to perform the conversion of markdown to static HTML via the internal Rails app + # helper method. This allows us to avoid using the Rails API or environment in this script, + # which makes developing and running the unit tests for this script much faster, + # because they can use 'fast_spec_helper' which does not require the entire Rails environment. + + # NOTE 2: We pass the input file path as a command line argument, and receive the output + # tempfile path as a return value. This is simplest in the case where we are invoking Ruby. + cmd = %(rails runner #{__dir__}/render_static_html.rb #{markdown_yml_tempfile_path}) + cmd_output = run_external_cmd(cmd) + # NOTE: Running under a debugger can add extra output, only take the last line + static_html_tempfile_path = cmd_output.split("\n").last + + output("Reading generated static HTML from tempfile #{static_html_tempfile_path}...") + YAML.load_file(static_html_tempfile_path) + end + + def generate_wysiwyg_html_and_json(markdown_yml_tempfile_path) + output("Generating WYSIWYG HTML and prosemirror JSON from markdown examples...") + + # NOTE: Unlike when we invoke a Ruby script, here we pass the input and output file paths + # via environment variables. This is because it's not straightforward/clean to pass command line + # arguments when we are invoking `yarn jest ...` + ENV['INPUT_MARKDOWN_YML_PATH'] = markdown_yml_tempfile_path + + # Dir::Tmpname.create requires a block, but we are using the non-block form to get the path + # via the return value, so we pass an empty block to avoid an error. + wysiwyg_html_and_json_tempfile_path = Dir::Tmpname.create(WYSIWYG_HTML_AND_JSON_TEMPFILE_BASENAME) {} + ENV['OUTPUT_WYSIWYG_HTML_AND_JSON_TEMPFILE_PATH'] = wysiwyg_html_and_json_tempfile_path + + cmd = %(yarn jest --testMatch '**/render_wysiwyg_html_and_json.js' #{__dir__}/render_wysiwyg_html_and_json.js) + run_external_cmd(cmd) + + output("Reading generated WYSIWYG HTML and prosemirror JSON from tempfile " \ + "#{wysiwyg_html_and_json_tempfile_path}...") + YAML.load_file(wysiwyg_html_and_json_tempfile_path) + end + + def write_html_yml(all_examples, static_html_hash, wysiwyg_html_and_json_hash) + generate_and_write_for_all_examples(all_examples, ES_HTML_YML_PATH) do |example, hash| + hash[example.fetch(:name)] = { + 'canonical' => example.fetch(:html), + 'static' => static_html_hash.fetch(example.fetch(:name)), + 'wysiwyg' => wysiwyg_html_and_json_hash.fetch(example.fetch(:name)).fetch('html') + } + end + end + + def write_prosemirror_json_yml(all_examples, wysiwyg_html_and_json_hash) + generate_and_write_for_all_examples(all_examples, ES_PROSEMIRROR_JSON_YML_PATH) do |example, hash| + hash[example.fetch(:name)] = wysiwyg_html_and_json_hash.fetch(example.fetch(:name)).fetch('json') + end + end + + def generate_and_write_for_all_examples(all_examples, output_file_path, literal_scalars: true, &generator_block) + output("Writing #{output_file_path}...") + generated_examples_hash = all_examples.each_with_object({}, &generator_block) + + yaml_string = dump_yaml_with_formatting(generated_examples_hash, literal_scalars: literal_scalars) + write_file(output_file_path, yaml_string) + end + + # Construct an AST so we can control YAML formatting for + # YAML block scalar literals and key quoting. + # + # Note that when Psych dumps the markdown to YAML, it will + # automatically use the default "clip" behavior of the Block Chomping Indicator (`|`) + # https://yaml.org/spec/1.2.2/#8112-block-chomping-indicator, + # when the markdown strings contain a trailing newline. The type of + # Block Chomping Indicator is automatically determined, you cannot specify it + # manually. + def dump_yaml_with_formatting(hash, literal_scalars:) + visitor = Psych::Visitors::YAMLTree.create + visitor << hash + ast = visitor.tree + + # Force all scalars to have literal formatting (using Block Chomping Indicator instead of quotes) + if literal_scalars + ast.grep(Psych::Nodes::Scalar).each do |node| + node.style = Psych::Nodes::Scalar::LITERAL + end + end + + # Do not quote the keys + ast.grep(Psych::Nodes::Mapping).each do |node| + node.children.each_slice(2) do |k, _| + k.quoted = false + k.style = Psych::Nodes::Scalar::ANY + end + end + + ast.to_yaml + end + end +end |