Add latest changes from gitlab-org/gitlab@master

6 files changed, 644 insertions, 0 deletions

diff --git a/scripts/lib/glfm/constants.rb b/scripts/lib/glfm/constants.rb
index 70f38e45cf5..e5917fc5cdb 100644
--- a/scripts/lib/glfm/constants.rb
+++ b/scripts/lib/glfm/constants.rb
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'pathname'
+
 module Glfm
   module Constants
     # Root dir containing all specification files
@@ -15,8 +17,16 @@ module Glfm
     specification_input_glfm_path = specification_path.join('input/gitlab_flavored_markdown')
     GLFM_INTRO_TXT_PATH = specification_input_glfm_path.join('glfm_intro.txt')
     GLFM_EXAMPLES_TXT_PATH = specification_input_glfm_path.join('glfm_canonical_examples.txt')
+    GLFM_EXAMPLE_STATUS_YML_PATH = specification_input_glfm_path.join('glfm_example_status.yml')
     GLFM_SPEC_TXT_PATH = specification_path.join('output/spec.txt')
 
+    # Example Snapshot (ES) files
+    es_fixtures_path = File.expand_path("../../../spec/fixtures/glfm/example_snapshots", __dir__)
+    ES_EXAMPLES_INDEX_YML_PATH = File.join(es_fixtures_path, 'examples_index.yml')
+    ES_MARKDOWN_YML_PATH = File.join(es_fixtures_path, 'markdown.yml')
+    ES_HTML_YML_PATH = File.join(es_fixtures_path, 'html.yml')
+    ES_PROSEMIRROR_JSON_YML_PATH = File.join(es_fixtures_path, 'prosemirror_json.yml')
+
     # Other constants used for processing files
     GLFM_SPEC_TXT_HEADER = <<~GLFM_SPEC_TXT_HEADER
       ---
@@ -26,5 +36,8 @@ module Glfm
     GLFM_SPEC_TXT_HEADER
     INTRODUCTION_HEADER_LINE_TEXT = /\A# Introduction\Z/.freeze
     END_TESTS_COMMENT_LINE_TEXT = /\A<!-- END TESTS -->\Z/.freeze
+    MARKDOWN_TEMPFILE_BASENAME = %w[MARKDOWN_TEMPFILE_ .yml].freeze
+    STATIC_HTML_TEMPFILE_BASENAME = %w[STATIC_HTML_TEMPFILE_ .yml].freeze
+    WYSIWYG_HTML_AND_JSON_TEMPFILE_BASENAME = %w[WYSIWYG_HTML_AND_JSON_TEMPFILE_ .yml].freeze
   end
 end
diff --git a/scripts/lib/glfm/parse_examples.rb b/scripts/lib/glfm/parse_examples.rb
new file mode 100644
index 00000000000..1c6afb800c3
--- /dev/null
+++ b/scripts/lib/glfm/parse_examples.rb
@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+# This module contains a Ruby port of Python logic from the `get_tests` method of the
+# `spec_test.py` script (see copy of original code in a comment at the bottom of this file):
+# https://github.com/github/cmark-gfm/blob/5dfedc7/test/spec_tests.py#L82
+#
+# The logic and structure is as unchanged as possible from the original Python - no
+# cleanup or refactoring was done.
+#
+# Changes from the original logic were made to follow Ruby/GitLab syntax, idioms, and linting rules.
+#
+# Additional logic was also added to:
+# 1. Capture all nested headers, not just the most recent.
+# 2. Raise an exception if an unexpected state is encountered.
+#
+# Comments indicate where changes or additions were made.
+module Glfm
+  module ParseExamples
+    REGULAR_TEXT = 0
+    MARKDOWN_EXAMPLE = 1
+    HTML_OUTPUT = 2
+    EXAMPLE_BACKTICKS_LENGTH = 32
+    EXAMPLE_BACKTICKS_STRING = '`' * EXAMPLE_BACKTICKS_LENGTH
+
+    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+    def parse_examples(spec_txt_lines)
+      line_number = 0
+      start_line = 0
+      example_number = 0
+      markdown_lines = []
+      html_lines = []
+      state = REGULAR_TEXT # 0 regular text, 1 markdown example, 2 html output
+      extensions = []
+      headertext = '' # most recent header text
+      headers = [] # all nested headers since last H2 - new logic compared to original Python code
+      tests = []
+
+      h1_regex = /\A# / # new logic compared to original Python code
+      h2_regex = /\A## / # new logic compared to original Python code
+      header_regex = /\A#+ / # Added beginning of line anchor to original Python code
+
+      spec_txt_lines.each do |line|
+        line_number += 1
+        stripped_line = line.strip
+        if stripped_line.start_with?("#{EXAMPLE_BACKTICKS_STRING} example")
+          # If beginning line of an example block...
+          state = MARKDOWN_EXAMPLE
+          extensions = stripped_line[(EXAMPLE_BACKTICKS_LENGTH + " example".length)..].split
+        elsif stripped_line == EXAMPLE_BACKTICKS_STRING
+          # Else if end line of an example block...
+          state = REGULAR_TEXT
+          example_number += 1
+          end_line = line_number
+          unless extensions.include?('disabled')
+            tests <<
+              {
+                markdown: markdown_lines.join.tr('→', "\t"),
+                html: html_lines.join.tr('→', "\t"),
+                example: example_number,
+                start_line: start_line,
+                end_line: end_line,
+                section: headertext,
+                extensions: extensions,
+                headers: headers.dup # new logic compared to original Python code
+              }
+            start_line = 0
+            markdown_lines = []
+            html_lines = []
+          end
+        elsif stripped_line == "."
+          # Else if the example divider line...
+          state = HTML_OUTPUT
+          # Else if we are currently in a markdown example...
+        elsif state == MARKDOWN_EXAMPLE
+          start_line = line_number - 1 if start_line == 0
+
+          markdown_lines.append(line)
+        elsif state == HTML_OUTPUT
+          # Else if we are currently in the html output...
+          html_lines.append(line)
+        elsif state == REGULAR_TEXT && line =~ header_regex
+          # Else if we are in regular text and it is a header line
+          # NOTE: This assumes examples are only nested up to 2 levels deep (H2)
+
+          # Extract the header text from the line
+          headertext = line.gsub(header_regex, '').strip
+
+          # reset the headers array if we found a new H1
+          headers = [] if line =~ h1_regex # new logic compared to original Python code
+
+          # pop the last entry from the headers array if we found a new H2
+          headers.pop if headers.length == 2 && line =~ h2_regex # new logic compared to original Python code
+
+          # push the new header text to the headers array
+          headers << headertext # New logic compared to original Python code
+        else
+          # Else if we are in regular text...
+
+          # This else block is new logic compared to original Python code
+
+          # Sanity check for state machine
+          raise 'Unexpected state encountered when parsing examples' unless state == REGULAR_TEXT
+
+          # no-op - skips any other non-header regular text lines
+        end
+      end
+      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
+      tests
+    end
+  end
+end
+
+# Original `get_tests` method from spec_test.py:
+# rubocop:disable Style/BlockComments
+# rubocop:disable Style/AsciiComments
+=begin
+def get_tests(specfile):
+    line_number = 0
+    start_line = 0
+    end_line = 0
+    example_number = 0
+    markdown_lines = []
+    html_lines = []
+    state = 0  # 0 regular text, 1 markdown example, 2 html output
+    extensions = []
+    headertext = ''
+    tests = []
+
+    header_re = re.compile('#+ ')
+
+    with open(specfile, 'r', encoding='utf-8', newline='\n') as specf:
+        for line in specf:
+            line_number = line_number + 1
+            l = line.strip()
+            if l.startswith("`" * 32 + " example"):
+                state = 1
+                extensions = l[32 + len(" example"):].split()
+            elif l == "`" * 32:
+                state = 0
+                example_number = example_number + 1
+                end_line = line_number
+                if 'disabled' not in extensions:
+                    tests.append({
+                        "markdown":''.join(markdown_lines).replace('→',"\t"),
+                        "html":''.join(html_lines).replace('→',"\t"),
+                        "example": example_number,
+                        "start_line": start_line,
+                        "end_line": end_line,
+                        "section": headertext,
+                        "extensions": extensions})
+                start_line = 0
+                markdown_lines = []
+                html_lines = []
+            elif l == ".":
+                state = 2
+            elif state == 1:
+                if start_line == 0:
+                    start_line = line_number - 1
+                markdown_lines.append(line)
+            elif state == 2:
+                html_lines.append(line)
+            elif state == 0 and re.match(header_re, line):
+                headertext = header_re.sub('', line).strip()
+    return tests
+
+=end
+# rubocop:enable Style/BlockComments
+# rubocop:enable Style/AsciiComments
diff --git a/scripts/lib/glfm/render_static_html.rb b/scripts/lib/glfm/render_static_html.rb
new file mode 100644
index 00000000000..d533e586fe1
--- /dev/null
+++ b/scripts/lib/glfm/render_static_html.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative 'constants'
+require_relative 'shared'
+
+# Purpose:
+# - Reads a set of markdown examples from a hash which has been serialized to disk
+# - Converts each example to static HTML using the `markdown` helper
+# - Writes the HTML for each example to a hash which is serialized to disk
+#
+# It should be invoked via `rails runner` from the Rails root directory.
+# It is intended to be invoked from the `update_example_snapshots.rb` script class.
+module Glfm
+  class RenderStaticHtml
+    include Constants
+    include Shared
+
+    def process
+      markdown_yml_path = ARGV[0]
+      markdown_hash = YAML.load_file(markdown_yml_path)
+
+      # NOTE: We COULD parallelize this loop like the Javascript WYSIWYG example generation does,
+      # but it wouldn't save much time. Most of the time is spent loading the Rails environment
+      # via `rails runner`. In initial testing, this loop only took ~7 seconds while the entire
+      # script took ~20 seconds. Unfortunately, there's no easy way to execute
+      # `ApplicationController.helpers.markdown` without using `rails runner`
+      static_html_hash = markdown_hash.transform_values do |markdown|
+        ApplicationController.helpers.markdown(markdown)
+      end
+
+      static_html_tempfile_path = Dir::Tmpname.create(STATIC_HTML_TEMPFILE_BASENAME) do |path|
+        tmpfile = File.open(path, 'w')
+        YAML.dump(static_html_hash, tmpfile)
+        tmpfile.close
+      end
+
+      # Write the path to the output file to stdout
+      print static_html_tempfile_path
+    end
+  end
+end
+
+# current_user must be in global scope for `markdown` helper to work. Currently it's not supported
+# to pass it in the context.
+def current_user
+  # TODO: This will likely need to be a more realistic user object for some of the GLFM examples
+  User.new
+end
+
+Glfm::RenderStaticHtml.new.process
diff --git a/scripts/lib/glfm/render_wysiwyg_html_and_json.js b/scripts/lib/glfm/render_wysiwyg_html_and_json.js
new file mode 100644
index 00000000000..58b440d7ab2
--- /dev/null
+++ b/scripts/lib/glfm/render_wysiwyg_html_and_json.js
@@ -0,0 +1,152 @@
+import fs from 'fs';
+import { DOMSerializer } from 'prosemirror-model';
+import jsYaml from 'js-yaml';
+// TODO: DRY up duplication with spec/frontend/content_editor/services/markdown_serializer_spec.js
+//  See https://gitlab.com/groups/gitlab-org/-/epics/7719#plan
+import Blockquote from '~/content_editor/extensions/blockquote';
+import Bold from '~/content_editor/extensions/bold';
+import BulletList from '~/content_editor/extensions/bullet_list';
+import Code from '~/content_editor/extensions/code';
+import CodeBlockHighlight from '~/content_editor/extensions/code_block_highlight';
+import DescriptionItem from '~/content_editor/extensions/description_item';
+import DescriptionList from '~/content_editor/extensions/description_list';
+import Details from '~/content_editor/extensions/details';
+import DetailsContent from '~/content_editor/extensions/details_content';
+import Division from '~/content_editor/extensions/division';
+import Emoji from '~/content_editor/extensions/emoji';
+import Figure from '~/content_editor/extensions/figure';
+import FigureCaption from '~/content_editor/extensions/figure_caption';
+import FootnoteDefinition from '~/content_editor/extensions/footnote_definition';
+import FootnoteReference from '~/content_editor/extensions/footnote_reference';
+import FootnotesSection from '~/content_editor/extensions/footnotes_section';
+import HardBreak from '~/content_editor/extensions/hard_break';
+import Heading from '~/content_editor/extensions/heading';
+import HorizontalRule from '~/content_editor/extensions/horizontal_rule';
+import Image from '~/content_editor/extensions/image';
+import InlineDiff from '~/content_editor/extensions/inline_diff';
+import Italic from '~/content_editor/extensions/italic';
+import Link from '~/content_editor/extensions/link';
+import ListItem from '~/content_editor/extensions/list_item';
+import OrderedList from '~/content_editor/extensions/ordered_list';
+import Strike from '~/content_editor/extensions/strike';
+import Table from '~/content_editor/extensions/table';
+import TableCell from '~/content_editor/extensions/table_cell';
+import TableHeader from '~/content_editor/extensions/table_header';
+import TableRow from '~/content_editor/extensions/table_row';
+import TaskItem from '~/content_editor/extensions/task_item';
+import TaskList from '~/content_editor/extensions/task_list';
+import createMarkdownDeserializer from '~/content_editor/services/remark_markdown_deserializer';
+import { createTestEditor } from 'jest/content_editor/test_utils';
+import { setTestTimeout } from 'jest/__helpers__/timeout';
+
+const tiptapEditor = createTestEditor({
+  extensions: [
+    Blockquote,
+    Bold,
+    BulletList,
+    Code,
+    CodeBlockHighlight,
+    DescriptionItem,
+    DescriptionList,
+    Details,
+    DetailsContent,
+    Division,
+    Emoji,
+    FootnoteDefinition,
+    FootnoteReference,
+    FootnotesSection,
+    Figure,
+    FigureCaption,
+    HardBreak,
+    Heading,
+    HorizontalRule,
+    Image,
+    InlineDiff,
+    Italic,
+    Link,
+    ListItem,
+    OrderedList,
+    Strike,
+    Table,
+    TableCell,
+    TableHeader,
+    TableRow,
+    TaskItem,
+    TaskList,
+  ],
+});
+
+async function renderMarkdownToHTMLAndJSON(markdown, schema, deserializer) {
+  let prosemirrorDocument;
+  try {
+    const { document } = await deserializer.deserialize({ schema, content: markdown });
+    prosemirrorDocument = document;
+  } catch (e) {
+    const errorMsg = `Error - check implementation:\n${e.message}`;
+    return {
+      html: errorMsg,
+      json: errorMsg,
+    };
+  }
+
+  const documentFragment = DOMSerializer.fromSchema(schema).serializeFragment(
+    prosemirrorDocument.content,
+  );
+  const htmlString = documentFragment.firstChild.outerHTML;
+
+  const json = prosemirrorDocument.toJSON();
+  const jsonString = JSON.stringify(json, null, 2);
+  return { html: htmlString, json: jsonString };
+}
+
+function renderHtmlAndJsonForAllExamples(markdownExamples) {
+  const { schema } = tiptapEditor;
+  const deserializer = createMarkdownDeserializer();
+  const exampleNames = Object.keys(markdownExamples);
+
+  return exampleNames.reduce(async (promisedExamples, exampleName) => {
+    const markdown = markdownExamples[exampleName];
+    const htmlAndJson = await renderMarkdownToHTMLAndJSON(markdown, schema, deserializer);
+    const examples = await promisedExamples;
+    examples[exampleName] = htmlAndJson;
+    return examples;
+  }, Promise.resolve({}));
+}
+
+/* eslint-disable no-undef */
+jest.mock('~/emoji');
+
+// The purpose of this file is to deserialize markdown examples
+// to WYSIWYG HTML and to prosemirror documents in JSON form, using
+// the logic implemented as part of the Content Editor.
+//
+// It reads an input YAML file containing all the markdown examples,
+// and outputs a YAML files containing the rendered HTML and JSON
+// corresponding each markdown example.
+//
+// The input and output file paths are provides as command line arguments.
+//
+// Although it is implemented as a Jest test, it is not a unit test. We use
+// Jest because that is the simplest environment in which to execute the
+// relevant Content Editor logic.
+//
+//
+// This script should be invoked via jest with the a command similar to the following:
+// yarn jest --testMatch '**/render_wysiwyg_html_and_json.js' ./scripts/lib/glfm/render_wysiwyg_html_and_json.js
+it('serializes html to prosemirror json', async () => {
+  setTestTimeout(20000);
+
+  const inputMarkdownTempfilePath = process.env.INPUT_MARKDOWN_YML_PATH;
+  expect(inputMarkdownTempfilePath).not.toBeUndefined();
+  const outputWysiwygHtmlAndJsonTempfilePath =
+    process.env.OUTPUT_WYSIWYG_HTML_AND_JSON_TEMPFILE_PATH;
+  expect(outputWysiwygHtmlAndJsonTempfilePath).not.toBeUndefined();
+  /* eslint-enable no-undef */
+
+  const markdownExamples = jsYaml.safeLoad(fs.readFileSync(inputMarkdownTempfilePath), {});
+
+  const htmlAndJsonExamples = await renderHtmlAndJsonForAllExamples(markdownExamples);
+
+  const htmlAndJsonExamplesYamlString = jsYaml.safeDump(htmlAndJsonExamples, {});
+  fs.writeFileSync(outputWysiwygHtmlAndJsonTempfilePath, htmlAndJsonExamplesYamlString);
+});
diff --git a/scripts/lib/glfm/shared.rb b/scripts/lib/glfm/shared.rb
index 92ca067f9f8..f11c66eb8be 100644
--- a/scripts/lib/glfm/shared.rb
+++ b/scripts/lib/glfm/shared.rb
@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 require 'fileutils'
+require 'open3'
 
 module Glfm
   module Shared
@@ -24,5 +25,19 @@ module Glfm
     def output(string)
       puts string
     end
+
+    def run_external_cmd(cmd)
+      # noinspection RubyMismatchedArgumentType
+      rails_root = File.expand_path('../../../', __dir__)
+
+      # See https://stackoverflow.com/a/20001569/25192
+      stdout_and_stderr_str, status = Open3.capture2e(cmd, chdir: rails_root)
+
+      return stdout_and_stderr_str if status.success?
+
+      warn("Error running command `#{cmd}`\n")
+      warn(stdout_and_stderr_str)
+      raise
+    end
   end
 end
diff --git a/scripts/lib/glfm/update_example_snapshots.rb b/scripts/lib/glfm/update_example_snapshots.rb
new file mode 100644
index 00000000000..42ba305565d
--- /dev/null
+++ b/scripts/lib/glfm/update_example_snapshots.rb
@@ -0,0 +1,245 @@
+# frozen_string_literal: true
+require 'fileutils'
+require 'open-uri'
+require 'yaml'
+require 'psych'
+require 'tempfile'
+require 'open3'
+require_relative 'constants'
+require_relative 'shared'
+require_relative 'parse_examples'
+
+# IMPORTANT NOTE: See https://docs.gitlab.com/ee/development/gitlab_flavored_markdown/specification_guide/
+# for details on the implementation and usage of this script. This developers guide
+# contains diagrams and documentation of this script,
+# including explanations and examples of all files it reads and writes.
+module Glfm
+  class UpdateExampleSnapshots
+    include Constants
+    include Shared
+    include ParseExamples
+
+    # skip_static_and_wysiwyg can be used to skip the backend/frontend html and prosemirror JSON
+    # generation which depends on external calls. This allows for faster processing in unit tests
+    # which do not require it.
+    def process(skip_static_and_wysiwyg: false)
+      output('Updating example snapshots...')
+
+      output('(Skipping static HTML generation)') if skip_static_and_wysiwyg
+
+      glfm_spec_txt_lines, _glfm_examples_status_lines = read_input_files
+
+      # Parse all the examples from `spec.txt`, using a Ruby port of the Python `get_tests`
+      # function the from original CommonMark/GFM `spec_test.py` script.
+      all_examples = parse_examples(glfm_spec_txt_lines)
+
+      add_example_names(all_examples)
+      write_snapshot_example_files(all_examples, skip_static_and_wysiwyg: skip_static_and_wysiwyg)
+    end
+
+    private
+
+    def read_input_files
+      [
+        GLFM_SPEC_TXT_PATH,
+        GLFM_EXAMPLE_STATUS_YML_PATH
+      ].map do |path|
+        output("Reading #{path}...")
+        File.open(path).readlines
+      end
+    end
+
+    def add_example_names(all_examples)
+      # NOTE: This method assumes:
+      # 1. Section 2 is the first section which contains examples
+      # 2. Examples are always nested exactly than 2 levels deep in an H2
+      # 3. We assume that the Appendix doesn't ever contain any examples, so it doesn't show up
+      #    in the H1 header count. So, even though due to the concatenation it appears before the
+      #    GitLab examples sections, it doesn't result in their header counts being off by +1.
+
+      h1_count = 1 # examples start in H1 section 2; section 1 is the overview with no examples.
+      h2_count = 0
+      previous_h1 = ''
+      previous_h2 = ''
+      index_within_h2 = 0
+      all_examples.each do |example|
+        headers = example[:headers]
+
+        if headers[0] != previous_h1
+          h1_count += 1
+          h2_count = 0
+          previous_h1 = headers[0]
+        end
+
+        if headers[1] != previous_h2
+          h2_count += 1
+          previous_h2 = headers[1]
+          index_within_h2 = 0
+        end
+
+        index_within_h2 += 1
+
+        # convert headers array to lowercase string with underscores, and double underscores between headers
+        formatted_headers_text = headers.join('__').tr('-', '_').tr(' ', '_').downcase
+
+        hierarchy_level = "#{h1_count.to_s.rjust(2, '0')}_#{h2_count.to_s.rjust(2, '0')}"
+        position_within_section = index_within_h2.to_s.rjust(2, '0')
+        name = "#{hierarchy_level}__#{formatted_headers_text}__#{position_within_section}"
+        converted_name = name.tr('(', '').tr(')', '') # remove any parens from the name
+        example[:name] = converted_name
+      end
+    end
+
+    def write_snapshot_example_files(all_examples, skip_static_and_wysiwyg:)
+      write_examples_index_yml(all_examples)
+
+      write_markdown_yml(all_examples)
+
+      if skip_static_and_wysiwyg
+        output("Skipping static/WYSIWYG HTML and prosemirror JSON generation...")
+        return
+      end
+
+      markdown_yml_tempfile_path = write_markdown_yml_tempfile
+      static_html_hash = generate_static_html(markdown_yml_tempfile_path)
+      wysiwyg_html_and_json_hash = generate_wysiwyg_html_and_json(markdown_yml_tempfile_path)
+
+      write_html_yml(all_examples, static_html_hash, wysiwyg_html_and_json_hash)
+
+      write_prosemirror_json_yml(all_examples, wysiwyg_html_and_json_hash)
+    end
+
+    def write_examples_index_yml(all_examples)
+      generate_and_write_for_all_examples(
+        all_examples, ES_EXAMPLES_INDEX_YML_PATH, literal_scalars: false
+      ) do |example, hash|
+        hash[example.fetch(:name)] = {
+          'spec_txt_example_position' => example.fetch(:example),
+          'source_specification' =>
+            if example[:extensions].empty?
+              'commonmark'
+            elsif example[:extensions].include?('gitlab')
+              'gitlab'
+            else
+              'github'
+            end
+        }
+      end
+    end
+
+    def write_markdown_yml(all_examples)
+      generate_and_write_for_all_examples(all_examples, ES_MARKDOWN_YML_PATH) do |example, hash|
+        hash[example.fetch(:name)] = example.fetch(:markdown)
+      end
+    end
+
+    def write_markdown_yml_tempfile
+      # NOTE: We must copy the markdown YAML file to a separate temporary file for the
+      # `render_static_html.rb` script to read it, because the script is run in a
+      # separate process, and during unit testing we are unable to substitute the mock
+      # StringIO when reading the input file in the subprocess.
+      Dir::Tmpname.create(MARKDOWN_TEMPFILE_BASENAME) do |path|
+        io = File.open(ES_MARKDOWN_YML_PATH)
+        io.seek(0) # rewind the file. This is necessary when testing with a mock StringIO
+        contents = io.read
+        write_file(path, contents)
+      end
+    end
+
+    def generate_static_html(markdown_yml_tempfile_path)
+      output("Generating static HTML from markdown examples...")
+
+      # NOTE 1: We shell out to perform the conversion of markdown to static HTML via the internal Rails app
+      # helper method. This allows us to avoid using the Rails API or environment in this script,
+      # which makes developing and running the unit tests for this script much faster,
+      # because they can use 'fast_spec_helper' which does not require the entire Rails environment.
+
+      # NOTE 2: We pass the input file path as a command line argument, and receive the output
+      # tempfile path as a return value. This is simplest in the case where we are invoking Ruby.
+      cmd = %(rails runner #{__dir__}/render_static_html.rb #{markdown_yml_tempfile_path})
+      cmd_output = run_external_cmd(cmd)
+      # NOTE: Running under a debugger can add extra output, only take the last line
+      static_html_tempfile_path = cmd_output.split("\n").last
+
+      output("Reading generated static HTML from tempfile #{static_html_tempfile_path}...")
+      YAML.load_file(static_html_tempfile_path)
+    end
+
+    def generate_wysiwyg_html_and_json(markdown_yml_tempfile_path)
+      output("Generating WYSIWYG HTML and prosemirror JSON from markdown examples...")
+
+      # NOTE: Unlike when we invoke a Ruby script, here we pass the input and output file paths
+      # via environment variables. This is because it's not straightforward/clean to pass command line
+      # arguments when we are invoking `yarn jest ...`
+      ENV['INPUT_MARKDOWN_YML_PATH'] = markdown_yml_tempfile_path
+
+      # Dir::Tmpname.create requires a block, but we are using the non-block form to get the path
+      # via the return value, so we pass an empty block to avoid an error.
+      wysiwyg_html_and_json_tempfile_path = Dir::Tmpname.create(WYSIWYG_HTML_AND_JSON_TEMPFILE_BASENAME) {}
+      ENV['OUTPUT_WYSIWYG_HTML_AND_JSON_TEMPFILE_PATH'] = wysiwyg_html_and_json_tempfile_path
+
+      cmd = %(yarn jest --testMatch '**/render_wysiwyg_html_and_json.js' #{__dir__}/render_wysiwyg_html_and_json.js)
+      run_external_cmd(cmd)
+
+      output("Reading generated WYSIWYG HTML and prosemirror JSON from tempfile " \
+        "#{wysiwyg_html_and_json_tempfile_path}...")
+      YAML.load_file(wysiwyg_html_and_json_tempfile_path)
+    end
+
+    def write_html_yml(all_examples, static_html_hash, wysiwyg_html_and_json_hash)
+      generate_and_write_for_all_examples(all_examples, ES_HTML_YML_PATH) do |example, hash|
+        hash[example.fetch(:name)] = {
+          'canonical' => example.fetch(:html),
+          'static' => static_html_hash.fetch(example.fetch(:name)),
+          'wysiwyg' => wysiwyg_html_and_json_hash.fetch(example.fetch(:name)).fetch('html')
+        }
+      end
+    end
+
+    def write_prosemirror_json_yml(all_examples, wysiwyg_html_and_json_hash)
+      generate_and_write_for_all_examples(all_examples, ES_PROSEMIRROR_JSON_YML_PATH) do |example, hash|
+        hash[example.fetch(:name)] = wysiwyg_html_and_json_hash.fetch(example.fetch(:name)).fetch('json')
+      end
+    end
+
+    def generate_and_write_for_all_examples(all_examples, output_file_path, literal_scalars: true, &generator_block)
+      output("Writing #{output_file_path}...")
+      generated_examples_hash = all_examples.each_with_object({}, &generator_block)
+
+      yaml_string = dump_yaml_with_formatting(generated_examples_hash, literal_scalars: literal_scalars)
+      write_file(output_file_path, yaml_string)
+    end
+
+    # Construct an AST so we can control YAML formatting for
+    # YAML block scalar literals and key quoting.
+    #
+    # Note that when Psych dumps the markdown to YAML, it will
+    # automatically use the default "clip" behavior of the Block Chomping Indicator (`|`)
+    # https://yaml.org/spec/1.2.2/#8112-block-chomping-indicator,
+    # when the markdown strings contain a trailing newline. The type of
+    # Block Chomping Indicator is automatically determined, you cannot specify it
+    # manually.
+    def dump_yaml_with_formatting(hash, literal_scalars:)
+      visitor = Psych::Visitors::YAMLTree.create
+      visitor << hash
+      ast = visitor.tree
+
+      # Force all scalars to have literal formatting (using Block Chomping Indicator instead of quotes)
+      if literal_scalars
+        ast.grep(Psych::Nodes::Scalar).each do |node|
+          node.style = Psych::Nodes::Scalar::LITERAL
+        end
+      end
+
+      # Do not quote the keys
+      ast.grep(Psych::Nodes::Mapping).each do |node|
+        node.children.each_slice(2) do |k, _|
+          k.quoted = false
+          k.style = Psych::Nodes::Scalar::ANY
+        end
+      end
+
+      ast.to_yaml
+    end
+  end
+end