1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
|
# frozen_string_literal: true
require 'fileutils'
require 'open-uri'
require 'yaml'
require 'psych'
require 'tempfile'
require 'open3'
require 'active_support/core_ext/enumerable'
require_relative 'constants'
require_relative 'shared'
require_relative 'parse_examples'
# IMPORTANT NOTE: See https://docs.gitlab.com/ee/development/gitlab_flavored_markdown/specification_guide/#update-example-snapshotsrb-script
# for details on the implementation and usage of this script. This developers guide
# contains diagrams and documentation of this script,
# including explanations and examples of all files it reads and writes.
#
# Also note that this script is intentionally written in a pure-functional (not OO) style,
# with no dependencies on Rails or the GitLab libraries. These choices are intended to make
# it faster and easier to test and debug.
module Glfm
class UpdateExampleSnapshots
include Constants
include Shared
include ParseExamples
# skip_static_and_wysiwyg can be used to skip the backend/frontend html and prosemirror JSON
# generation which depends on external calls. This allows for faster processing in unit tests
# which do not require it.
def process(skip_static_and_wysiwyg: false)
output('Updating example snapshots...')
output("Reading #{ES_SNAPSHOT_SPEC_MD_PATH}...")
es_snapshot_spec_md_lines = File.open(ES_SNAPSHOT_SPEC_MD_PATH).readlines
# Parse all the examples from `snapshot_spec.md`, using a Ruby port of the Python `get_tests`
# function the from original CommonMark/GFM `spec_test.py` script.
all_examples = parse_examples(es_snapshot_spec_md_lines)
add_example_names(all_examples)
reject_disabled_examples(all_examples)
write_snapshot_example_files(all_examples, skip_static_and_wysiwyg: skip_static_and_wysiwyg)
end
private
def add_example_names(all_examples)
# NOTE: This method and the parse_examples method assume:
# 1. Section 2 is the first section which contains examples
# 2. Examples are always nested in an H2 or an H3, never directly in an H1
# 3. There may exist headings with no examples (e.g. "Motivation" in the GLFM spec.txt)
# 4. The Appendix doesn't ever contain any examples, so it doesn't show up
# in the H1 header count. So, even though due to the concatenation it appears before the
# GitLab examples sections, it doesn't result in their header counts being off by +1.
# 5. If an example contains the 'disabled' string extension, it is skipped (and will thus
# result in a skip in the `spec_example_position`). This behavior is taken from the
# GFM `spec_test.py` script (but it's NOT in the original CommonMark `spec_test.py`).
# 6. If a section contains ONLY disabled examples, the section numbering will still be
# incremented to match the rendered HTML specification section numbering.
# 7. Every H2 or H3 must contain at least one example, but it is allowed that they are
# all disabled.
h1_count = 1 # examples start in H1 section 2; section 1 is the overview with no examples.
h2_count = 0
h3_count = 0
previous_h1 = ''
previous_h2 = ''
previous_h3 = ''
index_within_current_heading = 0
all_examples.each do |example|
headers = example[:headers]
if headers[0] != previous_h1
h1_count += 1
h2_count = 0
h3_count = 0
previous_h1 = headers[0]
end
if headers[1] != previous_h2
h2_count += 1
h3_count = 0
previous_h2 = headers[1]
index_within_current_heading = 0
end
if headers[2] && headers[2] != previous_h3
h3_count += 1
previous_h3 = headers[2]
index_within_current_heading = 0
end
index_within_current_heading += 1
# convert headers array to lowercase string with underscores, and double underscores between headers
formatted_headers_text = headers.join('__').tr('-', '_').tr(' ', '_').downcase
hierarchy_level =
"#{h1_count.to_s.rjust(2, '0')}_" \
"#{h2_count.to_s.rjust(2, '0')}_" \
"#{h3_count.to_s.rjust(2, '0')}"
position_within_section = index_within_current_heading.to_s.rjust(3, '0')
name = "#{hierarchy_level}__#{formatted_headers_text}__#{position_within_section}"
converted_name = name.tr('(', '').tr(')', '') # remove any parens from the name
example[:name] = converted_name
end
end
def reject_disabled_examples(all_examples)
all_examples.reject! { |example| example[:disabled] }
end
def write_snapshot_example_files(all_examples, skip_static_and_wysiwyg:)
output("Reading #{GLFM_EXAMPLE_STATUS_YML_PATH}...")
glfm_examples_statuses = YAML.safe_load(File.open(GLFM_EXAMPLE_STATUS_YML_PATH), symbolize_names: true) || {}
validate_glfm_example_status_yml(glfm_examples_statuses)
write_examples_index_yml(all_examples)
validate_glfm_config_file_example_names(all_examples)
write_markdown_yml(all_examples)
if skip_static_and_wysiwyg
output("Skipping static/WYSIWYG HTML and prosemirror JSON generation...")
return
end
# NOTE: We pass the INPUT_MARKDOWN_YML_PATH and INPUT_METADATA_YML_PATH via
# environment variables to the static/wysiwyg HTML generation scripts. This is because they
# are implemented as subprocesses which invoke rspec/jest scripts, and rspec/jest do not make
# it straightforward to pass arguments via the command line.
ENV['INPUT_MARKDOWN_YML_PATH'], ENV['INPUT_METADATA_YML_PATH'] = copy_tempfiles_for_subprocesses
static_html_hash = generate_static_html
wysiwyg_html_and_json_hash = generate_wysiwyg_html_and_json
write_html_yml(all_examples, static_html_hash, wysiwyg_html_and_json_hash, glfm_examples_statuses)
write_prosemirror_json_yml(all_examples, wysiwyg_html_and_json_hash, glfm_examples_statuses)
end
def validate_glfm_example_status_yml(glfm_examples_statuses)
glfm_examples_statuses.each do |example_name, statuses|
next unless statuses &&
statuses[:skip_update_example_snapshots] &&
statuses.any? { |key, value| key.to_s.include?('skip_update_example_snapshot_') && !!value }
raise "Error: '#{example_name}' must not have any 'skip_update_example_snapshot_*' values specified " \
"if 'skip_update_example_snapshots' is truthy"
end
end
def validate_glfm_config_file_example_names(all_examples)
valid_example_names = all_examples.pluck(:name).map(&:to_sym) # rubocop:disable CodeReuse/ActiveRecord
# We are re-reading GLFM_EXAMPLE_STATUS_YML_PATH here, but that's OK, it's a small file, and rereading it
# allows us to handle it in the same loop as the other manually-curated config files.
[
GLFM_EXAMPLE_STATUS_YML_PATH,
GLFM_EXAMPLE_METADATA_YML_PATH,
GLFM_EXAMPLE_NORMALIZATIONS_YML_PATH
].each do |path|
output("Reading #{path}...")
io = File.open(path)
config_file_examples = YAML.safe_load(io, symbolize_names: true, aliases: true)
# Skip validation if the config file is empty
next unless config_file_examples
config_file_example_names = config_file_examples.keys
# Validate that all example names exist in the config file refer to an existing example in `examples_index.yml`,
# unless it starts with the special prefix `00_`, which is preserved for usage as YAML anchors.
invalid_name = config_file_example_names.detect do |name|
!name.start_with?('00_') && valid_example_names.exclude?(name)
end
next unless invalid_name
# NOTE: The extra spaces before punctuation in the error message allows for easier copy/pasting of the paths.
err_msg =
<<~TXT
Error in input specification config file #{path} :
Config file entry named #{invalid_name}
does not have a corresponding example entry in
#{ES_EXAMPLES_INDEX_YML_PATH} .
Please delete or rename this config file entry.
If this entry is being used as a YAML anchor, please rename it to start with '00_'.
TXT
raise err_msg
end
end
def write_examples_index_yml(all_examples)
generate_and_write_for_all_examples(
all_examples, ES_EXAMPLES_INDEX_YML_PATH, literal_scalars: false
) do |example, hash|
name = example.fetch(:name).to_sym
hash[name] = {
'spec_example_position' => example.fetch(:example),
'source_specification' => source_specification_for_extensions(example.fetch(:extensions))
}
end
end
def source_specification_for_extensions(extensions)
unprocessed_extensions = extensions.map(&:to_sym)
unprocessed_extensions.delete(:disabled)
source_specification =
if unprocessed_extensions.empty?
'commonmark'
elsif unprocessed_extensions.include?(:gitlab)
unprocessed_extensions.delete(:gitlab)
'gitlab'
else
'github'
end
# We should only be left with at most one extension, which is an optional name for the example
raise "Error: Invalid extension(s) found: #{unprocessed_extensions.join(', ')}" if unprocessed_extensions.size > 1
source_specification
end
def write_markdown_yml(all_examples)
generate_and_write_for_all_examples(all_examples, ES_MARKDOWN_YML_PATH) do |example, hash|
name = example.fetch(:name).to_sym
hash[name] = example.fetch(:markdown)
end
end
def copy_tempfiles_for_subprocesses
# NOTE: We must copy the input YAML files used by the `render_static_html.rb`
# and `render_wysiwyg_html_and_json.js` scripts to a separate temporary file in order for
# the scripts to read them, because the scripts are run in
# separate subprocesses, and during unit testing we are unable to substitute the mock
# StringIO when reading the input files in the subprocess.
{
ES_MARKDOWN_YML_PATH => MARKDOWN_TEMPFILE_BASENAME,
GLFM_EXAMPLE_METADATA_YML_PATH => METADATA_TEMPFILE_BASENAME
}.map do |original_file_path, tempfile_basename|
Dir::Tmpname.create(tempfile_basename) do |path|
io = File.open(original_file_path)
io.seek(0) # rewind the file. This is necessary when testing with a mock StringIO
contents = io.read
write_file(path, contents)
end
end
end
def generate_static_html
output("Generating static HTML from markdown examples...")
# NOTE 1: We shell out to perform the conversion of markdown to static HTML by invoking a
# separate subprocess. This allows us to avoid using the Rails API or environment in this
# script, which makes developing and running the unit tests for this script much faster,
# because they can use 'fast_spec_helper' which does not require the entire Rails environment.
# NOTE 2: We run this as an RSpec process, for the same reasons we run via Jest process below:
# because that's the easiest way to ensure a reliable, fully-configured environment in which
# to execute the markdown-processing logic. Also, in the static/backend case, Rspec
# provides the easiest and most reliable way to generate example data via Factorybot
# creation of stable model records. This ensures consistent snapshot values across
# machines/environments.
# Dir::Tmpname.create requires a block, but we are using the non-block form to get the path
# via the return value, so we pass an empty block to avoid an error.
static_html_tempfile_path = Dir::Tmpname.create(STATIC_HTML_TEMPFILE_BASENAME) {}
ENV['OUTPUT_STATIC_HTML_TEMPFILE_PATH'] = static_html_tempfile_path
cmd = %(bin/rspec #{__dir__}/render_static_html.rb)
run_external_cmd(cmd)
output("Reading generated static HTML from tempfile #{static_html_tempfile_path}...")
YAML.safe_load(File.open(static_html_tempfile_path), symbolize_names: true)
end
def generate_wysiwyg_html_and_json
output("Generating WYSIWYG HTML and prosemirror JSON from markdown examples...")
# Dir::Tmpname.create requires a block, but we are using the non-block form to get the path
# via the return value, so we pass an empty block to avoid an error.
wysiwyg_html_and_json_tempfile_path = Dir::Tmpname.create(WYSIWYG_HTML_AND_JSON_TEMPFILE_BASENAME) {}
ENV['OUTPUT_WYSIWYG_HTML_AND_JSON_TEMPFILE_PATH'] = wysiwyg_html_and_json_tempfile_path
cmd = "yarn jest:scripts #{__dir__}/render_wysiwyg_html_and_json.js"
run_external_cmd(cmd)
output("Reading generated WYSIWYG HTML and prosemirror JSON from tempfile " \
"#{wysiwyg_html_and_json_tempfile_path}...")
YAML.safe_load(File.open(wysiwyg_html_and_json_tempfile_path), symbolize_names: true)
end
def write_html_yml(all_examples, static_html_hash, wysiwyg_html_and_json_hash, glfm_examples_statuses)
generate_and_write_for_all_examples(
all_examples, ES_HTML_YML_PATH, glfm_examples_statuses: glfm_examples_statuses
) do |example, hash, existing_hash|
name = example.fetch(:name).to_sym
example_statuses = glfm_examples_statuses[name] || {}
static = if example_statuses[:skip_update_example_snapshot_html_static]
existing_hash.dig(name, :static)
else
static_html_hash[name]
end
wysiwyg = if example_statuses[:skip_update_example_snapshot_html_wysiwyg]
existing_hash.dig(name, :wysiwyg)
else
wysiwyg_html_and_json_hash.dig(name, :html)
end
hash[name] = {
'canonical' => example.fetch(:html),
'static' => static,
'wysiwyg' => wysiwyg
}.compact # Do not assign nil values
end
end
def write_prosemirror_json_yml(all_examples, wysiwyg_html_and_json_hash, glfm_examples_statuses)
generate_and_write_for_all_examples(
all_examples, ES_PROSEMIRROR_JSON_YML_PATH, glfm_examples_statuses: glfm_examples_statuses
) do |example, hash, existing_hash|
name = example.fetch(:name).to_sym
json = if glfm_examples_statuses.dig(name, :skip_update_example_snapshot_prosemirror_json)
existing_hash[name]
else
wysiwyg_html_and_json_hash.dig(name, :json)
end
# Do not assign nil values
hash[name] = json if json
end
end
def generate_and_write_for_all_examples(
all_examples, output_file_path, glfm_examples_statuses: {}, literal_scalars: true
)
preserve_existing = !glfm_examples_statuses.empty?
output("#{preserve_existing ? 'Creating/Updating' : 'Creating/Overwriting'} #{output_file_path}...")
existing_hash = preserve_existing ? YAML.safe_load(File.open(output_file_path), symbolize_names: true) : {}
output_hash = all_examples.each_with_object({}) do |example, hash|
name = example.fetch(:name).to_sym
if (reason = glfm_examples_statuses.dig(name, :skip_update_example_snapshots))
# Output the reason for skipping the example, but only once, not multiple times for each file
output("Skipping '#{name}'. Reason: #{reason}") unless glfm_examples_statuses.dig(name, :already_printed)
# We just store the `:already_printed` flag in the hash entry itself. Then we
# don't need an instance variable to keep the state, and this can remain a pure function ;)
glfm_examples_statuses[name][:already_printed] = true
# Copy over the existing example only if it exists and preserve_existing is true, otherwise omit this example
hash[name] = existing_hash[name] if existing_hash[name]
next
end
yield(example, hash, existing_hash)
end
yaml_string = dump_yaml_with_formatting(output_hash, literal_scalars: literal_scalars)
write_file(output_file_path, yaml_string)
end
end
end
|