diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-08-30 21:09:50 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-08-30 21:09:50 +0300 |
commit | 6619ed911ffab93b90756bf392d2925fdc0c1ee2 (patch) | |
tree | 79d122438dd8e1cb672f8bb52b9d573bd49bdd77 /scripts/lib | |
parent | 4b1fc3dc32e768499d81ed64ea7ed497c1785c48 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'scripts/lib')
-rw-r--r-- | scripts/lib/glfm/render_static_html.rb | 35 | ||||
-rw-r--r-- | scripts/lib/glfm/shared.rb | 34 | ||||
-rw-r--r-- | scripts/lib/glfm/update_example_snapshots.rb | 78 |
3 files changed, 77 insertions, 70 deletions
diff --git a/scripts/lib/glfm/render_static_html.rb b/scripts/lib/glfm/render_static_html.rb index 1ab4ebdfd5f..25abc75e270 100644 --- a/scripts/lib/glfm/render_static_html.rb +++ b/scripts/lib/glfm/render_static_html.rb @@ -29,27 +29,28 @@ RSpec.describe 'Render Static HTML', :api, type: :request do # rubocop:disable R include Glfm::Constants include Glfm::Shared - let(:user) { create(:user, :admin, username: 'glfm_user') } + # TODO: Remove duplication of fixtures & logic with spec/support/shared_contexts/markdown_snapshot_shared_examples.rb - before do - stub_licensed_features(group_wikis: true) - - group = create(:group, name: 'glfm_group') - group.add_owner(user) - - project = create(:project, :repository, creator: user, group: group, name: 'glfm_project') + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group, name: 'glfm_group').tap { |group| group.add_owner(user) } } + let_it_be(:project) do # NOTE: We hardcode the IDs on all fixtures to prevent variability in the # rendered HTML/Prosemirror JSON, and to minimize the need for normalization: # https://docs.gitlab.com/ee/development/gitlab_flavored_markdown/specification_guide/#normalization - create(:project_snippet, id: 88888, project: project) # project snippet - create(:snippet, id: 99999) # personal snippet + create(:project, :repository, creator: user, group: group, name: 'glfm_project', id: 77777) + end + + let_it_be(:project_snippet) { create(:project_snippet, id: 88888, project: project) } + let_it_be(:personal_snippet) { create(:snippet, id: 99999) } + before do + stub_licensed_features(group_wikis: true) sign_in(user) end it 'can create a project dependency graph using factories' do - markdown_hash = YAML.load_file(ENV.fetch('INPUT_MARKDOWN_YML_PATH')) + markdown_hash = YAML.safe_load(File.open(ENV.fetch('INPUT_MARKDOWN_YML_PATH')), symbolize_names: true) # NOTE: We cannot parallelize this loop like the Javascript WYSIWYG example generation does, # because the rspec `post` API cannot be parallized (it is not thread-safe, it can't find @@ -58,12 +59,14 @@ RSpec.describe 'Render Static HTML', :api, type: :request do # rubocop:disable R api_url = api "/markdown" post api_url, params: { text: markdown, gfm: true } + # noinspection RubyResolve + expect(response).to be_successful returned_html_value = begin - parsed_response = Gitlab::Json.parse(response.body) - # The response may contain the HTML in either the `body` or `html` keys - parsed_response['body'] || parsed_response['html'] + parsed_response = Gitlab::Json.parse(response.body, symbolize_names: true) + # Some responses have the HTML in the `html` key, others in the `body` key. + parsed_response[:body] || parsed_response[:html] rescue JSON::ParserError # if we got a parsing error, just return the raw response body for debugging purposes. response.body @@ -79,7 +82,7 @@ RSpec.describe 'Render Static HTML', :api, type: :request do # rubocop:disable R def write_output_file(static_html_hash) tmpfile = File.open(ENV.fetch('OUTPUT_STATIC_HTML_TEMPFILE_PATH'), 'w') - YAML.dump(static_html_hash, tmpfile) - tmpfile.close + yaml_string = dump_yaml_with_formatting(static_html_hash) + write_file(tmpfile, yaml_string) end end diff --git a/scripts/lib/glfm/shared.rb b/scripts/lib/glfm/shared.rb index f11c66eb8be..b529d9ba94f 100644 --- a/scripts/lib/glfm/shared.rb +++ b/scripts/lib/glfm/shared.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'fileutils' require 'open3' +require 'active_support/core_ext/hash/keys' module Glfm module Shared @@ -39,5 +40,38 @@ module Glfm warn(stdout_and_stderr_str) raise end + + # Construct an AST so we can control YAML formatting for + # YAML block scalar literals and key quoting. + # + # Note that when Psych dumps the markdown to YAML, it will + # automatically use the default "clip" behavior of the Block Chomping Indicator (`|`) + # https://yaml.org/spec/1.2.2/#8112-block-chomping-indicator, + # when the markdown strings contain a trailing newline. The type of + # Block Chomping Indicator is automatically determined, you cannot specify it + # manually. + def dump_yaml_with_formatting(hash, literal_scalars: false) + stringified_keys_hash = hash.deep_stringify_keys + visitor = Psych::Visitors::YAMLTree.create + visitor << stringified_keys_hash + ast = visitor.tree + + # Force all scalars to have literal formatting (using Block Chomping Indicator instead of quotes) + if literal_scalars + ast.grep(Psych::Nodes::Scalar).each do |node| + node.style = Psych::Nodes::Scalar::LITERAL + end + end + + # Do not quote the keys + ast.grep(Psych::Nodes::Mapping).each do |node| + node.children.each_slice(2) do |k, _| + k.quoted = false + k.style = Psych::Nodes::Scalar::PLAIN + end + end + + ast.to_yaml + end end end diff --git a/scripts/lib/glfm/update_example_snapshots.rb b/scripts/lib/glfm/update_example_snapshots.rb index 05a5c988a28..6ff063c1690 100644 --- a/scripts/lib/glfm/update_example_snapshots.rb +++ b/scripts/lib/glfm/update_example_snapshots.rb @@ -125,7 +125,7 @@ module Glfm def write_snapshot_example_files(all_examples, skip_static_and_wysiwyg:) output("Reading #{GLFM_EXAMPLE_STATUS_YML_PATH}...") - glfm_examples_statuses = YAML.safe_load(File.open(GLFM_EXAMPLE_STATUS_YML_PATH)) + glfm_examples_statuses = YAML.safe_load(File.open(GLFM_EXAMPLE_STATUS_YML_PATH), symbolize_names: true) validate_glfm_example_status_yml(glfm_examples_statuses) write_examples_index_yml(all_examples) @@ -153,8 +153,8 @@ module Glfm def validate_glfm_example_status_yml(glfm_examples_statuses) glfm_examples_statuses.each do |example_name, statuses| next unless statuses && - statuses['skip_update_example_snapshots'] && - statuses.any? { |key, value| key.include?('skip_update_example_snapshot_') && !!value } + statuses[:skip_update_example_snapshots] && + statuses.any? { |key, value| key.to_s.include?('skip_update_example_snapshot_') && !!value } raise "Error: '#{example_name}' must not have any 'skip_update_example_snapshot_*' values specified " \ "if 'skip_update_example_snapshots' is truthy" @@ -165,7 +165,8 @@ module Glfm generate_and_write_for_all_examples( all_examples, ES_EXAMPLES_INDEX_YML_PATH, literal_scalars: false ) do |example, hash| - hash[example.fetch(:name)] = { + name = example.fetch(:name).to_sym + hash[name] = { 'spec_txt_example_position' => example.fetch(:example), 'source_specification' => if example[:extensions].empty? @@ -181,7 +182,8 @@ module Glfm def write_markdown_yml(all_examples) generate_and_write_for_all_examples(all_examples, ES_MARKDOWN_YML_PATH) do |example, hash| - hash[example.fetch(:name)] = example.fetch(:markdown) + name = example.fetch(:name).to_sym + hash[name] = example.fetch(:markdown) end end @@ -225,7 +227,7 @@ module Glfm run_external_cmd(cmd) output("Reading generated static HTML from tempfile #{static_html_tempfile_path}...") - YAML.load_file(static_html_tempfile_path) + YAML.safe_load(File.open(static_html_tempfile_path), symbolize_names: true) end def generate_wysiwyg_html_and_json @@ -241,26 +243,26 @@ module Glfm output("Reading generated WYSIWYG HTML and prosemirror JSON from tempfile " \ "#{wysiwyg_html_and_json_tempfile_path}...") - YAML.load_file(wysiwyg_html_and_json_tempfile_path) + YAML.safe_load(File.open(wysiwyg_html_and_json_tempfile_path), symbolize_names: true) end def write_html_yml(all_examples, static_html_hash, wysiwyg_html_and_json_hash, glfm_examples_statuses) generate_and_write_for_all_examples( - all_examples, ES_HTML_YML_PATH, glfm_examples_statuses + all_examples, ES_HTML_YML_PATH, glfm_examples_statuses: glfm_examples_statuses ) do |example, hash, existing_hash| - name = example.fetch(:name) + name = example.fetch(:name).to_sym example_statuses = glfm_examples_statuses[name] || {} - static = if example_statuses['skip_update_example_snapshot_html_static'] - existing_hash.dig(name, 'static') + static = if example_statuses[:skip_update_example_snapshot_html_static] + existing_hash.dig(name, :static) else static_html_hash[name] end - wysiwyg = if example_statuses['skip_update_example_snapshot_html_wysiwyg'] - existing_hash.dig(name, 'wysiwyg') + wysiwyg = if example_statuses[:skip_update_example_snapshot_html_wysiwyg] + existing_hash.dig(name, :wysiwyg) else - wysiwyg_html_and_json_hash.dig(name, 'html') + wysiwyg_html_and_json_hash.dig(name, :html) end hash[name] = { @@ -273,14 +275,14 @@ module Glfm def write_prosemirror_json_yml(all_examples, wysiwyg_html_and_json_hash, glfm_examples_statuses) generate_and_write_for_all_examples( - all_examples, ES_PROSEMIRROR_JSON_YML_PATH, glfm_examples_statuses + all_examples, ES_PROSEMIRROR_JSON_YML_PATH, glfm_examples_statuses: glfm_examples_statuses ) do |example, hash, existing_hash| - name = example.fetch(:name) + name = example.fetch(:name).to_sym - json = if glfm_examples_statuses.dig(name, 'skip_update_example_snapshot_prosemirror_json') + json = if glfm_examples_statuses.dig(name, :skip_update_example_snapshot_prosemirror_json) existing_hash[name] else - wysiwyg_html_and_json_hash.dig(name, 'json') + wysiwyg_html_and_json_hash.dig(name, :json) end # Do not assign nil values @@ -289,15 +291,15 @@ module Glfm end def generate_and_write_for_all_examples( - all_examples, output_file_path, glfm_examples_statuses = {}, literal_scalars: true + all_examples, output_file_path, glfm_examples_statuses: {}, literal_scalars: true ) preserve_existing = !glfm_examples_statuses.empty? output("#{preserve_existing ? 'Creating/Updating' : 'Creating/Overwriting'} #{output_file_path}...") - existing_hash = preserve_existing ? YAML.safe_load(File.open(output_file_path)) : {} + existing_hash = preserve_existing ? YAML.safe_load(File.open(output_file_path), symbolize_names: true) : {} output_hash = all_examples.each_with_object({}) do |example, hash| - name = example.fetch(:name) - if (reason = glfm_examples_statuses.dig(name, 'skip_update_example_snapshots')) + name = example.fetch(:name).to_sym + if (reason = glfm_examples_statuses.dig(name, :skip_update_example_snapshots)) # Output the reason for skipping the example, but only once, not multiple times for each file output("Skipping '#{name}'. Reason: #{reason}") unless glfm_examples_statuses.dig(name, :already_printed) # We just store the `:already_printed` flag in the hash entry itself. Then we @@ -317,37 +319,5 @@ module Glfm yaml_string = dump_yaml_with_formatting(output_hash, literal_scalars: literal_scalars) write_file(output_file_path, yaml_string) end - - # Construct an AST so we can control YAML formatting for - # YAML block scalar literals and key quoting. - # - # Note that when Psych dumps the markdown to YAML, it will - # automatically use the default "clip" behavior of the Block Chomping Indicator (`|`) - # https://yaml.org/spec/1.2.2/#8112-block-chomping-indicator, - # when the markdown strings contain a trailing newline. The type of - # Block Chomping Indicator is automatically determined, you cannot specify it - # manually. - def dump_yaml_with_formatting(hash, literal_scalars:) - visitor = Psych::Visitors::YAMLTree.create - visitor << hash - ast = visitor.tree - - # Force all scalars to have literal formatting (using Block Chomping Indicator instead of quotes) - if literal_scalars - ast.grep(Psych::Nodes::Scalar).each do |node| - node.style = Psych::Nodes::Scalar::LITERAL - end - end - - # Do not quote the keys - ast.grep(Psych::Nodes::Mapping).each do |node| - node.children.each_slice(2) do |k, _| - k.quoted = false - k.style = Psych::Nodes::Scalar::ANY - end - end - - ast.to_yaml - end end end |