diff options
Diffstat (limited to 'scripts')
-rwxr-xr-x | scripts/changed-feature-flags | 75 | ||||
-rwxr-xr-x | scripts/dump_graphql_schema | 25 | ||||
-rw-r--r-- | scripts/frontend/eslint.js | 22 | ||||
-rwxr-xr-x | scripts/gitaly-test-spawn | 3 | ||||
-rwxr-xr-x | scripts/glfm/update-example-snapshots.rb | 5 | ||||
-rwxr-xr-x | scripts/glfm/update-specification.rb | 5 | ||||
-rw-r--r-- | scripts/lib/glfm/constants.rb | 43 | ||||
-rw-r--r-- | scripts/lib/glfm/parse_examples.rb | 169 | ||||
-rw-r--r-- | scripts/lib/glfm/render_static_html.rb | 50 | ||||
-rw-r--r-- | scripts/lib/glfm/render_wysiwyg_html_and_json.js | 152 | ||||
-rw-r--r-- | scripts/lib/glfm/shared.rb | 43 | ||||
-rw-r--r-- | scripts/lib/glfm/update_example_snapshots.rb | 245 | ||||
-rw-r--r-- | scripts/lib/glfm/update_specification.rb | 127 | ||||
-rwxr-xr-x | scripts/qa/quarantine-types-check | 62 | ||||
-rwxr-xr-x | scripts/review_apps/review-apps.sh | 4 | ||||
-rwxr-xr-x | scripts/setup-test-env | 1 | ||||
-rwxr-xr-x | scripts/trigger-build.rb | 136 | ||||
-rwxr-xr-x | scripts/used-feature-flags | 7 | ||||
-rw-r--r-- | scripts/utils.sh | 15 |
19 files changed, 1110 insertions, 79 deletions
diff --git a/scripts/changed-feature-flags b/scripts/changed-feature-flags index 3a4f18bd78f..86214e86788 100755 --- a/scripts/changed-feature-flags +++ b/scripts/changed-feature-flags @@ -3,10 +3,12 @@ require 'yaml' require 'optparse' +require 'pathname' require_relative 'api/default_options' # This script returns the desired feature flag state as a comma-separated string for the feature flags in the specified files. -# Each desired feature flag state is specified as 'feature-flag=state'. +# Each desired feature flag state is specified as 'feature-flag=state'. This allows us to run package-and-qa with the +# feature flag set to the desired state. # # For example, if the specified files included `config/feature_flags/development/ci_yaml_limit_size.yml` and the desired # state as specified by the second argument was enabled, the value returned would be `ci_yaml_limit_size=enabled` @@ -15,31 +17,84 @@ class GetFeatureFlagsFromFiles def initialize(options) @files = options.delete(:files) @state = options.delete(:state) + + abort("ERROR: Please specify the directory containing MR diffs.") if @files.to_s.empty? end + # Gets feature flags from definition files or diffs of deleted defition files + # + # @return [String] a comma-separated list of feature flags and their desired state def extracted_flags - files.each_with_object([]) do |file_path, all| - next unless file_path =~ %r{/feature_flags/(development|ops)/.*\.yml} - next unless File.exist?(file_path) + flags_list = diffs_dir.glob('**/*').each_with_object([]) do |file_path, flags| + ff_yaml = ff_yaml_for_file(file_path) + next if ff_yaml.nil? + break [] if ff_yaml.empty? + + flags << ff_yaml['name'] + end + flags_list = flags_list.map { |flag| "#{flag}=#{state}" } unless state.to_s.empty? + flags_list.join(',') + end - ff_yaml = YAML.safe_load(File.read(file_path)) - ff_to_add = "#{ff_yaml['name']}" - ff_to_add += "=#{state}" unless state.to_s.empty? + # Loads the YAML feature flag definition based on a diff of the definition file. The definition is loaded from the + # definition file itself, or from a diff of the deleted definition file. + # + # @param [Pathname] path the path to the diff + # @return [Hash] a hash containing the YAML data for the feature flag definition + def ff_yaml_for_file(path) + return unless File.expand_path(path).to_s =~ %r{/feature_flags/(development|ops)/.*\.yml} - all << ff_to_add - end.join(',') + if path.to_s.end_with?('yml.deleted.diff') + # Ignore deleted feature flag definitions if we want to enable/disable existing flags. + return if state != 'deleted' + + yaml_from_deleted_diff(path) + else + # If we want deleted definition files but find one that wasn't deleted, we return immediately to + # because non-deleted flags are tested in separate jobs from deleted flags, so we don't need to run + # a job with just deleted flags. + return [] if state == 'deleted' + + yaml_from_file(path, diffs_dir) + end end private attr_reader :files, :state + + # The absolute path to the directory of diffs + # + # @return [String] + def diffs_dir + @diffs_dir ||= Pathname.new(files).expand_path + end + + # Loads the YAML feature flag definition from a file corresponding to a diff of the definition file. + # + # @param [Pathname] file_path the path to the diff + # @param [Pathname] diffs_dir the path to the diffs directory + # @return [Hash] a hash containing the YAML data from the feature flag definition file corresponding to the diff + def yaml_from_file(file_path, diffs_dir) + real_file_path = File.join(Dir.pwd, file_path.to_s.delete_prefix(diffs_dir.to_s)).delete_suffix('.diff') + YAML.safe_load(File.read(real_file_path)) + end + + # Loads the YAML feature flag definition from a diff of the deleted feature flag definition file. + # + # @param [Pathname] file_path the path of the diff + # @return [Hash] a hash containing the YAML data for the feature flag definition from the diff + def yaml_from_deleted_diff(file_path) + cleaned_diff = File.read(file_path).gsub(/^[^a-z]+/, '') + YAML.safe_load(cleaned_diff) + end end if $0 == __FILE__ options = API::DEFAULT_OPTIONS.dup OptionParser.new do |opts| - opts.on("-f", "--files FILES", Array, "Comma-separated list of feature flag config files") do |value| + opts.on("-f", "--files FILES", String, "A directory containing diffs including feature flag definition change diffs") do |value| options[:files] = value end diff --git a/scripts/dump_graphql_schema b/scripts/dump_graphql_schema new file mode 100755 index 00000000000..7a8bd5652e1 --- /dev/null +++ b/scripts/dump_graphql_schema @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +GITLAB_DIR="$(cd "$(dirname "$0")/.." || exit ; pwd -P)" +DOCKER_IMAGE="registry.gitlab.com/gitlab-org/gitlab-build-images:apollo" +CONFIG_FILE="config/apollo.config.js" +SCHEMA_DUMP_PATH="tmp/tests/graphql/gitlab_schema_apollo.graphql" + +if [ "$CI" = true ]; then + echo "Not dumping the GraphQL schema in CI" +else + echo "Dumping GraphQL schema" + cd "$GITLAB_DIR" || exit 1 + if command -v docker >/dev/null; then + bundle exec rake gitlab:graphql:schema:dump && + docker run --rm -it \ + -v "$GITLAB_DIR/config":/config \ + -v "$GITLAB_DIR/tmp":/tmp \ + -v "$GITLAB_DIR/app":/app \ + -v "$GITLAB_DIR/ee":/ee \ + "$DOCKER_IMAGE" client:download-schema --config="$CONFIG_FILE" "$SCHEMA_DUMP_PATH" + else + echo "Docker must be installed to run this script." + exit 1 + fi +fi diff --git a/scripts/frontend/eslint.js b/scripts/frontend/eslint.js new file mode 100644 index 00000000000..2ba5976d1b9 --- /dev/null +++ b/scripts/frontend/eslint.js @@ -0,0 +1,22 @@ +const { spawn } = require('child_process'); + +const runEslint = () => { + const [, , ...args] = process.argv; + const child = spawn(`yarn`, ['internal:eslint', ...args], { + stdio: 'inherit', + }); + + child.on('exit', (code) => { + process.exitCode = code; + + if (code === 0) { + return; + } + console.log(` +If you are seeing @graphql-eslint offences, the local GraphQL schema dump might be outdated. +Consider updating it by running \`./scripts/dump_graphql_schema\`. + `); + }); +}; + +runEslint(); diff --git a/scripts/gitaly-test-spawn b/scripts/gitaly-test-spawn index eed79f75224..b9c78b88555 100755 --- a/scripts/gitaly-test-spawn +++ b/scripts/gitaly-test-spawn @@ -11,6 +11,9 @@ class GitalyTestSpawn def run install_gitaly_gems + # Run Praefect migrations + setup_praefect + # Optionally specify the path to the gitaly config toml as first argument. # Used by workhorse in test. spawn_gitaly(ARGV[0]) diff --git a/scripts/glfm/update-example-snapshots.rb b/scripts/glfm/update-example-snapshots.rb new file mode 100755 index 00000000000..f0e529aa006 --- /dev/null +++ b/scripts/glfm/update-example-snapshots.rb @@ -0,0 +1,5 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative '../lib/glfm/update_example_snapshots' +Glfm::UpdateExampleSnapshots.new.process(skip_static_and_wysiwyg: ENV['SKIP_STATIC_AND_WYSIWYG'] == 'true') diff --git a/scripts/glfm/update-specification.rb b/scripts/glfm/update-specification.rb new file mode 100755 index 00000000000..7b2c30be61c --- /dev/null +++ b/scripts/glfm/update-specification.rb @@ -0,0 +1,5 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative '../lib/glfm/update_specification' +Glfm::UpdateSpecification.new.process diff --git a/scripts/lib/glfm/constants.rb b/scripts/lib/glfm/constants.rb new file mode 100644 index 00000000000..e5917fc5cdb --- /dev/null +++ b/scripts/lib/glfm/constants.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require 'pathname' + +module Glfm + module Constants + # Root dir containing all specification files + specification_path = Pathname.new(File.expand_path("../../../glfm_specification", __dir__)) + + # GitHub Flavored Markdown specification file + GHFM_SPEC_TXT_URI = 'https://raw.githubusercontent.com/github/cmark-gfm/master/test/spec.txt' + GHFM_SPEC_VERSION = '0.29' + GHFM_SPEC_TXT_FILENAME = "ghfm_spec_v_#{GHFM_SPEC_VERSION}.txt" + GHFM_SPEC_TXT_PATH = specification_path.join('input/github_flavored_markdown', GHFM_SPEC_TXT_FILENAME) + + # GitLab Flavored Markdown specification files + specification_input_glfm_path = specification_path.join('input/gitlab_flavored_markdown') + GLFM_INTRO_TXT_PATH = specification_input_glfm_path.join('glfm_intro.txt') + GLFM_EXAMPLES_TXT_PATH = specification_input_glfm_path.join('glfm_canonical_examples.txt') + GLFM_EXAMPLE_STATUS_YML_PATH = specification_input_glfm_path.join('glfm_example_status.yml') + GLFM_SPEC_TXT_PATH = specification_path.join('output/spec.txt') + + # Example Snapshot (ES) files + es_fixtures_path = File.expand_path("../../../spec/fixtures/glfm/example_snapshots", __dir__) + ES_EXAMPLES_INDEX_YML_PATH = File.join(es_fixtures_path, 'examples_index.yml') + ES_MARKDOWN_YML_PATH = File.join(es_fixtures_path, 'markdown.yml') + ES_HTML_YML_PATH = File.join(es_fixtures_path, 'html.yml') + ES_PROSEMIRROR_JSON_YML_PATH = File.join(es_fixtures_path, 'prosemirror_json.yml') + + # Other constants used for processing files + GLFM_SPEC_TXT_HEADER = <<~GLFM_SPEC_TXT_HEADER + --- + title: GitLab Flavored Markdown (GLFM) Spec + version: alpha + ... + GLFM_SPEC_TXT_HEADER + INTRODUCTION_HEADER_LINE_TEXT = /\A# Introduction\Z/.freeze + END_TESTS_COMMENT_LINE_TEXT = /\A<!-- END TESTS -->\Z/.freeze + MARKDOWN_TEMPFILE_BASENAME = %w[MARKDOWN_TEMPFILE_ .yml].freeze + STATIC_HTML_TEMPFILE_BASENAME = %w[STATIC_HTML_TEMPFILE_ .yml].freeze + WYSIWYG_HTML_AND_JSON_TEMPFILE_BASENAME = %w[WYSIWYG_HTML_AND_JSON_TEMPFILE_ .yml].freeze + end +end diff --git a/scripts/lib/glfm/parse_examples.rb b/scripts/lib/glfm/parse_examples.rb new file mode 100644 index 00000000000..1c6afb800c3 --- /dev/null +++ b/scripts/lib/glfm/parse_examples.rb @@ -0,0 +1,169 @@ +# frozen_string_literal: true + +# This module contains a Ruby port of Python logic from the `get_tests` method of the +# `spec_test.py` script (see copy of original code in a comment at the bottom of this file): +# https://github.com/github/cmark-gfm/blob/5dfedc7/test/spec_tests.py#L82 +# +# The logic and structure is as unchanged as possible from the original Python - no +# cleanup or refactoring was done. +# +# Changes from the original logic were made to follow Ruby/GitLab syntax, idioms, and linting rules. +# +# Additional logic was also added to: +# 1. Capture all nested headers, not just the most recent. +# 2. Raise an exception if an unexpected state is encountered. +# +# Comments indicate where changes or additions were made. +module Glfm + module ParseExamples + REGULAR_TEXT = 0 + MARKDOWN_EXAMPLE = 1 + HTML_OUTPUT = 2 + EXAMPLE_BACKTICKS_LENGTH = 32 + EXAMPLE_BACKTICKS_STRING = '`' * EXAMPLE_BACKTICKS_LENGTH + + # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity + def parse_examples(spec_txt_lines) + line_number = 0 + start_line = 0 + example_number = 0 + markdown_lines = [] + html_lines = [] + state = REGULAR_TEXT # 0 regular text, 1 markdown example, 2 html output + extensions = [] + headertext = '' # most recent header text + headers = [] # all nested headers since last H2 - new logic compared to original Python code + tests = [] + + h1_regex = /\A# / # new logic compared to original Python code + h2_regex = /\A## / # new logic compared to original Python code + header_regex = /\A#+ / # Added beginning of line anchor to original Python code + + spec_txt_lines.each do |line| + line_number += 1 + stripped_line = line.strip + if stripped_line.start_with?("#{EXAMPLE_BACKTICKS_STRING} example") + # If beginning line of an example block... + state = MARKDOWN_EXAMPLE + extensions = stripped_line[(EXAMPLE_BACKTICKS_LENGTH + " example".length)..].split + elsif stripped_line == EXAMPLE_BACKTICKS_STRING + # Else if end line of an example block... + state = REGULAR_TEXT + example_number += 1 + end_line = line_number + unless extensions.include?('disabled') + tests << + { + markdown: markdown_lines.join.tr('→', "\t"), + html: html_lines.join.tr('→', "\t"), + example: example_number, + start_line: start_line, + end_line: end_line, + section: headertext, + extensions: extensions, + headers: headers.dup # new logic compared to original Python code + } + start_line = 0 + markdown_lines = [] + html_lines = [] + end + elsif stripped_line == "." + # Else if the example divider line... + state = HTML_OUTPUT + # Else if we are currently in a markdown example... + elsif state == MARKDOWN_EXAMPLE + start_line = line_number - 1 if start_line == 0 + + markdown_lines.append(line) + elsif state == HTML_OUTPUT + # Else if we are currently in the html output... + html_lines.append(line) + elsif state == REGULAR_TEXT && line =~ header_regex + # Else if we are in regular text and it is a header line + # NOTE: This assumes examples are only nested up to 2 levels deep (H2) + + # Extract the header text from the line + headertext = line.gsub(header_regex, '').strip + + # reset the headers array if we found a new H1 + headers = [] if line =~ h1_regex # new logic compared to original Python code + + # pop the last entry from the headers array if we found a new H2 + headers.pop if headers.length == 2 && line =~ h2_regex # new logic compared to original Python code + + # push the new header text to the headers array + headers << headertext # New logic compared to original Python code + else + # Else if we are in regular text... + + # This else block is new logic compared to original Python code + + # Sanity check for state machine + raise 'Unexpected state encountered when parsing examples' unless state == REGULAR_TEXT + + # no-op - skips any other non-header regular text lines + end + end + # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity + + tests + end + end +end + +# Original `get_tests` method from spec_test.py: +# rubocop:disable Style/BlockComments +# rubocop:disable Style/AsciiComments +=begin +def get_tests(specfile): + line_number = 0 + start_line = 0 + end_line = 0 + example_number = 0 + markdown_lines = [] + html_lines = [] + state = 0 # 0 regular text, 1 markdown example, 2 html output + extensions = [] + headertext = '' + tests = [] + + header_re = re.compile('#+ ') + + with open(specfile, 'r', encoding='utf-8', newline='\n') as specf: + for line in specf: + line_number = line_number + 1 + l = line.strip() + if l.startswith("`" * 32 + " example"): + state = 1 + extensions = l[32 + len(" example"):].split() + elif l == "`" * 32: + state = 0 + example_number = example_number + 1 + end_line = line_number + if 'disabled' not in extensions: + tests.append({ + "markdown":''.join(markdown_lines).replace('→',"\t"), + "html":''.join(html_lines).replace('→',"\t"), + "example": example_number, + "start_line": start_line, + "end_line": end_line, + "section": headertext, + "extensions": extensions}) + start_line = 0 + markdown_lines = [] + html_lines = [] + elif l == ".": + state = 2 + elif state == 1: + if start_line == 0: + start_line = line_number - 1 + markdown_lines.append(line) + elif state == 2: + html_lines.append(line) + elif state == 0 and re.match(header_re, line): + headertext = header_re.sub('', line).strip() + return tests + +=end +# rubocop:enable Style/BlockComments +# rubocop:enable Style/AsciiComments diff --git a/scripts/lib/glfm/render_static_html.rb b/scripts/lib/glfm/render_static_html.rb new file mode 100644 index 00000000000..d533e586fe1 --- /dev/null +++ b/scripts/lib/glfm/render_static_html.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require_relative 'constants' +require_relative 'shared' + +# Purpose: +# - Reads a set of markdown examples from a hash which has been serialized to disk +# - Converts each example to static HTML using the `markdown` helper +# - Writes the HTML for each example to a hash which is serialized to disk +# +# It should be invoked via `rails runner` from the Rails root directory. +# It is intended to be invoked from the `update_example_snapshots.rb` script class. +module Glfm + class RenderStaticHtml + include Constants + include Shared + + def process + markdown_yml_path = ARGV[0] + markdown_hash = YAML.load_file(markdown_yml_path) + + # NOTE: We COULD parallelize this loop like the Javascript WYSIWYG example generation does, + # but it wouldn't save much time. Most of the time is spent loading the Rails environment + # via `rails runner`. In initial testing, this loop only took ~7 seconds while the entire + # script took ~20 seconds. Unfortunately, there's no easy way to execute + # `ApplicationController.helpers.markdown` without using `rails runner` + static_html_hash = markdown_hash.transform_values do |markdown| + ApplicationController.helpers.markdown(markdown) + end + + static_html_tempfile_path = Dir::Tmpname.create(STATIC_HTML_TEMPFILE_BASENAME) do |path| + tmpfile = File.open(path, 'w') + YAML.dump(static_html_hash, tmpfile) + tmpfile.close + end + + # Write the path to the output file to stdout + print static_html_tempfile_path + end + end +end + +# current_user must be in global scope for `markdown` helper to work. Currently it's not supported +# to pass it in the context. +def current_user + # TODO: This will likely need to be a more realistic user object for some of the GLFM examples + User.new +end + +Glfm::RenderStaticHtml.new.process diff --git a/scripts/lib/glfm/render_wysiwyg_html_and_json.js b/scripts/lib/glfm/render_wysiwyg_html_and_json.js new file mode 100644 index 00000000000..58b440d7ab2 --- /dev/null +++ b/scripts/lib/glfm/render_wysiwyg_html_and_json.js @@ -0,0 +1,152 @@ +import fs from 'fs'; +import { DOMSerializer } from 'prosemirror-model'; +import jsYaml from 'js-yaml'; +// TODO: DRY up duplication with spec/frontend/content_editor/services/markdown_serializer_spec.js +// See https://gitlab.com/groups/gitlab-org/-/epics/7719#plan +import Blockquote from '~/content_editor/extensions/blockquote'; +import Bold from '~/content_editor/extensions/bold'; +import BulletList from '~/content_editor/extensions/bullet_list'; +import Code from '~/content_editor/extensions/code'; +import CodeBlockHighlight from '~/content_editor/extensions/code_block_highlight'; +import DescriptionItem from '~/content_editor/extensions/description_item'; +import DescriptionList from '~/content_editor/extensions/description_list'; +import Details from '~/content_editor/extensions/details'; +import DetailsContent from '~/content_editor/extensions/details_content'; +import Division from '~/content_editor/extensions/division'; +import Emoji from '~/content_editor/extensions/emoji'; +import Figure from '~/content_editor/extensions/figure'; +import FigureCaption from '~/content_editor/extensions/figure_caption'; +import FootnoteDefinition from '~/content_editor/extensions/footnote_definition'; +import FootnoteReference from '~/content_editor/extensions/footnote_reference'; +import FootnotesSection from '~/content_editor/extensions/footnotes_section'; +import HardBreak from '~/content_editor/extensions/hard_break'; +import Heading from '~/content_editor/extensions/heading'; +import HorizontalRule from '~/content_editor/extensions/horizontal_rule'; +import Image from '~/content_editor/extensions/image'; +import InlineDiff from '~/content_editor/extensions/inline_diff'; +import Italic from '~/content_editor/extensions/italic'; +import Link from '~/content_editor/extensions/link'; +import ListItem from '~/content_editor/extensions/list_item'; +import OrderedList from '~/content_editor/extensions/ordered_list'; +import Strike from '~/content_editor/extensions/strike'; +import Table from '~/content_editor/extensions/table'; +import TableCell from '~/content_editor/extensions/table_cell'; +import TableHeader from '~/content_editor/extensions/table_header'; +import TableRow from '~/content_editor/extensions/table_row'; +import TaskItem from '~/content_editor/extensions/task_item'; +import TaskList from '~/content_editor/extensions/task_list'; +import createMarkdownDeserializer from '~/content_editor/services/remark_markdown_deserializer'; +import { createTestEditor } from 'jest/content_editor/test_utils'; +import { setTestTimeout } from 'jest/__helpers__/timeout'; + +const tiptapEditor = createTestEditor({ + extensions: [ + Blockquote, + Bold, + BulletList, + Code, + CodeBlockHighlight, + DescriptionItem, + DescriptionList, + Details, + DetailsContent, + Division, + Emoji, + FootnoteDefinition, + FootnoteReference, + FootnotesSection, + Figure, + FigureCaption, + HardBreak, + Heading, + HorizontalRule, + Image, + InlineDiff, + Italic, + Link, + ListItem, + OrderedList, + Strike, + Table, + TableCell, + TableHeader, + TableRow, + TaskItem, + TaskList, + ], +}); + +async function renderMarkdownToHTMLAndJSON(markdown, schema, deserializer) { + let prosemirrorDocument; + try { + const { document } = await deserializer.deserialize({ schema, content: markdown }); + prosemirrorDocument = document; + } catch (e) { + const errorMsg = `Error - check implementation:\n${e.message}`; + return { + html: errorMsg, + json: errorMsg, + }; + } + + const documentFragment = DOMSerializer.fromSchema(schema).serializeFragment( + prosemirrorDocument.content, + ); + const htmlString = documentFragment.firstChild.outerHTML; + + const json = prosemirrorDocument.toJSON(); + const jsonString = JSON.stringify(json, null, 2); + return { html: htmlString, json: jsonString }; +} + +function renderHtmlAndJsonForAllExamples(markdownExamples) { + const { schema } = tiptapEditor; + const deserializer = createMarkdownDeserializer(); + const exampleNames = Object.keys(markdownExamples); + + return exampleNames.reduce(async (promisedExamples, exampleName) => { + const markdown = markdownExamples[exampleName]; + const htmlAndJson = await renderMarkdownToHTMLAndJSON(markdown, schema, deserializer); + const examples = await promisedExamples; + examples[exampleName] = htmlAndJson; + return examples; + }, Promise.resolve({})); +} + +/* eslint-disable no-undef */ +jest.mock('~/emoji'); + +// The purpose of this file is to deserialize markdown examples +// to WYSIWYG HTML and to prosemirror documents in JSON form, using +// the logic implemented as part of the Content Editor. +// +// It reads an input YAML file containing all the markdown examples, +// and outputs a YAML files containing the rendered HTML and JSON +// corresponding each markdown example. +// +// The input and output file paths are provides as command line arguments. +// +// Although it is implemented as a Jest test, it is not a unit test. We use +// Jest because that is the simplest environment in which to execute the +// relevant Content Editor logic. +// +// +// This script should be invoked via jest with the a command similar to the following: +// yarn jest --testMatch '**/render_wysiwyg_html_and_json.js' ./scripts/lib/glfm/render_wysiwyg_html_and_json.js +it('serializes html to prosemirror json', async () => { + setTestTimeout(20000); + + const inputMarkdownTempfilePath = process.env.INPUT_MARKDOWN_YML_PATH; + expect(inputMarkdownTempfilePath).not.toBeUndefined(); + const outputWysiwygHtmlAndJsonTempfilePath = + process.env.OUTPUT_WYSIWYG_HTML_AND_JSON_TEMPFILE_PATH; + expect(outputWysiwygHtmlAndJsonTempfilePath).not.toBeUndefined(); + /* eslint-enable no-undef */ + + const markdownExamples = jsYaml.safeLoad(fs.readFileSync(inputMarkdownTempfilePath), {}); + + const htmlAndJsonExamples = await renderHtmlAndJsonForAllExamples(markdownExamples); + + const htmlAndJsonExamplesYamlString = jsYaml.safeDump(htmlAndJsonExamples, {}); + fs.writeFileSync(outputWysiwygHtmlAndJsonTempfilePath, htmlAndJsonExamplesYamlString); +}); diff --git a/scripts/lib/glfm/shared.rb b/scripts/lib/glfm/shared.rb new file mode 100644 index 00000000000..f11c66eb8be --- /dev/null +++ b/scripts/lib/glfm/shared.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true +require 'fileutils' +require 'open3' + +module Glfm + module Shared + def write_file(file_path, file_content_string) + FileUtils.mkdir_p(File.dirname(file_path)) + # NOTE: We don't use the block form of File.open because we want to be able to easily + # mock it for testing. + io = File.open(file_path, 'w') + io.binmode + io.write(file_content_string) + # NOTE: We are using #fsync + #close_write instead of just #close`, in order to unit test + # with a real StringIO and not just a mock object. + io.fsync + io.close_write + end + + # All script output goes through this method. This makes it easy to mock in order to prevent + # output from being printed to the console during tests. We don't want to directly mock + # Kernel#puts, because that could interfere or cause spurious test failures when #puts is used + # for debugging. And for some reason RuboCop says `rubocop:disable Rails/Output` would be + # redundant here, so not including it. + def output(string) + puts string + end + + def run_external_cmd(cmd) + # noinspection RubyMismatchedArgumentType + rails_root = File.expand_path('../../../', __dir__) + + # See https://stackoverflow.com/a/20001569/25192 + stdout_and_stderr_str, status = Open3.capture2e(cmd, chdir: rails_root) + + return stdout_and_stderr_str if status.success? + + warn("Error running command `#{cmd}`\n") + warn(stdout_and_stderr_str) + raise + end + end +end diff --git a/scripts/lib/glfm/update_example_snapshots.rb b/scripts/lib/glfm/update_example_snapshots.rb new file mode 100644 index 00000000000..42ba305565d --- /dev/null +++ b/scripts/lib/glfm/update_example_snapshots.rb @@ -0,0 +1,245 @@ +# frozen_string_literal: true +require 'fileutils' +require 'open-uri' +require 'yaml' +require 'psych' +require 'tempfile' +require 'open3' +require_relative 'constants' +require_relative 'shared' +require_relative 'parse_examples' + +# IMPORTANT NOTE: See https://docs.gitlab.com/ee/development/gitlab_flavored_markdown/specification_guide/ +# for details on the implementation and usage of this script. This developers guide +# contains diagrams and documentation of this script, +# including explanations and examples of all files it reads and writes. +module Glfm + class UpdateExampleSnapshots + include Constants + include Shared + include ParseExamples + + # skip_static_and_wysiwyg can be used to skip the backend/frontend html and prosemirror JSON + # generation which depends on external calls. This allows for faster processing in unit tests + # which do not require it. + def process(skip_static_and_wysiwyg: false) + output('Updating example snapshots...') + + output('(Skipping static HTML generation)') if skip_static_and_wysiwyg + + glfm_spec_txt_lines, _glfm_examples_status_lines = read_input_files + + # Parse all the examples from `spec.txt`, using a Ruby port of the Python `get_tests` + # function the from original CommonMark/GFM `spec_test.py` script. + all_examples = parse_examples(glfm_spec_txt_lines) + + add_example_names(all_examples) + write_snapshot_example_files(all_examples, skip_static_and_wysiwyg: skip_static_and_wysiwyg) + end + + private + + def read_input_files + [ + GLFM_SPEC_TXT_PATH, + GLFM_EXAMPLE_STATUS_YML_PATH + ].map do |path| + output("Reading #{path}...") + File.open(path).readlines + end + end + + def add_example_names(all_examples) + # NOTE: This method assumes: + # 1. Section 2 is the first section which contains examples + # 2. Examples are always nested exactly than 2 levels deep in an H2 + # 3. We assume that the Appendix doesn't ever contain any examples, so it doesn't show up + # in the H1 header count. So, even though due to the concatenation it appears before the + # GitLab examples sections, it doesn't result in their header counts being off by +1. + + h1_count = 1 # examples start in H1 section 2; section 1 is the overview with no examples. + h2_count = 0 + previous_h1 = '' + previous_h2 = '' + index_within_h2 = 0 + all_examples.each do |example| + headers = example[:headers] + + if headers[0] != previous_h1 + h1_count += 1 + h2_count = 0 + previous_h1 = headers[0] + end + + if headers[1] != previous_h2 + h2_count += 1 + previous_h2 = headers[1] + index_within_h2 = 0 + end + + index_within_h2 += 1 + + # convert headers array to lowercase string with underscores, and double underscores between headers + formatted_headers_text = headers.join('__').tr('-', '_').tr(' ', '_').downcase + + hierarchy_level = "#{h1_count.to_s.rjust(2, '0')}_#{h2_count.to_s.rjust(2, '0')}" + position_within_section = index_within_h2.to_s.rjust(2, '0') + name = "#{hierarchy_level}__#{formatted_headers_text}__#{position_within_section}" + converted_name = name.tr('(', '').tr(')', '') # remove any parens from the name + example[:name] = converted_name + end + end + + def write_snapshot_example_files(all_examples, skip_static_and_wysiwyg:) + write_examples_index_yml(all_examples) + + write_markdown_yml(all_examples) + + if skip_static_and_wysiwyg + output("Skipping static/WYSIWYG HTML and prosemirror JSON generation...") + return + end + + markdown_yml_tempfile_path = write_markdown_yml_tempfile + static_html_hash = generate_static_html(markdown_yml_tempfile_path) + wysiwyg_html_and_json_hash = generate_wysiwyg_html_and_json(markdown_yml_tempfile_path) + + write_html_yml(all_examples, static_html_hash, wysiwyg_html_and_json_hash) + + write_prosemirror_json_yml(all_examples, wysiwyg_html_and_json_hash) + end + + def write_examples_index_yml(all_examples) + generate_and_write_for_all_examples( + all_examples, ES_EXAMPLES_INDEX_YML_PATH, literal_scalars: false + ) do |example, hash| + hash[example.fetch(:name)] = { + 'spec_txt_example_position' => example.fetch(:example), + 'source_specification' => + if example[:extensions].empty? + 'commonmark' + elsif example[:extensions].include?('gitlab') + 'gitlab' + else + 'github' + end + } + end + end + + def write_markdown_yml(all_examples) + generate_and_write_for_all_examples(all_examples, ES_MARKDOWN_YML_PATH) do |example, hash| + hash[example.fetch(:name)] = example.fetch(:markdown) + end + end + + def write_markdown_yml_tempfile + # NOTE: We must copy the markdown YAML file to a separate temporary file for the + # `render_static_html.rb` script to read it, because the script is run in a + # separate process, and during unit testing we are unable to substitute the mock + # StringIO when reading the input file in the subprocess. + Dir::Tmpname.create(MARKDOWN_TEMPFILE_BASENAME) do |path| + io = File.open(ES_MARKDOWN_YML_PATH) + io.seek(0) # rewind the file. This is necessary when testing with a mock StringIO + contents = io.read + write_file(path, contents) + end + end + + def generate_static_html(markdown_yml_tempfile_path) + output("Generating static HTML from markdown examples...") + + # NOTE 1: We shell out to perform the conversion of markdown to static HTML via the internal Rails app + # helper method. This allows us to avoid using the Rails API or environment in this script, + # which makes developing and running the unit tests for this script much faster, + # because they can use 'fast_spec_helper' which does not require the entire Rails environment. + + # NOTE 2: We pass the input file path as a command line argument, and receive the output + # tempfile path as a return value. This is simplest in the case where we are invoking Ruby. + cmd = %(rails runner #{__dir__}/render_static_html.rb #{markdown_yml_tempfile_path}) + cmd_output = run_external_cmd(cmd) + # NOTE: Running under a debugger can add extra output, only take the last line + static_html_tempfile_path = cmd_output.split("\n").last + + output("Reading generated static HTML from tempfile #{static_html_tempfile_path}...") + YAML.load_file(static_html_tempfile_path) + end + + def generate_wysiwyg_html_and_json(markdown_yml_tempfile_path) + output("Generating WYSIWYG HTML and prosemirror JSON from markdown examples...") + + # NOTE: Unlike when we invoke a Ruby script, here we pass the input and output file paths + # via environment variables. This is because it's not straightforward/clean to pass command line + # arguments when we are invoking `yarn jest ...` + ENV['INPUT_MARKDOWN_YML_PATH'] = markdown_yml_tempfile_path + + # Dir::Tmpname.create requires a block, but we are using the non-block form to get the path + # via the return value, so we pass an empty block to avoid an error. + wysiwyg_html_and_json_tempfile_path = Dir::Tmpname.create(WYSIWYG_HTML_AND_JSON_TEMPFILE_BASENAME) {} + ENV['OUTPUT_WYSIWYG_HTML_AND_JSON_TEMPFILE_PATH'] = wysiwyg_html_and_json_tempfile_path + + cmd = %(yarn jest --testMatch '**/render_wysiwyg_html_and_json.js' #{__dir__}/render_wysiwyg_html_and_json.js) + run_external_cmd(cmd) + + output("Reading generated WYSIWYG HTML and prosemirror JSON from tempfile " \ + "#{wysiwyg_html_and_json_tempfile_path}...") + YAML.load_file(wysiwyg_html_and_json_tempfile_path) + end + + def write_html_yml(all_examples, static_html_hash, wysiwyg_html_and_json_hash) + generate_and_write_for_all_examples(all_examples, ES_HTML_YML_PATH) do |example, hash| + hash[example.fetch(:name)] = { + 'canonical' => example.fetch(:html), + 'static' => static_html_hash.fetch(example.fetch(:name)), + 'wysiwyg' => wysiwyg_html_and_json_hash.fetch(example.fetch(:name)).fetch('html') + } + end + end + + def write_prosemirror_json_yml(all_examples, wysiwyg_html_and_json_hash) + generate_and_write_for_all_examples(all_examples, ES_PROSEMIRROR_JSON_YML_PATH) do |example, hash| + hash[example.fetch(:name)] = wysiwyg_html_and_json_hash.fetch(example.fetch(:name)).fetch('json') + end + end + + def generate_and_write_for_all_examples(all_examples, output_file_path, literal_scalars: true, &generator_block) + output("Writing #{output_file_path}...") + generated_examples_hash = all_examples.each_with_object({}, &generator_block) + + yaml_string = dump_yaml_with_formatting(generated_examples_hash, literal_scalars: literal_scalars) + write_file(output_file_path, yaml_string) + end + + # Construct an AST so we can control YAML formatting for + # YAML block scalar literals and key quoting. + # + # Note that when Psych dumps the markdown to YAML, it will + # automatically use the default "clip" behavior of the Block Chomping Indicator (`|`) + # https://yaml.org/spec/1.2.2/#8112-block-chomping-indicator, + # when the markdown strings contain a trailing newline. The type of + # Block Chomping Indicator is automatically determined, you cannot specify it + # manually. + def dump_yaml_with_formatting(hash, literal_scalars:) + visitor = Psych::Visitors::YAMLTree.create + visitor << hash + ast = visitor.tree + + # Force all scalars to have literal formatting (using Block Chomping Indicator instead of quotes) + if literal_scalars + ast.grep(Psych::Nodes::Scalar).each do |node| + node.style = Psych::Nodes::Scalar::LITERAL + end + end + + # Do not quote the keys + ast.grep(Psych::Nodes::Mapping).each do |node| + node.children.each_slice(2) do |k, _| + k.quoted = false + k.style = Psych::Nodes::Scalar::ANY + end + end + + ast.to_yaml + end + end +end diff --git a/scripts/lib/glfm/update_specification.rb b/scripts/lib/glfm/update_specification.rb new file mode 100644 index 00000000000..73c23d40de5 --- /dev/null +++ b/scripts/lib/glfm/update_specification.rb @@ -0,0 +1,127 @@ +# frozen_string_literal: true +require 'fileutils' +require 'open-uri' +require 'pathname' +require_relative 'constants' +require_relative 'shared' + +module Glfm + class UpdateSpecification + include Constants + include Shared + + def process + output('Updating specification...') + ghfm_spec_txt_lines = load_ghfm_spec_txt + glfm_spec_txt_string = build_glfm_spec_txt(ghfm_spec_txt_lines) + write_glfm_spec_txt(glfm_spec_txt_string) + end + + private + + def load_ghfm_spec_txt + # We only re-download the GitHub Flavored Markdown specification if the + # UPDATE_GHFM_SPEC_TXT environment variable is set to true, which should only + # ever be done manually and locally, never in CI. This provides some security + # protection against a possible injection attack vector, if the GitHub-hosted + # version of the spec is ever temporarily compromised with an injection attack. + # + # This also avoids doing external network access to download the file + # in CI jobs, which can avoid potentially flaky builds if the GitHub-hosted + # version of the file is temporarily unavailable. + if ENV['UPDATE_GHFM_SPEC_TXT'] == 'true' + download_and_write_ghfm_spec_txt + else + read_existing_ghfm_spec_txt + end + end + + def read_existing_ghfm_spec_txt + output("Reading existing #{GHFM_SPEC_TXT_PATH}...") + File.open(GHFM_SPEC_TXT_PATH).readlines + end + + def download_and_write_ghfm_spec_txt + output("Downloading #{GHFM_SPEC_TXT_URI}...") + ghfm_spec_txt_uri_io = URI.open(GHFM_SPEC_TXT_URI) + + # Read IO stream into an array of lines for easy processing later + ghfm_spec_txt_lines = ghfm_spec_txt_uri_io.readlines + raise "Unable to read lines from #{GHFM_SPEC_TXT_URI}" if ghfm_spec_txt_lines.empty? + + # Make sure the GHFM spec version has not changed + validate_expected_spec_version!(ghfm_spec_txt_lines[2]) + + # Reset IO stream and re-read into a single string for easy writing + # noinspection RubyNilAnalysis + ghfm_spec_txt_uri_io.seek(0) + ghfm_spec_txt_string = ghfm_spec_txt_uri_io.read + raise "Unable to read string from #{GHFM_SPEC_TXT_URI}" unless ghfm_spec_txt_string + + output("Writing #{GHFM_SPEC_TXT_PATH}...") + GHFM_SPEC_TXT_PATH.dirname.mkpath + write_file(GHFM_SPEC_TXT_PATH, ghfm_spec_txt_string) + + ghfm_spec_txt_lines + end + + def validate_expected_spec_version!(version_line) + return if version_line =~ /\Aversion: #{GHFM_SPEC_VERSION}\Z/o + + raise "GitHub Flavored Markdown spec.txt version mismatch! " \ + "Expected 'version: #{GHFM_SPEC_VERSION}', got '#{version_line}'" + end + + def build_glfm_spec_txt(ghfm_spec_txt_lines) + glfm_spec_txt_lines = ghfm_spec_txt_lines.dup + replace_header(glfm_spec_txt_lines) + replace_intro_section(glfm_spec_txt_lines) + insert_examples_txt(glfm_spec_txt_lines) + glfm_spec_txt_lines.join('') + end + + def replace_header(spec_txt_lines) + spec_txt_lines[0, spec_txt_lines.index("...\n") + 1] = GLFM_SPEC_TXT_HEADER + end + + def replace_intro_section(spec_txt_lines) + glfm_intro_txt_lines = File.open(GLFM_INTRO_TXT_PATH).readlines + raise "Unable to read lines from #{GLFM_INTRO_TXT_PATH}" if glfm_intro_txt_lines.empty? + + ghfm_intro_header_begin_index = spec_txt_lines.index do |line| + line =~ INTRODUCTION_HEADER_LINE_TEXT + end + raise "Unable to locate introduction header line in #{GHFM_SPEC_TXT_PATH}" if ghfm_intro_header_begin_index.nil? + + # Find the index of the next header after the introduction header, starting from the index + # of the introduction header this is the length of the intro section + ghfm_intro_section_length = spec_txt_lines[ghfm_intro_header_begin_index + 1..].index do |line| + line.start_with?('# ') + end + + # Replace the intro section with the GitLab flavored Markdown intro section + spec_txt_lines[ghfm_intro_header_begin_index, ghfm_intro_section_length] = glfm_intro_txt_lines + end + + def insert_examples_txt(spec_txt_lines) + glfm_examples_txt_lines = File.open(GLFM_EXAMPLES_TXT_PATH).readlines + raise "Unable to read lines from #{GLFM_EXAMPLES_TXT_PATH}" if glfm_examples_txt_lines.empty? + + ghfm_end_tests_comment_index = spec_txt_lines.index do |line| + line =~ END_TESTS_COMMENT_LINE_TEXT + end + raise "Unable to locate 'END TESTS' comment line in #{GHFM_SPEC_TXT_PATH}" if ghfm_end_tests_comment_index.nil? + + # Insert the GLFM examples before the 'END TESTS' comment line + spec_txt_lines[ghfm_end_tests_comment_index - 1] = ["\n", glfm_examples_txt_lines, "\n"].flatten + + spec_txt_lines + end + + def write_glfm_spec_txt(glfm_spec_txt_string) + output("Writing #{GLFM_SPEC_TXT_PATH}...") + FileUtils.mkdir_p(Pathname.new(GLFM_SPEC_TXT_PATH).dirname) + write_file(GLFM_SPEC_TXT_PATH, glfm_spec_txt_string) + end + end +end diff --git a/scripts/qa/quarantine-types-check b/scripts/qa/quarantine-types-check new file mode 100755 index 00000000000..44d329a3590 --- /dev/null +++ b/scripts/qa/quarantine-types-check @@ -0,0 +1,62 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'json' + +QUARANTINE_TYPES = %w[stale bug investigating flaky broken test_environment waiting_on].freeze + +missing_issues = [] +quarantine_type_errors = [] +invalid_type_message = %"\n*** The following quarantined tests have invalid types:\n\n%s\n" +missing_issue_message = %"\n*** The following quarantined tests are missing issue links:\n\n%s\n" + +test_metadata_file = ARGV.shift + +unless test_metadata_file + puts "usage: #{__FILE__} <test_metadata_file>" + exit 1 +end + +file = File.read(test_metadata_file) +data_hash = JSON.parse(file) + +unless data_hash['examples'].count > 1 + puts "\nRspec output does not contain examples. Check test-metadata.json file.\n" + exit 1 +end + +puts "\nAnalyzing quarantined test data...\n" + +tests = data_hash['examples'] + +tests.each do |test| + if test['quarantine'] + unless QUARANTINE_TYPES.include?(test['quarantine']['type']) + quarantine_type_errors.push( + <<~TYPE_ERRORS + ==> #{test['full_description']} + in file: #{test['id']} + with type: "#{test['quarantine']['type']}" + TYPE_ERRORS + ) + end + + missing_issues.push(" ==> #{test['id']} - #{test['full_description']}\n") unless test['quarantine']['issue'] + end +end + +if quarantine_type_errors.empty? && missing_issues.empty? + puts "\nNo errors found." +else + puts "\n*** Quarantine format violations detected! ***\n" + + unless quarantine_type_errors.empty? + puts invalid_type_message % quarantine_type_errors.join("\n") + puts "*** Please use one of the following quarantine types for the tests listed above.\n" + puts " #{QUARANTINE_TYPES}\n" + end + + puts missing_issue_message % missing_issues unless missing_issues.empty? + puts "See https://about.gitlab.com/handbook/engineering/quality/quality-engineering/debugging-qa-test-failures/#quarantining-tests" + exit 1 +end diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh index f529c8eaafe..829e806e378 100755 --- a/scripts/review_apps/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -359,11 +359,11 @@ EOF function verify_deploy() { echoinfo "Verifying deployment at ${CI_ENVIRONMENT_URL}" - if retry "test_url \"${CI_ENVIRONMENT_URL}\" curl_output.txt"; then + if retry "test_url \"${CI_ENVIRONMENT_URL}\""; then echoinfo "Review app is deployed to ${CI_ENVIRONMENT_URL}" return 0 else - echoerr "Review app is not available at ${CI_ENVIRONMENT_URL}. See curl_output.txt artifact for detail." + echoerr "Review app is not available at ${CI_ENVIRONMENT_URL}: see the logs from cURL above for more details" return 1 fi } diff --git a/scripts/setup-test-env b/scripts/setup-test-env index a81aaa5cda3..97762e1cafa 100755 --- a/scripts/setup-test-env +++ b/scripts/setup-test-env @@ -53,6 +53,7 @@ require 'omniauth' require 'omniauth-github' require 'etc' require_dependency File.expand_path('../lib/gitlab/access', __dir__) +require_dependency File.expand_path('../lib/gitlab/utils', __dir__) require_dependency File.expand_path('../config/initializers/1_settings', __dir__) diff --git a/scripts/trigger-build.rb b/scripts/trigger-build.rb index a3356c664d1..28b52fa916a 100755 --- a/scripts/trigger-build.rb +++ b/scripts/trigger-build.rb @@ -33,13 +33,13 @@ module Trigger ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN'] end - def invoke!(post_comment: false, downstream_job_name: nil) + def invoke!(downstream_job_name: nil) pipeline_variables = variables puts "Triggering downstream pipeline on #{downstream_project_path}" puts "with variables #{pipeline_variables}" - pipeline = gitlab_client(:downstream).run_trigger( + pipeline = downstream_client.run_trigger( downstream_project_path, trigger_token, ref, @@ -48,18 +48,17 @@ module Trigger puts "Triggered downstream pipeline: #{pipeline.web_url}\n" puts "Waiting for downstream pipeline status" - Trigger::CommitComment.post!(pipeline, gitlab_client(:upstream)) if post_comment downstream_job = if downstream_job_name - gitlab_client(:downstream).pipeline_jobs(downstream_project_path, pipeline.id).auto_paginate.find do |potential_job| + downstream_client.pipeline_jobs(downstream_project_path, pipeline.id).auto_paginate.find do |potential_job| potential_job.name == downstream_job_name end end if downstream_job - Trigger::Job.new(downstream_project_path, downstream_job.id, gitlab_client(:downstream)) + Trigger::Job.new(downstream_project_path, downstream_job.id, downstream_client) else - Trigger::Pipeline.new(downstream_project_path, pipeline.id, gitlab_client(:downstream)) + Trigger::Pipeline.new(downstream_project_path, pipeline.id, downstream_client) end end @@ -80,28 +79,35 @@ module Trigger private - # Override to trigger and work with pipeline on different GitLab instance - # type: :downstream -> downstream build and pipeline status - # type: :upstream -> this project, e.g. for posting comments - def gitlab_client(type) - # By default, always use the same client - @gitlab_client ||= Gitlab.client( + def com_gitlab_client + @com_gitlab_client ||= Gitlab.client( endpoint: 'https://gitlab.com/api/v4', private_token: self.class.access_token ) end + # This client is used for downstream build and pipeline status + # Can be overridden + def downstream_client + com_gitlab_client + end + # Must be overridden def downstream_project_path raise NotImplementedError end # Must be overridden - def ref + def ref_param_name raise NotImplementedError end # Can be overridden + def primary_ref + 'main' + end + + # Can be overridden def trigger_token ENV['CI_JOB_TOKEN'] end @@ -116,6 +122,27 @@ module Trigger ENV[version_file]&.strip || File.read(version_file).strip end + # Can be overridden + def trigger_stable_branch_if_detected? + false + end + + def stable_branch? + ENV['CI_COMMIT_REF_NAME'] =~ /^[\d-]+-stable(-ee)?$/ + end + + def fallback_ref + if trigger_stable_branch_if_detected? && stable_branch? + ENV['CI_COMMIT_REF_NAME'].delete_suffix('-ee') + else + primary_ref + end + end + + def ref + ENV.fetch(ref_param_name, fallback_ref) + end + def base_variables # Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results, # and fallback to CI_COMMIT_SHA for the `detached` pipelines. @@ -146,8 +173,16 @@ module Trigger ENV.fetch('OMNIBUS_PROJECT_PATH', 'gitlab-org/build/omnibus-gitlab-mirror') end - def ref - ENV.fetch('OMNIBUS_BRANCH', 'master') + def ref_param_name + 'OMNIBUS_BRANCH' + end + + def primary_ref + 'master' + end + + def trigger_stable_branch_if_detected? + true end def extra_variables @@ -184,10 +219,16 @@ module Trigger private - def ref - return ENV['CI_COMMIT_REF_NAME'] if ENV['CI_COMMIT_REF_NAME'] =~ /^[\d-]+-stable(-ee)?$/ + def ref_param_name + 'CNG_BRANCH' + end + + def primary_ref + 'master' + end - ENV.fetch('CNG_BRANCH', 'master') + def trigger_stable_branch_if_detected? + true end def extra_variables @@ -243,12 +284,12 @@ module Trigger # Remove a remote branch in gitlab-docs. # def cleanup! - environment = gitlab_client(:downstream).environments(downstream_project_path, name: downstream_environment).first + environment = com_gitlab_client.environments(downstream_project_path, name: downstream_environment).first return unless environment - environment = gitlab_client(:downstream).stop_environment(downstream_project_path, environment.id) + environment = com_gitlab_client.stop_environment(downstream_project_path, environment.id) if environment.state == 'stopped' - puts "=> Downstream environment '#{downstream_environment}' stopped" + puts "=> Downstream environment '#{downstream_environment}' stopped." else puts "=> Downstream environment '#{downstream_environment}' failed to stop." end @@ -272,8 +313,8 @@ module Trigger ENV.fetch('DOCS_PROJECT_PATH', 'gitlab-org/gitlab-docs') end - def ref - ENV.fetch('DOCS_BRANCH', 'main') + def ref_param_name + 'DOCS_BRANCH' end # `gitlab-org/gitlab-docs` pipeline trigger "Triggered from gitlab-org/gitlab 'review-docs-deploy' job" @@ -317,26 +358,21 @@ module Trigger class DatabaseTesting < Base IDENTIFIABLE_NOTE_TAG = 'gitlab-org/database-team/gitlab-com-database-testing:identifiable-note' - def self.access_token - ENV['GITLABCOM_DATABASE_TESTING_ACCESS_TOKEN'] - end - - def invoke!(post_comment: false, downstream_job_name: nil) + def invoke!(downstream_job_name: nil) pipeline = super - gitlab = gitlab_client(:upstream) project_path = variables['TOP_UPSTREAM_SOURCE_PROJECT'] merge_request_id = variables['TOP_UPSTREAM_MERGE_REQUEST_IID'] comment = "<!-- #{IDENTIFIABLE_NOTE_TAG} --> \nStarted database testing [pipeline](https://ops.gitlab.net/#{downstream_project_path}/-/pipelines/#{pipeline.id}) " \ "(limited access). This comment will be updated once the pipeline has finished running." # Look for an existing note - db_testing_notes = gitlab.merge_request_notes(project_path, merge_request_id).auto_paginate.select do |note| + db_testing_notes = com_gitlab_client.merge_request_notes(project_path, merge_request_id).auto_paginate.select do |note| note.body.include?(IDENTIFIABLE_NOTE_TAG) end if db_testing_notes.empty? # This is the first note - note = gitlab.create_merge_request_note(project_path, merge_request_id, comment) + note = com_gitlab_client.create_merge_request_note(project_path, merge_request_id, comment) puts "Posted comment to:\n" puts "https://gitlab.com/#{project_path}/-/merge_requests/#{merge_request_id}#note_#{note.id}" @@ -345,19 +381,15 @@ module Trigger private - def gitlab_client(type) - @gitlab_clients ||= { - downstream: Gitlab.client( - endpoint: 'https://ops.gitlab.net/api/v4', - private_token: self.class.access_token - ), - upstream: Gitlab.client( - endpoint: 'https://gitlab.com/api/v4', - private_token: Base.access_token - ) - } + def ops_gitlab_client + @ops_gitlab_client ||= Gitlab.client( + endpoint: 'https://ops.gitlab.net/api/v4', + private_token: ENV['GITLABCOM_DATABASE_TESTING_ACCESS_TOKEN'] + ) + end - @gitlab_clients[type] + def downstream_client + ops_gitlab_client end def trigger_token @@ -377,20 +409,12 @@ module Trigger } end - def ref - ENV['GITLABCOM_DATABASE_TESTING_TRIGGER_REF'] || 'master' + def ref_param_name + 'GITLABCOM_DATABASE_TESTING_TRIGGER_REF' end - end - - class CommitComment - def self.post!(downstream_pipeline, gitlab_client) - gitlab_client.create_commit_comment( - ENV['CI_PROJECT_PATH'], - Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'], - "The [`#{ENV['CI_JOB_NAME']}`](#{ENV['CI_JOB_URL']}) job from pipeline #{ENV['CI_PIPELINE_URL']} triggered #{downstream_pipeline.web_url} downstream.") - rescue Gitlab::Error::Error => error - puts "Ignoring the following error: #{error}" + def primary_ref + 'master' end end @@ -458,7 +482,7 @@ end if $0 == __FILE__ case ARGV[0] when 'omnibus' - Trigger::Omnibus.new.invoke!(post_comment: true, downstream_job_name: 'Trigger:qa-test').wait! + Trigger::Omnibus.new.invoke!(downstream_job_name: 'Trigger:qa-test').wait! when 'cng' Trigger::CNG.new.invoke!.wait! when 'gitlab-com-database-testing' diff --git a/scripts/used-feature-flags b/scripts/used-feature-flags index 89ea99c6984..0966795f451 100755 --- a/scripts/used-feature-flags +++ b/scripts/used-feature-flags @@ -70,13 +70,6 @@ flags_paths.each do |flags_path| next end - # Dynamic feature flag names for redirect to latest CI templates - # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63144/diffs#fa2193ace3f6a02f7ef9995ef9bc519eca92c4ee_57_84 - if feature_flag_name.start_with?('redirect_to_latest_template_') - puts "Skipping the #{feature_flag_name} feature flag since it starts with 'redirect_to_latest_template_'." - next - end - all_flags[feature_flag_name] = File.exist?(File.join('tmp', 'feature_flags', feature_flag_name + '.used')) end end diff --git a/scripts/utils.sh b/scripts/utils.sh index e896fe40e06..ae071b98b43 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -15,16 +15,18 @@ function retry() { function test_url() { local url="${1}" - local curl_output="${2}" local status - status=$(curl -s -o "${curl_output}" -L -w ''%{http_code}'' "${url}") + status=$(curl --output /dev/null -L -s -w ''%{http_code}'' "${url}") if [[ $status == "200" ]]; then return 0 + else + # We display the error in the job to allow for better debugging + curl -L --fail --output /dev/null "${url}" + echo -e "\nExpected HTTP status 200: received ${status}\n" + return 1 fi - - return 1 } function bundle_install_script() { @@ -59,9 +61,14 @@ function setup_db_user_only() { source scripts/create_postgres_user.sh } +function setup_db_praefect() { + createdb -h postgres -U postgres --encoding=UTF8 --echo praefect_test +} + function setup_db() { run_timed_command "setup_db_user_only" run_timed_command_with_metric "bundle exec rake db:drop db:create db:structure:load db:migrate gitlab:db:setup_ee" "setup_db" + run_timed_command "setup_db_praefect" } function install_api_client_dependencies_with_apk() { |