Add latest changes from gitlab-org/gitlab@15-1-stable-eev15.1.0-rc42

12 files changed, 232 insertions, 50 deletions

diff --git a/scripts/frontend/postinstall.js b/scripts/frontend/postinstall.js
index 94977e459e3..50052bb806e 100644
--- a/scripts/frontend/postinstall.js
+++ b/scripts/frontend/postinstall.js
@@ -1,3 +1,4 @@
+const { execSync } = require('child_process');
 const chalk = require('chalk');
 
 // check that fsevents is available if we're on macOS
@@ -20,3 +21,8 @@ if (process.platform === 'darwin') {
 }
 
 console.log(`${chalk.green('success')} Dependency postinstall check passed.`);
+
+// Apply any patches to our packages
+// See https://gitlab.com/gitlab-org/gitlab/-/issues/336138
+execSync('node_modules/.bin/patch-package --error-on-fail');
+console.log(`${chalk.green('success')} Packages successfully patched.`);
diff --git a/scripts/glfm/run-snapshot-tests.sh b/scripts/glfm/run-snapshot-tests.sh
new file mode 100755
index 00000000000..70fdae60edc
--- /dev/null
+++ b/scripts/glfm/run-snapshot-tests.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC2059
+
+set -o errexit # AKA -e - exit immediately on errors (http://mywiki.wooledge.org/BashFAQ/105)
+
+# https://stackoverflow.com/a/28938235
+BCyan='\033[1;36m' # Bold Cyan
+BRed='\033[1;31m' # Bold Red
+BGreen='\033[1;32m' # Bold Green
+BBlue='\033[1;34m' # Bold Blue
+Color_Off='\033[0m' # Text Reset
+
+function onexit_err() {
+  local exit_status=${1:-$?}
+  printf "\n❌❌❌ ${BRed}GLFM snapshot tests failed!${Color_Off} ❌❌❌\n"
+  exit "${exit_status}"
+}
+trap onexit_err ERR
+set -o errexit
+
+printf "${BCyan}"
+printf "\nStarting GLFM snapshot example tests. See https://docs.gitlab.com/ee/development/gitlab_flavored_markdown/specification_guide/#run-snapshot-testssh-script for more details.\n\n"
+printf "Set 'FOCUSED_MARKDOWN_EXAMPLES=example_name_1[,...]' for focused examples, with example name(s) from https://docs.gitlab.com/ee/development/gitlab_flavored_markdown/specification_guide/#glfm_specificationexample_snapshotsexamples_indexyml.\n"
+printf "${Color_Off}"
+
+# This section can be uncommented as soon as this is merged: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89953
+#printf "\n${BBlue}Running frontend 'yarn jest spec/frontend/content_editor/markdown_snapshot_spec.js'...${Color_Off}\n\n"
+#yarn jest spec/frontend/content_editor/markdown_snapshot_spec.js
+#printf "\n${BBlue}'yarn jest spec/frontend/content_editor/markdown_snapshot_spec.js' passed!${Color_Off}\n\n"
+
+printf "\n${BBlue}Running backend 'bundle exec rspec spec/requests/api/markdown_snapshot_spec.rb'...${Color_Off}\n\n"
+bundle exec rspec spec/requests/api/markdown_snapshot_spec.rb
+printf "\n${BBlue}'bundle exec rspec spec/requests/api/markdown_snapshot_spec.rb' passed!${Color_Off}\n\n"
+
+printf "\n✅✅✅ ${BGreen}All GLFM snapshot example tests passed successfully!${Color_Off} ✅✅✅\n"
diff --git a/scripts/lib/glfm/render_static_html.rb b/scripts/lib/glfm/render_static_html.rb
index d533e586fe1..a0bda05b41e 100644
--- a/scripts/lib/glfm/render_static_html.rb
+++ b/scripts/lib/glfm/render_static_html.rb
@@ -19,13 +19,15 @@ module Glfm
       markdown_yml_path = ARGV[0]
       markdown_hash = YAML.load_file(markdown_yml_path)
 
+      context = build_context
+
       # NOTE: We COULD parallelize this loop like the Javascript WYSIWYG example generation does,
       # but it wouldn't save much time. Most of the time is spent loading the Rails environment
       # via `rails runner`. In initial testing, this loop only took ~7 seconds while the entire
       # script took ~20 seconds. Unfortunately, there's no easy way to execute
-      # `ApplicationController.helpers.markdown` without using `rails runner`
+      # `Banzai.render_and_post_process` without using `rails runner`
       static_html_hash = markdown_hash.transform_values do |markdown|
-        ApplicationController.helpers.markdown(markdown)
+        Banzai.render_and_post_process(markdown, context)
       end
 
       static_html_tempfile_path = Dir::Tmpname.create(STATIC_HTML_TEMPFILE_BASENAME) do |path|
@@ -37,14 +39,42 @@ module Glfm
       # Write the path to the output file to stdout
       print static_html_tempfile_path
     end
-  end
-end
 
-# current_user must be in global scope for `markdown` helper to work. Currently it's not supported
-# to pass it in the context.
-def current_user
-  # TODO: This will likely need to be a more realistic user object for some of the GLFM examples
-  User.new
+    private
+
+    def build_context
+      user_username = 'glfm_user_username'
+      user = User.find_by_username(user_username) ||
+        User.create!(
+          email: "glfm_user_email@example.com",
+          name: "glfm_user_name",
+          password: "glfm_user_password",
+          username: user_username
+        )
+
+      # Ensure that we never try to hit Gitaly, even if we
+      # reload the project
+      Project.define_method(:skip_disk_validation) do
+        true
+      end
+
+      project_name = 'glfm_project_name'
+      project = Project.find_by_name(project_name) ||
+        Project.create!(
+          creator: user,
+          description: "glfm_project_description",
+          name: project_name,
+          namespace: user.namespace,
+          path: 'glfm_project_path'
+        )
+
+      {
+        only_path: false,
+        current_user: user,
+        project: project
+      }
+    end
+  end
 end
 
 Glfm::RenderStaticHtml.new.process
diff --git a/scripts/lib/glfm/update_example_snapshots.rb b/scripts/lib/glfm/update_example_snapshots.rb
index 42ba305565d..9ffa54cd5d4 100644
--- a/scripts/lib/glfm/update_example_snapshots.rb
+++ b/scripts/lib/glfm/update_example_snapshots.rb
@@ -13,6 +13,10 @@ require_relative 'parse_examples'
 # for details on the implementation and usage of this script. This developers guide
 # contains diagrams and documentation of this script,
 # including explanations and examples of all files it reads and writes.
+#
+# Also note that this script is intentionally written in a pure-functional (not OO) style,
+# with no dependencies on Rails or the GitLab libraries. These choices are intended to make
+# it faster and easier to test and debug.
 module Glfm
   class UpdateExampleSnapshots
     include Constants
@@ -27,28 +31,20 @@ module Glfm
 
       output('(Skipping static HTML generation)') if skip_static_and_wysiwyg
 
-      glfm_spec_txt_lines, _glfm_examples_status_lines = read_input_files
+      output("Reading #{GLFM_SPEC_TXT_PATH}...")
+      glfm_spec_txt_lines = File.open(GLFM_SPEC_TXT_PATH).readlines
 
       # Parse all the examples from `spec.txt`, using a Ruby port of the Python `get_tests`
       # function the from original CommonMark/GFM `spec_test.py` script.
       all_examples = parse_examples(glfm_spec_txt_lines)
 
       add_example_names(all_examples)
+
       write_snapshot_example_files(all_examples, skip_static_and_wysiwyg: skip_static_and_wysiwyg)
     end
 
     private
 
-    def read_input_files
-      [
-        GLFM_SPEC_TXT_PATH,
-        GLFM_EXAMPLE_STATUS_YML_PATH
-      ].map do |path|
-        output("Reading #{path}...")
-        File.open(path).readlines
-      end
-    end
-
     def add_example_names(all_examples)
       # NOTE: This method assumes:
       # 1. Section 2 is the first section which contains examples
@@ -83,7 +79,7 @@ module Glfm
         formatted_headers_text = headers.join('__').tr('-', '_').tr(' ', '_').downcase
 
         hierarchy_level = "#{h1_count.to_s.rjust(2, '0')}_#{h2_count.to_s.rjust(2, '0')}"
-        position_within_section = index_within_h2.to_s.rjust(2, '0')
+        position_within_section = index_within_h2.to_s.rjust(3, '0')
         name = "#{hierarchy_level}__#{formatted_headers_text}__#{position_within_section}"
         converted_name = name.tr('(', '').tr(')', '') # remove any parens from the name
         example[:name] = converted_name
@@ -91,6 +87,10 @@ module Glfm
     end
 
     def write_snapshot_example_files(all_examples, skip_static_and_wysiwyg:)
+      output("Reading #{GLFM_EXAMPLE_STATUS_YML_PATH}...")
+      glfm_examples_statuses = YAML.safe_load(File.open(GLFM_EXAMPLE_STATUS_YML_PATH))
+      validate_glfm_example_status_yml(glfm_examples_statuses)
+
       write_examples_index_yml(all_examples)
 
       write_markdown_yml(all_examples)
@@ -104,9 +104,20 @@ module Glfm
       static_html_hash = generate_static_html(markdown_yml_tempfile_path)
       wysiwyg_html_and_json_hash = generate_wysiwyg_html_and_json(markdown_yml_tempfile_path)
 
-      write_html_yml(all_examples, static_html_hash, wysiwyg_html_and_json_hash)
+      write_html_yml(all_examples, static_html_hash, wysiwyg_html_and_json_hash, glfm_examples_statuses)
+
+      write_prosemirror_json_yml(all_examples, wysiwyg_html_and_json_hash, glfm_examples_statuses)
+    end
+
+    def validate_glfm_example_status_yml(glfm_examples_statuses)
+      glfm_examples_statuses.each do |example_name, statuses|
+        next unless statuses &&
+          statuses['skip_update_example_snapshots'] &&
+          statuses.any? { |key, value| key.include?('skip_update_example_snapshot_') && !!value }
 
-      write_prosemirror_json_yml(all_examples, wysiwyg_html_and_json_hash)
+        raise "Error: '#{example_name}' must not have any 'skip_update_example_snapshot_*' values specified " \
+                "if 'skip_update_example_snapshots' is truthy"
+      end
     end
 
     def write_examples_index_yml(all_examples)
@@ -186,27 +197,77 @@ module Glfm
       YAML.load_file(wysiwyg_html_and_json_tempfile_path)
     end
 
-    def write_html_yml(all_examples, static_html_hash, wysiwyg_html_and_json_hash)
-      generate_and_write_for_all_examples(all_examples, ES_HTML_YML_PATH) do |example, hash|
-        hash[example.fetch(:name)] = {
+    def write_html_yml(all_examples, static_html_hash, wysiwyg_html_and_json_hash, glfm_examples_statuses)
+      generate_and_write_for_all_examples(
+        all_examples, ES_HTML_YML_PATH, glfm_examples_statuses
+      ) do |example, hash, existing_hash|
+        name = example.fetch(:name)
+        example_statuses = glfm_examples_statuses[name] || {}
+
+        static = if example_statuses['skip_update_example_snapshot_html_static']
+                   existing_hash.dig(name, 'static')
+                 else
+                   static_html_hash[name]
+                 end
+
+        wysiwyg = if example_statuses['skip_update_example_snapshot_html_wysiwyg']
+                    existing_hash.dig(name, 'wysiwyg')
+                  else
+                    wysiwyg_html_and_json_hash.dig(name, 'html')
+                  end
+
+        hash[name] = {
           'canonical' => example.fetch(:html),
-          'static' => static_html_hash.fetch(example.fetch(:name)),
-          'wysiwyg' => wysiwyg_html_and_json_hash.fetch(example.fetch(:name)).fetch('html')
-        }
+          'static' => static,
+          'wysiwyg' => wysiwyg
+        }.compact # Do not assign nil values
       end
     end
 
-    def write_prosemirror_json_yml(all_examples, wysiwyg_html_and_json_hash)
-      generate_and_write_for_all_examples(all_examples, ES_PROSEMIRROR_JSON_YML_PATH) do |example, hash|
-        hash[example.fetch(:name)] = wysiwyg_html_and_json_hash.fetch(example.fetch(:name)).fetch('json')
+    def write_prosemirror_json_yml(all_examples, wysiwyg_html_and_json_hash, glfm_examples_statuses)
+      generate_and_write_for_all_examples(
+        all_examples, ES_PROSEMIRROR_JSON_YML_PATH, glfm_examples_statuses
+      ) do |example, hash, existing_hash|
+        name = example.fetch(:name)
+
+        json = if glfm_examples_statuses.dig(name, 'skip_update_example_snapshot_prosemirror_json')
+                 existing_hash.dig(name)
+               else
+                 wysiwyg_html_and_json_hash.dig(name, 'json')
+               end
+
+        # Do not assign nil values
+        hash[name] = json if json
       end
     end
 
-    def generate_and_write_for_all_examples(all_examples, output_file_path, literal_scalars: true, &generator_block)
-      output("Writing #{output_file_path}...")
-      generated_examples_hash = all_examples.each_with_object({}, &generator_block)
+    def generate_and_write_for_all_examples(
+      all_examples, output_file_path, glfm_examples_statuses = {}, literal_scalars: true
+    )
+      preserve_existing = !glfm_examples_statuses.empty?
+      output("#{preserve_existing ? 'Creating/Updating' : 'Creating/Overwriting'} #{output_file_path}...")
+      existing_hash = preserve_existing ? YAML.safe_load(File.open(output_file_path)) : {}
+
+      output_hash = all_examples.each_with_object({}) do |example, hash|
+        name = example.fetch(:name)
+        if (reason = glfm_examples_statuses.dig(name, 'skip_update_example_snapshots'))
+          # Output the reason for skipping the example, but only once, not multiple times for each file
+          output("Skipping '#{name}'. Reason: #{reason}") unless glfm_examples_statuses.dig(name, :already_printed)
+          # We just store the `:already_printed` flag in the hash entry itself. Then we
+          # don't need an instance variable to keep the state, and this can remain a pure function ;)
+          glfm_examples_statuses[name][:already_printed] = true
+
+          # Copy over the existing example only if it exists and preserve_existing is true, otherwise omit this example
+          # noinspection RubyScope
+          hash[name] = existing_hash[name] if existing_hash[name]
+
+          next
+        end
+
+        yield(example, hash, existing_hash)
+      end
 
-      yaml_string = dump_yaml_with_formatting(generated_examples_hash, literal_scalars: literal_scalars)
+      yaml_string = dump_yaml_with_formatting(output_hash, literal_scalars: literal_scalars)
       write_file(output_file_path, yaml_string)
     end
 
diff --git a/scripts/license-check.sh b/scripts/license-check.sh
new file mode 100755
index 00000000000..c2d5cfd8cde
--- /dev/null
+++ b/scripts/license-check.sh
@@ -0,0 +1,32 @@
+#!/bin/sh
+#
+# This script runs the LicenseFinder gem to verify that all licenses are
+# compliant. However, bundler v2.2+ and LicenseFinder do not play well
+# together when:
+#
+# 1. There are native gems installed (e.g. nokogiri, grpc, and google-protobuf).
+# 2. `Gemfile.lock` doesn't list the platform-specific gems that were installed.
+#
+# A full explanation is here:
+# https://github.com/pivotal/LicenseFinder/issues/828#issuecomment-953359134
+#
+# To work around the issue, we configure bundler to install gems for the
+# current Ruby platform, which causes Gemfile and Gemfile.lock to be
+# updated with the platform-specific gems. This allows LicenseFinder to
+# run properly. After it finishes, we clean up the mess.
+
+PROJECT_PATH=${1:-`pwd`}
+
+echo "Using project path ${PROJECT_PATH}"
+
+GEMFILE_DIFF=`git diff Gemfile Gemfile.lock`
+
+if [ ! -z "$GEMFILE_DIFF" ]; then
+  echo "LicenseFinder needs to lock the Gemfile to the current platform, but Gemfile or Gemfile.lock has changes."
+  exit 1
+fi
+
+BUNDLE_DEPLOYMENT=false BUNDLE_FROZEN=false bundle lock --add-platform `ruby -e "puts RUBY_PLATFORM"`
+bundle exec license_finder --decisions-file config/dependency_decisions.yml --project-path ${PROJECT_PATH}
+
+git checkout -q Gemfile Gemfile.lock
diff --git a/scripts/lint-doc.sh b/scripts/lint-doc.sh
index 5a3f439009d..afc04da19a7 100755
--- a/scripts/lint-doc.sh
+++ b/scripts/lint-doc.sh
@@ -106,10 +106,10 @@ then
   echo "Merge request pipeline (detached) detected. Testing all files."
 else
   MERGE_BASE=$(git merge-base ${CI_MERGE_REQUEST_TARGET_BRANCH_SHA} ${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA})
-  if git diff --diff-filter=d --name-only "${MERGE_BASE}..${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}" | grep -E "\.vale|\.markdownlint|lint-doc\.sh"
+  if git diff --diff-filter=d --name-only "${MERGE_BASE}..${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}" | grep -E "\.vale|\.markdownlint|lint-doc\.sh|docs\.gitlab-ci\.yml"
   then
     MD_DOC_PATH=${MD_DOC_PATH:-doc}
-    echo "Vale, Markdownlint, or lint-doc.sh configuration changed. Testing all files."
+    echo "Vale, Markdownlint, lint-doc.sh, or pipeline configuration changed. Testing all files."
   else
     MD_DOC_PATH=$(git diff --diff-filter=d --name-only "${MERGE_BASE}..${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}" -- 'doc/*.md')
     if [ -n "${MD_DOC_PATH}" ]
diff --git a/scripts/lint-yaml.sh b/scripts/lint-yaml.sh
new file mode 100755
index 00000000000..c3bd7f2817e
--- /dev/null
+++ b/scripts/lint-yaml.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if ! command -v yamllint > /dev/null; then
+  echo "ERROR: yamllint is not installed. For more information, see https://yamllint.readthedocs.io/en/stable/index.html."
+  exit 1
+fi
+
+yamllint --strict -f colored "$@"
diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh
index 4b528696322..d68432d9ec0 100644
--- a/scripts/prepare_build.sh
+++ b/scripts/prepare_build.sh
@@ -17,8 +17,11 @@ else
   cp config/database.yml.postgresql config/database.yml
 fi
 
-if [ -f config/database_geo.yml.postgresql ]; then
-  cp config/database_geo.yml.postgresql config/database_geo.yml
+# Remove Geo database setting if `ee/` directory does not exist. When it does
+# not exist, it runs the GitLab test suite "as if FOSS", meaning the jobs run
+# in the context of gitlab-org/gitlab-foss where the Geo is not available.
+if [ ! -d "ee/" ] ; then
+  sed -i '/geo:/,/^$/d' config/database.yml
 fi
 
 # Set user to a non-superuser to ensure we test permissions
@@ -27,11 +30,6 @@ sed -i 's/username: root/username: gitlab/g' config/database.yml
 sed -i 's/localhost/postgres/g' config/database.yml
 sed -i 's/username: git/username: postgres/g' config/database.yml
 
-if [ -f config/database_geo.yml ]; then
-  sed -i 's/localhost/postgres/g' config/database_geo.yml
-  sed -i 's/username: git/username: postgres/g' config/database_geo.yml
-fi
-
 cp config/cable.yml.example config/cable.yml
 sed -i 's|url:.*$|url: redis://redis:6379|g' config/cable.yml
 
diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh
index 36bab05067a..8a868edce3a 100755
--- a/scripts/review_apps/review-apps.sh
+++ b/scripts/review_apps/review-apps.sh
@@ -296,12 +296,21 @@ function deploy() {
 
   create_application_secret
 
+cat > review_apps.values.yml <<EOF
+  gitlab:
+    webservice:
+      extraEnv:
+        REVIEW_APPS_ENABLED: "true"
+        REVIEW_APPS_MERGE_REQUEST_IID: "${CI_MERGE_REQUEST_IID}"
+EOF
+
 HELM_CMD=$(cat << EOF
   helm upgrade \
     --namespace="${namespace}" \
     --create-namespace \
     --install \
     --wait \
+    -f review_apps.values.yml \
     --timeout "${HELM_INSTALL_TIMEOUT:-20m}" \
     --set ci.branch="${CI_COMMIT_REF_NAME}" \
     --set ci.commit.sha="${CI_COMMIT_SHORT_SHA}" \
diff --git a/scripts/static-analysis b/scripts/static-analysis
index 317652eb075..14d6a3deccb 100755
--- a/scripts/static-analysis
+++ b/scripts/static-analysis
@@ -46,7 +46,7 @@ class StaticAnalysis
     (Gitlab.ee? ? Task.new(%w[bin/rake gettext:updated_check], 360) : nil),
     Task.new(%w[yarn run lint:prettier], 160),
     Task.new(%w[bin/rake gettext:lint], 85),
-    Task.new(%W[bundle exec license_finder --decisions-file config/dependency_decisions.yml --project-path #{project_path}], 20),
+    Task.new(%W[scripts/license-check.sh #{project_path}], 20),
     Task.new(%w[bin/rake lint:static_verification], 35),
     Task.new(%w[scripts/rubocop-max-files-in-cache-check], 20),
     Task.new(%w[bin/rake config_lint], 10),
diff --git a/scripts/utils.sh b/scripts/utils.sh
index ae071b98b43..4f339bbc850 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -38,7 +38,7 @@ function bundle_install_script() {
     exit 1;
   fi;
 
-  gem install bundler --no-document --conservative --version 2.3.6
+  gem install bundler --no-document --conservative --version 2.3.15
   bundle --version
   bundle config set path "$(pwd)/vendor"
   bundle config set clean 'true'
@@ -67,7 +67,7 @@ function setup_db_praefect() {
 
 function setup_db() {
   run_timed_command "setup_db_user_only"
-  run_timed_command_with_metric "bundle exec rake db:drop db:create db:structure:load db:migrate gitlab:db:setup_ee" "setup_db"
+  run_timed_command_with_metric "bundle exec rake db:drop db:create db:schema:load db:migrate" "setup_db"
   run_timed_command "setup_db_praefect"
 }
 
diff --git a/scripts/verify-tff-mapping b/scripts/verify-tff-mapping
index b3462f74faa..9eb1d43c65b 100755
--- a/scripts/verify-tff-mapping
+++ b/scripts/verify-tff-mapping
@@ -67,13 +67,13 @@ tests = [
   {
     explanation: 'FOSS factory should map to factories spec',
     source: 'spec/factories/users.rb',
-    expected: ['spec/factories_spec.rb']
+    expected: ['spec/models/factories_spec.rb']
   },
 
   {
     explanation: 'EE factory should map to factories spec',
     source: 'ee/spec/factories/users.rb',
-    expected: ['spec/factories_spec.rb']
+    expected: ['spec/models/factories_spec.rb']
   },
 
   {