Add latest changes from gitlab-org/gitlab@15-8-stable-eev15.8.0-rc42

18 files changed, 146 insertions, 74 deletions

diff --git a/scripts/allowed_warnings.txt b/scripts/allowed_warnings.txt
new file mode 100644
index 00000000000..19bd5d51a20
--- /dev/null
+++ b/scripts/allowed_warnings.txt
@@ -0,0 +1,15 @@
+# List of ignored warnings used by `fail_on_warnings` in `scripts/utils.sh`.
+# Each line is a regexp and represents a match used by
+# `grep --invert-match --extended-regexp --file`.
+# Comments and empty lines are ignored.
+
+# https://github.com/browserslist/browserslist/blob/d0ec62eb48c41c218478cd3ac28684df051cc865/node.js#L329
+# warns if caniuse-lite package is older than 6 months. Ignore this
+# warning message so that GitLab backports don't fail.
+Browserslist: caniuse-lite is outdated\. Please run next command `yarn upgrade`
+
+# https://github.com/mime-types/mime-types-data/pull/50#issuecomment-1060908930
+Type application/netcdf is already registered as a variant of application/netcdf\.
+
+# This warning is emitted by scripts/static-analysis.
+\*\*\*\* .+ had the following warning\(s\):
diff --git a/scripts/build_qa_image b/scripts/build_qa_image
index 3728608e32c..c0001181a51 100755
--- a/scripts/build_qa_image
+++ b/scripts/build_qa_image
@@ -3,7 +3,7 @@
 QA_IMAGE_NAME="gitlab-ee-qa"
 QA_BUILD_TARGET="ee"
 
-if [[ "${CI_PROJECT_NAME}" == "gitlabhq" || "${CI_PROJECT_NAME}" == "gitlab-foss" ]]; then
+if [[ "${CI_PROJECT_NAME}" == "gitlabhq" || "${CI_PROJECT_NAME}" == "gitlab-foss" || "${FOSS_ONLY}" == "1" ]]; then
   QA_IMAGE_NAME="gitlab-ce-qa"
   QA_BUILD_TARGET="foss"
 fi
@@ -28,7 +28,7 @@ if [ "${QA_IMAGE_NAME}" == "gitlab-ee-qa" ]; then
   DESTINATIONS="${DESTINATIONS} --tag $QA_IMAGE_FOR_AUTO_DEPLOY"
 fi
 
-echo "Building QA image for destinations: ${DESTINATIONS}"
+echo "Building QA image for '${QA_BUILD_TARGET}' for destinations: ${DESTINATIONS}"
 
 docker buildx build \
   --cache-to=type=inline \
diff --git a/scripts/decomposition/generate-loose-foreign-key b/scripts/decomposition/generate-loose-foreign-key
index 3f4c510020a..2e91aa662de 100755
--- a/scripts/decomposition/generate-loose-foreign-key
+++ b/scripts/decomposition/generate-loose-foreign-key
@@ -253,7 +253,7 @@ end
 # Show only cross-schema foreign keys
 if $options[:cross_schema]
   all_foreign_keys.select! do |definition|
-    Gitlab::Database::GitlabSchema.table_schema(definition.from_table) != Gitlab::Database::GitlabSchema.table_schema(definition.to_table)
+    Gitlab::Database::GitlabSchema.table_schema!(definition.from_table) != Gitlab::Database::GitlabSchema.table_schema!(definition.to_table)
   end
 end
 
diff --git a/scripts/frontend/extract_gettext_all.js b/scripts/frontend/extract_gettext_all.js
index 0a5e2b06971..922aa85241f 100644
--- a/scripts/frontend/extract_gettext_all.js
+++ b/scripts/frontend/extract_gettext_all.js
@@ -6,7 +6,7 @@ const {
   decorateExtractorWithHelpers,
 } = require('gettext-extractor-vue');
 const vue2TemplateCompiler = require('vue-template-compiler');
-const ensureSingleLine = require('../../app/assets/javascripts/locale/ensure_single_line');
+const ensureSingleLine = require('../../app/assets/javascripts/locale/ensure_single_line.cjs');
 
 const args = argumentsParser
   .option('-f, --file <file>', 'Extract message from one single file')
diff --git a/scripts/gitlab_component_helpers.sh b/scripts/gitlab_component_helpers.sh
index 0d72f940036..c46dbb57a58 100644
--- a/scripts/gitlab_component_helpers.sh
+++ b/scripts/gitlab_component_helpers.sh
@@ -87,12 +87,10 @@ function upload_package() {
 
 function read_curl_package() {
   local package_url="${1}"
-  local token_header="${CURL_TOKEN_HEADER}"
-  local token="${CI_JOB_TOKEN}"
 
   echoinfo "Downloading from ${package_url} ..."
 
-  curl --fail --silent --retry 3 --header "${token_header}: ${token}" "${package_url}"
+  curl --fail --silent --retry 3 "${package_url}"
 }
 
 function extract_package() {
diff --git a/scripts/glfm/run-spec-tests.sh b/scripts/glfm/run-spec-tests.sh
new file mode 100755
index 00000000000..b60f6b05051
--- /dev/null
+++ b/scripts/glfm/run-spec-tests.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC2059
+
+set -o errexit # AKA -e - exit immediately on errors (http://mywiki.wooledge.org/BashFAQ/105)
+
+# https://stackoverflow.com/a/28938235
+BCyan='\033[1;36m' # Bold Cyan
+BRed='\033[1;31m' # Bold Red
+Color_Off='\033[0m' # Text Reset
+
+function onexit_err() {
+  local exit_status=${1:-$?}
+  printf "\n❌❌❌ ${BRed}GLFM spec tests failed!${Color_Off} ❌❌❌\n"
+  exit "${exit_status}"
+}
+trap onexit_err ERR
+set -o errexit
+
+printf "${BCyan}"
+printf "\nThis script is not yet implemented!\n"
+printf "\nSee https://docs.gitlab.com/ee/development/gitlab_flavored_markdown/specification_guide/#run-spec-testssh-script for more details.\n\n"
+printf "${Color_Off}"
diff --git a/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb b/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb
index 3d4570f74e5..ef8872ecbb0 100644
--- a/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb
+++ b/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb
@@ -54,7 +54,7 @@ module Glfm
       # Ensure that the diff output is flushed and output before we raise and exit.
       $stderr.flush
 
-      raise('ERROR: The generated files are not up to date.')
+      raise('ERROR: The generated files are not up to date.  The specification or examples may need to be updated. See https://docs.gitlab.com/ee/development/gitlab_flavored_markdown/specification_guide/#workflows')
     end
   end
 end
diff --git a/scripts/lint-doc-quality.sh b/scripts/lint-doc-quality.sh
deleted file mode 100755
index 9d8409a7c80..00000000000
--- a/scripts/lint-doc-quality.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-
-echo '=> Generating code quality artifact...'
-echo
-
-# Generate code quality artifact for Vale warnings only on changed files.
-# Only works on merged results pipelines, so first checks if a merged results CI variable is present.
-# If not present, runs on all files.
-
-if [ -z "${CI_MERGE_REQUEST_TARGET_BRANCH_SHA}" ]
-then
-  MD_DOC_PATH=${MD_DOC_PATH:-doc}
-  echo "Merge request pipeline (detached) detected. Testing all files."
-else
-  MERGE_BASE=$(git merge-base "${CI_MERGE_REQUEST_TARGET_BRANCH_SHA}" "${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}")
-  MD_DOC_PATH=$(git diff --diff-filter=d --name-only "${MERGE_BASE}..${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}" -- 'doc/*.md')
-  if [ -n "${MD_DOC_PATH}" ]
-  then
-    echo -e "Merged results pipeline detected. Testing only the following files: ${MD_DOC_PATH}"
-  fi
-fi
-
-echo "vale --output=doc/.vale/vale-json.tmpl --minAlertLevel warning ${MD_DOC_PATH} > gl-code-quality-report-docs.json"
-vale --output=doc/.vale/vale-json.tmpl --minAlertLevel warning ${MD_DOC_PATH} > gl-code-quality-report-docs.json
diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb
index 2d87c18d7d2..f459cb9c0ea 100755
--- a/scripts/review_apps/automated_cleanup.rb
+++ b/scripts/review_apps/automated_cleanup.rb
@@ -213,7 +213,7 @@ module ReviewApps
       gitlab.environments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc', states: 'stopped', search: ENVIRONMENT_PREFIX[environment_type]).auto_paginate do |environment|
         next if skip_environment?(environment: environment, checked_environments: checked_environments, last_updated_threshold: last_updated_threshold, environment_type: environment_type)
 
-        yield environment if delete_environment(environment)
+        yield environment if delete_environment(environment) && block_given?
 
         checked_environments << environment.slug
       end
diff --git a/scripts/review_apps/base-config.yaml b/scripts/review_apps/base-config.yaml
index 0981aafec22..4d0f0ffec42 100644
--- a/scripts/review_apps/base-config.yaml
+++ b/scripts/review_apps/base-config.yaml
@@ -99,7 +99,7 @@ gitlab:
         cpu: 746m
         memory: 2809Mi
       limits:
-        cpu: 1300m
+        cpu: 1400m
         memory: 4214Mi
     minReplicas: 1
     maxReplicas: 1
diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh
index e185ed43e38..6a72b19c631 100755
--- a/scripts/review_apps/review-apps.sh
+++ b/scripts/review_apps/review-apps.sh
@@ -299,21 +299,21 @@ HELM_CMD=$(cat << EOF
     --set global.appConfig.sentry.dsn="${REVIEW_APPS_SENTRY_DSN}" \
     --set global.appConfig.sentry.environment="review" \
     --set gitlab.migrations.image.repository="${gitlab_toolbox_image_repository}" \
-    --set gitlab.migrations.image.tag="${CI_COMMIT_REF_SLUG}" \
+    --set gitlab.migrations.image.tag="${CI_COMMIT_SHA}" \
     --set gitlab.gitaly.image.repository="${gitlab_gitaly_image_repository}" \
     --set gitlab.gitaly.image.tag="${gitaly_image_tag}" \
     --set gitlab.gitlab-shell.image.repository="${gitlab_shell_image_repository}" \
     --set gitlab.gitlab-shell.image.tag="v${GITLAB_SHELL_VERSION}" \
     --set gitlab.sidekiq.annotations.commit="${CI_COMMIT_SHORT_SHA}" \
     --set gitlab.sidekiq.image.repository="${gitlab_sidekiq_image_repository}" \
-    --set gitlab.sidekiq.image.tag="${CI_COMMIT_REF_SLUG}" \
+    --set gitlab.sidekiq.image.tag="${CI_COMMIT_SHA}" \
     --set gitlab.webservice.annotations.commit="${CI_COMMIT_SHORT_SHA}" \
     --set gitlab.webservice.image.repository="${gitlab_webservice_image_repository}" \
-    --set gitlab.webservice.image.tag="${CI_COMMIT_REF_SLUG}" \
+    --set gitlab.webservice.image.tag="${CI_COMMIT_SHA}" \
     --set gitlab.webservice.workhorse.image="${gitlab_workhorse_image_repository}" \
-    --set gitlab.webservice.workhorse.tag="${CI_COMMIT_REF_SLUG}" \
+    --set gitlab.webservice.workhorse.tag="${CI_COMMIT_SHA}" \
     --set gitlab.toolbox.image.repository="${gitlab_toolbox_image_repository}" \
-    --set gitlab.toolbox.image.tag="${CI_COMMIT_REF_SLUG}"
+    --set gitlab.toolbox.image.tag="${CI_COMMIT_SHA}"
 EOF
 )
 
@@ -362,20 +362,46 @@ function verify_deploy() {
 
   if [[ "${deployed}" == "true" ]]; then
     echoinfo "[$(date '+%H:%M:%S')] Review app is deployed to ${CI_ENVIRONMENT_URL}"
-    return 0
   else
     echoerr "[$(date '+%H:%M:%S')] Review app is not available at ${CI_ENVIRONMENT_URL}: see the logs from cURL above for more details"
     return 1
   fi
 }
 
+# We need to be able to access the GitLab API to run this method.
+# Since we are creating a personal access token in `disable_sign_ups`,
+# This method should be executed after it.
+function verify_commit_sha() {
+  local verify_success="false"
+
+  for i in {1..60}; do # try for 2 minutes in case review-apps containers are restarting
+    echoinfo "[$(date '+%H:%M:%S')] Checking the correct commit is deployed in the review-app:"
+    echo "Expected commit sha: ${CI_COMMIT_SHA}"
+
+    review_app_revision=$(curl --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" "${CI_ENVIRONMENT_URL}/api/v4/metadata" | jq -r .revision)
+    echo "review-app revision: ${review_app_revision}"
+
+    if [[ "${CI_COMMIT_SHA}" == "${review_app_revision}"* ]]; then
+      verify_success="true"
+      break
+    fi
+
+    sleep 2
+  done
+
+  if [[ "${verify_success}" != "true" ]]; then
+    echoerr "[$(date '+%H:%M:%S')] Review app revision is not the same as the current commit!"
+    return 1
+  fi
+
+  return 0
+}
+
 function display_deployment_debug() {
   local namespace="${CI_ENVIRONMENT_SLUG}"
 
   # Install dig to inspect DNS entries
-  #
-  # Silent install: see https://stackoverflow.com/a/52642167/1620195
-  apt-get -qq update && apt-get -qq install -y dnsutils < /dev/null > /dev/null
+  apk add -q bind-tools
 
   echoinfo "[debugging data] Check review-app webservice DNS entry:"
   dig +short $(echo "${CI_ENVIRONMENT_URL}" | sed 's~http[s]*://~~g')
diff --git a/scripts/rspec_check_order_dependence b/scripts/rspec_check_order_dependence
index 91d4c5938a7..00f1176e251 100755
--- a/scripts/rspec_check_order_dependence
+++ b/scripts/rspec_check_order_dependence
@@ -40,6 +40,8 @@ done
 
 set -xe
 
+export RSPEC_WARN_MISSING_FEATURE_CATEGORY=0
+
 bin/rspec --order defined "${RSPEC_ARGS[@]}" "$@"
 RSPEC_ORDER=reverse bin/rspec "${RSPEC_ARGS[@]}" "$@"
 bin/rspec --order random "${RSPEC_ARGS[@]}" "$@"
diff --git a/scripts/setup/as-if-jh.sh b/scripts/setup/as-if-jh.sh
index 38c3ac9b913..445a988b996 100755
--- a/scripts/setup/as-if-jh.sh
+++ b/scripts/setup/as-if-jh.sh
@@ -19,10 +19,9 @@ download_jh_path() {
 
     echoinfo "Downloading ${path}"
 
-    curl --location -o "${output}" -H "Private-Token: ${ADD_JH_FILES_TOKEN}" "https://gitlab.com/api/v4/projects/${GITLAB_JH_MIRROR_PROJECT}/repository/archive?sha=${JH_BRANCH}&path=${path}"
+    curl --location --output "${output}" --header "Private-Token: ${ADD_JH_FILES_TOKEN}" --get --data-urlencode "sha=${JH_BRANCH}" --data-urlencode "path=${path}" "https://gitlab.com/api/v4/projects/${GITLAB_JH_MIRROR_PROJECT}/repository/archive"
 
-    tar -zxf "${output}"
+    tar -zxf "${output}" --strip-component 1
     rm "${output}"
-    mv gitlab-"${JH_BRANCH}"-*/"${path}" ./
   done
 }
diff --git a/scripts/setup/find-jh-branch.rb b/scripts/setup/find-jh-branch.rb
index 5b36aa7a1f4..67e83423d9c 100755
--- a/scripts/setup/find-jh-branch.rb
+++ b/scripts/setup/find-jh-branch.rb
@@ -92,7 +92,7 @@ class FindJhBranch
   def gitlab
     @gitlab ||= Gitlab.client(
       endpoint: API::DEFAULT_OPTIONS[:endpoint],
-      private_token: API::DEFAULT_OPTIONS[:api_token] || ''
+      private_token: ENV['ADD_JH_FILES_TOKEN'] || ''
     )
   end
 end
diff --git a/scripts/static-analysis b/scripts/static-analysis
index 9a0057d8f4d..0d03dd42c73 100755
--- a/scripts/static-analysis
+++ b/scripts/static-analysis
@@ -7,14 +7,7 @@ require_relative '../lib/gitlab/popen'
 require_relative '../lib/gitlab/popen/runner'
 
 class StaticAnalysis
-  ALLOWED_WARNINGS = [
-    # https://github.com/browserslist/browserslist/blob/d0ec62eb48c41c218478cd3ac28684df051cc865/node.js#L329
-    # warns if caniuse-lite package is older than 6 months. Ignore this
-    # warning message so that GitLab backports don't fail.
-    "Browserslist: caniuse-lite is outdated. Please run next command `yarn upgrade`",
-    # https://github.com/mime-types/mime-types-data/pull/50#issuecomment-1060908930
-    "Type application/netcdf is already registered as a variant of application/netcdf."
-  ].freeze
+  # `ALLOWED_WARNINGS` moved to scripts/allowed_warnings.txt
 
   Task = Struct.new(:command, :duration) do
     def cmd
@@ -94,12 +87,12 @@ class StaticAnalysis
     if static_analysis.all_success_and_clean?
       puts 'All static analyses passed successfully.'
     elsif static_analysis.all_success?
-      puts 'All static analyses passed successfully, but we have warnings:'
+      puts 'All static analyses passed successfully with warnings.'
       puts
 
       emit_warnings(static_analysis)
 
-      exit 2 if warning_count(static_analysis).nonzero?
+      # We used to exit 2 on warnings but `fail_on_warnings` takes care of it now.
     else
       puts 'Some static analyses failed:'
 
@@ -112,11 +105,11 @@ class StaticAnalysis
 
   def emit_warnings(static_analysis)
     static_analysis.warned_results.each do |result|
-      puts
-      puts "**** #{result.cmd.join(' ')} had the following warning(s):"
-      puts
-      puts result.stderr
-      puts
+      warn
+      warn "**** #{result.cmd.join(' ')} had the following warning(s):"
+      warn
+      warn result.stderr
+      warn
     end
   end
 
@@ -131,11 +124,6 @@ class StaticAnalysis
     end
   end
 
-  def warning_count(static_analysis)
-    static_analysis.warned_results
-      .count { |result| !ALLOWED_WARNINGS.include?(result.stderr.strip) } # rubocop:disable Rails/NegateInclude
-  end
-
   def tasks_to_run(node_total)
     total_time = TASKS_WITH_DURATIONS_SECONDS.sum(&:duration).to_f
     ideal_time_per_node = total_time / node_total
diff --git a/scripts/trigger-build.rb b/scripts/trigger-build.rb
index 411e5ed13c6..033c2e55329 100755
--- a/scripts/trigger-build.rb
+++ b/scripts/trigger-build.rb
@@ -184,6 +184,20 @@ module Trigger
       true
     end
 
+    def gitlab_ref_slug
+      if ENV['CI_COMMIT_TAG']
+        ENV['CI_COMMIT_REF_NAME']
+      else
+        ENV['CI_COMMIT_SHA']
+      end
+    end
+
+    def base_variables
+      super.merge(
+        'GITLAB_REF_SLUG' => gitlab_ref_slug
+      )
+    end
+
     def extra_variables
       {
         "TRIGGER_BRANCH" => ref,
diff --git a/scripts/utils.sh b/scripts/utils.sh
index 92f647958fe..c71de666ac6 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -109,21 +109,38 @@ function install_junit_merge_gem() {
 
 function fail_on_warnings() {
   local cmd="$*"
-  local warnings
-  warnings="$(mktemp)"
+  local warning_file
+  warning_file="$(mktemp)"
+
+  local allowed_warning_file
+  allowed_warning_file="$(mktemp)"
 
-  eval "$cmd 2>$warnings"
+  eval "$cmd 2>$warning_file"
   local ret=$?
 
-  if test -s "$warnings";
+  # Filter out comments and empty lines from allowed warnings file.
+  grep --invert-match --extended-regexp "^#|^$" scripts/allowed_warnings.txt > "$allowed_warning_file"
+
+  local warnings
+  # Filter out allowed warnings from stderr.
+  # Turn grep errors into warnings so we fail later.
+  warnings=$(grep --invert-match --extended-regexp --file "$allowed_warning_file" "$warning_file" 2>&1 || true)
+
+  rm -f "$allowed_warning_file"
+
+  if [ "$warnings" != "" ]
   then
     echoerr "There were warnings:"
-    cat "$warnings"
-    rm "$warnings"
+    echoerr "======================== Filtered warnings ====================================="
+    echo "$warnings" >&2
+    echoerr "======================= Unfiltered warnings ===================================="
+    cat "$warning_file" >&2
+    echoerr "================================================================================"
+    rm -f "$warning_file"
     return 1
   fi
 
-  rm "$warnings"
+  rm -f "$warning_file"
 
   return $ret
 }
diff --git a/scripts/verify-tff-mapping b/scripts/verify-tff-mapping
index 302e50bf34f..08d9d7a33fd 100755
--- a/scripts/verify-tff-mapping
+++ b/scripts/verify-tff-mapping
@@ -188,6 +188,20 @@ tests = [
     explanation: 'GLFM spec and config files for CE and EE should map to respective markdown snapshot specs',
     source: 'glfm_specification/foo',
     expected: ['spec/requests/api/markdown_snapshot_spec.rb', 'ee/spec/requests/api/markdown_snapshot_spec.rb']
+  },
+
+  {
+    explanation: 'https://gitlab.com/gitlab-org/quality/engineering-productivity/master-broken-incidents/-/issues/287#note_1192008962',
+    # Note: The metrics seem to be changed every year or so, so this test will fail once a year or so.
+    #       You will need to change the metric below for another metric present in the project.
+    source: 'ee/config/metrics/counts_all/20221114065035_delete_merge_request.yml',
+    expected: ['ee/spec/config/metrics/every_metric_definition_spec.rb']
+  },
+
+  {
+    explanation: 'https://gitlab.com/gitlab-org/quality/engineering-productivity/master-broken-incidents/-/issues/287#note_1192008962',
+    source: 'ee/lib/ee/gitlab/usage_data_counters/known_events/common.yml',
+    expected: ['ee/spec/config/metrics/every_metric_definition_spec.rb']
   }
 ]