Add latest changes from gitlab-org/gitlab@14-4-stable-eev14.4.0-rc42

11 files changed, 194 insertions, 38 deletions

diff --git a/scripts/frontend/start_storybook.sh b/scripts/frontend/start_storybook.sh
new file mode 100755
index 00000000000..7ae0a21b15b
--- /dev/null
+++ b/scripts/frontend/start_storybook.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+bold=$(tput bold)
+normal=$(tput sgr0)
+
+echo -e "Storybook provides a mock server that allows creating stories for components that make HTTP requests."
+echo -e "${bold}Storybook will fail to start if it can’t find the fixtures used by the mock server.${normal}\n"
+read -rp "Would you like to generate/update the frontend fixtures used by the mock server (y/N)? " answer
+
+if [[ "$answer" =~ ^(Y|y)$ ]] ; then
+  bundle exec rake frontend:mock_server_fixtures
+fi
+
+if ! [[ -d storybook/node_modules ]] ; then
+  yarn storybook:install
+fi
+
+yarn --cwd ./storybook start
diff --git a/scripts/frontend/startup_css/startup_css_changed.sh b/scripts/frontend/startup_css/startup_css_changed.sh
index 2713d752974..db6fb575d1d 100755
--- a/scripts/frontend/startup_css/startup_css_changed.sh
+++ b/scripts/frontend/startup_css/startup_css_changed.sh
@@ -7,7 +7,13 @@ echo ""
 echo "https://gitlab.com/gitlab-org/gitlab/-/issues/331812"
 echo "-----------------------------------------------------------"
 
-startup_glob="*stylesheets/startup*"
+startup_glob="app/assets/stylesheets/startup*"
+
+if ! [ "$FOSS_ONLY" ]
+then
+  startup_glob="*${startup_glob}"
+fi
+
 
 echo "Staging changes to '${startup_glob}' so we can check for untracked files..."
 git add "${startup_glob}"
diff --git a/scripts/lint-doc.sh b/scripts/lint-doc.sh
index 0157f0c0812..e99b8a47301 100755
--- a/scripts/lint-doc.sh
+++ b/scripts/lint-doc.sh
@@ -67,7 +67,7 @@ fi
 
 # Do not use 'README.md', instead use 'index.md'
 # Number of 'README.md's as of 2021-08-17
-NUMBER_READMES=13
+NUMBER_READMES=0
 FIND_READMES=$(find doc/ -name "README.md" | wc -l)
 echo '=> Checking for new README.md files...'
 echo
@@ -81,6 +81,22 @@ then
   ((ERRORCODE++))
 fi
 
+# Do not use dashes (-) in directory names, use underscores (_) instead.
+# Number of directories with dashes as of 2021-09-17
+NUMBER_DASHES=2
+FIND_DASHES=$(find doc -type d -name "*-*" | wc -l)
+echo '=> Checking for directory names containing dashes...'
+echo
+if [ ${FIND_DASHES} -ne $NUMBER_DASHES ]
+then
+  echo
+  echo '  ✖ ERROR: The number of directory names containing dashes has changed. Use underscores instead of dashes for the directory names.' >&2
+  echo '  ✖        If removing a directory containing dashes, update NUMBER_DASHES in lint-doc.sh.' >&2
+  echo '  https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#work-with-directories-and-files'
+  echo
+  ((ERRORCODE++))
+fi
+
 # Run Vale and Markdownlint only on changed files. Only works on merged results
 # pipelines, so first checks if a merged results CI variable is present. If not present,
 # runs test on all files.
diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh
index f3b9ac56082..4b528696322 100644
--- a/scripts/prepare_build.sh
+++ b/scripts/prepare_build.sh
@@ -38,18 +38,6 @@ sed -i 's|url:.*$|url: redis://redis:6379|g' config/cable.yml
 cp config/resque.yml.example config/resque.yml
 sed -i 's|url:.*$|url: redis://redis:6379|g' config/resque.yml
 
-cp config/redis.cache.yml.example config/redis.cache.yml
-sed -i 's|url:.*$|url: redis://redis:6379/10|g' config/redis.cache.yml
-
-cp config/redis.queues.yml.example config/redis.queues.yml
-sed -i 's|url:.*$|url: redis://redis:6379/11|g' config/redis.queues.yml
-
-cp config/redis.shared_state.yml.example config/redis.shared_state.yml
-sed -i 's|url:.*$|url: redis://redis:6379/12|g' config/redis.shared_state.yml
-
-cp config/redis.trace_chunks.yml.example config/redis.trace_chunks.yml
-sed -i 's|url:.*$|url: redis://redis:6379/13|g' config/redis.trace_chunks.yml
-
 if [ "$SETUP_DB" != "false" ]; then
   setup_db
 elif getent hosts postgres; then
diff --git a/scripts/rails-update-bisect b/scripts/rails-update-bisect
new file mode 100755
index 00000000000..77a01e2dadc
--- /dev/null
+++ b/scripts/rails-update-bisect
@@ -0,0 +1,39 @@
+#!/bin/bash
+SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
+GITLAB_DIR="$(dirname "$SCRIPT_DIR")"
+
+# Because this script is intended to be run with `git bisect run`,
+# we are returning `-1` status code to alert `git bisect` of failures.
+#
+# See: https://git-scm.com/docs/git-bisect#_bisect_run
+#
+abort_bisect () {
+  exit -1
+}
+
+if [ $# -eq 0 ]; then
+  echo "No arguments supplied. Please provide spec file(s) as first argument(s)"
+  abort_bisect
+fi
+
+[[ -z "${RAILS_FOLDER}" ]] && { echo >&2 "RAILS_FOLDER env variable is not set"; abort_bisect; }
+
+if ! grep -q -E "gem 'rails'.+RAILS_VERSION.+RAILS_FOLDER" $GITLAB_DIR/Gemfile; then
+  echo "Gemfile is not modified"
+  echo "Please alter the gem 'rails' line in Gemfile with:"
+  echo "gem 'rails', ENV['RAILS_VERSION'], path: ENV['RAILS_FOLDER']"
+  abort_bisect
+fi
+
+export RAILS_VERSION=$(cat $RAILS_FOLDER/RAILS_VERSION)
+
+cd $GITLAB_DIR && \
+echo "Updating dependencies... this could take a while." && \
+bundle update rails --quiet
+
+test $? -eq 0 || { echo >&2 "bundle update has failed"; abort_bisect; }
+
+# Stop spring if it's installed
+command -v spring >/dev/null 2>&1 && spring stop
+
+bin/rspec $@
diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb
index 90dc0fd418e..e6efbca9e86 100755
--- a/scripts/review_apps/automated_cleanup.rb
+++ b/scripts/review_apps/automated_cleanup.rb
@@ -8,6 +8,10 @@ class AutomatedCleanup
   attr_reader :project_path, :gitlab_token
 
   DEPLOYMENTS_PER_PAGE = 100
+  ENVIRONMENT_PREFIX = {
+    review_app: 'review/',
+    docs_review_app: 'review-docs/'
+  }.freeze
   IGNORED_HELM_ERRORS = [
     'transport is closing',
     'error upgrading connection',
@@ -62,13 +66,14 @@ class AutomatedCleanup
 
     releases_to_delete = []
 
+    # Delete environments via deployments
     gitlab.deployments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc').auto_paginate do |deployment|
       break if Time.parse(deployment.created_at) < deployments_look_back_threshold
 
       environment = deployment.environment
 
       next unless environment
-      next unless environment.name.start_with?('review/')
+      next unless environment.name.start_with?(ENVIRONMENT_PREFIX[:review_app])
       next if checked_environments.include?(environment.slug)
 
       last_deploy = deployment.created_at
@@ -92,6 +97,10 @@ class AutomatedCleanup
       checked_environments << environment.slug
     end
 
+    delete_stopped_environments(environment_type: :review_app, checked_environments: checked_environments, last_updated_threshold: delete_threshold) do |environment|
+      releases_to_delete << Tooling::Helm3Client::Release.new(environment.slug, 1, environment.updated_at, nil, nil, review_apps_namespace)
+    end
+
     delete_helm_releases(releases_to_delete)
   end
 
@@ -102,14 +111,12 @@ class AutomatedCleanup
     stop_threshold = threshold_time(days: days_for_stop)
     delete_threshold = threshold_time(days: days_for_delete)
 
-    max_delete_count = 1000
-    delete_count = 0
-
+    # Delete environments via deployments
     gitlab.deployments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc').auto_paginate do |deployment|
       environment = deployment.environment
 
       next unless environment
-      next unless environment.name.start_with?('review-docs/')
+      next unless environment.name.start_with?(ENVIRONMENT_PREFIX[:docs_review_app])
       next if checked_environments.include?(environment.slug)
 
       last_deploy = deployment.created_at
@@ -120,15 +127,12 @@ class AutomatedCleanup
         stop_environment(environment, deployment) if environment_state && environment_state != 'stopped'
       end
 
-      if deployed_at < delete_threshold
-        delete_environment(environment, deployment)
-        delete_count += 1
-
-        break if delete_count > max_delete_count
-      end
+      delete_environment(environment, deployment) if deployed_at < delete_threshold
 
       checked_environments << environment.slug
     end
+
+    delete_stopped_environments(environment_type: :docs_review_app, checked_environments: checked_environments, last_updated_threshold: delete_threshold)
   end
 
   def perform_helm_releases_cleanup!(days:)
@@ -171,8 +175,9 @@ class AutomatedCleanup
     nil
   end
 
-  def delete_environment(environment, deployment)
-    print_release_state(subject: 'Review app', release_name: environment.slug, release_date: deployment.created_at, action: 'deleting')
+  def delete_environment(environment, deployment = nil)
+    release_date = deployment ? deployment.created_at : environment.updated_at
+    print_release_state(subject: 'Review app', release_name: environment.slug, release_date: release_date, action: 'deleting')
     gitlab.delete_environment(project_path, environment.id)
 
   rescue Gitlab::Error::Forbidden
@@ -187,6 +192,24 @@ class AutomatedCleanup
     puts "Review app '#{environment.name}' / '#{environment.slug}' (##{environment.id}) is forbidden: skipping it"
   end
 
+  def delete_stopped_environments(environment_type:, checked_environments:, last_updated_threshold:)
+    gitlab.environments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc', states: 'stopped', search: ENVIRONMENT_PREFIX[environment_type]).auto_paginate do |environment|
+      next if skip_environment?(environment: environment, checked_environments: checked_environments, last_updated_threshold: last_updated_threshold, environment_type: environment_type)
+
+      yield environment if delete_environment(environment)
+
+      checked_environments << environment.slug
+    end
+  end
+
+  def skip_environment?(environment:, checked_environments:, last_updated_threshold:, environment_type:)
+    return true unless environment.name.start_with?(ENVIRONMENT_PREFIX[environment_type])
+    return true if checked_environments.include?(environment.slug)
+    return true if Time.parse(environment.updated_at) > last_updated_threshold
+
+    false
+  end
+
   def helm_releases
     args = ['--all', '--date']
 
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh
index 797d9188f81..accc52a7ece 100644
--- a/scripts/rspec_helpers.sh
+++ b/scripts/rspec_helpers.sh
@@ -98,7 +98,7 @@ function rspec_simple_job() {
 }
 
 function rspec_db_library_code() {
-  local db_files="spec/lib/gitlab/database/ spec/support/helpers/database/"
+  local db_files="spec/lib/gitlab/database/"
 
   rspec_simple_job "-- ${db_files}"
 }
@@ -109,14 +109,18 @@ function rspec_paralellized_job() {
   local test_level="${job_name[1]}"
   local report_name=$(echo "${CI_JOB_NAME}" | sed -E 's|[/ ]|_|g') # e.g. 'rspec unit pg12 1/24' would become 'rspec_unit_pg12_1_24'
   local rspec_opts="${1}"
-  local spec_folder_prefix=""
+  local spec_folder_prefixes=""
 
   if [[ "${test_tool}" =~ "-ee" ]]; then
-    spec_folder_prefix="ee/"
+    spec_folder_prefixes="'ee/'"
   fi
 
   if [[ "${test_tool}" =~ "-jh" ]]; then
-    spec_folder_prefix="jh/"
+    spec_folder_prefixes="'jh/'"
+  fi
+
+  if [[ "${test_tool}" =~ "-all" ]]; then
+    spec_folder_prefixes="['', 'ee/', 'jh/']"
   fi
 
   export KNAPSACK_LOG_LEVEL="debug"
@@ -131,7 +135,7 @@ function rspec_paralellized_job() {
   cp "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "${KNAPSACK_REPORT_PATH}"
 
   if [[ -z "${KNAPSACK_TEST_FILE_PATTERN}" ]]; then
-    pattern=$(ruby -r./tooling/quality/test_level.rb -e "puts Quality::TestLevel.new(%(${spec_folder_prefix})).pattern(:${test_level})")
+    pattern=$(ruby -r./tooling/quality/test_level.rb -e "puts Quality::TestLevel.new(${spec_folder_prefixes}).pattern(:${test_level})")
     export KNAPSACK_TEST_FILE_PATTERN="${pattern}"
   fi
 
diff --git a/scripts/rubocop-max-files-in-cache-check b/scripts/rubocop-max-files-in-cache-check
new file mode 100755
index 00000000000..5b422d0a0f4
--- /dev/null
+++ b/scripts/rubocop-max-files-in-cache-check
@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'yaml'
+
+MINIMUM_MAX_FILES_IN_CACHE_MARGIN = 1.05
+RECOMMENDED_MAX_FILES_IN_CACHE_MARGIN = 1.25
+RUBOCOP_LIST_TARGET_FILES_COMMAND = 'bundle exec rubocop --list-target-files | wc -l'
+
+RuboCopMaxFilesInCacheIsTooSmall = Class.new(StandardError)
+
+rubocop_target_files_count = `#{RUBOCOP_LIST_TARGET_FILES_COMMAND}`.strip.to_i
+
+raise Error, "#{RUBOCOP_LIST_TARGET_FILES_COMMAND} failed with status #{$?}!" if rubocop_target_files_count == 0
+
+rubocop_target_files_count = rubocop_target_files_count.to_i
+rubocop_current_max_files_in_cache = YAML.load_file(File.expand_path('../.rubocop.yml', __dir__)).dig('AllCops', 'MaxFilesInCache').to_i
+minimum_max_files_in_cache = (rubocop_target_files_count * MINIMUM_MAX_FILES_IN_CACHE_MARGIN).round(-3)
+
+# We want AllCops.MaxFilesInCache to be at least 5% above the actual files count at any time to give us enough time to increase it accordingly
+if rubocop_current_max_files_in_cache <= minimum_max_files_in_cache
+  recommended_max_files_in_cache = (rubocop_target_files_count * RECOMMENDED_MAX_FILES_IN_CACHE_MARGIN).round(-3)
+  raise RuboCopMaxFilesInCacheIsTooSmall, "Current count of RuboCop target file is #{rubocop_target_files_count} but AllCops.MaxFilesInCache is set to #{rubocop_current_max_files_in_cache}. We recommend to increase it to #{recommended_max_files_in_cache}."
+else
+  puts "Current count of RuboCop target file is #{rubocop_target_files_count} and AllCops.MaxFilesInCache is set to #{rubocop_current_max_files_in_cache}. All good."
+  exit(0)
+end
diff --git a/scripts/static-analysis b/scripts/static-analysis
index de5a1b407f9..f50e4a24b58 100755
--- a/scripts/static-analysis
+++ b/scripts/static-analysis
@@ -54,6 +54,7 @@ class StaticAnalysis
     Task.new(%w[bin/rake gettext:lint], 85),
     Task.new(%W[bundle exec license_finder --decisions-file config/dependency_decisions.yml --project-path #{project_path}], 20),
     Task.new(%w[bin/rake lint:static_verification], 35),
+    Task.new(%w[scripts/rubocop-max-files-in-cache-check], 20),
     Task.new(%w[bin/rake config_lint], 10),
     Task.new(%w[bin/rake gitlab:sidekiq:all_queues_yml:check], 15),
     (Gitlab.ee? ? Task.new(%w[bin/rake gitlab:sidekiq:sidekiq_queues_yml:check], 11) : nil),
@@ -65,7 +66,9 @@ class StaticAnalysis
   ].compact.freeze
 
   def run_tasks!(options = {})
-    node_assignment = tasks_to_run((ENV['CI_NODE_TOTAL'] || 1).to_i)[(ENV['CI_NODE_INDEX'] || 1).to_i - 1]
+    total_nodes = (ENV['CI_NODE_TOTAL'] || 1).to_i
+    current_node_number = (ENV['CI_NODE_INDEX'] || 1).to_i
+    node_assignment = tasks_to_run(total_nodes)[current_node_number - 1]
 
     if options[:dry_run]
       puts "Dry-run mode!"
diff --git a/scripts/utils.sh b/scripts/utils.sh
index d2e8c151438..faabc151963 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -60,7 +60,7 @@ function setup_db_user_only() {
 
 function setup_db() {
   run_timed_command "setup_db_user_only"
-  run_timed_command "bundle exec rake db:drop db:create db:structure:load db:migrate gitlab:db:setup_ee"
+  run_timed_command_with_metric "bundle exec rake db:drop db:create db:structure:load db:migrate gitlab:db:setup_ee" "setup_db"
 }
 
 function install_api_client_dependencies_with_apk() {
@@ -78,15 +78,27 @@ function install_tff_gem() {
 
 function run_timed_command() {
   local cmd="${1}"
+  local metric_name="${2}"
+  local timed_metric_file
   local start=$(date +%s)
+
   echosuccess "\$ ${cmd}"
   eval "${cmd}"
+
   local ret=$?
   local end=$(date +%s)
   local runtime=$((end-start))
 
   if [[ $ret -eq 0 ]]; then
     echosuccess "==> '${cmd}' succeeded in ${runtime} seconds."
+
+    if [[ -n "${metric_name}" ]]; then
+      timed_metric_file=$(timed_metric_file $metric_name)
+      echo "# TYPE ${metric_name} gauge" > "${timed_metric_file}"
+      echo "# UNIT ${metric_name} seconds" >> "${timed_metric_file}"
+      echo "${metric_name} ${runtime}" >> "${timed_metric_file}"
+    fi
+
     return 0
   else
     echoerr "==> '${cmd}' failed (${ret}) in ${runtime} seconds."
@@ -94,6 +106,26 @@ function run_timed_command() {
   fi
 }
 
+function run_timed_command_with_metric() {
+  local cmd="${1}"
+  local metric_name="${2}"
+  local metrics_file=${METRICS_FILE:-metrics.txt}
+
+  run_timed_command "${cmd}" "${metric_name}"
+
+  local ret=$?
+
+  cat $(timed_metric_file $metric_name) >> "${metrics_file}"
+
+  return $ret
+}
+
+function timed_metric_file() {
+  local metric_name="${1}"
+
+  echo "$(pwd)/tmp/duration_${metric_name}.txt"
+}
+
 function echoerr() {
   local header="${2}"
 
diff --git a/scripts/verify-tff-mapping b/scripts/verify-tff-mapping
index ca73d4f5f7a..ee86f9ecde5 100755
--- a/scripts/verify-tff-mapping
+++ b/scripts/verify-tff-mapping
@@ -102,14 +102,14 @@ tests = [
 
   {
     explanation: 'Migration should map to its non-timestamped spec',
-    source: 'db/migrate/20191023152913_add_default_and_free_plans.rb',
-    expected: ['spec/migrations/add_default_and_free_plans_spec.rb']
+    source: 'db/migrate/20200116175538_update_timestamp_softwarelicensespolicy.rb',
+    expected: ['spec/migrations/update_timestamp_softwarelicensespolicy_spec.rb']
   },
 
   {
     explanation: 'Migration should map to its timestamped spec',
-    source: 'db/post_migrate/20190924152703_migrate_issue_trackers_data.rb',
-    expected: ['spec/migrations/20190924152703_migrate_issue_trackers_data_spec.rb']
+    source: 'db/post_migrate/20210915022415_cleanup_bigint_conversion_for_ci_builds.rb',
+    expected: ['spec/migrations/20210915022415_cleanup_bigint_conversion_for_ci_builds_spec.rb']
   },
 
   {