Add latest changes from gitlab-org/gitlab@13-11-stable-eev13.11.0-rc43

11 files changed, 243 insertions, 72 deletions

diff --git a/scripts/gitaly-test-build b/scripts/gitaly-test-build
index bb561e2906a..849c08df527 100755
--- a/scripts/gitaly-test-build
+++ b/scripts/gitaly-test-build
@@ -13,6 +13,8 @@ class GitalyTestBuild
   include GitalyTest
 
   def run
+    set_bundler_config
+
     abort 'gitaly build failed' unless build_gitaly
 
     ensure_gitlab_shell_secret!
diff --git a/scripts/gitaly-test-spawn b/scripts/gitaly-test-spawn
index 8547d0b13e4..7cb9ea803f8 100755
--- a/scripts/gitaly-test-spawn
+++ b/scripts/gitaly-test-spawn
@@ -9,6 +9,7 @@ class GitalyTestSpawn
   include GitalyTest
 
   def run
+    set_bundler_config
     install_gitaly_gems if ENV['CI']
     check_gitaly_config!
 
diff --git a/scripts/gitaly_test.rb b/scripts/gitaly_test.rb
index 2262870eb96..b7ea9cd628e 100644
--- a/scripts/gitaly_test.rb
+++ b/scripts/gitaly_test.rb
@@ -12,11 +12,11 @@ require 'logger'
 
 module GitalyTest
   LOGGER = begin
-             default_name = ENV['CI'] ? 'DEBUG' : 'WARN'
-             level_name = ENV['GITLAB_TESTING_LOG_LEVEL']&.upcase
-             level = Logger.const_get(level_name || default_name, true) # rubocop: disable Gitlab/ConstGetInheritFalse
-             Logger.new(STDOUT, level: level, formatter: ->(_, _, _, msg) { msg })
-           end
+    default_name = ENV['CI'] ? 'DEBUG' : 'WARN'
+    level_name = ENV['GITLAB_TESTING_LOG_LEVEL']&.upcase
+    level = Logger.const_get(level_name || default_name, true) # rubocop: disable Gitlab/ConstGetInheritFalse
+    Logger.new(STDOUT, level: level, formatter: ->(_, _, _, msg) { msg })
+  end
 
   def tmp_tests_gitaly_dir
     File.expand_path('../tmp/tests/gitaly', __dir__)
@@ -34,16 +34,19 @@ module GitalyTest
     File.join(tmp_tests_gitaly_dir, 'ruby', 'Gemfile')
   end
 
+  def gemfile_dir
+    File.dirname(gemfile)
+  end
+
   def gitlab_shell_secret_file
     File.join(tmp_tests_gitlab_shell_dir, '.gitlab_shell_secret')
   end
 
   def env
-    env_hash = {
+    {
       'HOME' => File.expand_path('tmp/tests'),
       'GEM_PATH' => Gem.path.join(':'),
-      'BUNDLE_APP_CONFIG' => File.join(File.dirname(gemfile), '.bundle/config'),
-      'BUNDLE_FLAGS' => "--jobs=4 --retry=3",
+      'BUNDLE_APP_CONFIG' => File.join(gemfile_dir, '.bundle'),
       'BUNDLE_INSTALL_FLAGS' => nil,
       'BUNDLE_GEMFILE' => gemfile,
       'RUBYOPT' => nil,
@@ -51,14 +54,19 @@ module GitalyTest
       # Git hooks can't run during tests as the internal API is not running.
       'GITALY_TESTING_NO_GIT_HOOKS' => "1"
     }
+  end
+
+  # rubocop:disable GitlabSecurity/SystemCommandInjection
+  def set_bundler_config
+    system('bundle config set --local jobs 4', chdir: gemfile_dir)
+    system('bundle config set --local retry 3', chdir: gemfile_dir)
 
     if ENV['CI']
       bundle_path = File.expand_path('../vendor/gitaly-ruby', __dir__)
-      env_hash['BUNDLE_FLAGS'] += " --path=#{bundle_path}"
+      system('bundle', 'config', 'set', '--local', 'path', bundle_path, chdir: gemfile_dir)
     end
-
-    env_hash
   end
+  # rubocop:enable GitlabSecurity/SystemCommandInjection
 
   def config_path(service)
     case service
diff --git a/scripts/lint-rugged b/scripts/lint-rugged
index 038fd5199c2..ae5eddbe3b4 100755
--- a/scripts/lint-rugged
+++ b/scripts/lint-rugged
@@ -29,6 +29,7 @@ ALLOWED = [
   'config/initializers/lograge.rb',
   'lib/gitlab/grape_logging/loggers/perf_logger.rb',
   'lib/gitlab/instrumentation_helper.rb',
+  'lib/gitlab/sidekiq_middleware/instrumentation_logger.rb',
   'lib/gitlab/rugged_instrumentation.rb',
   'lib/peek/views/rugged.rb'
 ].freeze
diff --git a/scripts/perf/query_limiting_report.rb b/scripts/perf/query_limiting_report.rb
new file mode 100755
index 00000000000..2f263eeb567
--- /dev/null
+++ b/scripts/perf/query_limiting_report.rb
@@ -0,0 +1,167 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+####
+# Prints a report which helps reconcile occurrences of the `QueryLimiting.disable(ISSUE_LINK)`
+# allowlist block against the corresponding open issues.
+#
+# If everything is consistent, the script should ideally not report any issues or code lines,
+# other than possibly remaining "calls with no issue iid" which use variables/etc.
+#
+# - See https://gitlab.com/gitlab-org/gitlab/-/issues/325640
+# - See https://gitlab.com/groups/gitlab-org/-/epics/5670
+
+require 'rubygems'
+require 'gitlab'
+require 'optparse'
+
+class QueryLimitingReport
+  GITLAB_PROJECT_ID = 278964 # gitlab-org/gitlab project
+  ISSUES_SEARCH_LABEL = 'querylimiting-disable'
+  CODE_LINES_SEARCH_STRING = 'QueryLimiting.disable'
+  PAGINATION_LIMIT = 500
+
+  DEFAULT_OPTIONS = {
+    api_token: ENV['API_TOKEN']
+  }.freeze
+
+  def initialize(options)
+    @options = options
+
+    Gitlab.configure do |config|
+      config.endpoint = 'https://gitlab.com/api/v4'
+      config.private_token = options.fetch(:api_token)
+    end
+  end
+
+  def execute
+    # PLAN:
+    # Read all issues matching criteria and extract array of issue iids
+    # Find all code references and extract issue iids
+    # Print list of all issues without code references
+    # Print list of all code references issue iids that don't have search label
+    # Print list of all code references with no issue iids (i.e. dynamic or variable argument)
+
+    total_issues = find_issues_by_label(ISSUES_SEARCH_LABEL)
+    issues = total_issues.select { |issue| issue[:state] == 'opened' }
+    code_lines = find_code_lines
+
+    code_lines_grouped = code_lines.group_by { |code_line| code_line[:has_issue_iid] }
+    code_lines_without_issue_iid = code_lines_grouped[false]
+    code_lines_with_issue_iid = code_lines_grouped[true]
+
+    all_issue_iids_in_code_lines = code_lines_with_issue_iid.map { |line| line[:issue_iid] }
+
+    issues_without_code_references = issues.reject do |issue|
+      all_issue_iids_in_code_lines.include?(issue[:iid])
+    end
+
+    all_issue_iids = issues.map { |issue| issue[:iid] }
+    code_lines_with_missing_issues = code_lines_with_issue_iid.reject do |code_line|
+      all_issue_iids.include?(code_line[:issue_iid])
+    end
+
+    puts "\n\n\nREPORT:"
+
+    puts "\n\nFound #{total_issues.length} total issues with '#{ISSUES_SEARCH_LABEL}' search label, #{issues.length} are still opened..."
+    puts "\n\nFound #{code_lines.length} total occurrences of '#{CODE_LINES_SEARCH_STRING}' in code..."
+
+    puts "\n" + '-' * 80
+
+    puts "\n\nIssues without any '#{CODE_LINES_SEARCH_STRING}' code references (#{issues_without_code_references.length} total):"
+    pp issues_without_code_references
+
+    puts "\n" + '-' * 80
+
+    puts "\n\n'#{CODE_LINES_SEARCH_STRING}' calls with references to an issue which doesn't have '#{ISSUES_SEARCH_LABEL}' search label (#{code_lines_with_missing_issues.length} total):"
+    pp code_lines_with_missing_issues
+
+    puts "\n" + '-' * 80
+
+    puts "\n\n'#{CODE_LINES_SEARCH_STRING}' calls with no issue iid (#{code_lines_without_issue_iid&.length || 0} total):"
+    pp code_lines_without_issue_iid
+  end
+
+  private
+
+  attr_reader :options
+
+  def find_issues_by_label(label)
+    issues = []
+
+    puts("Finding issues by label #{label}...")
+    paginated_issues = Gitlab.issues(GITLAB_PROJECT_ID, 'labels' => label)
+    paginated_issues.paginate_with_limit(PAGINATION_LIMIT) do |item|
+      item_hash = item.to_hash
+
+      issue_iid = item_hash.fetch('iid')
+      issue = {
+        iid: issue_iid,
+        state: item_hash.fetch('state'),
+        title: item_hash.fetch('title'),
+        issue_url: "https://gitlab.com/gitlab-org/gitlab/issues/#{issue_iid}"
+      }
+
+      issues << issue
+    end
+
+    issues
+  end
+
+  def find_code_lines
+    code_lines = []
+
+    puts("Finding code lines...")
+    paginated_blobs = Gitlab.search_in_project(GITLAB_PROJECT_ID, 'blobs', CODE_LINES_SEARCH_STRING)
+    paginated_blobs.paginate_with_limit(PAGINATION_LIMIT) do |item|
+      item_hash = item.to_hash
+
+      filename = item_hash.fetch('filename')
+      next if filename !~ /\.rb\Z/
+
+      file_contents = Gitlab.file_contents(GITLAB_PROJECT_ID, filename)
+      file_lines = file_contents.split("\n")
+
+      file_lines.each_index do |index|
+        line = file_lines[index]
+        if line =~ /#{CODE_LINES_SEARCH_STRING}/
+          issue_iid = line.slice(%r{issues/(\d+)\D}, 1)
+          line_number = index + 1
+          code_line = {
+            file_location: "#{filename}:#{line_number}",
+            filename: filename,
+            line_number: line_number,
+            line: line,
+            issue_iid: issue_iid.to_i,
+            has_issue_iid: !issue_iid.nil?
+          }
+          code_lines << code_line
+        end
+      end
+    end
+
+    code_lines.sort_by! { |line| "#{line[:filename]}-#{line[:line_number].to_s.rjust(4, '0')}" }
+    code_lines.map do |line|
+      line.delete(:filename)
+      line.delete(:line_number)
+      line
+    end
+  end
+end
+
+if $0 == __FILE__
+  options = QueryLimitingReport::DEFAULT_OPTIONS.dup
+
+  OptionParser.new do |opts|
+    opts.on("-t", "--api-token API_TOKEN", String, "A value API token with the `read_api` scope. Can be set as an env variable 'API_TOKEN'.") do |value|
+      options[:api_token] = value
+    end
+
+    opts.on("-h", "--help", "Prints this help") do
+      puts opts
+      exit
+    end
+  end.parse!
+
+  QueryLimitingReport.new(options).execute
+end
diff --git a/scripts/review_apps/base-config.yaml b/scripts/review_apps/base-config.yaml
index cf55fca7452..7daf3f80efc 100644
--- a/scripts/review_apps/base-config.yaml
+++ b/scripts/review_apps/base-config.yaml
@@ -70,10 +70,10 @@ gitlab:
     resources:
       requests:
         cpu: 746m
-        memory: 1873M
+        memory: 2809M
       limits:
         cpu: 1119m
-        memory: 2809M
+        memory: 4214M
     deployment:
       readinessProbe:
         initialDelaySeconds: 5  # Default is 0
@@ -83,10 +83,10 @@ gitlab:
       resources:
         requests:
           cpu: 400m
-          memory: 50M
+          memory: 75M
         limits:
           cpu: 600m
-          memory: 75M
+          memory: 113M
       readinessProbe:
         initialDelaySeconds: 5  # Default is 0
         periodSeconds: 15  # Default is 10
@@ -137,10 +137,10 @@ postgresql:
     enabled: false
   resources:
     requests:
-      cpu: 550m
+      cpu: 600m
       memory: 1000M
     limits:
-      cpu: 825m
+      cpu: 1300m
       memory: 1500M
 prometheus:
   install: false
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh
index d9957fb6ced..e23231e429b 100644
--- a/scripts/rspec_helpers.sh
+++ b/scripts/rspec_helpers.sh
@@ -97,6 +97,10 @@ function rspec_paralellized_job() {
     spec_folder_prefix="ee/"
   fi
 
+  if [[ "${test_tool}" =~ "-jh" ]]; then
+    spec_folder_prefix="jh/"
+  fi
+
   export KNAPSACK_LOG_LEVEL="debug"
   export KNAPSACK_REPORT_PATH="knapsack/${report_name}_report.json"
 
@@ -109,7 +113,7 @@ function rspec_paralellized_job() {
   cp "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "${KNAPSACK_REPORT_PATH}"
 
   if [[ -z "${KNAPSACK_TEST_FILE_PATTERN}" ]]; then
-    pattern=$(ruby -r./lib/quality/test_level.rb -e "puts Quality::TestLevel.new(%(${spec_folder_prefix})).pattern(:${test_level})")
+    pattern=$(ruby -r./tooling/quality/test_level.rb -e "puts Quality::TestLevel.new(%(${spec_folder_prefix})).pattern(:${test_level})")
     export KNAPSACK_TEST_FILE_PATTERN="${pattern}"
   fi
 
diff --git a/scripts/static-analysis b/scripts/static-analysis
index 2442455e630..136b2966244 100755
--- a/scripts/static-analysis
+++ b/scripts/static-analysis
@@ -33,7 +33,7 @@ class StaticAnalysis
     %w[bin/rake gitlab:sidekiq:all_queues_yml:check] => 13,
     (Gitlab.ee? ? %w[bin/rake gitlab:sidekiq:sidekiq_queues_yml:check] : nil) => 13,
     %w[bin/rake config_lint] => 11,
-    %w[yarn run lint:stylelint] => 9,
+    %w[yarn run internal:stylelint] => 9,
     %w[scripts/lint-conflicts.sh] => 0.59,
     %w[yarn run block-dependencies] => 0.35,
     %w[scripts/lint-rugged] => 0.23,
diff --git a/scripts/trigger-build b/scripts/trigger-build
index 29d53609026..3a9301bda3b 100755
--- a/scripts/trigger-build
+++ b/scripts/trigger-build
@@ -136,13 +136,15 @@ module Trigger
 
     def extra_variables
       # Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results
-      # and fallback to CI_COMMIT_SHA for the `detached` pipelines.
-      # We also set IMAGE_TAG so the GitLab and QA docker images are tagged with
-      # that SHA.
+      # and fallback to CI_COMMIT_SHA (merged result commit) for the non-MR pipelines.
+      # See https://docs.gitlab.com/ee/development/testing_guide/end_to_end/index.html#with-pipeline-for-merged-results.
+      # We also set IMAGE_TAG so the GitLab Docker image is tagged with that SHA.
       source_sha = Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA']
       {
         'GITLAB_VERSION' => source_sha,
         'IMAGE_TAG' => source_sha,
+        'QA_IMAGE' => "#{ENV['CI_REGISTRY']}/#{ENV['CI_PROJECT_PATH']}/gitlab-ee-qa:#{ENV['CI_COMMIT_REF_SLUG']}",
+        'SKIP_QA_DOCKER' => 'true',
         'ALTERNATIVE_SOURCES' => 'true',
         'SECURITY_SOURCES' => Trigger.security? ? 'true' : 'false',
         'ee' => Trigger.ee? ? 'true' : 'false',
@@ -214,15 +216,7 @@ module Trigger
     => If something doesn't work, drop a line in the #docs chat channel.
     MSG
 
-    # Create a remote branch in gitlab-docs and immediately cancel the pipeline
-    # to avoid race conditions, since a triggered pipeline will also run right
-    # after the branch creation. This only happens the very first time a branch
-    # is created and will be skipped in subsequent runs. Read more in
-    # https://gitlab.com/gitlab-org/gitlab-docs/issues/154.
-    #
     def deploy!
-      create_remote_branch!
-      cancel_latest_pipeline!
       invoke!.wait!
       display_success_message
     end
@@ -231,31 +225,52 @@ module Trigger
     # Remove a remote branch in gitlab-docs.
     #
     def cleanup!
-      gitlab_client(:downstream).delete_branch(downstream_project_path, ref)
-      puts "=> Remote branch '#{downstream_project_path}' deleted"
+      environment = gitlab_client(:downstream).environments(downstream_project_path, name: downstream_environment).first
+      return unless environment
+
+      environment = gitlab_client(:downstream).stop_environment(downstream_project_path, environment.id)
+      if environment.state == 'stopped'
+        puts "=> Downstream environment '#{downstream_environment}' stopped"
+      else
+        puts "=> Downstream environment '#{downstream_environment}' failed to stop."
+      end
     end
 
     private
 
+    def downstream_environment
+      "review/#{ref}#{review_slug}"
+    end
+
+    # We prepend the `-` here because we cannot use variable substitution in `environment.name`/`environment.url`
+    # Some projects (e.g. `omnibus-gitlab`) use this script for branch pipelines, so we fallback to using `CI_COMMIT_REF_SLUG` for those cases.
+    def review_slug
+      identifier = ENV['CI_MERGE_REQUEST_IID'] || ENV['CI_COMMIT_REF_SLUG']
+
+      "-#{project_slug}-#{identifier}"
+    end
+
     def downstream_project_path
       ENV['DOCS_PROJECT_PATH'] || 'gitlab-org/gitlab-docs'
     end
 
     def ref
-      if ENV['CI_MERGE_REQUEST_IID'].nil?
-        "docs-preview-#{slug}-#{ENV['CI_COMMIT_REF_SLUG']}"
-      else
-        "docs-preview-#{slug}-#{ENV['CI_MERGE_REQUEST_IID']}"
-      end
+      ENV['DOCS_BRANCH'] || 'master'
+    end
+
+    # `gitlab-org/gitlab-docs` pipeline trigger "Triggered from gitlab-org/gitlab 'review-docs-deploy' job"
+    def trigger_token
+      ENV['DOCS_TRIGGER_TOKEN']
     end
 
     def extra_variables
       {
-        "BRANCH_#{slug.upcase}" => ENV['CI_COMMIT_REF_NAME']
+        "BRANCH_#{project_slug.upcase}" => ENV['CI_COMMIT_REF_NAME'],
+        "REVIEW_SLUG" => review_slug
       }
     end
 
-    def slug
+    def project_slug
       case ENV['CI_PROJECT_PATH']
       when 'gitlab-org/gitlab-foss'
         'ce'
@@ -270,37 +285,14 @@ module Trigger
       end
     end
 
+    # app_url is the URL of the `gitlab-docs` Review App URL defined in
+    # https://gitlab.com/gitlab-org/gitlab-docs/-/blob/b38038132cf82a24271bbb294dead7c2f529e275/.gitlab-ci.yml#L383
     def app_url
-      "http://#{ref}.#{ENV['DOCS_REVIEW_APPS_DOMAIN']}/#{slug}"
-    end
-
-    def create_remote_branch!
-      gitlab_client(:downstream).create_branch(downstream_project_path, ref, 'master')
-      puts "=> Remote branch '#{ref}' created"
-    rescue Gitlab::Error::BadRequest
-      puts "=> Remote branch '#{ref}' already exists!"
-    end
-
-    def cancel_latest_pipeline!
-      pipelines = nil
-
-      # Wait until the pipeline is started
-      loop do
-        sleep 1
-        puts "=> Waiting for pipeline to start..."
-        pipelines = gitlab_client(:downstream).pipelines(downstream_project_path, { ref: ref })
-        break if pipelines.any?
-      end
-
-      # Get the first pipeline ID which should be the only one for the branch
-      pipeline_id = pipelines.first.id
-
-      # Cancel the pipeline
-      gitlab_client(:downstream).cancel_pipeline(downstream_project_path, pipeline_id)
+      "http://#{ref}#{review_slug}.#{ENV['DOCS_REVIEW_APPS_DOMAIN']}/#{project_slug}"
     end
 
     def display_success_message
-      format(SUCCESS_MESSAGE, app_url: app_url)
+      puts format(SUCCESS_MESSAGE, app_url: app_url)
     end
   end
 
diff --git a/scripts/utils.sh b/scripts/utils.sh
index c598afc4582..2e9839e4df8 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -26,13 +26,9 @@ function install_api_client_dependencies_with_apk() {
   apk add --update openssl curl jq
 }
 
-function install_api_client_dependencies_with_apt() {
-  apt update && apt install jq -y
-}
-
 function install_gitlab_gem() {
   gem install httparty --no-document --version 0.18.1
-  gem install gitlab --no-document --version 4.14.1
+  gem install gitlab --no-document --version 4.17.0
 }
 
 function install_tff_gem() {
diff --git a/scripts/verify-tff-mapping b/scripts/verify-tff-mapping
index 5bc4b23f99f..e18a14e17d6 100755
--- a/scripts/verify-tff-mapping
+++ b/scripts/verify-tff-mapping
@@ -8,7 +8,7 @@ require 'set'
 # The verification depend on the presence of actual test files,
 # so they would fail if one of the test files mentioned here is deleted.
 # To minimize the chance of this test failing due to unrelated changes,
-# the test files are chosen to be critical files that are unlikely to be deleted in a typical Merge Request
+# the test files are chosen to be critical files that are unlikely to be deleted in a typical merge request
 tests = [
   {
     explanation: 'EE code should map to respective spec',