Add latest changes from gitlab-org/gitlab@14-2-stable-eev14.2.0-rc42

409 files changed, 9005 insertions, 2859 deletions

diff --git a/lib/after_commit_queue.rb b/lib/after_commit_queue.rb
index aea4231205d..2698d7adbd7 100644
--- a/lib/after_commit_queue.rb
+++ b/lib/after_commit_queue.rb
@@ -15,7 +15,7 @@ module AfterCommitQueue
   end
 
   def run_after_commit_or_now(&block)
-    if Gitlab::Database.inside_transaction?
+    if Gitlab::Database.main.inside_transaction?
       if ActiveRecord::Base.connection.current_transaction.records&.include?(self)
         run_after_commit(&block)
       else
diff --git a/lib/api/api.rb b/lib/api/api.rb
index f9e89191a36..40f1b2fa9d3 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -153,10 +153,14 @@ module API
       mount ::API::Branches
       mount ::API::BroadcastMessages
       mount ::API::BulkImports
+      mount ::API::Ci::JobArtifacts
+      mount ::API::Ci::Jobs
       mount ::API::Ci::Pipelines
       mount ::API::Ci::PipelineSchedules
       mount ::API::Ci::Runner
       mount ::API::Ci::Runners
+      mount ::API::Ci::Triggers
+      mount ::API::Ci::Variables
       mount ::API::Commits
       mount ::API::CommitStatuses
       mount ::API::ContainerRegistryEvent
@@ -184,14 +188,13 @@ module API
       mount ::API::GroupMilestones
       mount ::API::Groups
       mount ::API::GroupContainerRepositories
+      mount ::API::GroupDebianDistributions
       mount ::API::GroupVariables
       mount ::API::ImportBitbucketServer
       mount ::API::ImportGithub
       mount ::API::IssueLinks
       mount ::API::Invitations
       mount ::API::Issues
-      mount ::API::JobArtifacts
-      mount ::API::Jobs
       mount ::API::Keys
       mount ::API::Labels
       mount ::API::Lint
@@ -268,14 +271,12 @@ module API
       mount ::API::Tags
       mount ::API::Templates
       mount ::API::Todos
-      mount ::API::Triggers
       mount ::API::Unleash
       mount ::API::UsageData
       mount ::API::UsageDataQueries
       mount ::API::UsageDataNonSqlMetrics
       mount ::API::UserCounts
       mount ::API::Users
-      mount ::API::Variables
       mount ::API::Version
       mount ::API::Wikis
     end
diff --git a/lib/api/appearance.rb b/lib/api/appearance.rb
index fe498bf611b..1eaa4167a7d 100644
--- a/lib/api/appearance.rb
+++ b/lib/api/appearance.rb
@@ -48,3 +48,5 @@ module API
     end
   end
 end
+
+API::Appearance.prepend_mod
diff --git a/lib/api/badges.rb b/lib/api/badges.rb
index 04f155be4e1..d7c850c2f40 100644
--- a/lib/api/badges.rb
+++ b/lib/api/badges.rb
@@ -8,7 +8,7 @@ module API
 
     helpers ::API::Helpers::BadgesHelpers
 
-    feature_category :continuous_integration
+    feature_category :projects
 
     helpers do
       def find_source_if_admin(source_type)
diff --git a/lib/api/bulk_imports.rb b/lib/api/bulk_imports.rb
index 189851cee65..0705a8285c1 100644
--- a/lib/api/bulk_imports.rb
+++ b/lib/api/bulk_imports.rb
@@ -8,7 +8,10 @@ module API
 
     helpers do
       def bulk_imports
-        @bulk_imports ||= ::BulkImports::ImportsFinder.new(user: current_user, status: params[:status]).execute
+        @bulk_imports ||= ::BulkImports::ImportsFinder.new(
+          user: current_user,
+          status: params[:status]
+        ).execute
       end
 
       def bulk_import
@@ -16,7 +19,11 @@ module API
       end
 
       def bulk_import_entities
-        @bulk_import_entities ||= ::BulkImports::EntitiesFinder.new(user: current_user, bulk_import: bulk_import, status: params[:status]).execute
+        @bulk_import_entities ||= ::BulkImports::EntitiesFinder.new(
+          user: current_user,
+          bulk_import: bulk_import,
+          status: params[:status]
+        ).execute
       end
 
       def bulk_import_entity
@@ -27,13 +34,44 @@ module API
     before { authenticate! }
 
     resource :bulk_imports do
+      desc 'Start a new GitLab Migration' do
+        detail 'This feature was introduced in GitLab 14.2.'
+      end
+      params do
+        requires :configuration, type: Hash, desc: 'The source GitLab instance configuration' do
+          requires :url, type: String, desc: 'Source GitLab instance URL'
+          requires :access_token, type: String, desc: 'Access token to the source GitLab instance'
+        end
+        requires :entities, type: Array, desc: 'List of entities to import' do
+          requires :source_type, type: String, desc: 'Source entity type (only `group_entity` is supported)',
+                   values: %w[group_entity]
+          requires :source_full_path, type: String, desc: 'Source full path of the entity to import'
+          requires :destination_name, type: String, desc: 'Destination name for the entity'
+          requires :destination_namespace, type: String, desc: 'Destination namespace for the entity'
+        end
+      end
+      post do
+        response = BulkImportService.new(
+          current_user,
+          params[:entities],
+          url: params[:configuration][:url],
+          access_token: params[:configuration][:access_token]
+        ).execute
+
+        if response.success?
+          present response.payload, with: Entities::BulkImport
+        else
+          render_api_error!(response.message, response.http_status)
+        end
+      end
+
       desc 'List all GitLab Migrations' do
         detail 'This feature was introduced in GitLab 14.1.'
       end
       params do
         use :pagination
         optional :status, type: String, values: BulkImport.all_human_statuses,
-                 desc: 'Return GitLab Migrations with specified status'
+          desc: 'Return GitLab Migrations with specified status'
       end
       get do
         present paginate(bulk_imports), with: Entities::BulkImport
@@ -45,10 +83,13 @@ module API
       params do
         use :pagination
         optional :status, type: String, values: ::BulkImports::Entity.all_human_statuses,
-                 desc: "Return all GitLab Migrations' entities with specified status"
+          desc: "Return all GitLab Migrations' entities with specified status"
       end
       get :entities do
-        entities = ::BulkImports::EntitiesFinder.new(user: current_user, status: params[:status]).execute
+        entities = ::BulkImports::EntitiesFinder.new(
+          user: current_user,
+          status: params[:status]
+        ).execute
 
         present paginate(entities), with: Entities::BulkImports::Entity
       end
@@ -69,7 +110,7 @@ module API
       params do
         requires :import_id, type: Integer, desc: "The ID of user's GitLab Migration"
         optional :status, type: String, values: ::BulkImports::Entity.all_human_statuses,
-                 desc: 'Return import entities with specified status'
+          desc: 'Return import entities with specified status'
         use :pagination
       end
       get ':import_id/entities' do
diff --git a/lib/api/ci/helpers/runner.rb b/lib/api/ci/helpers/runner.rb
new file mode 100644
index 00000000000..b9662b822fb
--- /dev/null
+++ b/lib/api/ci/helpers/runner.rb
@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module API
+  module Ci
+    module Helpers
+      module Runner
+        include Gitlab::Utils::StrongMemoize
+
+        prepend_mod_with('API::Ci::Helpers::Runner') # rubocop: disable Cop/InjectEnterpriseEditionModule
+
+        JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'
+        JOB_TOKEN_PARAM = :token
+
+        def runner_registration_token_valid?
+          ActiveSupport::SecurityUtils.secure_compare(params[:token], Gitlab::CurrentSettings.runners_registration_token)
+        end
+
+        def runner_registrar_valid?(type)
+          Feature.disabled?(:runner_registration_control) || Gitlab::CurrentSettings.valid_runner_registrars.include?(type)
+        end
+
+        def authenticate_runner!
+          forbidden! unless current_runner
+
+          current_runner
+            .heartbeat(get_runner_details_from_request)
+        end
+
+        def get_runner_details_from_request
+          return get_runner_ip unless params['info'].present?
+
+          attributes_for_keys(%w(name version revision platform architecture), params['info'])
+            .merge(get_runner_config_from_request)
+            .merge(get_runner_ip)
+        end
+
+        def get_runner_ip
+          { ip_address: ip_address }
+        end
+
+        def current_runner
+          token = params[:token]
+
+          if token
+            ::Gitlab::Database::LoadBalancing::RackMiddleware
+              .stick_or_unstick(env, :runner, token)
+          end
+
+          strong_memoize(:current_runner) do
+            ::Ci::Runner.find_by_token(token.to_s)
+          end
+        end
+
+        # HTTP status codes to terminate the job on GitLab Runner:
+        # - 403
+        def authenticate_job!(require_running: true)
+          job = current_job
+
+          # 404 is not returned here because we want to terminate the job if it's
+          # running. A 404 can be returned from anywhere in the networking stack which is why
+          # we are explicit about a 403, we should improve this in
+          # https://gitlab.com/gitlab-org/gitlab/-/issues/327703
+          forbidden! unless job
+
+          forbidden! unless job_token_valid?(job)
+
+          forbidden!('Project has been deleted!') if job.project.nil? || job.project.pending_delete?
+          forbidden!('Job has been erased!') if job.erased?
+
+          if require_running
+            job_forbidden!(job, 'Job is not running') unless job.running?
+          end
+
+          job.runner&.heartbeat(get_runner_ip)
+
+          job
+        end
+
+        def current_job
+          id = params[:id]
+
+          if id
+            ::Gitlab::Database::LoadBalancing::RackMiddleware
+              .stick_or_unstick(env, :build, id)
+          end
+
+          strong_memoize(:current_job) do
+            ::Ci::Build.find_by_id(id)
+          end
+        end
+
+        def job_token_valid?(job)
+          token = (params[JOB_TOKEN_PARAM] || env[JOB_TOKEN_HEADER]).to_s
+          token && job.valid_token?(token)
+        end
+
+        def job_forbidden!(job, reason)
+          header 'Job-Status', job.status
+          forbidden!(reason)
+        end
+
+        def set_application_context
+          return unless current_job
+
+          Gitlab::ApplicationContext.push(
+            user: -> { current_job.user },
+            project: -> { current_job.project }
+          )
+        end
+
+        def track_ci_minutes_usage!(_build, _runner)
+          # noop: overridden in EE
+        end
+
+        private
+
+        def get_runner_config_from_request
+          { config: attributes_for_keys(%w(gpus), params.dig('info', 'config')) }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/ci/job_artifacts.rb b/lib/api/ci/job_artifacts.rb
new file mode 100644
index 00000000000..6431436b50d
--- /dev/null
+++ b/lib/api/ci/job_artifacts.rb
@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module API
+  module Ci
+    class JobArtifacts < ::API::Base
+      before { authenticate_non_get! }
+
+      feature_category :build_artifacts
+
+      # EE::API::Ci::JobArtifacts would override the following helpers
+      helpers do
+        def authorize_download_artifacts!
+          authorize_read_builds!
+        end
+      end
+
+      prepend_mod_with('API::Ci::JobArtifacts') # rubocop: disable Cop/InjectEnterpriseEditionModule
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+        desc 'Download the artifacts archive from a job' do
+          detail 'This feature was introduced in GitLab 8.10'
+        end
+        params do
+          requires :ref_name, type: String, desc: 'The ref from repository'
+          requires :job,      type: String, desc: 'The name for the job'
+        end
+        route_setting :authentication, job_token_allowed: true
+        get ':id/jobs/artifacts/:ref_name/download',
+          requirements: { ref_name: /.+/ } do
+            authorize_download_artifacts!
+
+            latest_build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name])
+            authorize_read_job_artifacts!(latest_build)
+
+            present_carrierwave_file!(latest_build.artifacts_file)
+          end
+
+        desc 'Download a specific file from artifacts archive from a ref' do
+          detail 'This feature was introduced in GitLab 11.5'
+        end
+        params do
+          requires :ref_name, type: String, desc: 'The ref from repository'
+          requires :job, type: String, desc: 'The name for the job'
+          requires :artifact_path, type: String, desc: 'Artifact path'
+        end
+        route_setting :authentication, job_token_allowed: true
+        get ':id/jobs/artifacts/:ref_name/raw/*artifact_path',
+          format: false,
+          requirements: { ref_name: /.+/ } do
+            authorize_download_artifacts!
+
+            build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name])
+            authorize_read_job_artifacts!(build)
+
+            path = Gitlab::Ci::Build::Artifacts::Path
+              .new(params[:artifact_path])
+
+            bad_request! unless path.valid?
+
+            send_artifacts_entry(build.artifacts_file, path)
+          end
+
+        desc 'Download the artifacts archive from a job' do
+          detail 'This feature was introduced in GitLab 8.5'
+        end
+        params do
+          requires :job_id, type: Integer, desc: 'The ID of a job'
+        end
+        route_setting :authentication, job_token_allowed: true
+        get ':id/jobs/:job_id/artifacts' do
+          authorize_download_artifacts!
+
+          build = find_build!(params[:job_id])
+          authorize_read_job_artifacts!(build)
+
+          present_carrierwave_file!(build.artifacts_file)
+        end
+
+        desc 'Download a specific file from artifacts archive' do
+          detail 'This feature was introduced in GitLab 10.0'
+        end
+        params do
+          requires :job_id, type: Integer, desc: 'The ID of a job'
+          requires :artifact_path, type: String, desc: 'Artifact path'
+        end
+        route_setting :authentication, job_token_allowed: true
+        get ':id/jobs/:job_id/artifacts/*artifact_path', format: false do
+          authorize_download_artifacts!
+
+          build = find_build!(params[:job_id])
+          authorize_read_job_artifacts!(build)
+
+          not_found! unless build.available_artifacts?
+
+          path = Gitlab::Ci::Build::Artifacts::Path
+            .new(params[:artifact_path])
+
+          bad_request! unless path.valid?
+
+          send_artifacts_entry(build.artifacts_file, path)
+        end
+
+        desc 'Keep the artifacts to prevent them from being deleted' do
+          success ::API::Entities::Ci::Job
+        end
+        params do
+          requires :job_id, type: Integer, desc: 'The ID of a job'
+        end
+        post ':id/jobs/:job_id/artifacts/keep' do
+          authorize_update_builds!
+
+          build = find_build!(params[:job_id])
+          authorize!(:update_build, build)
+          break not_found!(build) unless build.artifacts?
+
+          build.keep_artifacts!
+
+          status 200
+          present build, with: ::API::Entities::Ci::Job
+        end
+
+        desc 'Delete the artifacts files from a job' do
+          detail 'This feature was introduced in GitLab 11.9'
+        end
+        params do
+          requires :job_id, type: Integer, desc: 'The ID of a job'
+        end
+        delete ':id/jobs/:job_id/artifacts' do
+          authorize_destroy_artifacts!
+          build = find_build!(params[:job_id])
+          authorize!(:destroy_artifacts, build)
+
+          build.erase_erasable_artifacts!
+
+          status :no_content
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/ci/jobs.rb b/lib/api/ci/jobs.rb
new file mode 100644
index 00000000000..eea1637c32a
--- /dev/null
+++ b/lib/api/ci/jobs.rb
@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+module API
+  module Ci
+    class Jobs < ::API::Base
+      include PaginationParams
+      before { authenticate! }
+
+      resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+        params do
+          requires :id, type: String, desc: 'The ID of a project'
+        end
+
+        helpers do
+          params :optional_scope do
+            optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show',
+                             values: ::CommitStatus::AVAILABLE_STATUSES,
+                             coerce_with: ->(scope) {
+                               case scope
+                               when String
+                                 [scope]
+                               when ::Hash
+                                 scope.values
+                               when ::Array
+                                 scope
+                               else
+                                 ['unknown']
+                               end
+                             }
+          end
+        end
+
+        desc 'Get a projects jobs' do
+          success Entities::Ci::Job
+        end
+        params do
+          use :optional_scope
+          use :pagination
+        end
+        # rubocop: disable CodeReuse/ActiveRecord
+        get ':id/jobs', feature_category: :continuous_integration do
+          authorize_read_builds!
+
+          builds = user_project.builds.order('id DESC')
+          builds = filter_builds(builds, params[:scope])
+
+          builds = builds.preload(:user, :job_artifacts_archive, :job_artifacts, :runner, :tags, pipeline: :project)
+          present paginate(builds), with: Entities::Ci::Job
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        desc 'Get a specific job of a project' do
+          success Entities::Ci::Job
+        end
+        params do
+          requires :job_id, type: Integer, desc: 'The ID of a job'
+        end
+        get ':id/jobs/:job_id', feature_category: :continuous_integration do
+          authorize_read_builds!
+
+          build = find_build!(params[:job_id])
+
+          present build, with: Entities::Ci::Job
+        end
+
+        # TODO: We should use `present_disk_file!` and leave this implementation for backward compatibility (when build trace
+        #       is saved in the DB instead of file). But before that, we need to consider how to replace the value of
+        #       `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse.
+        desc 'Get a trace of a specific job of a project'
+        params do
+          requires :job_id, type: Integer, desc: 'The ID of a job'
+        end
+        get ':id/jobs/:job_id/trace', feature_category: :continuous_integration do
+          authorize_read_builds!
+
+          build = find_build!(params[:job_id])
+
+          authorize_read_build_trace!(build) if build
+
+          header 'Content-Disposition', "infile; filename=\"#{build.id}.log\""
+          content_type 'text/plain'
+          env['api.format'] = :binary
+
+          # The trace can be nil bu body method expects a string as an argument.
+          trace = build.trace.raw || ''
+          body trace
+        end
+
+        desc 'Cancel a specific job of a project' do
+          success Entities::Ci::Job
+        end
+        params do
+          requires :job_id, type: Integer, desc: 'The ID of a job'
+        end
+        post ':id/jobs/:job_id/cancel', feature_category: :continuous_integration do
+          authorize_update_builds!
+
+          build = find_build!(params[:job_id])
+          authorize!(:update_build, build)
+
+          build.cancel
+
+          present build, with: Entities::Ci::Job
+        end
+
+        desc 'Retry a specific build of a project' do
+          success Entities::Ci::Job
+        end
+        params do
+          requires :job_id, type: Integer, desc: 'The ID of a build'
+        end
+        post ':id/jobs/:job_id/retry', feature_category: :continuous_integration do
+          authorize_update_builds!
+
+          build = find_build!(params[:job_id])
+          authorize!(:update_build, build)
+          break forbidden!('Job is not retryable') unless build.retryable?
+
+          build = ::Ci::Build.retry(build, current_user)
+
+          present build, with: Entities::Ci::Job
+        end
+
+        desc 'Erase job (remove artifacts and the trace)' do
+          success Entities::Ci::Job
+        end
+        params do
+          requires :job_id, type: Integer, desc: 'The ID of a build'
+        end
+        post ':id/jobs/:job_id/erase', feature_category: :continuous_integration do
+          authorize_update_builds!
+
+          build = find_build!(params[:job_id])
+          authorize!(:erase_build, build)
+          break forbidden!('Job is not erasable!') unless build.erasable?
+
+          build.erase(erased_by: current_user)
+          present build, with: Entities::Ci::Job
+        end
+
+        desc 'Trigger an actionable job (manual, delayed, etc)' do
+          success Entities::Ci::JobBasic
+          detail 'This feature was added in GitLab 8.11'
+        end
+        params do
+          requires :job_id, type: Integer, desc: 'The ID of a Job'
+        end
+
+        post ":id/jobs/:job_id/play", feature_category: :continuous_integration do
+          authorize_read_builds!
+
+          job = find_job!(params[:job_id])
+
+          authorize!(:play_job, job)
+
+          bad_request!("Unplayable Job") unless job.playable?
+
+          job.play(current_user)
+
+          status 200
+
+          if job.is_a?(::Ci::Build)
+            present job, with: Entities::Ci::Job
+          else
+            present job, with: Entities::Ci::Bridge
+          end
+        end
+      end
+
+      resource :job do
+        desc 'Get current project using job token' do
+          success Entities::Ci::Job
+        end
+        route_setting :authentication, job_token_allowed: true
+        get '', feature_category: :continuous_integration do
+          validate_current_authenticated_job
+
+          present current_authenticated_job, with: Entities::Ci::Job
+        end
+      end
+
+      helpers do
+        # rubocop: disable CodeReuse/ActiveRecord
+        def filter_builds(builds, scope)
+          return builds if scope.nil? || scope.empty?
+
+          available_statuses = ::CommitStatus::AVAILABLE_STATUSES
+
+          unknown = scope - available_statuses
+          render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty?
+
+          builds.where(status: available_statuses && scope)
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        def validate_current_authenticated_job
+          # current_authenticated_job will be nil if user is using
+          # a valid authentication (like PRIVATE-TOKEN) that is not CI_JOB_TOKEN
+          not_found!('Job') unless current_authenticated_job
+        end
+      end
+    end
+  end
+end
+
+API::Ci::Jobs.prepend_mod_with('API::Ci::Jobs')
diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb
index 339c0e779f9..4d6d38f2dce 100644
--- a/lib/api/ci/pipelines.rb
+++ b/lib/api/ci/pipelines.rb
@@ -44,7 +44,7 @@ module API
           optional :ref,      type: String, desc: 'The ref of pipelines'
           optional :sha,      type: String, desc: 'The sha of pipelines'
           optional :yaml_errors, type: Boolean, desc: 'Returns pipelines with invalid configurations'
-          optional :name,     type: String, desc: 'The name of the user who triggered pipelines'
+          optional :name,     type: String, desc: '(deprecated) The name of the user who triggered pipelines'
           optional :username, type: String, desc: 'The username of the user who triggered pipelines'
           optional :updated_before, type: DateTime, desc: 'Return pipelines updated before the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ'
           optional :updated_after, type: DateTime, desc: 'Return pipelines updated after the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ'
@@ -52,13 +52,14 @@ module API
                               desc: 'Order pipelines'
           optional :sort,     type: String, values: %w[asc desc], default: 'desc',
                               desc: 'Sort pipelines'
+          optional :source,   type: String, values: ::Ci::Pipeline.sources.keys
         end
         get ':id/pipelines' do
           authorize! :read_pipeline, user_project
           authorize! :read_build, user_project
 
           pipelines = ::Ci::PipelinesFinder.new(user_project, current_user, params).execute
-          present paginate(pipelines), with: Entities::Ci::PipelineBasic
+          present paginate(pipelines), with: Entities::Ci::PipelineBasic, project: user_project
         end
 
         desc 'Create a new pipeline' do
@@ -78,12 +79,11 @@ module API
             .merge(variables_attributes: params[:variables])
             .except(:variables)
 
-          new_pipeline = ::Ci::CreatePipelineService.new(user_project,
-                                                         current_user,
-                                                         pipeline_params)
-                             .execute(:api, ignore_skip_ci: true, save_on_errors: false)
+          response = ::Ci::CreatePipelineService.new(user_project, current_user, pipeline_params)
+            .execute(:api, ignore_skip_ci: true, save_on_errors: false)
+          new_pipeline = response.payload
 
-          if new_pipeline.persisted?
+          if response.success?
             present new_pipeline, with: Entities::Ci::Pipeline
           else
             render_validation_error!(new_pipeline)
@@ -188,6 +188,19 @@ module API
           present pipeline.test_reports, with: TestReportEntity, details: true
         end
 
+        desc 'Gets the test report summary for a given pipeline' do
+          detail 'This feature was introduced in GitLab 14.2'
+          success TestReportSummaryEntity
+        end
+        params do
+          requires :pipeline_id, type: Integer, desc: 'The pipeline ID'
+        end
+        get ':id/pipelines/:pipeline_id/test_report_summary' do
+          authorize! :read_build, pipeline
+
+          present pipeline.test_report_summary, with: TestReportSummaryEntity
+        end
+
         desc 'Deletes a pipeline' do
           detail 'This feature was introduced in GitLab 11.6'
           http_codes [[204, 'Pipeline was deleted'], [403, 'Forbidden']]
diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb
index 0bac6fe2054..aabcf34952c 100644
--- a/lib/api/ci/runner.rb
+++ b/lib/api/ci/runner.rb
@@ -3,12 +3,10 @@
 module API
   module Ci
     class Runner < ::API::Base
-      helpers ::API::Helpers::Runner
+      helpers ::API::Ci::Helpers::Runner
 
       content_type :txt, 'text/plain'
 
-      feature_category :runner
-
       resource :runners do
         desc 'Registers a new Runner' do
           success Entities::Ci::RunnerRegistrationDetails
@@ -26,7 +24,7 @@ module API
           optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: %q(List of Runner's tags)
           optional :maximum_timeout, type: Integer, desc: 'Maximum timeout set when this Runner will handle the job'
         end
-        post '/' do
+        post '/', feature_category: :runner do
           attributes = attributes_for_keys([:description, :active, :locked, :run_untagged, :tag_list, :access_level, :maximum_timeout])
             .merge(get_runner_details_from_request)
 
@@ -59,7 +57,7 @@ module API
         params do
           requires :token, type: String, desc: %q(Runner's authentication token)
         end
-        delete '/' do
+        delete '/', feature_category: :runner do
           authenticate_runner!
 
           destroy_conditionally!(current_runner)
@@ -71,7 +69,7 @@ module API
         params do
           requires :token, type: String, desc: %q(Runner's authentication token)
         end
-        post '/verify' do
+        post '/verify', feature_category: :runner do
           authenticate_runner!
           status 200
           body "200"
@@ -123,7 +121,7 @@ module API
         formatter :build_json, ->(object, _) { object }
         parser :build_json, ::Grape::Parser::Json
 
-        post '/request' do
+        post '/request', feature_category: :continuous_integration do
           authenticate_runner!
 
           unless current_runner.active?
@@ -177,7 +175,7 @@ module API
           end
           optional :exit_code, type: Integer, desc: %q(Job's exit code)
         end
-        put '/:id' do
+        put '/:id', feature_category: :continuous_integration do
           job = authenticate_job!
 
           Gitlab::Metrics.add_event(:update_build)
@@ -204,7 +202,7 @@ module API
           requires :id, type: Integer, desc: %q(Job's ID)
           optional :token, type: String, desc: %q(Job's authentication token)
         end
-        patch '/:id/trace' do
+        patch '/:id/trace', feature_category: :continuous_integration do
           job = authenticate_job!
 
           error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range')
@@ -249,7 +247,7 @@ module API
           optional :artifact_type, type: String, desc: %q(The type of artifact),
                                   default: 'archive', values: ::Ci::JobArtifact.file_types.keys
         end
-        post '/:id/artifacts/authorize' do
+        post '/:id/artifacts/authorize', feature_category: :build_artifacts do
           not_allowed! unless Gitlab.config.artifacts.enabled
           require_gitlab_workhorse!
 
@@ -285,7 +283,7 @@ module API
                                     default: 'zip', values: ::Ci::JobArtifact.file_formats.keys
           optional :metadata, type: ::API::Validations::Types::WorkhorseFile, desc: %(The artifact metadata to store (generated by Multipart middleware))
         end
-        post '/:id/artifacts' do
+        post '/:id/artifacts', feature_category: :build_artifacts do
           not_allowed! unless Gitlab.config.artifacts.enabled
           require_gitlab_workhorse!
 
@@ -314,7 +312,7 @@ module API
           optional :token, type: String, desc: %q(Job's authentication token)
           optional :direct_download, default: false, type: Boolean, desc: %q(Perform direct download from remote storage instead of proxying artifacts)
         end
-        get '/:id/artifacts' do
+        get '/:id/artifacts', feature_category: :build_artifacts do
           job = authenticate_job!(require_running: false)
 
           present_carrierwave_file!(job.artifacts_file, supports_direct_download: params[:direct_download])
diff --git a/lib/api/ci/triggers.rb b/lib/api/ci/triggers.rb
new file mode 100644
index 00000000000..6a2b16e1568
--- /dev/null
+++ b/lib/api/ci/triggers.rb
@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module API
+  module Ci
+    class Triggers < ::API::Base
+      include PaginationParams
+
+      HTTP_GITLAB_EVENT_HEADER = "HTTP_#{WebHookService::GITLAB_EVENT_HEADER}".underscore.upcase
+
+      feature_category :continuous_integration
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+        desc 'Trigger a GitLab project pipeline' do
+          success Entities::Ci::Pipeline
+        end
+        params do
+          requires :ref, type: String, desc: 'The commit sha or name of a branch or tag', allow_blank: false
+          requires :token, type: String, desc: 'The unique token of trigger or job token'
+          optional :variables, type: Hash, desc: 'The list of variables to be injected into build'
+        end
+        post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do
+          Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20758')
+
+          forbidden! if gitlab_pipeline_hook_request?
+
+          # validate variables
+          params[:variables] = params[:variables].to_h
+          unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) }
+            render_api_error!('variables needs to be a map of key-valued strings', 400)
+          end
+
+          project = find_project(params[:id])
+          not_found! unless project
+
+          result = ::Ci::PipelineTriggerService.new(project, nil, params).execute
+          not_found! unless result
+
+          if result.error?
+            render_api_error!(result[:message], result[:http_status])
+          else
+            present result[:pipeline], with: Entities::Ci::Pipeline
+          end
+        end
+
+        desc 'Get triggers list' do
+          success Entities::Trigger
+        end
+        params do
+          use :pagination
+        end
+        # rubocop: disable CodeReuse/ActiveRecord
+        get ':id/triggers' do
+          authenticate!
+          authorize! :admin_build, user_project
+
+          triggers = user_project.triggers.includes(:trigger_requests)
+
+          present paginate(triggers), with: Entities::Trigger, current_user: current_user
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        desc 'Get specific trigger of a project' do
+          success Entities::Trigger
+        end
+        params do
+          requires :trigger_id, type: Integer, desc: 'The trigger ID'
+        end
+        get ':id/triggers/:trigger_id' do
+          authenticate!
+          authorize! :admin_build, user_project
+
+          trigger = user_project.triggers.find(params.delete(:trigger_id))
+          break not_found!('Trigger') unless trigger
+
+          present trigger, with: Entities::Trigger, current_user: current_user
+        end
+
+        desc 'Create a trigger' do
+          success Entities::Trigger
+        end
+        params do
+          requires :description, type: String, desc: 'The trigger description'
+        end
+        post ':id/triggers' do
+          authenticate!
+          authorize! :admin_build, user_project
+
+          trigger = user_project.triggers.create(
+            declared_params(include_missing: false).merge(owner: current_user))
+
+          if trigger.valid?
+            present trigger, with: Entities::Trigger, current_user: current_user
+          else
+            render_validation_error!(trigger)
+          end
+        end
+
+        desc 'Update a trigger' do
+          success Entities::Trigger
+        end
+        params do
+          requires :trigger_id, type: Integer,  desc: 'The trigger ID'
+          optional :description, type: String,  desc: 'The trigger description'
+        end
+        put ':id/triggers/:trigger_id' do
+          authenticate!
+          authorize! :admin_build, user_project
+
+          trigger = user_project.triggers.find(params.delete(:trigger_id))
+          break not_found!('Trigger') unless trigger
+
+          authorize! :admin_trigger, trigger
+
+          if trigger.update(declared_params(include_missing: false))
+            present trigger, with: Entities::Trigger, current_user: current_user
+          else
+            render_validation_error!(trigger)
+          end
+        end
+
+        desc 'Delete a trigger' do
+          success Entities::Trigger
+        end
+        params do
+          requires :trigger_id, type: Integer, desc: 'The trigger ID'
+        end
+        delete ':id/triggers/:trigger_id' do
+          authenticate!
+          authorize! :admin_build, user_project
+
+          trigger = user_project.triggers.find(params.delete(:trigger_id))
+          break not_found!('Trigger') unless trigger
+
+          destroy_conditionally!(trigger)
+        end
+      end
+
+      helpers do
+        def gitlab_pipeline_hook_request?
+          request.get_header(HTTP_GITLAB_EVENT_HEADER) == WebHookService.hook_to_event(:pipeline_hooks)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/api/ci/variables.rb b/lib/api/ci/variables.rb
new file mode 100644
index 00000000000..9c04d5e9923
--- /dev/null
+++ b/lib/api/ci/variables.rb
@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+module API
+  module Ci
+    class Variables < ::API::Base
+      include PaginationParams
+
+      before { authenticate! }
+      before { authorize! :admin_build, user_project }
+
+      feature_category :pipeline_authoring
+
+      helpers ::API::Helpers::VariablesHelpers
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+
+      resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+        desc 'Get project variables' do
+          success Entities::Ci::Variable
+        end
+        params do
+          use :pagination
+        end
+        get ':id/variables' do
+          variables = user_project.variables
+          present paginate(variables), with: Entities::Ci::Variable
+        end
+
+        desc 'Get a specific variable from a project' do
+          success Entities::Ci::Variable
+        end
+        params do
+          requires :key, type: String, desc: 'The key of the variable'
+        end
+        # rubocop: disable CodeReuse/ActiveRecord
+        get ':id/variables/:key' do
+          variable = find_variable(user_project, params)
+          not_found!('Variable') unless variable
+
+          present variable, with: Entities::Ci::Variable
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        desc 'Create a new variable in a project' do
+          success Entities::Ci::Variable
+        end
+        params do
+          requires :key, type: String, desc: 'The key of the variable'
+          requires :value, type: String, desc: 'The value of the variable'
+          optional :protected, type: Boolean, desc: 'Whether the variable is protected'
+          optional :masked, type: Boolean, desc: 'Whether the variable is masked'
+          optional :variable_type, type: String, values: ::Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var'
+          optional :environment_scope, type: String, desc: 'The environment_scope of the variable'
+        end
+        post ':id/variables' do
+          variable = ::Ci::ChangeVariableService.new(
+            container: user_project,
+            current_user: current_user,
+            params: { action: :create, variable_params: declared_params(include_missing: false) }
+          ).execute
+
+          if variable.valid?
+            present variable, with: Entities::Ci::Variable
+          else
+            render_validation_error!(variable)
+          end
+        end
+
+        desc 'Update an existing variable from a project' do
+          success Entities::Ci::Variable
+        end
+        params do
+          optional :key, type: String, desc: 'The key of the variable'
+          optional :value, type: String, desc: 'The value of the variable'
+          optional :protected, type: Boolean, desc: 'Whether the variable is protected'
+          optional :masked, type: Boolean, desc: 'Whether the variable is masked'
+          optional :variable_type, type: String, values: ::Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file'
+          optional :environment_scope, type: String, desc: 'The environment_scope of the variable'
+          optional :filter, type: Hash, desc: 'Available filters: [environment_scope]. Example: filter[environment_scope]=production'
+        end
+        # rubocop: disable CodeReuse/ActiveRecord
+        put ':id/variables/:key' do
+          variable = find_variable(user_project, params)
+          not_found!('Variable') unless variable
+
+          variable = ::Ci::ChangeVariableService.new(
+            container: user_project,
+            current_user: current_user,
+            params: { action: :update, variable: variable, variable_params: declared_params(include_missing: false).except(:key, :filter) }
+          ).execute
+
+          if variable.valid?
+            present variable, with: Entities::Ci::Variable
+          else
+            render_validation_error!(variable)
+          end
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        desc 'Delete an existing variable from a project' do
+          success Entities::Ci::Variable
+        end
+        params do
+          requires :key, type: String, desc: 'The key of the variable'
+          optional :filter, type: Hash, desc: 'Available filters: [environment_scope]. Example: filter[environment_scope]=production'
+        end
+        # rubocop: disable CodeReuse/ActiveRecord
+        delete ':id/variables/:key' do
+          variable = find_variable(user_project, params)
+          not_found!('Variable') unless variable
+
+          ::Ci::ChangeVariableService.new(
+            container: user_project,
+            current_user: current_user,
+            params: { action: :destroy, variable: variable }
+          ).execute
+
+          no_content!
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+      end
+    end
+  end
+end
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index 541a37b0abe..5d8985455ad 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -47,7 +47,7 @@ module API
         path = params[:path]
         before = params[:until]
         after = params[:since]
-        ref = params[:ref_name].presence || user_project.try(:default_branch) || 'master' unless params[:all]
+        ref = params[:ref_name].presence || user_project.default_branch unless params[:all]
         offset = (params[:page] - 1) * params[:per_page]
         all = params[:all]
         with_stats = params[:with_stats]
diff --git a/lib/api/concerns/packages/debian_distribution_endpoints.rb b/lib/api/concerns/packages/debian_distribution_endpoints.rb
index 4670c3e3521..798e583b87a 100644
--- a/lib/api/concerns/packages/debian_distribution_endpoints.rb
+++ b/lib/api/concerns/packages/debian_distribution_endpoints.rb
@@ -80,6 +80,8 @@ module API
               use :optional_distribution_params
             end
             get '/' do
+              authorize_read_package!(project_or_group)
+
               distribution_params = declared_params(include_missing: false)
               distributions = ::Packages::Debian::DistributionsFinder.new(project_or_group, distribution_params).execute
 
@@ -96,6 +98,8 @@ module API
               requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename'
             end
             get '/:codename' do
+              authorize_read_package!(project_or_group)
+
               distribution = ::Packages::Debian::DistributionsFinder.new(project_or_group, codename: params[:codename]).execute.last!
 
               present distribution, with: ::API::Entities::Packages::Debian::Distribution
diff --git a/lib/api/concerns/packages/debian_package_endpoints.rb b/lib/api/concerns/packages/debian_package_endpoints.rb
index 7740ba6bfa6..0acc015f366 100644
--- a/lib/api/concerns/packages/debian_package_endpoints.rb
+++ b/lib/api/concerns/packages/debian_package_endpoints.rb
@@ -6,8 +6,6 @@ module API
       module DebianPackageEndpoints
         extend ActiveSupport::Concern
 
-        LETTER_REGEX = %r{(lib)?[a-z0-9]}.freeze
-        PACKAGE_REGEX = API::NO_SLASH_URL_PART_REGEX
         DISTRIBUTION_REQUIREMENTS = {
           distribution: ::Packages::Debian::DISTRIBUTION_REGEX
         }.freeze
@@ -15,14 +13,6 @@ module API
           component: ::Packages::Debian::COMPONENT_REGEX,
           architecture: ::Packages::Debian::ARCHITECTURE_REGEX
         }.freeze
-        COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS = {
-          component: ::Packages::Debian::COMPONENT_REGEX,
-          letter: LETTER_REGEX,
-          source_package: PACKAGE_REGEX
-        }.freeze
-        FILE_NAME_REQUIREMENTS = {
-          file_name: API::NO_SLASH_URL_PART_REGEX
-        }.freeze
 
         included do
           feature_category :package_registry
@@ -31,109 +21,106 @@ module API
           helpers ::API::Helpers::Packages::BasicAuthHelpers
           include ::API::Helpers::Authentication
 
-          namespace 'packages/debian' do
-            authenticate_with do |accept|
-              accept.token_types(:personal_access_token, :deploy_token, :job_token)
-                    .sent_through(:http_basic_auth)
+          helpers do
+            params :shared_package_file_params do
+              requires :distribution, type: String, desc: 'The Debian Codename or Suite', regexp: Gitlab::Regex.debian_distribution_regex
+              requires :letter, type: String, desc: 'The Debian Classification (first-letter or lib-first-letter)'
+              requires :package_name, type: String, desc: 'The Debian Source Package Name', regexp: Gitlab::Regex.debian_package_name_regex
+              requires :package_version, type: String, desc: 'The Debian Source Package Version', regexp: Gitlab::Regex.debian_version_regex
+              requires :file_name, type: String, desc: 'The Debian File Name'
             end
 
-            helpers do
-              def present_release_file
-                distribution = ::Packages::Debian::DistributionsFinder.new(project_or_group, codename_or_suite: params[:distribution]).execute.last!
-
-                present_carrierwave_file!(distribution.file)
-              end
+            def distribution_from!(container)
+              ::Packages::Debian::DistributionsFinder.new(container, codename_or_suite: params[:distribution]).execute.last!
             end
 
-            format :txt
-            content_type :txt, 'text/plain'
+            def present_package_file!
+              not_found! unless params[:package_name].start_with?(params[:letter])
 
-            params do
-              requires :distribution, type: String, desc: 'The Debian Codename', regexp: Gitlab::Regex.debian_distribution_regex
+              package_file = distribution_from!(user_project).package_files.with_file_name(params[:file_name]).last!
+
+              present_carrierwave_file!(package_file.file)
             end
+          end
 
-            namespace 'dists/*distribution', requirements: DISTRIBUTION_REQUIREMENTS do
-              # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release.gpg
-              desc 'The Release file signature' do
-                detail 'This feature was introduced in GitLab 13.5'
-              end
+          authenticate_with do |accept|
+            accept.token_types(:personal_access_token, :deploy_token, :job_token)
+                  .sent_through(:http_basic_auth)
+          end
 
-              route_setting :authentication, authenticate_non_public: true
-              get 'Release.gpg' do
-                not_found!
-              end
+          rescue_from ArgumentError do |e|
+            render_api_error!(e.message, 400)
+          end
 
-              # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release
-              desc 'The unsigned Release file' do
-                detail 'This feature was introduced in GitLab 13.5'
-              end
+          rescue_from ActiveRecord::RecordInvalid do |e|
+            render_api_error!(e.message, 400)
+          end
 
-              route_setting :authentication, authenticate_non_public: true
-              get 'Release' do
-                present_release_file
-              end
+          format :txt
+          content_type :txt, 'text/plain'
 
-              # GET {projects|groups}/:id/packages/debian/dists/*distribution/InRelease
-              desc 'The signed Release file' do
-                detail 'This feature was introduced in GitLab 13.5'
-              end
+          params do
+            requires :distribution, type: String, desc: 'The Debian Codename or Suite', regexp: Gitlab::Regex.debian_distribution_regex
+          end
 
-              route_setting :authentication, authenticate_non_public: true
-              get 'InRelease' do
-                # Signature to be added in 7.3 of https://gitlab.com/groups/gitlab-org/-/epics/6057#note_582697034
-                present_release_file
-              end
+          namespace 'dists/*distribution', requirements: DISTRIBUTION_REQUIREMENTS do
+            # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release.gpg
+            desc 'The Release file signature' do
+              detail 'This feature was introduced in GitLab 13.5'
+            end
 
-              params do
-                requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex
-                requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex
-              end
+            route_setting :authentication, authenticate_non_public: true
+            get 'Release.gpg' do
+              distribution_from!(project_or_group).file_signature
+            end
 
-              namespace ':component/binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do
-                # GET {projects|groups}/:id/packages/debian/dists/*distribution/:component/binary-:architecture/Packages
-                desc 'The binary files index' do
-                  detail 'This feature was introduced in GitLab 13.5'
-                end
-
-                route_setting :authentication, authenticate_non_public: true
-                get 'Packages' do
-                  relation = "::Packages::Debian::#{project_or_group.class.name}ComponentFile".constantize
-
-                  component_file = relation
-                    .preload_distribution
-                    .with_container(project_or_group)
-                    .with_codename_or_suite(params[:distribution])
-                    .with_component_name(params[:component])
-                    .with_file_type(:packages)
-                    .with_architecture_name(params[:architecture])
-                    .with_compression_type(nil)
-                    .order_created_asc
-                    .last!
-
-                  present_carrierwave_file!(component_file.file)
-                end
-              end
+            # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release
+            desc 'The unsigned Release file' do
+              detail 'This feature was introduced in GitLab 13.5'
+            end
+
+            route_setting :authentication, authenticate_non_public: true
+            get 'Release' do
+              present_carrierwave_file!(distribution_from!(project_or_group).file)
+            end
+
+            # GET {projects|groups}/:id/packages/debian/dists/*distribution/InRelease
+            desc 'The signed Release file' do
+              detail 'This feature was introduced in GitLab 13.5'
+            end
+
+            route_setting :authentication, authenticate_non_public: true
+            get 'InRelease' do
+              present_carrierwave_file!(distribution_from!(project_or_group).signed_file)
             end
 
             params do
               requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex
-              requires :letter, type: String, desc: 'The Debian Classification (first-letter or lib-first-letter)'
-              requires :source_package, type: String, desc: 'The Debian Source Package Name', regexp: Gitlab::Regex.debian_package_name_regex
+              requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex
             end
 
-            namespace 'pool/:component/:letter/:source_package', requirements: COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS do
-              # GET {projects|groups}/:id/packages/debian/pool/:component/:letter/:source_package/:file_name
-              params do
-                requires :file_name, type: String, desc: 'The Debian File Name'
-              end
-              desc 'The package' do
+            namespace ':component/binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do
+              # GET {projects|groups}/:id/packages/debian/dists/*distribution/:component/binary-:architecture/Packages
+              desc 'The binary files index' do
                 detail 'This feature was introduced in GitLab 13.5'
               end
 
               route_setting :authentication, authenticate_non_public: true
-              get ':file_name', requirements: FILE_NAME_REQUIREMENTS do
-                # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286
-                'TODO File'
+              get 'Packages' do
+                relation = "::Packages::Debian::#{project_or_group.class.name}ComponentFile".constantize
+
+                component_file = relation
+                  .preload_distribution
+                  .with_container(project_or_group)
+                  .with_codename_or_suite(params[:distribution])
+                  .with_component_name(params[:component])
+                  .with_file_type(:packages)
+                  .with_architecture_name(params[:architecture])
+                  .with_compression_type(nil)
+                  .order_created_asc
+                  .last!
+
+                present_carrierwave_file!(component_file.file)
               end
             end
           end
diff --git a/lib/api/debian_group_packages.rb b/lib/api/debian_group_packages.rb
index 191ed42a5b8..29f5047230a 100644
--- a/lib/api/debian_group_packages.rb
+++ b/lib/api/debian_group_packages.rb
@@ -2,35 +2,50 @@
 
 module API
   class DebianGroupPackages < ::API::Base
-    params do
-      requires :id, type: String, desc: 'The ID of a group'
-    end
+    PACKAGE_FILE_REQUIREMENTS = ::API::DebianProjectPackages::PACKAGE_FILE_REQUIREMENTS.merge(
+      project_id: %r{[0-9]+}.freeze
+    ).freeze
 
     resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
-      rescue_from ArgumentError do |e|
-        render_api_error!(e.message, 400)
-      end
+      helpers do
+        def user_project
+          @project ||= find_project!(params[:project_id])
+        end
 
-      rescue_from ActiveRecord::RecordInvalid do |e|
-        render_api_error!(e.message, 400)
+        def project_or_group
+          user_group
+        end
       end
 
-      before do
+      after_validation do
         require_packages_enabled!
 
-        not_found! unless ::Feature.enabled?(:debian_packages, user_group)
+        not_found! unless ::Feature.enabled?(:debian_group_packages, user_group)
 
         authorize_read_package!(user_group)
       end
 
-      namespace ':id/-' do
-        helpers do
-          def project_or_group
-            user_group
-          end
-        end
+      params do
+        requires :id, type: String, desc: 'The ID of a group'
+      end
 
+      namespace ':id/-/packages/debian' do
         include ::API::Concerns::Packages::DebianPackageEndpoints
+
+        # GET groups/:id/packages/debian/pool/:distribution/:project_id/:letter/:package_name/:package_version/:file_name
+        params do
+          requires :project_id, type: Integer, desc: 'The Project Id'
+          use :shared_package_file_params
+        end
+
+        desc 'The package' do
+          detail 'This feature was introduced in GitLab 14.2'
+        end
+
+        route_setting :authentication, authenticate_non_public: true
+        get 'pool/:distribution/:project_id/:letter/:package_name/:package_version/:file_name', requirements: PACKAGE_FILE_REQUIREMENTS do
+          present_package_file!
+        end
       end
     end
   end
diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb
index 70ddf9dea37..497ce2f4356 100644
--- a/lib/api/debian_project_packages.rb
+++ b/lib/api/debian_project_packages.rb
@@ -2,17 +2,23 @@
 
 module API
   class DebianProjectPackages < ::API::Base
-    params do
-      requires :id, type: String, desc: 'The ID of a project'
-    end
+    PACKAGE_FILE_REQUIREMENTS = {
+      id: API::NO_SLASH_URL_PART_REGEX,
+      distribution: ::Packages::Debian::DISTRIBUTION_REGEX,
+      letter: ::Packages::Debian::LETTER_REGEX,
+      package_name: API::NO_SLASH_URL_PART_REGEX,
+      package_version: API::NO_SLASH_URL_PART_REGEX,
+      file_name: API::NO_SLASH_URL_PART_REGEX
+    }.freeze
+    FILE_NAME_REQUIREMENTS = {
+      file_name: API::NO_SLASH_URL_PART_REGEX
+    }.freeze
 
     resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
-      rescue_from ArgumentError do |e|
-        render_api_error!(e.message, 400)
-      end
-
-      rescue_from ActiveRecord::RecordInvalid do |e|
-        render_api_error!(e.message, 400)
+      helpers do
+        def project_or_group
+          user_project
+        end
       end
 
       after_validation do
@@ -23,20 +29,32 @@ module API
         authorize_read_package!
       end
 
-      namespace ':id' do
-        helpers do
-          def project_or_group
-            user_project
-          end
-        end
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
 
+      namespace ':id/packages/debian' do
         include ::API::Concerns::Packages::DebianPackageEndpoints
 
+        # GET projects/:id/packages/debian/pool/:distribution/:letter/:package_name/:package_version/:file_name
+        params do
+          use :shared_package_file_params
+        end
+
+        desc 'The package' do
+          detail 'This feature was introduced in GitLab 14.2'
+        end
+
+        route_setting :authentication, authenticate_non_public: true
+        get 'pool/:distribution/:letter/:package_name/:package_version/:file_name', requirements: PACKAGE_FILE_REQUIREMENTS do
+          present_package_file!
+        end
+
         params do
           requires :file_name, type: String, desc: 'The file name'
         end
 
-        namespace 'packages/debian/:file_name', requirements: FILE_NAME_REQUIREMENTS do
+        namespace ':file_name', requirements: FILE_NAME_REQUIREMENTS do
           format :txt
           content_type :json, Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
 
diff --git a/lib/api/entities/ci/job_request/dependency.rb b/lib/api/entities/ci/job_request/dependency.rb
index 2c6ed417714..2672a4a245b 100644
--- a/lib/api/entities/ci/job_request/dependency.rb
+++ b/lib/api/entities/ci/job_request/dependency.rb
@@ -6,7 +6,7 @@ module API
       module JobRequest
         class Dependency < Grape::Entity
           expose :id, :name, :token
-          expose :artifacts_file, using: Entities::Ci::JobArtifactFile, if: ->(job, _) { job.artifacts? }
+          expose :artifacts_file, using: Entities::Ci::JobArtifactFile, if: ->(job, _) { job.available_artifacts? }
         end
       end
     end
diff --git a/lib/api/entities/ci/pipeline_basic.rb b/lib/api/entities/ci/pipeline_basic.rb
index f4f2356c812..8086062dc9b 100644
--- a/lib/api/entities/ci/pipeline_basic.rb
+++ b/lib/api/entities/ci/pipeline_basic.rb
@@ -7,6 +7,8 @@ module API
         expose :id, :project_id, :sha, :ref, :status
         expose :created_at, :updated_at
 
+        expose :source, if: ->(pipeline, options) { ::Feature.enabled?(:pipeline_source_filter, options[:project], default_enabled: :yaml) }
+
         expose :web_url do |pipeline, _options|
           Gitlab::Routing.url_helpers.project_pipeline_url(pipeline.project, pipeline)
         end
diff --git a/lib/api/entities/error_tracking.rb b/lib/api/entities/error_tracking.rb
index c762c274486..a38e00ca295 100644
--- a/lib/api/entities/error_tracking.rb
+++ b/lib/api/entities/error_tracking.rb
@@ -8,6 +8,7 @@ module API
         expose :project_name
         expose :sentry_external_url
         expose :api_url
+        expose :integrated
       end
     end
   end
diff --git a/lib/api/entities/issue_basic.rb b/lib/api/entities/issue_basic.rb
index 6c332870228..ab248523028 100644
--- a/lib/api/entities/issue_basic.rb
+++ b/lib/api/entities/issue_basic.rb
@@ -23,7 +23,7 @@ module API
       expose :issue_type,
              as: :type,
              format_with: :upcase,
-             documentation: { type: "String", desc: "One of #{::Issue.issue_types.keys.map(&:upcase)}" }
+             documentation: { type: "String", desc: "One of #{::WorkItem::Type.base_types.keys.map(&:upcase)}" }
 
       expose :assignee, using: ::API::Entities::UserBasic do |issue|
         issue.assignees.first
diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb
index f5f565e5b07..890b42ed8c8 100644
--- a/lib/api/entities/project.rb
+++ b/lib/api/entities/project.rb
@@ -71,6 +71,7 @@ module API
       expose(:pages_access_level) { |project, options| project.project_feature.string_access_level(:pages) }
       expose(:operations_access_level) { |project, options| project.project_feature.string_access_level(:operations) }
       expose(:analytics_access_level) { |project, options| project.project_feature.string_access_level(:analytics) }
+      expose(:container_registry_access_level) { |project, options| project.project_feature.string_access_level(:container_registry) }
 
       expose :emails_disabled
       expose :shared_runners_enabled
diff --git a/lib/api/entities/project_with_access.rb b/lib/api/entities/project_with_access.rb
index c53a712a879..ac89cb52e43 100644
--- a/lib/api/entities/project_with_access.rb
+++ b/lib/api/entities/project_with_access.rb
@@ -26,8 +26,10 @@ module API
       # rubocop: disable CodeReuse/ActiveRecord
       def self.preload_relation(projects_relation, options = {})
         relation = super(projects_relation, options)
-        project_ids = relation.select('projects.id')
-        namespace_ids = relation.select(:namespace_id)
+        # use reselect to override the existing select and
+        # prevent an error `subquery has too many columns`
+        project_ids = relation.reselect('projects.id')
+        namespace_ids = relation.reselect(:namespace_id)
 
         options[:project_members] = options[:current_user]
           .project_members
diff --git a/lib/api/environments.rb b/lib/api/environments.rb
index 57e548183b0..e50da4264b5 100644
--- a/lib/api/environments.rb
+++ b/lib/api/environments.rb
@@ -77,7 +77,7 @@ module API
 
       desc "Delete multiple stopped review apps" do
         detail "Remove multiple stopped review environments older than a specific age"
-        success Entities::Environment
+        success Entities::EnvironmentBasic
       end
       params do
         optional :before, type: Time, desc: "The timestamp before which environments can be deleted. Defaults to 30 days ago.", default: -> { 30.days.ago }
@@ -90,8 +90,8 @@ module API
         result = ::Environments::ScheduleToDeleteReviewAppsService.new(user_project, current_user, params).execute
 
         response = {
-          scheduled_entries: Entities::Environment.represent(result.scheduled_entries),
-          unprocessable_entries: Entities::Environment.represent(result.unprocessable_entries)
+          scheduled_entries: Entities::EnvironmentBasic.represent(result.scheduled_entries),
+          unprocessable_entries: Entities::EnvironmentBasic.represent(result.unprocessable_entries)
         }
 
         if result.success?
diff --git a/lib/api/error_tracking.rb b/lib/api/error_tracking.rb
index 0e44c8b1081..3abf2831bd3 100644
--- a/lib/api/error_tracking.rb
+++ b/lib/api/error_tracking.rb
@@ -32,6 +32,7 @@ module API
       end
       params do
         requires :active, type: Boolean, desc: 'Specifying whether to enable or disable error tracking settings', allow_blank: false
+        optional :integrated, type: Boolean, desc: 'Specifying whether to enable or disable integrated error tracking'
       end
 
       patch ':id/error_tracking/settings/' do
@@ -45,6 +46,10 @@ module API
           error_tracking_setting_attributes: { enabled: params[:active] }
         }
 
+        unless params[:integrated].nil?
+          update_params[:error_tracking_setting_attributes][:integrated] = params[:integrated]
+        end
+
         result = ::Projects::Operations::UpdateService.new(user_project, current_user, update_params).execute
 
         if result[:status] == :success
diff --git a/lib/api/error_tracking_collector.rb b/lib/api/error_tracking_collector.rb
index 08ff8d2e4d1..13e8e476808 100644
--- a/lib/api/error_tracking_collector.rb
+++ b/lib/api/error_tracking_collector.rb
@@ -13,6 +13,7 @@ module API
     before do
       not_found!('Project') unless project
       not_found! unless feature_enabled?
+      not_found! unless active_client_key?
     end
 
     helpers do
@@ -21,8 +22,24 @@ module API
       end
 
       def feature_enabled?
-        ::Feature.enabled?(:integrated_error_tracking, project) &&
-          project.error_tracking_setting&.enabled?
+        project.error_tracking_setting&.enabled? &&
+          project.error_tracking_setting&.integrated_client?
+      end
+
+      def find_client_key(public_key)
+        return unless public_key.present?
+
+        project.error_tracking_client_keys.active.find_by_public_key(public_key)
+      end
+
+      def active_client_key?
+        begin
+          public_key = ::ErrorTracking::Collector::SentryAuthParser.parse(request)[:public_key]
+        rescue StandardError
+          bad_request!('Failed to parse sentry request')
+        end
+
+        find_client_key(public_key)
       end
     end
 
@@ -46,7 +63,7 @@ module API
       begin
         parsed_request = ::ErrorTracking::Collector::SentryRequestParser.parse(request)
       rescue StandardError
-        render_api_error!('Failed to parse sentry request', 400)
+        bad_request!('Failed to parse sentry request')
       end
 
       type = parsed_request[:request_type]
@@ -67,6 +84,9 @@ module API
           .execute
       end
 
+      # Collector should never return any information back.
+      # Because DSN and public key are designed for public use,
+      # it is safe only for submission of new events.
       no_content!
     end
   end
diff --git a/lib/api/group_debian_distributions.rb b/lib/api/group_debian_distributions.rb
new file mode 100644
index 00000000000..01a8774bd97
--- /dev/null
+++ b/lib/api/group_debian_distributions.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module API
+  class GroupDebianDistributions < ::API::Base
+    params do
+      requires :id, type: String, desc: 'The ID of a group'
+    end
+
+    resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      rescue_from ArgumentError do |e|
+        render_api_error!(e.message, 400)
+      end
+
+      rescue_from ActiveRecord::RecordInvalid do |e|
+        render_api_error!(e.message, 400)
+      end
+
+      after_validation do
+        require_packages_enabled!
+
+        not_found! unless ::Feature.enabled?(:debian_group_packages, user_group)
+      end
+
+      namespace ':id/-' do
+        helpers do
+          def project_or_group
+            user_group
+          end
+        end
+
+        include ::API::Concerns::Packages::DebianDistributionEndpoints
+      end
+    end
+  end
+end
diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb
index 8d52a0a5b4e..13daf05fc78 100644
--- a/lib/api/group_variables.rb
+++ b/lib/api/group_variables.rb
@@ -8,7 +8,7 @@ module API
     before { authorize! :admin_group, user_group }
     feature_category :continuous_integration
 
-    helpers Helpers::VariablesHelpers
+    helpers ::API::Helpers::VariablesHelpers
 
     params do
       requires :id, type: String, desc: 'The ID of a group'
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 9b6b28733ff..0896357cc73 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -35,7 +35,8 @@ module API
           :all_available,
           :custom_attributes,
           :owned, :min_access_level,
-          :include_parent_descendants
+          :include_parent_descendants,
+          :search
         )
 
         find_params[:parent] = if params[:top_level_only]
@@ -48,7 +49,6 @@ module API
           find_params.fetch(:all_available, current_user&.can_read_all_resources?)
 
         groups = GroupsFinder.new(current_user, find_params).execute
-        groups = groups.search(params[:search], include_parents: true) if params[:search].present?
         groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present?
 
         order_groups(groups)
@@ -128,10 +128,6 @@ module API
         groups.reorder(group_without_similarity_options) # rubocop: disable CodeReuse/ActiveRecord
       end
 
-      def order_by_similarity?
-        params[:order_by] == 'similarity' && params[:search].present?
-      end
-
       def group_without_similarity_options
         order_options = { params[:order_by] => params[:sort] }
         order_options['name'] = order_options.delete('similarity') if order_options.has_key?('similarity')
@@ -141,7 +137,7 @@ module API
 
       # rubocop: disable CodeReuse/ActiveRecord
       def handle_similarity_order(group, projects)
-        if params[:search].present? && Feature.enabled?(:similarity_search, group, default_enabled: true)
+        if params[:search].present?
           projects.sorted_by_similarity_desc(params[:search])
         else
           order_options = { name: :asc }
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 3398d5da7f5..9c347148fd0 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -577,6 +577,10 @@ module API
       Gitlab::AppLogger.warn("Redis tracking event failed for event: #{event_name}, message: #{error.message}")
     end
 
+    def order_by_similarity?(allow_unauthorized: true)
+      params[:order_by] == 'similarity' && params[:search].present? && (allow_unauthorized || current_user.present?)
+    end
+
     protected
 
     def project_finder_params_visibility_ce
diff --git a/lib/api/helpers/groups_helpers.rb b/lib/api/helpers/groups_helpers.rb
index e38213532ba..72bdb32d38c 100644
--- a/lib/api/helpers/groups_helpers.rb
+++ b/lib/api/helpers/groups_helpers.rb
@@ -23,7 +23,7 @@ module API
         optional :mentions_disabled, type: Boolean, desc: 'Disable a group from getting mentioned'
         optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group'
         optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
-        optional :default_branch_protection, type: Integer, values: ::Gitlab::Access.protection_values, desc: 'Determine if developers can push to master'
+        optional :default_branch_protection, type: Integer, values: ::Gitlab::Access.protection_values, desc: 'Determine if developers can push to default branch'
         optional :shared_runners_setting, type: String, values: ::Namespace::SHARED_RUNNERS_SETTINGS, desc: 'Enable/disable shared runners for the group and its subgroups and projects'
       end
 
diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb
index bd0c2501220..e72bbb931f0 100644
--- a/lib/api/helpers/members_helpers.rb
+++ b/lib/api/helpers/members_helpers.rb
@@ -54,6 +54,14 @@ module API
         source.add_user(user, params[:access_level], current_user: current_user, expires_at: params[:expires_at])
       end
 
+      def track_areas_of_focus(member, areas_of_focus)
+        return unless areas_of_focus
+
+        areas_of_focus.each do |area_of_focus|
+          Gitlab::Tracking.event(::Members::CreateService.name, 'area_of_focus', label: area_of_focus, property: member.id.to_s)
+        end
+      end
+
       def present_members(members)
         present members, with: Entities::Member, current_user: current_user, show_seat_info: params[:show_seat_info]
       end
diff --git a/lib/api/helpers/packages/dependency_proxy_helpers.rb b/lib/api/helpers/packages/dependency_proxy_helpers.rb
index 989c4e1761b..b8ae1dddd7e 100644
--- a/lib/api/helpers/packages/dependency_proxy_helpers.rb
+++ b/lib/api/helpers/packages/dependency_proxy_helpers.rb
@@ -5,11 +5,17 @@ module API
     module Packages
       module DependencyProxyHelpers
         REGISTRY_BASE_URLS = {
-          npm: 'https://registry.npmjs.org/'
+          npm: 'https://registry.npmjs.org/',
+          pypi: 'https://pypi.org/simple/'
+        }.freeze
+
+        APPLICATION_SETTING_NAMES = {
+          npm: 'npm_package_requests_forwarding',
+          pypi: 'pypi_package_requests_forwarding'
         }.freeze
 
         def redirect_registry_request(forward_to_registry, package_type, options)
-          if forward_to_registry && redirect_registry_request_available?
+          if forward_to_registry && redirect_registry_request_available?(package_type)
             ::Gitlab::Tracking.event(self.options[:for].name, "#{package_type}_request_forward")
             redirect(registry_url(package_type, options))
           else
@@ -25,11 +31,20 @@ module API
           case package_type
           when :npm
             "#{base_url}#{options[:package_name]}"
+          when :pypi
+            "#{base_url}#{options[:package_name]}/"
           end
         end
 
-        def redirect_registry_request_available?
-          ::Gitlab::CurrentSettings.current_application_settings.npm_package_requests_forwarding
+        def redirect_registry_request_available?(package_type)
+          application_setting_name = APPLICATION_SETTING_NAMES[package_type]
+
+          raise ArgumentError, "Can't find application setting for package_type #{package_type}" unless application_setting_name
+
+          ::Gitlab::CurrentSettings
+            .current_application_settings
+            .attributes
+            .fetch(application_setting_name, false)
         end
       end
     end
diff --git a/lib/api/helpers/packages/npm.rb b/lib/api/helpers/packages/npm.rb
index 2d556f889bf..ce5db52fdbc 100644
--- a/lib/api/helpers/packages/npm.rb
+++ b/lib/api/helpers/packages/npm.rb
@@ -49,28 +49,20 @@ module API
             when :project
               params[:id]
             when :instance
-              namespace_path = namespace_path_from_package_name
+              package_name = params[:package_name]
+              namespace_path = ::Packages::Npm.scope_of(package_name)
               next unless namespace_path
 
               namespace = Namespace.top_most
                                    .by_path(namespace_path)
               next unless namespace
 
-              finder = ::Packages::Npm::PackageFinder.new(params[:package_name], namespace: namespace)
+              finder = ::Packages::Npm::PackageFinder.new(package_name, namespace: namespace)
 
               finder.last&.project_id
             end
           end
         end
-
-        # from "@scope/package-name" return "scope" or nil
-        def namespace_path_from_package_name
-          package_name = params[:package_name]
-          return unless package_name.starts_with?('@')
-          return unless package_name.include?('/')
-
-          package_name.match(Gitlab::Regex.npm_package_name_regex)&.captures&.first
-        end
       end
     end
   end
diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb
index 272452bd8db..becd25595a6 100644
--- a/lib/api/helpers/projects_helpers.rb
+++ b/lib/api/helpers/projects_helpers.rb
@@ -35,13 +35,14 @@ module API
         optional :pages_access_level, type: String, values: %w(disabled private enabled public), desc: 'Pages access level. One of `disabled`, `private`, `enabled` or `public`'
         optional :operations_access_level, type: String, values: %w(disabled private enabled), desc: 'Operations access level. One of `disabled`, `private` or `enabled`'
         optional :analytics_access_level, type: String, values: %w(disabled private enabled), desc: 'Analytics access level. One of `disabled`, `private` or `enabled`'
+        optional :container_registry_access_level, type: String, values: %w(disabled private enabled), desc: 'Controls visibility of the container registry. One of `disabled`, `private` or `enabled`. `private` will make the container registry accessible only to project members (reporter role and above). `enabled` will make the container registry accessible to everyone who has access to the project. `disabled` will disable the container registry'
 
         optional :emails_disabled, type: Boolean, desc: 'Disable email notifications'
         optional :show_default_award_emojis, type: Boolean, desc: 'Show default award emojis'
         optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project'
         optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push'
         optional :remove_source_branch_after_merge, type: Boolean, desc: 'Remove the source branch by default after merge'
-        optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project'
+        optional :container_registry_enabled, type: Boolean, desc: 'Deprecated: Use :container_registry_access_level instead. Flag indication if the container registry is enabled for that project'
         optional :container_expiration_policy_attributes, type: Hash do
           use :optional_container_expiration_policy_params
         end
@@ -124,7 +125,7 @@ module API
           :ci_config_path,
           :ci_default_git_depth,
           :ci_forward_deployment_enabled,
-          :container_registry_enabled,
+          :container_registry_access_level,
           :container_expiration_policy_attributes,
           :default_branch,
           :description,
@@ -132,7 +133,10 @@ module API
           :forking_access_level,
           :issues_access_level,
           :lfs_enabled,
+          :merge_pipelines_enabled,
           :merge_requests_access_level,
+          :merge_requests_template,
+          :merge_trains_enabled,
           :merge_method,
           :name,
           :only_allow_merge_if_all_discussions_are_resolved,
@@ -166,7 +170,8 @@ module API
           :jobs_enabled,
           :merge_requests_enabled,
           :wiki_enabled,
-          :snippets_enabled
+          :snippets_enabled,
+          :container_registry_enabled
         ]
       end
 
diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb
deleted file mode 100644
index a022d1a56ac..00000000000
--- a/lib/api/helpers/runner.rb
+++ /dev/null
@@ -1,121 +0,0 @@
-# frozen_string_literal: true
-
-module API
-  module Helpers
-    module Runner
-      include Gitlab::Utils::StrongMemoize
-
-      prepend_mod_with('API::Helpers::Runner') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
-      JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'
-      JOB_TOKEN_PARAM = :token
-
-      def runner_registration_token_valid?
-        ActiveSupport::SecurityUtils.secure_compare(params[:token], Gitlab::CurrentSettings.runners_registration_token)
-      end
-
-      def runner_registrar_valid?(type)
-        Feature.disabled?(:runner_registration_control) || Gitlab::CurrentSettings.valid_runner_registrars.include?(type)
-      end
-
-      def authenticate_runner!
-        forbidden! unless current_runner
-
-        current_runner
-          .heartbeat(get_runner_details_from_request)
-      end
-
-      def get_runner_details_from_request
-        return get_runner_ip unless params['info'].present?
-
-        attributes_for_keys(%w(name version revision platform architecture), params['info'])
-          .merge(get_runner_config_from_request)
-          .merge(get_runner_ip)
-      end
-
-      def get_runner_ip
-        { ip_address: ip_address }
-      end
-
-      def current_runner
-        token = params[:token]
-
-        if token
-          ::Gitlab::Database::LoadBalancing::RackMiddleware
-            .stick_or_unstick(env, :runner, token)
-        end
-
-        strong_memoize(:current_runner) do
-          ::Ci::Runner.find_by_token(token.to_s)
-        end
-      end
-
-      # HTTP status codes to terminate the job on GitLab Runner:
-      # - 403
-      def authenticate_job!(require_running: true)
-        job = current_job
-
-        # 404 is not returned here because we want to terminate the job if it's
-        # running. A 404 can be returned from anywhere in the networking stack which is why
-        # we are explicit about a 403, we should improve this in
-        # https://gitlab.com/gitlab-org/gitlab/-/issues/327703
-        forbidden! unless job
-
-        forbidden! unless job_token_valid?(job)
-
-        forbidden!('Project has been deleted!') if job.project.nil? || job.project.pending_delete?
-        forbidden!('Job has been erased!') if job.erased?
-
-        if require_running
-          job_forbidden!(job, 'Job is not running') unless job.running?
-        end
-
-        job.runner&.heartbeat(get_runner_ip)
-
-        job
-      end
-
-      def current_job
-        id = params[:id]
-
-        if id
-          ::Gitlab::Database::LoadBalancing::RackMiddleware
-            .stick_or_unstick(env, :build, id)
-        end
-
-        strong_memoize(:current_job) do
-          ::Ci::Build.find_by_id(id)
-        end
-      end
-
-      def job_token_valid?(job)
-        token = (params[JOB_TOKEN_PARAM] || env[JOB_TOKEN_HEADER]).to_s
-        token && job.valid_token?(token)
-      end
-
-      def job_forbidden!(job, reason)
-        header 'Job-Status', job.status
-        forbidden!(reason)
-      end
-
-      def set_application_context
-        return unless current_job
-
-        Gitlab::ApplicationContext.push(
-          user: -> { current_job.user },
-          project: -> { current_job.project }
-        )
-      end
-
-      def track_ci_minutes_usage!(_build, _runner)
-        # noop: overridden in EE
-      end
-
-      private
-
-      def get_runner_config_from_request
-        { config: attributes_for_keys(%w(gpus), params.dig('info', 'config')) }
-      end
-    end
-  end
-end
diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb
index a06b052847d..d740c626557 100644
--- a/lib/api/internal/base.rb
+++ b/lib/api/internal/base.rb
@@ -165,9 +165,9 @@ module API
         # Check whether an SSH key is known to GitLab
         #
         get '/authorized_keys', feature_category: :source_code_management do
-          fingerprint = Gitlab::InsecureKeyFingerprint.new(params.fetch(:key)).fingerprint
+          fingerprint = Gitlab::InsecureKeyFingerprint.new(params.fetch(:key)).fingerprint_sha256
 
-          key = Key.find_by_fingerprint(fingerprint)
+          key = Key.find_by_fingerprint_sha256(fingerprint)
           not_found!('Key') if key.nil?
           present key, with: Entities::SSHKey
         end
diff --git a/lib/api/invitations.rb b/lib/api/invitations.rb
index 46d8c0c958d..1f437ad5bd3 100644
--- a/lib/api/invitations.rb
+++ b/lib/api/invitations.rb
@@ -24,6 +24,7 @@ module API
           requires :access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'A valid access level (defaults: `30`, developer access level)'
           optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY'
           optional :invite_source, type: String, desc: 'Source that triggered the member creation process', default: 'invitations-api'
+          optional :areas_of_focus, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Areas the inviter wants the member to focus upon'
         end
         post ":id/invitations" do
           params[:source] = find_source(source_type, params[:id])
@@ -54,11 +55,11 @@ module API
           success Entities::Member
         end
         params do
-          requires :email, type: String, desc: 'The email address of the invitation.'
-          optional :access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'A valid access level (defaults: `30`, developer access level).'
-          optional :expires_at, type: DateTime, desc: 'Date string in ISO 8601 format (`YYYY-MM-DDTHH:MM:SSZ`).'
+          requires :email, type: String, desc: 'The email address of the invitation'
+          optional :access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'A valid access level (defaults: `30`, developer access level)'
+          optional :expires_at, type: DateTime, desc: 'Date string in ISO 8601 format (`YYYY-MM-DDTHH:MM:SSZ`)'
         end
-        put ":id/invitations/:email", requirements: { email: /[^\/]+/ } do
+        put ":id/invitations/:email", requirements: { email: %r{[^/]+} } do
           source = find_source(source_type, params.delete(:id))
           invite_email = params[:email]
           authorize_admin_source!(source_type, source)
@@ -87,7 +88,7 @@ module API
         params do
           requires :email, type: String, desc: 'The email address of the invitation'
         end
-        delete ":id/invitations/:email", requirements: { email: /[^\/]+/ } do
+        delete ":id/invitations/:email", requirements: { email: %r{[^/]+} } do
           source = find_source(source_type, params[:id])
           invite_email = params[:email]
           authorize_admin_source!(source_type, source)
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 54013d0e7b4..a6565f913e3 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -74,7 +74,7 @@ module API
                  desc: 'Return issues sorted in `asc` or `desc` order.'
         optional :due_date, type: String, values: %w[0 overdue week month next_month_and_previous_two_weeks] << '',
                  desc: 'Return issues that have no due date (`0`), or whose due date is this week, this month, between two weeks ago and next month, or which are overdue. Accepts: `overdue`, `week`, `month`, `next_month_and_previous_two_weeks`, `0`'
-        optional :issue_type, type: String, values: Issue.issue_types.keys, desc: "The type of the issue. Accepts: #{Issue.issue_types.keys.join(', ')}"
+        optional :issue_type, type: String, values: WorkItem::Type.base_types.keys, desc: "The type of the issue. Accepts: #{WorkItem::Type.base_types.keys.join(', ')}"
 
         use :issues_stats_params
         use :pagination
@@ -91,7 +91,7 @@ module API
         optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY'
         optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential'
         optional :discussion_locked, type: Boolean, desc: " Boolean parameter indicating if the issue's discussion is locked"
-        optional :issue_type, type: String, values: Issue.issue_types.keys, desc: "The type of the issue. Accepts: #{Issue.issue_types.keys.join(', ')}"
+        optional :issue_type, type: String, values: WorkItem::Type.base_types.keys, desc: "The type of the issue. Accepts: #{WorkItem::Type.base_types.keys.join(', ')}"
 
         use :optional_issue_params_ee
       end
diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb
deleted file mode 100644
index beda4433e4f..00000000000
--- a/lib/api/job_artifacts.rb
+++ /dev/null
@@ -1,141 +0,0 @@
-# frozen_string_literal: true
-
-module API
-  class JobArtifacts < ::API::Base
-    before { authenticate_non_get! }
-
-    feature_category :build_artifacts
-
-    # EE::API::JobArtifacts would override the following helpers
-    helpers do
-      def authorize_download_artifacts!
-        authorize_read_builds!
-      end
-    end
-
-    prepend_mod_with('API::JobArtifacts') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
-    params do
-      requires :id, type: String, desc: 'The ID of a project'
-    end
-    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
-      desc 'Download the artifacts archive from a job' do
-        detail 'This feature was introduced in GitLab 8.10'
-      end
-      params do
-        requires :ref_name, type: String, desc: 'The ref from repository'
-        requires :job,      type: String, desc: 'The name for the job'
-      end
-      route_setting :authentication, job_token_allowed: true
-      get ':id/jobs/artifacts/:ref_name/download',
-        requirements: { ref_name: /.+/ } do
-        authorize_download_artifacts!
-
-        latest_build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name])
-        authorize_read_job_artifacts!(latest_build)
-
-        present_carrierwave_file!(latest_build.artifacts_file)
-      end
-
-      desc 'Download a specific file from artifacts archive from a ref' do
-        detail 'This feature was introduced in GitLab 11.5'
-      end
-      params do
-        requires :ref_name, type: String, desc: 'The ref from repository'
-        requires :job, type: String, desc: 'The name for the job'
-        requires :artifact_path, type: String, desc: 'Artifact path'
-      end
-      route_setting :authentication, job_token_allowed: true
-      get ':id/jobs/artifacts/:ref_name/raw/*artifact_path',
-          format: false,
-          requirements: { ref_name: /.+/ } do
-        authorize_download_artifacts!
-
-        build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name])
-        authorize_read_job_artifacts!(build)
-
-        path = Gitlab::Ci::Build::Artifacts::Path
-                 .new(params[:artifact_path])
-
-        bad_request! unless path.valid?
-
-        send_artifacts_entry(build.artifacts_file, path)
-      end
-
-      desc 'Download the artifacts archive from a job' do
-        detail 'This feature was introduced in GitLab 8.5'
-      end
-      params do
-        requires :job_id, type: Integer, desc: 'The ID of a job'
-      end
-      route_setting :authentication, job_token_allowed: true
-      get ':id/jobs/:job_id/artifacts' do
-        authorize_download_artifacts!
-
-        build = find_build!(params[:job_id])
-        authorize_read_job_artifacts!(build)
-
-        present_carrierwave_file!(build.artifacts_file)
-      end
-
-      desc 'Download a specific file from artifacts archive' do
-        detail 'This feature was introduced in GitLab 10.0'
-      end
-      params do
-        requires :job_id, type: Integer, desc: 'The ID of a job'
-        requires :artifact_path, type: String, desc: 'Artifact path'
-      end
-      route_setting :authentication, job_token_allowed: true
-      get ':id/jobs/:job_id/artifacts/*artifact_path', format: false do
-        authorize_download_artifacts!
-
-        build = find_build!(params[:job_id])
-        authorize_read_job_artifacts!(build)
-
-        not_found! unless build.available_artifacts?
-
-        path = Gitlab::Ci::Build::Artifacts::Path
-          .new(params[:artifact_path])
-
-        bad_request! unless path.valid?
-
-        send_artifacts_entry(build.artifacts_file, path)
-      end
-
-      desc 'Keep the artifacts to prevent them from being deleted' do
-        success ::API::Entities::Ci::Job
-      end
-      params do
-        requires :job_id, type: Integer, desc: 'The ID of a job'
-      end
-      post ':id/jobs/:job_id/artifacts/keep' do
-        authorize_update_builds!
-
-        build = find_build!(params[:job_id])
-        authorize!(:update_build, build)
-        break not_found!(build) unless build.artifacts?
-
-        build.keep_artifacts!
-
-        status 200
-        present build, with: ::API::Entities::Ci::Job
-      end
-
-      desc 'Delete the artifacts files from a job' do
-        detail 'This feature was introduced in GitLab 11.9'
-      end
-      params do
-        requires :job_id, type: Integer, desc: 'The ID of a job'
-      end
-      delete ':id/jobs/:job_id/artifacts' do
-        authorize_destroy_artifacts!
-        build = find_build!(params[:job_id])
-        authorize!(:destroy_artifacts, build)
-
-        build.erase_erasable_artifacts!
-
-        status :no_content
-      end
-    end
-  end
-end
diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb
deleted file mode 100644
index 723a5b0fa3a..00000000000
--- a/lib/api/jobs.rb
+++ /dev/null
@@ -1,204 +0,0 @@
-# frozen_string_literal: true
-
-module API
-  class Jobs < ::API::Base
-    include PaginationParams
-    before { authenticate! }
-
-    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
-      params do
-        requires :id, type: String, desc: 'The ID of a project'
-      end
-
-      helpers do
-        params :optional_scope do
-          optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show',
-                           values: ::CommitStatus::AVAILABLE_STATUSES,
-                           coerce_with: ->(scope) {
-                             case scope
-                             when String
-                               [scope]
-                             when ::Hash
-                               scope.values
-                             when ::Array
-                               scope
-                             else
-                               ['unknown']
-                             end
-                           }
-        end
-      end
-
-      desc 'Get a projects jobs' do
-        success Entities::Ci::Job
-      end
-      params do
-        use :optional_scope
-        use :pagination
-      end
-      # rubocop: disable CodeReuse/ActiveRecord
-      get ':id/jobs', feature_category: :continuous_integration do
-        authorize_read_builds!
-
-        builds = user_project.builds.order('id DESC')
-        builds = filter_builds(builds, params[:scope])
-
-        builds = builds.preload(:user, :job_artifacts_archive, :job_artifacts, :runner, :tags, pipeline: :project)
-        present paginate(builds), with: Entities::Ci::Job
-      end
-      # rubocop: enable CodeReuse/ActiveRecord
-
-      desc 'Get a specific job of a project' do
-        success Entities::Ci::Job
-      end
-      params do
-        requires :job_id, type: Integer, desc: 'The ID of a job'
-      end
-      get ':id/jobs/:job_id', feature_category: :continuous_integration do
-        authorize_read_builds!
-
-        build = find_build!(params[:job_id])
-
-        present build, with: Entities::Ci::Job
-      end
-
-      # TODO: We should use `present_disk_file!` and leave this implementation for backward compatibility (when build trace
-      #       is saved in the DB instead of file). But before that, we need to consider how to replace the value of
-      #       `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse.
-      desc 'Get a trace of a specific job of a project'
-      params do
-        requires :job_id, type: Integer, desc: 'The ID of a job'
-      end
-      get ':id/jobs/:job_id/trace', feature_category: :continuous_integration do
-        authorize_read_builds!
-
-        build = find_build!(params[:job_id])
-
-        authorize_read_build_trace!(build) if build
-
-        header 'Content-Disposition', "infile; filename=\"#{build.id}.log\""
-        content_type 'text/plain'
-        env['api.format'] = :binary
-
-        # The trace can be nil bu body method expects a string as an argument.
-        trace = build.trace.raw || ''
-        body trace
-      end
-
-      desc 'Cancel a specific job of a project' do
-        success Entities::Ci::Job
-      end
-      params do
-        requires :job_id, type: Integer, desc: 'The ID of a job'
-      end
-      post ':id/jobs/:job_id/cancel', feature_category: :continuous_integration do
-        authorize_update_builds!
-
-        build = find_build!(params[:job_id])
-        authorize!(:update_build, build)
-
-        build.cancel
-
-        present build, with: Entities::Ci::Job
-      end
-
-      desc 'Retry a specific build of a project' do
-        success Entities::Ci::Job
-      end
-      params do
-        requires :job_id, type: Integer, desc: 'The ID of a build'
-      end
-      post ':id/jobs/:job_id/retry', feature_category: :continuous_integration do
-        authorize_update_builds!
-
-        build = find_build!(params[:job_id])
-        authorize!(:update_build, build)
-        break forbidden!('Job is not retryable') unless build.retryable?
-
-        build = ::Ci::Build.retry(build, current_user)
-
-        present build, with: Entities::Ci::Job
-      end
-
-      desc 'Erase job (remove artifacts and the trace)' do
-        success Entities::Ci::Job
-      end
-      params do
-        requires :job_id, type: Integer, desc: 'The ID of a build'
-      end
-      post ':id/jobs/:job_id/erase', feature_category: :continuous_integration do
-        authorize_update_builds!
-
-        build = find_build!(params[:job_id])
-        authorize!(:erase_build, build)
-        break forbidden!('Job is not erasable!') unless build.erasable?
-
-        build.erase(erased_by: current_user)
-        present build, with: Entities::Ci::Job
-      end
-
-      desc 'Trigger an actionable job (manual, delayed, etc)' do
-        success Entities::Ci::JobBasic
-        detail 'This feature was added in GitLab 8.11'
-      end
-      params do
-        requires :job_id, type: Integer, desc: 'The ID of a Job'
-      end
-
-      post ":id/jobs/:job_id/play", feature_category: :continuous_integration do
-        authorize_read_builds!
-
-        job = find_job!(params[:job_id])
-
-        authorize!(:play_job, job)
-
-        bad_request!("Unplayable Job") unless job.playable?
-
-        job.play(current_user)
-
-        status 200
-
-        if job.is_a?(::Ci::Build)
-          present job, with: Entities::Ci::Job
-        else
-          present job, with: Entities::Ci::Bridge
-        end
-      end
-    end
-
-    resource :job do
-      desc 'Get current project using job token' do
-        success Entities::Ci::Job
-      end
-      route_setting :authentication, job_token_allowed: true
-      get '', feature_category: :continuous_integration do
-        validate_current_authenticated_job
-
-        present current_authenticated_job, with: Entities::Ci::Job
-      end
-    end
-
-    helpers do
-      # rubocop: disable CodeReuse/ActiveRecord
-      def filter_builds(builds, scope)
-        return builds if scope.nil? || scope.empty?
-
-        available_statuses = ::CommitStatus::AVAILABLE_STATUSES
-
-        unknown = scope - available_statuses
-        render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty?
-
-        builds.where(status: available_statuses && scope)
-      end
-      # rubocop: enable CodeReuse/ActiveRecord
-
-      def validate_current_authenticated_job
-        # current_authenticated_job will be nil if user is using
-        # a valid authentication (like PRIVATE-TOKEN) that is not CI_JOB_TOKEN
-        not_found!('Job') unless current_authenticated_job
-      end
-    end
-  end
-end
-
-API::Jobs.prepend_mod_with('API::Jobs')
diff --git a/lib/api/members.rb b/lib/api/members.rb
index 70e13e8d4ae..7130635281a 100644
--- a/lib/api/members.rb
+++ b/lib/api/members.rb
@@ -94,6 +94,7 @@ module API
           requires :user_id, types: [Integer, String], desc: 'The user ID of the new member or multiple IDs separated by commas.'
           optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY'
           optional :invite_source, type: String, desc: 'Source that triggered the member creation process', default: 'members-api'
+          optional :areas_of_focus, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Areas the inviter wants the member to focus upon'
         end
         # rubocop: disable CodeReuse/ActiveRecord
         post ":id/members" do
@@ -119,7 +120,12 @@ module API
               not_allowed! # This currently can only be reached in EE
             elsif member.valid? && member.persisted?
               present_members(member)
-              Gitlab::Tracking.event(::Members::CreateService.name, 'create_member', label: params[:invite_source], property: 'existing_user', user: current_user)
+              Gitlab::Tracking.event(::Members::CreateService.name,
+                                     'create_member',
+                                     label: params[:invite_source],
+                                     property: 'existing_user',
+                                     user: current_user)
+              track_areas_of_focus(member, params[:areas_of_focus])
             else
               render_validation_error!(member)
             end
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index a9617482557..7ab57982907 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -404,6 +404,7 @@ module API
         pipeline = ::MergeRequests::CreatePipelineService
           .new(project: user_project, current_user: current_user, params: { allow_duplicate: true })
           .execute(find_merge_request_with_access(params[:merge_request_iid]))
+          .payload
 
         if pipeline.nil?
           not_allowed!
diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb
index 9d41c2f148f..c2d839571a6 100644
--- a/lib/api/namespaces.rb
+++ b/lib/api/namespaces.rb
@@ -27,12 +27,15 @@ module API
       end
       params do
         optional :search, type: String, desc: "Search query for namespaces"
+        optional :owned_only, type: Boolean, desc: "Owned namespaces only"
 
         use :pagination
         use :optional_list_params_ee
       end
       get do
-        namespaces = current_user.admin ? Namespace.all : current_user.namespaces
+        owned_only = params[:owned_only] == true
+
+        namespaces = current_user.admin ? Namespace.all : current_user.namespaces(owned_only: owned_only)
 
         namespaces = namespaces.include_route
 
diff --git a/lib/api/project_debian_distributions.rb b/lib/api/project_debian_distributions.rb
index 58edf51f4f7..f057251fb6b 100644
--- a/lib/api/project_debian_distributions.rb
+++ b/lib/api/project_debian_distributions.rb
@@ -19,8 +19,6 @@ module API
         require_packages_enabled!
 
         not_found! unless ::Feature.enabled?(:debian_packages, user_project)
-
-        authorize_read_package!
       end
 
       namespace ':id' do
diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb
index acf9bfece65..fe0e837c596 100644
--- a/lib/api/project_templates.rb
+++ b/lib/api/project_templates.rb
@@ -12,7 +12,7 @@ module API
 
     before { authenticate_non_get! }
 
-    feature_category :templates
+    feature_category :source_code_management
 
     params do
       requires :id, type: String, desc: 'The ID of a project'
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 3b1d239398f..28bcb382ecf 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -45,6 +45,20 @@ module API
                             end
       end
 
+      def support_order_by_similarity!(attrs)
+        return unless params[:order_by] == 'similarity'
+
+        if order_by_similarity?(allow_unauthorized: false)
+          # Limit to projects the current user is a member of.
+          # Do not include all public projects because it
+          # could cause long running queries
+          attrs[:non_public] = true
+          attrs[:sort] = params['order_by']
+        else
+          params[:order_by] = route.params['order_by'][:default]
+        end
+      end
+
       def delete_project(user_project)
         destroy_conditionally!(user_project) do
           ::Projects::DestroyService.new(user_project, current_user, {}).async_execute
@@ -93,8 +107,8 @@ module API
 
       params :sort_params do
         optional :order_by, type: String,
-                            values: %w[id name path created_at updated_at last_activity_at] + Helpers::ProjectsHelpers::STATISTICS_SORT_PARAMS,
-                            default: 'created_at', desc: "Return projects ordered by field. #{Helpers::ProjectsHelpers::STATISTICS_SORT_PARAMS.join(', ')} are only available to admins."
+                            values: %w[id name path created_at updated_at last_activity_at similarity] + Helpers::ProjectsHelpers::STATISTICS_SORT_PARAMS,
+                            default: 'created_at', desc: "Return projects ordered by field. #{Helpers::ProjectsHelpers::STATISTICS_SORT_PARAMS.join(', ')} are only available to admins. Similarity is available when searching and is limited to projects the user has access to."
         optional :sort, type: String, values: %w[asc desc], default: 'desc',
                         desc: 'Return projects sorted in ascending and descending order'
       end
@@ -131,16 +145,17 @@ module API
       end
 
       def load_projects
-        params = project_finder_params
-        verify_project_filters!(params)
+        project_params = project_finder_params
+        support_order_by_similarity!(project_params)
+        verify_project_filters!(project_params)
 
-        ProjectsFinder.new(current_user: current_user, params: params).execute
+        ProjectsFinder.new(current_user: current_user, params: project_params).execute
       end
 
       def present_projects(projects, options = {})
         verify_statistics_order_by_projects!
 
-        projects = reorder_projects(projects)
+        projects = reorder_projects(projects) unless order_by_similarity?(allow_unauthorized: false)
         projects = apply_filters(projects)
 
         records, options = paginate_with_strategies(projects, options[:request_scope]) do |projects|
@@ -572,6 +587,27 @@ module API
       end
       # rubocop: enable CodeReuse/ActiveRecord
 
+      desc 'Import members from another project' do
+        detail 'This feature was introduced in GitLab 14.2'
+      end
+      params do
+        requires :project_id, type: Integer, desc: 'The ID of the source project to import the members from.'
+      end
+      post ":id/import_project_members/:project_id", feature_category: :experimentation_expansion do
+        authorize! :admin_project, user_project
+
+        source_project = Project.find_by_id(params[:project_id])
+        not_found!('Project') unless source_project && can?(current_user, :read_project, source_project)
+
+        result = ::Members::ImportProjectTeamService.new(current_user, params).execute
+
+        if result
+          { status: result, message: 'Successfully imported' }
+        else
+          render_api_error!('Import failed', :unprocessable_entity)
+        end
+      end
+
       desc 'Workhorse authorize the file upload' do
         detail 'This feature was introduced in GitLab 13.11'
       end
diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb
index 7c5f8bb4d99..706c0702fce 100644
--- a/lib/api/pypi_packages.rb
+++ b/lib/api/pypi_packages.rb
@@ -10,6 +10,7 @@ module API
     helpers ::API::Helpers::PackagesManagerClientsHelpers
     helpers ::API::Helpers::RelatedResourcesHelpers
     helpers ::API::Helpers::Packages::BasicAuthHelpers
+    helpers ::API::Helpers::Packages::DependencyProxyHelpers
     include ::API::Helpers::Packages::BasicAuthHelpers::Constants
 
     feature_category :package_registry
@@ -40,7 +41,7 @@ module API
     end
 
     params do
-      requires :id, type: Integer, desc: 'The ID of a group'
+      requires :id, type: String, desc: 'The ID of a group'
     end
     resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       after_validation do
@@ -82,21 +83,26 @@ module API
 
           track_package_event('list_package', :pypi)
 
-          packages = Packages::Pypi::PackagesFinder.new(current_user, group, { package_name: params[:package_name] }).execute!
-          presenter = ::Packages::Pypi::PackagePresenter.new(packages, group)
+          packages = Packages::Pypi::PackagesFinder.new(current_user, group, { package_name: params[:package_name] }).execute
+          empty_packages = packages.empty?
 
-          # Adjusts grape output format
-          # to be HTML
-          content_type "text/html; charset=utf-8"
-          env['api.format'] = :binary
+          redirect_registry_request(empty_packages, :pypi, package_name: params[:package_name]) do
+            not_found!('Package') if empty_packages
+            presenter = ::Packages::Pypi::PackagePresenter.new(packages, group)
 
-          body presenter.body
+            # Adjusts grape output format
+            # to be HTML
+            content_type "text/html; charset=utf-8"
+            env['api.format'] = :binary
+
+            body presenter.body
+          end
         end
       end
     end
 
     params do
-      requires :id, type: Integer, desc: 'The ID of a project'
+      requires :id, type: String, desc: 'The ID of a project'
     end
 
     resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
@@ -142,15 +148,20 @@ module API
 
           track_package_event('list_package', :pypi, project: authorized_user_project, namespace: authorized_user_project.namespace)
 
-          packages = Packages::Pypi::PackagesFinder.new(current_user, authorized_user_project, { package_name: params[:package_name] }).execute!
-          presenter = ::Packages::Pypi::PackagePresenter.new(packages, authorized_user_project)
+          packages = Packages::Pypi::PackagesFinder.new(current_user, authorized_user_project, { package_name: params[:package_name] }).execute
+          empty_packages = packages.empty?
+
+          redirect_registry_request(empty_packages, :pypi, package_name: params[:package_name]) do
+            not_found!('Package') if empty_packages
+            presenter = ::Packages::Pypi::PackagePresenter.new(packages, authorized_user_project)
 
-          # Adjusts grape output format
-          # to be HTML
-          content_type "text/html; charset=utf-8"
-          env['api.format'] = :binary
+            # Adjusts grape output format
+            # to be HTML
+            content_type "text/html; charset=utf-8"
+            env['api.format'] = :binary
 
-          body presenter.body
+            body presenter.body
+          end
         end
 
         desc 'The PyPi Package upload endpoint' do
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index f274406e225..20320d1b7ae 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -29,14 +29,13 @@ module API
           not_found!
         end
 
-        def assign_blob_vars!
+        def assign_blob_vars!(limit:)
           authorize! :download_code, user_project
 
           @repo = user_project.repository
 
           begin
-            @blob = Gitlab::Git::Blob.raw(@repo, params[:sha])
-            @blob.load_all_data!(@repo)
+            @blob = Gitlab::Git::Blob.raw(@repo, params[:sha], limit: limit)
           rescue StandardError
             not_found! 'Blob'
           end
@@ -55,7 +54,7 @@ module API
         use :pagination
       end
       get ':id/repository/tree' do
-        ref = params[:ref] || user_project.try(:default_branch) || 'master'
+        ref = params[:ref] || user_project.default_branch
         path = params[:path] || nil
 
         commit = user_project.commit(ref)
@@ -71,7 +70,8 @@ module API
         requires :sha, type: String, desc: 'The commit hash'
       end
       get ':id/repository/blobs/:sha/raw' do
-        assign_blob_vars!
+        # Load metadata enough to ask Workhorse to load the whole blob
+        assign_blob_vars!(limit: 0)
 
         no_cache_headers
 
@@ -83,7 +83,7 @@ module API
         requires :sha, type: String, desc: 'The commit hash'
       end
       get ':id/repository/blobs/:sha' do
-        assign_blob_vars!
+        assign_blob_vars!(limit: -1)
 
         {
           size: @blob.size,
diff --git a/lib/api/rubygem_packages.rb b/lib/api/rubygem_packages.rb
index d7f9c584c67..9ef6ec03a41 100644
--- a/lib/api/rubygem_packages.rb
+++ b/lib/api/rubygem_packages.rb
@@ -101,7 +101,7 @@ module API
 
             package_file = nil
 
-            ActiveRecord::Base.transaction do
+            ApplicationRecord.transaction do
               package = ::Packages::CreateTemporaryPackageService.new(
                 user_project, current_user, declared_params.merge(build: current_authenticated_job)
               ).execute(:rubygems, name: ::Packages::Rubygems::TEMPORARY_PACKAGE_NAME)
diff --git a/lib/api/settings.rb b/lib/api/settings.rb
index 952bf09b1b1..aac195f0668 100644
--- a/lib/api/settings.rb
+++ b/lib/api/settings.rb
@@ -48,7 +48,7 @@ module API
       optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts"
       optional :default_ci_config_path, type: String, desc: 'The instance default CI/CD configuration file and path for new projects'
       optional :default_project_creation, type: Integer, values: ::Gitlab::Access.project_creation_values, desc: 'Determine if developers can create projects in the group'
-      optional :default_branch_protection, type: Integer, values: ::Gitlab::Access.protection_values, desc: 'Determine if developers can push to master'
+      optional :default_branch_protection, type: Integer, values: ::Gitlab::Access.protection_values, desc: 'Determine if developers can push to default branch'
       optional :default_group_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default group visibility'
       optional :default_project_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default project visibility'
       optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects'
diff --git a/lib/api/tags.rb b/lib/api/tags.rb
index 6c8e2c69a6d..395aacced78 100644
--- a/lib/api/tags.rb
+++ b/lib/api/tags.rb
@@ -59,8 +59,6 @@ module API
         optional :message,             type: String, desc: 'Specifying a message creates an annotated tag'
       end
       post ':id/repository/tags', :release_orchestration do
-        deprecate_release_notes unless params[:release_description].blank?
-
         authorize_admin_tag
 
         result = ::Tags::CreateService.new(user_project, current_user)
diff --git a/lib/api/templates.rb b/lib/api/templates.rb
index b7fb35eac03..a595129fd6a 100644
--- a/lib/api/templates.rb
+++ b/lib/api/templates.rb
@@ -4,17 +4,18 @@ module API
   class Templates < ::API::Base
     include PaginationParams
 
-    feature_category :templates
-
     GLOBAL_TEMPLATE_TYPES = {
       gitignores: {
-        gitlab_version: 8.8
+        gitlab_version: 8.8,
+        feature_category: :source_code_management
       },
       gitlab_ci_ymls: {
-        gitlab_version: 8.9
+        gitlab_version: 8.9,
+        feature_category: :continuous_integration
       },
       dockerfiles: {
-        gitlab_version: 8.15
+        gitlab_version: 8.15,
+        feature_category: :source_code_management
       }
     }.freeze
 
@@ -33,7 +34,7 @@ module API
       optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses'
       use :pagination
     end
-    get "templates/licenses" do
+    get "templates/licenses", feature_category: :source_code_management do
       popular = declared(params)[:popular]
       popular = to_boolean(popular) if popular.present?
 
@@ -49,7 +50,7 @@ module API
     params do
       requires :name, type: String, desc: 'The name of the template'
     end
-    get "templates/licenses/:name", requirements: { name: /[\w\.-]+/ } do
+    get "templates/licenses/:name", requirements: { name: /[\w\.-]+/ }, feature_category: :source_code_management do
       template = TemplateFinder.build(:licenses, nil, name: params[:name]).execute
 
       not_found!('License') unless template.present?
@@ -72,7 +73,7 @@ module API
       params do
         use :pagination
       end
-      get "templates/#{template_type}" do
+      get "templates/#{template_type}", feature_category: properties[:feature_category] do
         templates = ::Kaminari.paginate_array(TemplateFinder.build(template_type, nil).execute)
         present paginate(templates), with: Entities::TemplatesList
       end
@@ -84,7 +85,7 @@ module API
       params do
         requires :name, type: String, desc: 'The name of the template'
       end
-      get "templates/#{template_type}/:name", requirements: { name: /[\w\.-]+/ } do
+      get "templates/#{template_type}/:name", requirements: { name: /[\w\.-]+/ }, feature_category: properties[:feature_category] do
         finder = TemplateFinder.build(template_type, nil, name: declared(params)[:name])
         new_template = finder.execute
 
diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb
index 969122d7906..b8323304957 100644
--- a/lib/api/time_tracking_endpoints.rb
+++ b/lib/api/time_tracking_endpoints.rb
@@ -88,6 +88,7 @@ module API
         update_params = {
           spend_time: {
             duration: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration)),
+            summary: params.delete(:summary),
             user_id: current_user.id
           }
         }
diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb
deleted file mode 100644
index a359083a9d2..00000000000
--- a/lib/api/triggers.rb
+++ /dev/null
@@ -1,146 +0,0 @@
-# frozen_string_literal: true
-
-module API
-  class Triggers < ::API::Base
-    include PaginationParams
-
-    HTTP_GITLAB_EVENT_HEADER = "HTTP_#{WebHookService::GITLAB_EVENT_HEADER}".underscore.upcase
-
-    feature_category :continuous_integration
-
-    params do
-      requires :id, type: String, desc: 'The ID of a project'
-    end
-    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
-      desc 'Trigger a GitLab project pipeline' do
-        success Entities::Ci::Pipeline
-      end
-      params do
-        requires :ref, type: String, desc: 'The commit sha or name of a branch or tag', allow_blank: false
-        requires :token, type: String, desc: 'The unique token of trigger or job token'
-        optional :variables, type: Hash, desc: 'The list of variables to be injected into build'
-      end
-      post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do
-        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20758')
-
-        forbidden! if gitlab_pipeline_hook_request?
-
-        # validate variables
-        params[:variables] = params[:variables].to_h
-        unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) }
-          render_api_error!('variables needs to be a map of key-valued strings', 400)
-        end
-
-        project = find_project(params[:id])
-        not_found! unless project
-
-        result = ::Ci::PipelineTriggerService.new(project, nil, params).execute
-        not_found! unless result
-
-        if result.error?
-          render_api_error!(result[:message], result[:http_status])
-        else
-          present result[:pipeline], with: Entities::Ci::Pipeline
-        end
-      end
-
-      desc 'Get triggers list' do
-        success Entities::Trigger
-      end
-      params do
-        use :pagination
-      end
-      # rubocop: disable CodeReuse/ActiveRecord
-      get ':id/triggers' do
-        authenticate!
-        authorize! :admin_build, user_project
-
-        triggers = user_project.triggers.includes(:trigger_requests)
-
-        present paginate(triggers), with: Entities::Trigger, current_user: current_user
-      end
-      # rubocop: enable CodeReuse/ActiveRecord
-
-      desc 'Get specific trigger of a project' do
-        success Entities::Trigger
-      end
-      params do
-        requires :trigger_id, type: Integer, desc: 'The trigger ID'
-      end
-      get ':id/triggers/:trigger_id' do
-        authenticate!
-        authorize! :admin_build, user_project
-
-        trigger = user_project.triggers.find(params.delete(:trigger_id))
-        break not_found!('Trigger') unless trigger
-
-        present trigger, with: Entities::Trigger, current_user: current_user
-      end
-
-      desc 'Create a trigger' do
-        success Entities::Trigger
-      end
-      params do
-        requires :description, type: String, desc: 'The trigger description'
-      end
-      post ':id/triggers' do
-        authenticate!
-        authorize! :admin_build, user_project
-
-        trigger = user_project.triggers.create(
-          declared_params(include_missing: false).merge(owner: current_user))
-
-        if trigger.valid?
-          present trigger, with: Entities::Trigger, current_user: current_user
-        else
-          render_validation_error!(trigger)
-        end
-      end
-
-      desc 'Update a trigger' do
-        success Entities::Trigger
-      end
-      params do
-        requires :trigger_id, type: Integer,  desc: 'The trigger ID'
-        optional :description, type: String,  desc: 'The trigger description'
-      end
-      put ':id/triggers/:trigger_id' do
-        authenticate!
-        authorize! :admin_build, user_project
-
-        trigger = user_project.triggers.find(params.delete(:trigger_id))
-        break not_found!('Trigger') unless trigger
-
-        authorize! :admin_trigger, trigger
-
-        if trigger.update(declared_params(include_missing: false))
-          present trigger, with: Entities::Trigger, current_user: current_user
-        else
-          render_validation_error!(trigger)
-        end
-      end
-
-      desc 'Delete a trigger' do
-        success Entities::Trigger
-      end
-      params do
-        requires :trigger_id, type: Integer, desc: 'The trigger ID'
-      end
-      delete ':id/triggers/:trigger_id' do
-        authenticate!
-        authorize! :admin_build, user_project
-
-        trigger = user_project.triggers.find(params.delete(:trigger_id))
-        break not_found!('Trigger') unless trigger
-
-        destroy_conditionally!(trigger)
-      end
-    end
-
-    helpers do
-      def gitlab_pipeline_hook_request?
-        request.get_header(HTTP_GITLAB_EVENT_HEADER) == WebHookService.hook_to_event(:pipeline_hooks)
-      end
-    end
-  end
-end
diff --git a/lib/api/user_counts.rb b/lib/api/user_counts.rb
index 31c923a219a..634dd0f2179 100644
--- a/lib/api/user_counts.rb
+++ b/lib/api/user_counts.rb
@@ -6,15 +6,17 @@ module API
 
     resource :user_counts do
       desc 'Return the user specific counts' do
-        detail 'Open MR Count'
+        detail 'Assigned open issues, assigned MRs and pending todos count'
       end
       get do
         unauthorized! unless current_user
 
         {
           merge_requests: current_user.assigned_open_merge_requests_count, # @deprecated
+          assigned_issues: current_user.assigned_open_issues_count,
           assigned_merge_requests: current_user.assigned_open_merge_requests_count,
-          review_requested_merge_requests: current_user.review_requested_open_merge_requests_count
+          review_requested_merge_requests: current_user.review_requested_open_merge_requests_count,
+          todos: current_user.todos_pending_count
         }
       end
     end
diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb
index 29e4a79110f..310054c298a 100644
--- a/lib/api/v3/github.rb
+++ b/lib/api/v3/github.rb
@@ -214,6 +214,8 @@ module API
 
           update_project_feature_usage_for(user_project)
 
+          next [] unless user_project.repo_exists?
+
           branches = ::Kaminari.paginate_array(user_project.repository.branches.sort_by(&:name))
 
           present paginate(branches), with: ::API::Github::Entities::Branch, project: user_project
diff --git a/lib/api/variables.rb b/lib/api/variables.rb
deleted file mode 100644
index 75df0e050a6..00000000000
--- a/lib/api/variables.rb
+++ /dev/null
@@ -1,124 +0,0 @@
-# frozen_string_literal: true
-
-module API
-  class Variables < ::API::Base
-    include PaginationParams
-
-    before { authenticate! }
-    before { authorize! :admin_build, user_project }
-
-    feature_category :pipeline_authoring
-
-    helpers Helpers::VariablesHelpers
-
-    params do
-      requires :id, type: String, desc: 'The ID of a project'
-    end
-
-    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
-      desc 'Get project variables' do
-        success Entities::Ci::Variable
-      end
-      params do
-        use :pagination
-      end
-      get ':id/variables' do
-        variables = user_project.variables
-        present paginate(variables), with: Entities::Ci::Variable
-      end
-
-      desc 'Get a specific variable from a project' do
-        success Entities::Ci::Variable
-      end
-      params do
-        requires :key, type: String, desc: 'The key of the variable'
-      end
-      # rubocop: disable CodeReuse/ActiveRecord
-      get ':id/variables/:key' do
-        variable = find_variable(user_project, params)
-        not_found!('Variable') unless variable
-
-        present variable, with: Entities::Ci::Variable
-      end
-      # rubocop: enable CodeReuse/ActiveRecord
-
-      desc 'Create a new variable in a project' do
-        success Entities::Ci::Variable
-      end
-      params do
-        requires :key, type: String, desc: 'The key of the variable'
-        requires :value, type: String, desc: 'The value of the variable'
-        optional :protected, type: Boolean, desc: 'Whether the variable is protected'
-        optional :masked, type: Boolean, desc: 'Whether the variable is masked'
-        optional :variable_type, type: String, values: ::Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var'
-        optional :environment_scope, type: String, desc: 'The environment_scope of the variable'
-      end
-      post ':id/variables' do
-        variable = ::Ci::ChangeVariableService.new(
-          container: user_project,
-          current_user: current_user,
-          params: { action: :create, variable_params: declared_params(include_missing: false) }
-        ).execute
-
-        if variable.valid?
-          present variable, with: Entities::Ci::Variable
-        else
-          render_validation_error!(variable)
-        end
-      end
-
-      desc 'Update an existing variable from a project' do
-        success Entities::Ci::Variable
-      end
-      params do
-        optional :key, type: String, desc: 'The key of the variable'
-        optional :value, type: String, desc: 'The value of the variable'
-        optional :protected, type: Boolean, desc: 'Whether the variable is protected'
-        optional :masked, type: Boolean, desc: 'Whether the variable is masked'
-        optional :variable_type, type: String, values: ::Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file'
-        optional :environment_scope, type: String, desc: 'The environment_scope of the variable'
-        optional :filter, type: Hash, desc: 'Available filters: [environment_scope]. Example: filter[environment_scope]=production'
-      end
-      # rubocop: disable CodeReuse/ActiveRecord
-      put ':id/variables/:key' do
-        variable = find_variable(user_project, params)
-        not_found!('Variable') unless variable
-
-        variable = ::Ci::ChangeVariableService.new(
-          container: user_project,
-          current_user: current_user,
-          params: { action: :update, variable: variable, variable_params: declared_params(include_missing: false).except(:key, :filter) }
-        ).execute
-
-        if variable.valid?
-          present variable, with: Entities::Ci::Variable
-        else
-          render_validation_error!(variable)
-        end
-      end
-      # rubocop: enable CodeReuse/ActiveRecord
-
-      desc 'Delete an existing variable from a project' do
-        success Entities::Ci::Variable
-      end
-      params do
-        requires :key, type: String, desc: 'The key of the variable'
-        optional :filter, type: Hash, desc: 'Available filters: [environment_scope]. Example: filter[environment_scope]=production'
-      end
-      # rubocop: disable CodeReuse/ActiveRecord
-      delete ':id/variables/:key' do
-        variable = find_variable(user_project, params)
-        not_found!('Variable') unless variable
-
-        ::Ci::ChangeVariableService.new(
-          container: user_project,
-          current_user: current_user,
-          params: { action: :destroy, variable: variable }
-        ).execute
-
-        no_content!
-      end
-      # rubocop: enable CodeReuse/ActiveRecord
-    end
-  end
-end
diff --git a/lib/atlassian/jira_connect/client.rb b/lib/atlassian/jira_connect/client.rb
index ea83076c49b..3e2e6f1b9ba 100644
--- a/lib/atlassian/jira_connect/client.rb
+++ b/lib/atlassian/jira_connect/client.rb
@@ -81,7 +81,7 @@ module Atlassian
       end
 
       def store_dev_info(project:, commits: nil, branches: nil, merge_requests: nil, update_sequence_id: nil)
-        repo = Serializers::RepositoryEntity.represent(
+        repo = ::Atlassian::JiraConnect::Serializers::RepositoryEntity.represent(
           project,
           commits: commits,
           branches: branches,
diff --git a/lib/backup.rb b/lib/backup.rb
index 2712b33b4b4..91682645a9a 100644
--- a/lib/backup.rb
+++ b/lib/backup.rb
@@ -2,4 +2,43 @@
 
 module Backup
   Error = Class.new(StandardError)
+
+  class FileBackupError < Backup::Error
+    attr_reader :app_files_dir, :backup_tarball
+
+    def initialize(app_files_dir, backup_tarball)
+      @app_files_dir = app_files_dir
+      @backup_tarball = backup_tarball
+    end
+
+    def message
+      "Failed to create compressed file '#{backup_tarball}' when trying to backup the following paths: '#{app_files_dir}'"
+    end
+  end
+
+  class RepositoryBackupError < Backup::Error
+    attr_reader :container, :backup_repos_path
+
+    def initialize(container, backup_repos_path)
+      @container = container
+      @backup_repos_path = backup_repos_path
+    end
+
+    def message
+      "Failed to create compressed file '#{backup_repos_path}' when trying to backup the following paths: '#{container.disk_path}'"
+    end
+  end
+
+  class DatabaseBackupError < Backup::Error
+    attr_reader :config, :db_file_name
+
+    def initialize(config, db_file_name)
+      @config = config
+      @db_file_name = db_file_name
+    end
+
+    def message
+      "Failed to create compressed file '#{db_file_name}' when trying to backup the main database:\n - host: '#{config[:host]}'\n - port: '#{config[:port]}'\n - database: '#{config[:database]}'"
+    end
+  end
 end
diff --git a/lib/backup/gitaly_backup.rb b/lib/backup/gitaly_backup.rb
index c15b0ed6a1b..55fd68fd6e8 100644
--- a/lib/backup/gitaly_backup.rb
+++ b/lib/backup/gitaly_backup.rb
@@ -25,18 +25,21 @@ module Backup
       args += ['-parallel', @parallel.to_s] if type == :create && @parallel
       args += ['-parallel-storage', @parallel_storage.to_s] if type == :create && @parallel_storage
 
-      @read_io, @write_io = IO.pipe
-      @pid = Process.spawn(bin_path, command, '-path', backup_repos_path, *args, in: @read_io, out: @progress)
+      @stdin, stdout, @thread = Open3.popen2(ENV, bin_path, command, '-path', backup_repos_path, *args)
+
+      @out_reader = Thread.new do
+        IO.copy_stream(stdout, @progress)
+      end
     end
 
     def wait
       return unless started?
 
-      @write_io.close
-      Process.wait(@pid)
-      status = $?
+      @stdin.close
+      [@thread, @out_reader].each(&:join)
+      status =  @thread.value
 
-      @pid = nil
+      @thread = nil
 
       raise Error, "gitaly-backup exit status #{status.exitstatus}" if status.exitstatus != 0
     end
@@ -46,7 +49,7 @@ module Backup
 
       repository = repo_type.repository_for(container)
 
-      @write_io.puts({
+      @stdin.puts({
         storage_name: repository.storage,
         relative_path: repository.relative_path,
         gl_project_path: repository.gl_project_path,
@@ -61,7 +64,7 @@ module Backup
     private
 
     def started?
-      @pid.present?
+      @thread.present?
     end
 
     def backup_repos_path
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index 522a034a283..52810b0fb35 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -72,6 +72,17 @@ module Backup
       end
     end
 
+    def remove_tmp
+      # delete tmp inside backups
+      progress.print "Deleting backups/tmp ... "
+
+      if FileUtils.rm_rf(File.join(backup_path, "tmp"))
+        progress.puts "done".color(:green)
+      else
+        puts "deleting backups/tmp failed".color(:red)
+      end
+    end
+
     def remove_old
       # delete backups
       progress.print "Deleting old backups ... "
@@ -232,7 +243,7 @@ module Backup
     end
 
     def folders_to_backup
-      FOLDERS_TO_BACKUP.reject { |name| skipped?(name) }
+      FOLDERS_TO_BACKUP.select { |name| !skipped?(name) && Dir.exist?(File.join(backup_path, name)) }
     end
 
     def disabled_features
diff --git a/lib/banzai/filter/references/reference_cache.rb b/lib/banzai/filter/references/reference_cache.rb
index 24b8b4984cd..816ce973cad 100644
--- a/lib/banzai/filter/references/reference_cache.rb
+++ b/lib/banzai/filter/references/reference_cache.rb
@@ -28,20 +28,11 @@ module Banzai
           @references_per_parent[parent_type] ||= begin
             refs = Hash.new { |hash, key| hash[key] = Set.new }
 
-            nodes.each do |node|
-              prepare_node_for_scan(node).scan(regex) do
-                parent_path = if parent_type == :project
-                                full_project_path($~[:namespace], $~[:project])
-                              else
-                                full_group_path($~[:group])
-                              end
-
-                ident = filter.identifier($~)
-                refs[parent_path] << ident if ident
-              end
+            if Feature.enabled?(:milestone_reference_pattern, default_enabled: :yaml)
+              doc_search(refs)
+            else
+              node_search(nodes, refs)
             end
-
-            refs
           end
         end
 
@@ -172,6 +163,39 @@ module Banzai
 
         delegate :project, :group, :parent, :parent_type, to: :filter
 
+        # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/336268
+        def node_search(nodes, refs)
+          nodes.each do |node|
+            prepare_node_for_scan(node).scan(regex) do
+              parent_path = if parent_type == :project
+                              full_project_path($~[:namespace], $~[:project])
+                            else
+                              full_group_path($~[:group])
+                            end
+
+              ident = filter.identifier($~)
+              refs[parent_path] << ident if ident
+            end
+          end
+
+          refs
+        end
+
+        def doc_search(refs)
+          prepare_doc_for_scan(filter.doc).to_enum(:scan, regex).each do
+            parent_path = if parent_type == :project
+                            full_project_path($~[:namespace], $~[:project])
+                          else
+                            full_group_path($~[:group])
+                          end
+
+            ident = filter.identifier($~)
+            refs[parent_path] << ident if ident
+          end
+
+          refs
+        end
+
         def regex
           strong_memoize(:regex) do
             [
@@ -185,6 +209,13 @@ module Banzai
           Gitlab::SafeRequestStore["banzai_#{parent_type}_refs".to_sym] ||= {}
         end
 
+        def prepare_doc_for_scan(doc)
+          html = doc.to_html
+
+          filter.requires_unescaping? ? unescape_html_entities(html) : html
+        end
+
+        # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/336268
         def prepare_node_for_scan(node)
           html = node.to_html
 
diff --git a/lib/banzai/filter/table_of_contents_tag_filter.rb b/lib/banzai/filter/table_of_contents_tag_filter.rb
index 13d0a6a4cc7..4e80b543e2d 100644
--- a/lib/banzai/filter/table_of_contents_tag_filter.rb
+++ b/lib/banzai/filter/table_of_contents_tag_filter.rb
@@ -2,26 +2,31 @@
 
 module Banzai
   module Filter
-    # Using `[[_TOC_]]`, inserts a Table of Contents list.
-    # This syntax is based on the Gollum syntax. This way we have
-    # some consistency between with wiki and normal markdown.
-    # If there ever emerges a markdown standard, we can implement
-    # that here.
+    # Using `[[_TOC_]]` or `[TOC]` (both case insensitive), inserts a Table of Contents list.
     #
+    # `[[_TOC_]]` is based on the Gollum syntax. This way we have
+    # some consistency between with wiki and normal markdown.
     # The support for this has been removed from GollumTagsFilter
     #
+    # `[toc]` is a generally accepted form, used by Typora for example.
+    #
     # Based on Banzai::Filter::GollumTagsFilter
     class TableOfContentsTagFilter < HTML::Pipeline::Filter
-      TEXT_QUERY = %q(descendant-or-self::text()[ancestor::p and contains(., 'TOC')])
+      TEXT_QUERY = %q(descendant-or-self::text()[ancestor::p and contains(translate(., 'TOC', 'toc'), 'toc')])
 
       def call
         return doc if context[:no_header_anchors]
 
         doc.xpath(TEXT_QUERY).each do |node|
-          # A Gollum ToC tag is `[[_TOC_]]`, but due to MarkdownFilter running
-          # before this one, it will be converted into `[[<em>TOC</em>]]`, so it
-          # needs special-case handling
-          process_toc_tag(node) if toc_tag?(node)
+          if toc_tag?(node)
+            # Support [TOC] / [toc] tags, which don't have a wrapping <em>-tag
+            process_toc_tag(node)
+          elsif toc_tag_em?(node)
+            # Support Gollum like ToC tag (`[[_TOC_]]` / `[[_toc_]]`), which will be converted
+            # into `[[<em>TOC</em>]]` by the markdown filter, so it
+            # needs special-case handling
+            process_toc_tag_em(node)
+          end
         end
 
         doc
@@ -31,14 +36,25 @@ module Banzai
 
       # Replace an entire `[[<em>TOC</em>]]` node with the result generated by
       # TableOfContentsFilter
+      def process_toc_tag_em(node)
+        process_toc_tag(node.parent)
+      end
+
+      # Replace an entire `[TOC]` node with the result generated by
+      # TableOfContentsFilter
       def process_toc_tag(node)
-        node.parent.parent.replace(result[:toc].presence || '')
+        # we still need to go one step up to also replace the surrounding <p></p>
+        node.parent.replace(result[:toc].presence || '')
       end
 
-      def toc_tag?(node)
-        node.content == 'TOC' &&
+      def toc_tag_em?(node)
+        node.content.casecmp?('toc') &&
           node.parent.name == 'em' &&
-          node.parent.parent.text == '[[TOC]]'
+          node.parent.parent.text.casecmp?('[[toc]]')
+      end
+
+      def toc_tag?(node)
+        node.parent.text.casecmp?('[toc]')
       end
     end
   end
diff --git a/lib/error_tracking/collector/sentry_auth_parser.rb b/lib/error_tracking/collector/sentry_auth_parser.rb
new file mode 100644
index 00000000000..4945b8f73e1
--- /dev/null
+++ b/lib/error_tracking/collector/sentry_auth_parser.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module ErrorTracking
+  module Collector
+    class SentryAuthParser
+      def self.parse(request)
+        # Sentry client sends auth in X-Sentry-Auth header
+        #
+        # Example of content:
+        # "Sentry sentry_version=7, sentry_client=sentry-ruby/4.5.1, sentry_timestamp=1623923398,
+        #         sentry_key=afadk312..., sentry_secret=123456asd32131..."
+        auth = request.headers['X-Sentry-Auth']
+
+        # Sentry DSN contains key and secret.
+        # The key is required while secret is optional.
+        # We are going to use only the key since secret is deprecated.
+        public_key = auth[/sentry_key=(\w+)/, 1]
+
+        {
+          public_key: public_key
+        }
+      end
+    end
+  end
+end
diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb
index 055a3a771c2..8f6576c2206 100644
--- a/lib/extracts_path.rb
+++ b/lib/extracts_path.rb
@@ -47,16 +47,6 @@ module ExtractsPath
   end
   # rubocop:enable Gitlab/ModuleWithInstanceVariables
 
-  def lfs_blob_ids
-    blob_ids = tree.blobs.map(&:id)
-
-    # When current endpoint is a Blob then `tree.blobs` will be empty, it means we need to analyze
-    # the current Blob in order to determine if it's a LFS object
-    blob_ids = Array.wrap(@repo.blob_at(@commit.id, @path)&.id) if blob_ids.empty? # rubocop:disable Gitlab/ModuleWithInstanceVariables
-
-    @lfs_blob_ids = Gitlab::Git::Blob.batch_lfs_pointers(repository_container.repository, blob_ids).map(&:id) # rubocop:disable Gitlab/ModuleWithInstanceVariables
-  end
-
   private
 
   # Override in controllers to determine which actions are subject to the redirect
diff --git a/lib/feature.rb b/lib/feature.rb
index 453ecc8255a..f8d34e9c386 100644
--- a/lib/feature.rb
+++ b/lib/feature.rb
@@ -36,7 +36,7 @@ class Feature
     end
 
     def persisted_names
-      return [] unless Gitlab::Database.exists?
+      return [] unless Gitlab::Database.main.exists?
 
       # This loads names of all stored feature flags
       # and returns a stable Set in the following order:
@@ -56,7 +56,7 @@ class Feature
 
     # use `default_enabled: true` to default the flag to being `enabled`
     # unless set explicitly.  The default is `disabled`
-    # TODO: remove the `default_enabled:` and read it from the `defintion_yaml`
+    # TODO: remove the `default_enabled:` and read it from the `definition_yaml`
     # check: https://gitlab.com/gitlab-org/gitlab/-/issues/30228
     def enabled?(key, thing = nil, type: :development, default_enabled: false)
       if check_feature_flags_definition?
@@ -73,7 +73,7 @@ class Feature
 
       # During setup the database does not exist yet. So we haven't stored a value
       # for the feature yet and return the default.
-      return default_enabled unless Gitlab::Database.exists?
+      return default_enabled unless Gitlab::Database.main.exists?
 
       feature = get(key)
 
diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb
index e603a1dc8d2..a061a83e79c 100644
--- a/lib/feature/gitaly.rb
+++ b/lib/feature/gitaly.rb
@@ -15,7 +15,7 @@ class Feature
 
       def server_feature_flags(project = nil)
         # We need to check that both the DB connection and table exists
-        return {} unless ::Gitlab::Database.cached_table_exists?(FlipperFeature.table_name)
+        return {} unless ::Gitlab::Database.main.cached_table_exists?(FlipperFeature.table_name)
 
         Feature.persisted_names
           .select { |f| f.start_with?(PREFIX) }
diff --git a/lib/gem_extensions/active_record/association.rb b/lib/gem_extensions/active_record/association.rb
new file mode 100644
index 00000000000..91a9f45ce7e
--- /dev/null
+++ b/lib/gem_extensions/active_record/association.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module GemExtensions
+  module ActiveRecord
+    module Association
+      extend ActiveSupport::Concern
+
+      attr_reader :disable_joins
+
+      def initialize(owner, reflection)
+        super
+
+        @disable_joins = @reflection.options[:disable_joins] || false
+      end
+
+      def scope
+        if disable_joins
+          DisableJoins::Associations::AssociationScope.create.scope(self)
+        else
+          super
+        end
+      end
+
+      def association_scope
+        if klass
+          @association_scope ||= begin # rubocop:disable Gitlab/ModuleWithInstanceVariables
+            if disable_joins
+              DisableJoins::Associations::AssociationScope.scope(self)
+            else
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gem_extensions/active_record/associations/builder/has_many.rb b/lib/gem_extensions/active_record/associations/builder/has_many.rb
new file mode 100644
index 00000000000..7e51e632cc3
--- /dev/null
+++ b/lib/gem_extensions/active_record/associations/builder/has_many.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module GemExtensions
+  module ActiveRecord
+    module Associations
+      module Builder
+        module HasMany
+          extend ActiveSupport::Concern
+
+          class_methods do
+            def valid_options(options)
+              valid = super
+              valid += [:disable_joins] if options[:disable_joins] && options[:through]
+              valid
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gem_extensions/active_record/associations/builder/has_one.rb b/lib/gem_extensions/active_record/associations/builder/has_one.rb
new file mode 100644
index 00000000000..91765db8a5a
--- /dev/null
+++ b/lib/gem_extensions/active_record/associations/builder/has_one.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module GemExtensions
+  module ActiveRecord
+    module Associations
+      module Builder
+        module HasOne
+          extend ActiveSupport::Concern
+
+          class_methods do
+            def valid_options(options)
+              valid = super
+              valid += [:disable_joins] if options[:disable_joins] && options[:through]
+              valid
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gem_extensions/active_record/associations/has_many_through_association.rb b/lib/gem_extensions/active_record/associations/has_many_through_association.rb
new file mode 100644
index 00000000000..e7051e4d9cb
--- /dev/null
+++ b/lib/gem_extensions/active_record/associations/has_many_through_association.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module GemExtensions
+  module ActiveRecord
+    module Associations
+      module HasManyThroughAssociation
+        extend ActiveSupport::Concern
+
+        def find_target
+          return [] unless target_reflection_has_associated_record?
+          return scope.to_a if disable_joins
+
+          super
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gem_extensions/active_record/associations/has_one_through_association.rb b/lib/gem_extensions/active_record/associations/has_one_through_association.rb
new file mode 100644
index 00000000000..1487392a4ea
--- /dev/null
+++ b/lib/gem_extensions/active_record/associations/has_one_through_association.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module GemExtensions
+  module ActiveRecord
+    module Associations
+      module HasOneThroughAssociation
+        extend ActiveSupport::Concern
+
+        def find_target
+          return scope.first if disable_joins
+
+          super
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gem_extensions/active_record/associations/preloader/through_association.rb b/lib/gem_extensions/active_record/associations/preloader/through_association.rb
new file mode 100644
index 00000000000..16b53846a58
--- /dev/null
+++ b/lib/gem_extensions/active_record/associations/preloader/through_association.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module GemExtensions
+  module ActiveRecord
+    module Associations
+      module Preloader
+        module ThroughAssociation
+          extend ActiveSupport::Concern
+
+          def through_scope
+            scope = through_reflection.klass.unscoped
+            options = reflection.options
+
+            return scope if options[:disable_joins]
+
+            super
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gem_extensions/active_record/configurable_disable_joins.rb b/lib/gem_extensions/active_record/configurable_disable_joins.rb
new file mode 100644
index 00000000000..8e4c6bd6fc5
--- /dev/null
+++ b/lib/gem_extensions/active_record/configurable_disable_joins.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module GemExtensions
+  module ActiveRecord
+    module ConfigurableDisableJoins
+      extend ActiveSupport::Concern
+
+      def disable_joins
+        # rubocop:disable Gitlab/ModuleWithInstanceVariables
+        return @disable_joins.call if @disable_joins.is_a?(Proc)
+
+        @disable_joins
+        # rubocop:enable Gitlab/ModuleWithInstanceVariables
+      end
+    end
+  end
+end
diff --git a/lib/gem_extensions/active_record/delegate_cache.rb b/lib/gem_extensions/active_record/delegate_cache.rb
new file mode 100644
index 00000000000..63c93f7a2d3
--- /dev/null
+++ b/lib/gem_extensions/active_record/delegate_cache.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module GemExtensions
+  module ActiveRecord
+    module DelegateCache
+      def relation_delegate_class(klass)
+        @relation_delegate_cache2[klass] || super # rubocop:disable Gitlab/ModuleWithInstanceVariables
+      end
+
+      def initialize_relation_delegate_cache_disable_joins
+        @relation_delegate_cache2 = {} # rubocop:disable Gitlab/ModuleWithInstanceVariables
+
+        [
+          ::GemExtensions::ActiveRecord::DisableJoins::Relation
+        ].each do |klass|
+          delegate = Class.new(klass) do
+            include ::ActiveRecord::Delegation::ClassSpecificRelation
+          end
+          include_relation_methods(delegate)
+          mangled_name = klass.name.gsub("::", "_")
+          const_set mangled_name, delegate
+          private_constant mangled_name
+
+          @relation_delegate_cache2[klass] = delegate # rubocop:disable Gitlab/ModuleWithInstanceVariables
+        end
+      end
+
+      def inherited(child_class)
+        child_class.initialize_relation_delegate_cache_disable_joins
+        super
+      end
+    end
+  end
+end
diff --git a/lib/gem_extensions/active_record/disable_joins/associations/association_scope.rb b/lib/gem_extensions/active_record/disable_joins/associations/association_scope.rb
new file mode 100644
index 00000000000..1e4476330a2
--- /dev/null
+++ b/lib/gem_extensions/active_record/disable_joins/associations/association_scope.rb
@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module GemExtensions
+  module ActiveRecord
+    module DisableJoins
+      module Associations
+        class AssociationScope < ::ActiveRecord::Associations::AssociationScope # :nodoc:
+          def scope(association)
+            source_reflection = association.reflection
+            owner = association.owner
+            unscoped = association.klass.unscoped
+            reverse_chain = get_chain(source_reflection, association, unscoped.alias_tracker).reverse
+
+            previous_reflection, last_reflection, last_ordered, last_join_ids = last_scope_chain(reverse_chain, owner)
+
+            add_constraints(last_reflection, last_reflection.join_primary_key, last_join_ids, owner, last_ordered,
+                previous_reflection: previous_reflection)
+          end
+
+          private
+
+          def last_scope_chain(reverse_chain, owner)
+            # Pulled from https://github.com/rails/rails/pull/42448
+            # Fixes cases where the foreign key is not id
+            first_item = reverse_chain.shift
+            first_scope = [nil, first_item, false, [owner._read_attribute(first_item.join_foreign_key)]]
+
+            reverse_chain.inject(first_scope) do |(previous_reflection, reflection, ordered, join_ids), next_reflection|
+              key = reflection.join_primary_key
+              records = add_constraints(reflection, key, join_ids, owner, ordered, previous_reflection: previous_reflection)
+              foreign_key = next_reflection.join_foreign_key
+              record_ids = records.pluck(foreign_key) # rubocop:disable CodeReuse/ActiveRecord
+              records_ordered = records && records.order_values.any?
+
+              [reflection, next_reflection, records_ordered, record_ids]
+            end
+          end
+
+          def add_constraints(reflection, key, join_ids, owner, ordered, previous_reflection: nil)
+            scope = reflection.build_scope(reflection.aliased_table).where(key => join_ids) # rubocop:disable CodeReuse/ActiveRecord
+
+            # Pulled from https://github.com/rails/rails/pull/42590
+            # Fixes cases where used with an STI type
+            relation = reflection.klass.scope_for_association
+            scope.merge!(
+              relation.except(:select, :create_with, :includes, :preload, :eager_load, :joins, :left_outer_joins)
+            )
+
+            # Attempt to fix use case where we have a polymorphic relationship
+            # Build on an additional scope to filter by the polymorphic type
+            if reflection.type
+              polymorphic_class = previous_reflection.try(:klass) || owner.class
+
+              polymorphic_type = transform_value(polymorphic_class.polymorphic_name)
+              scope = apply_scope(scope, reflection.aliased_table, reflection.type, polymorphic_type)
+            end
+
+            scope = reflection.constraints.inject(scope) do |memo, scope_chain_item|
+              item = eval_scope(reflection, scope_chain_item, owner)
+              scope.unscope!(*item.unscope_values)
+              scope.where_clause += item.where_clause
+              scope.order_values = item.order_values | scope.order_values
+              scope
+            end
+
+            if scope.order_values.empty? && ordered
+              split_scope = DisableJoins::Relation.create(scope.klass, key, join_ids)
+              split_scope.where_clause += scope.where_clause
+              split_scope
+            else
+              scope
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gem_extensions/active_record/disable_joins/relation.rb b/lib/gem_extensions/active_record/disable_joins/relation.rb
new file mode 100644
index 00000000000..01eb6381a85
--- /dev/null
+++ b/lib/gem_extensions/active_record/disable_joins/relation.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module GemExtensions
+  module ActiveRecord
+    module DisableJoins
+      class Relation < ::ActiveRecord::Relation
+        attr_reader :ids, :key
+
+        def initialize(klass, key, ids)
+          @ids = ids.uniq
+          @key = key
+          super(klass)
+        end
+
+        def limit(value)
+          records.take(value) # rubocop:disable CodeReuse/ActiveRecord
+        end
+
+        def first(limit = nil)
+          if limit
+            records.limit(limit).first
+          else
+            records.first
+          end
+        end
+
+        def load
+          super
+          records = @records
+
+          records_by_id = records.group_by do |record|
+            record[key]
+          end
+
+          records = ids.flat_map { |id| records_by_id[id.to_i] }
+          records.compact!
+
+          @records = records
+        end
+      end
+    end
+  end
+end
diff --git a/lib/generators/gitlab/usage_metric/templates/database_instrumentation_class.rb.template b/lib/generators/gitlab/usage_metric/templates/database_instrumentation_class.rb.template
new file mode 100644
index 00000000000..74b1ed69a5c
--- /dev/null
+++ b/lib/generators/gitlab/usage_metric/templates/database_instrumentation_class.rb.template
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Usage
+    module Metrics
+      module Instrumentations
+        class <%= class_name %>Metric < DatabaseMetric
+          operation :<%= operation%>
+
+          relation do
+            # Insert ActiveRecord relation here
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/generators/gitlab/usage_metric/templates/instrumentation_class.rb.template b/lib/generators/gitlab/usage_metric/templates/generic_instrumentation_class.rb.template
index 603b6f3bc8a..fa6c18a289c 100644
--- a/lib/generators/gitlab/usage_metric/templates/instrumentation_class.rb.template
+++ b/lib/generators/gitlab/usage_metric/templates/generic_instrumentation_class.rb.template
@@ -4,8 +4,9 @@ module Gitlab
   module Usage
     module Metrics
       module Instrumentations
-        class <%= class_name %>Metric < <%= metric_superclass %>Metric
-          def value
+        class <%= class_name %>Metric < GenericMetric
+          value do
+            # Insert metric code logic here
           end
         end
       end
diff --git a/lib/generators/gitlab/usage_metric/usage_metric_generator.rb b/lib/generators/gitlab/usage_metric_generator.rb
index f7125fdc911..c0fdcf21f20 100644
--- a/lib/generators/gitlab/usage_metric/usage_metric_generator.rb
+++ b/lib/generators/gitlab/usage_metric_generator.rb
@@ -12,20 +12,25 @@ module Gitlab
     ALLOWED_SUPERCLASSES = {
       generic: 'Generic',
       database: 'Database',
-      redis_hll: 'RedisHLL'
+      redis: 'Redis'
     }.freeze
 
-    source_root File.expand_path('templates', __dir__)
+    ALLOWED_OPERATIONS = %w(count distinct_count estimate_batch_distinct_count).freeze
+
+    source_root File.expand_path('usage_metric/templates', __dir__)
 
     class_option :ee, type: :boolean, optional: true, default: false, desc: 'Indicates if instrumentation is for EE'
     class_option :type, type: :string, desc: "Metric type, must be one of: #{ALLOWED_SUPERCLASSES.keys.join(', ')}"
+    class_option :operation, type: :string, desc: "Metric operation, must be one of: #{ALLOWED_OPERATIONS.join(', ')}"
 
     argument :class_name, type: :string, desc: 'Instrumentation class name, e.g.: CountIssues'
 
     def create_class_files
       validate!
 
-      template "instrumentation_class.rb.template", file_path
+      template "database_instrumentation_class.rb.template", file_path if type == 'database'
+      template "generic_instrumentation_class.rb.template", file_path if type == 'generic'
+
       template "instrumentation_class_spec.rb.template", spec_file_path
     end
 
@@ -34,6 +39,7 @@ module Gitlab
     def validate!
       raise ArgumentError, "Type is required, valid options are #{ALLOWED_SUPERCLASSES.keys.join(', ')}" unless type.present?
       raise ArgumentError, "Unknown type '#{type}', valid options are #{ALLOWED_SUPERCLASSES.keys.join(', ')}" if metric_superclass.nil?
+      raise ArgumentError, "Unknown operation '#{operation}' valid operations are #{ALLOWED_OPERATIONS.join(', ')}" if type == 'database' && !ALLOWED_OPERATIONS.include?(operation)
     end
 
     def ee?
@@ -44,6 +50,10 @@ module Gitlab
       options[:type]
     end
 
+    def operation
+      options[:operation]
+    end
+
     def file_path
       dir = ee? ? EE_DIR : CE_DIR
 
diff --git a/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb b/lib/generators/post_deployment_migration/post_deployment_migration_generator.rb
index 568104cb30b..66ee0e2440f 100644
--- a/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb
+++ b/lib/generators/post_deployment_migration/post_deployment_migration_generator.rb
@@ -2,7 +2,7 @@
 
 require 'rails/generators'
 
-module Rails
+module PostDeploymentMigration
   class PostDeploymentMigrationGenerator < Rails::Generators::NamedBase
     def create_migration_file
       timestamp = Time.now.utc.strftime('%Y%m%d%H%M%S')
diff --git a/lib/gitlab.rb b/lib/gitlab.rb
index f10168623e9..d93d7acbaad 100644
--- a/lib/gitlab.rb
+++ b/lib/gitlab.rb
@@ -136,7 +136,6 @@ module Gitlab
 
   def self.process_name
     return 'sidekiq' if Gitlab::Runtime.sidekiq?
-    return 'action_cable' if Gitlab::Runtime.action_cable?
     return 'console' if Gitlab::Runtime.console?
     return 'test' if Rails.env.test?
 
diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
index 9a37a41ff81..f94696e3186 100644
--- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
+++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
@@ -38,36 +38,19 @@ module Gitlab
         # rubocop: disable CodeReuse/ActiveRecord
         def serialized_records
           strong_memoize(:serialized_records) do
-            # special case (legacy): 'Test' and 'Staging' stages should show Ci::Build records
-            if default_test_stage? || default_staging_stage?
-              ci_build_join = mr_metrics_table
-                .join(build_table)
-                .on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id]))
-                .join_sources
-
-              records = ordered_and_limited_query
-                .joins(ci_build_join)
-                .select(build_table[:id], *time_columns)
-
-              yield records if block_given?
-              ci_build_records = preload_ci_build_associations(records)
-
-              AnalyticsBuildSerializer.new.represent(ci_build_records.map { |e| e['build'] })
-            else
-              records = ordered_and_limited_query.select(*columns, *time_columns)
-
-              yield records if block_given?
-              records = preload_associations(records)
-
-              records.map do |record|
-                project = record.project
-                attributes = record.attributes.merge({
-                  project_path: project.path,
-                  namespace_path: project.namespace.route.path,
-                  author: record.author
-                })
-                serializer.represent(attributes)
-              end
+            records = ordered_and_limited_query.select(*columns, *time_columns)
+
+            yield records if block_given?
+            records = preload_associations(records)
+
+            records.map do |record|
+              project = record.project
+              attributes = record.attributes.merge({
+                project_path: project.path,
+                namespace_path: project.namespace.route.path,
+                author: record.author
+              })
+              serializer.represent(attributes)
             end
           end
         end
@@ -83,26 +66,10 @@ module Gitlab
           end
         end
 
-        def default_test_stage?
-          stage.matches_with_stage_params?(Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_test_stage)
-        end
-
-        def default_staging_stage?
-          stage.matches_with_stage_params?(Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_staging_stage)
-        end
-
         def serializer
           MAPPINGS.fetch(subject_class).fetch(:serializer_class).new
         end
 
-        # rubocop: disable CodeReuse/ActiveRecord
-        def preload_ci_build_associations(records)
-          results = records.map(&:attributes)
-
-          Gitlab::CycleAnalytics::Updater.update!(results, from: 'id', to: 'build', klass: ::Ci::Build.includes({ project: [:namespace], user: [], pipeline: [] }))
-        end
-        # rubocop: enable CodeReuse/ActiveRecord
-
         def ordered_and_limited_query
           strong_memoize(:ordered_and_limited_query) do
             order_by(query, sort, direction, columns).page(page).per(per_page).without_count
diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb
new file mode 100644
index 00000000000..94e20762368
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb
@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      class RequestParams
+        include ActiveModel::Model
+        include ActiveModel::Validations
+        include ActiveModel::Attributes
+        include Gitlab::Utils::StrongMemoize
+
+        MAX_RANGE_DAYS = 180.days.freeze
+        DEFAULT_DATE_RANGE = 29.days # 30 including Date.today
+
+        STRONG_PARAMS_DEFINITION = [
+          :created_before,
+          :created_after,
+          :author_username,
+          :milestone_title,
+          :sort,
+          :direction,
+          :page,
+          :stage_id,
+          :end_event_filter,
+          label_name: [].freeze,
+          assignee_username: [].freeze,
+          project_ids: [].freeze
+        ].freeze
+
+        FINDER_PARAM_NAMES = [
+          :assignee_username,
+          :author_username,
+          :milestone_title,
+          :label_name
+        ].freeze
+
+        attr_writer :project_ids
+
+        attribute :created_after, :datetime
+        attribute :created_before, :datetime
+        attribute :group
+        attribute :current_user
+        attribute :value_stream
+        attribute :sort
+        attribute :direction
+        attribute :page
+        attribute :project
+        attribute :stage_id
+        attribute :end_event_filter
+
+        FINDER_PARAM_NAMES.each do |param_name|
+          attribute param_name
+        end
+
+        validates :created_after, presence: true
+        validates :created_before, presence: true
+
+        validate :validate_created_before
+        validate :validate_date_range
+
+        def initialize(params = {})
+          super(params)
+
+          self.created_before = (self.created_before || Time.current).at_end_of_day
+          self.created_after = (created_after || default_created_after).at_beginning_of_day
+          self.end_event_filter ||= Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder::DEFAULT_END_EVENT_FILTER
+        end
+
+        def project_ids
+          Array(@project_ids)
+        end
+
+        def to_data_collector_params
+          {
+            current_user: current_user,
+            from: created_after,
+            to: created_before,
+            project_ids: project_ids,
+            sort: sort&.to_sym,
+            direction: direction&.to_sym,
+            page: page,
+            end_event_filter: end_event_filter.to_sym
+          }.merge(attributes.symbolize_keys.slice(*FINDER_PARAM_NAMES))
+        end
+
+        def to_data_attributes
+          {}.tap do |attrs|
+            attrs[:group] = group_data_attributes if group
+            attrs[:value_stream] = value_stream_data_attributes.to_json if value_stream
+            attrs[:created_after] = created_after.to_date.iso8601
+            attrs[:created_before] = created_before.to_date.iso8601
+            attrs[:projects] = group_projects(project_ids) if group && project_ids.present?
+            attrs[:labels] = label_name.to_json if label_name.present?
+            attrs[:assignees] = assignee_username.to_json if assignee_username.present?
+            attrs[:author] = author_username if author_username.present?
+            attrs[:milestone] = milestone_title if milestone_title.present?
+            attrs[:sort] = sort if sort.present?
+            attrs[:direction] = direction if direction.present?
+            attrs[:stage] = stage_data_attributes.to_json if stage_id.present?
+          end
+        end
+
+        private
+
+        def group_data_attributes
+          {
+            id: group.id,
+            name: group.name,
+            parent_id: group.parent_id,
+            full_path: group.full_path,
+            avatar_url: group.avatar_url
+          }
+        end
+
+        def value_stream_data_attributes
+          {
+            id: value_stream.id,
+            name: value_stream.name,
+            is_custom: value_stream.custom?
+          }
+        end
+
+        def group_projects(project_ids)
+          GroupProjectsFinder.new(
+            group: group,
+            current_user: current_user,
+            options: { include_subgroups: true },
+            project_ids_relation: project_ids
+          )
+            .execute
+            .with_route
+            .map { |project| project_data_attributes(project) }
+            .to_json
+        end
+
+        def project_data_attributes(project)
+          {
+            id: project.to_gid.to_s,
+            name: project.name,
+            path_with_namespace: project.path_with_namespace,
+            avatar_url: project.avatar_url
+          }
+        end
+
+        def stage_data_attributes
+          return unless stage
+
+          {
+            id: stage.id || stage.name,
+            title: stage.name
+          }
+        end
+
+        def validate_created_before
+          return if created_after.nil? || created_before.nil?
+
+          errors.add(:created_before, :invalid) if created_after > created_before
+        end
+
+        def validate_date_range
+          return if created_after.nil? || created_before.nil?
+
+          if (created_before - created_after) > MAX_RANGE_DAYS
+            errors.add(:created_after, s_('CycleAnalytics|The given date range is larger than 180 days'))
+          end
+        end
+
+        def default_created_after
+          if created_before
+            (created_before - DEFAULT_DATE_RANGE)
+          else
+            DEFAULT_DATE_RANGE.ago
+          end
+        end
+
+        def stage
+          return unless value_stream
+
+          strong_memoize(:stage) do
+            ::Analytics::CycleAnalytics::StageFinder.new(parent: project || group, stage_id: stage_id).execute if stage_id
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
index 11fe1dde12f..5648984ecbb 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
@@ -5,7 +5,7 @@ module Gitlab
     module CycleAnalytics
       module StageQueryHelpers
         def execute_query(query)
-          ActiveRecord::Base.connection.execute(query.to_sql)
+          ApplicationRecord.connection.execute(query.to_sql)
         end
 
         def zero_interval
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 13e78e72175..1afb2eda149 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -53,7 +53,7 @@ module Gitlab
           personal_access_token_check(password, project) ||
           deploy_token_check(login, password, project) ||
           user_with_password_for_git(login, password) ||
-          Gitlab::Auth::Result.new
+          Gitlab::Auth::Result::EMPTY
 
         rate_limit!(rate_limiter, success: result.success?, login: login)
         look_to_limit_user(result.actor)
@@ -202,13 +202,29 @@ module Gitlab
 
         return unless valid_scoped_token?(token, all_available_scopes)
 
-        return if project && token.user.project_bot? && !project.bots.include?(token.user)
+        if project && token.user.project_bot?
+          return unless token_bot_in_project?(token.user, project) || token_bot_in_group?(token.user, project)
+        end
 
         if can_user_login_with_non_expired_password?(token.user) || token.user.project_bot?
           Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scopes(token.scopes))
         end
       end
 
+      def token_bot_in_project?(user, project)
+        project.bots.include?(user)
+      end
+
+      # rubocop: disable CodeReuse/ActiveRecord
+
+      # A workaround for adding group-level automation is to add the bot user of a project access token as a group member.
+      # In order to make project access tokens work this way during git authentication, we need to add an additional check for group membership.
+      # This is a temporary workaround until service accounts are implemented.
+      def token_bot_in_group?(user, project)
+        project.group && project.group.members_with_parents.where(user_id: user.id).exists?
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
       def valid_oauth_token?(token)
         token && token.accessible? && valid_scoped_token?(token, [:api])
       end
diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb
index f54fa7504a3..a7312ac759a 100644
--- a/lib/gitlab/auth/auth_finders.rb
+++ b/lib/gitlab/auth/auth_finders.rb
@@ -298,7 +298,7 @@ module Gitlab
         when :api
           api_request?
         when :archive
-          archive_request? if Feature.enabled?(:allow_archive_as_web_access_format, default_enabled: :yaml)
+          archive_request?
         end
       end
 
diff --git a/lib/gitlab/auth/otp/strategies/forti_token_cloud.rb b/lib/gitlab/auth/otp/strategies/forti_token_cloud.rb
index 079d631e22a..7ef8a1076f4 100644
--- a/lib/gitlab/auth/otp/strategies/forti_token_cloud.rb
+++ b/lib/gitlab/auth/otp/strategies/forti_token_cloud.rb
@@ -35,7 +35,7 @@ module Gitlab
           end
 
           def access_token
-            Gitlab::Json.parse(access_token_create_response)['access_token']
+            Gitlab::Json.parse(access_token_create_response.body)['access_token']
           end
 
           def verify_otp(otp_code)
diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb
index da874524826..69525a281e9 100644
--- a/lib/gitlab/auth/result.rb
+++ b/lib/gitlab/auth/result.rb
@@ -2,7 +2,18 @@
 
 module Gitlab
   module Auth
-    Result = Struct.new(:actor, :project, :type, :authentication_abilities) do
+    class Result
+      attr_reader :actor, :project, :type, :authentication_abilities
+
+      def initialize(actor, project, type, authentication_abilities)
+        @actor = actor
+        @project = project
+        @type = type
+        @authentication_abilities = authentication_abilities
+      end
+
+      EMPTY = self.new(nil, nil, nil, nil).freeze
+
       def ci?(for_project)
         type == :ci &&
           project &&
@@ -21,6 +32,29 @@ module Gitlab
       def failed?
         !success?
       end
+
+      def auth_user
+        actor.is_a?(User) ? actor : nil
+      end
+      alias_method :user, :auth_user
+
+      def deploy_token
+        actor.is_a?(DeployToken) ? actor : nil
+      end
+
+      def can?(action)
+        actor&.can?(action)
+      end
+
+      def can_perform_action_on_project?(action, given_project)
+        Ability.allowed?(actor, action, given_project)
+      end
+
+      def authentication_abilities_include?(ability)
+        return false if authentication_abilities.blank?
+
+        authentication_abilities.include?(ability)
+      end
     end
   end
 end
diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb
index 9f4d6557023..0826887dd0a 100644
--- a/lib/gitlab/background_migration.rb
+++ b/lib/gitlab/background_migration.rb
@@ -31,7 +31,7 @@ module Gitlab
         queue.each do |job|
           migration_class, migration_args = job.args
 
-          next unless job.queue == self.queue
+          next unless job.klass == 'BackgroundMigrationWorker'
           next unless migration_class == steal_class
           next if block_given? && !(yield job)
 
@@ -60,11 +60,14 @@ module Gitlab
     end
 
     def self.remaining
-      scheduled = Sidekiq::ScheduledSet.new.count do |job|
-        job.queue == self.queue
-      end
+      enqueued = Sidekiq::Queue.new(self.queue)
+      scheduled = Sidekiq::ScheduledSet.new
 
-      scheduled + Sidekiq::Queue.new(self.queue).size
+      [enqueued, scheduled].sum do |set|
+        set.count do |job|
+          job.klass == 'BackgroundMigrationWorker'
+        end
+      end
     end
 
     def self.exists?(migration_class, additional_queues = [])
@@ -105,13 +108,11 @@ module Gitlab
     end
 
     def self.enqueued_job?(queues, migration_class)
-      queues.each do |queue|
-        queue.each do |job|
-          return true if job.queue == self.queue && job.args.first == migration_class
+      queues.any? do |queue|
+        queue.any? do |job|
+          job.klass == 'BackgroundMigrationWorker' && job.args.first == migration_class
         end
       end
-
-      false
     end
   end
 end
diff --git a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb
index a0d0791b6af..b0a8c3a8cbb 100644
--- a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb
+++ b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb
@@ -27,6 +27,17 @@ module Gitlab
         eligible_mrs.each_slice(10) do |slice|
           MergeRequest.where(id: slice).update_all(draft: true)
         end
+
+        mark_job_as_succeeded(start_id, end_id)
+      end
+
+      private
+
+      def mark_job_as_succeeded(*arguments)
+        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
+          'BackfillDraftStatusOnMergeRequests',
+          arguments
+        )
       end
     end
   end
diff --git a/lib/gitlab/background_migration/backfill_integrations_type_new.rb b/lib/gitlab/background_migration/backfill_integrations_type_new.rb
new file mode 100644
index 00000000000..d1a939af58e
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_integrations_type_new.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Backfills the new `integrations.type_new` column, which contains
+    # the real class name, rather than the legacy class name in `type`
+    # which is mapped via `Gitlab::Integrations::StiType`.
+    class BackfillIntegrationsTypeNew
+      include Gitlab::Database::DynamicModelHelpers
+
+      def perform(start_id, stop_id, batch_table, batch_column, sub_batch_size, pause_ms)
+        parent_batch_relation = define_batchable_model(batch_table)
+          .where(batch_column => start_id..stop_id)
+
+        parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch|
+          process_sub_batch(sub_batch)
+
+          sleep(pause_ms * 0.001) if pause_ms > 0
+        end
+      end
+
+      private
+
+      def connection
+        ActiveRecord::Base.connection
+      end
+
+      def process_sub_batch(sub_batch)
+        # Extract the start/stop IDs from the current sub-batch
+        sub_start_id, sub_stop_id = sub_batch.pluck(Arel.sql('MIN(id), MAX(id)')).first
+
+        # This matches the mapping from the INSERT trigger added in
+        # db/migrate/20210721135638_add_triggers_to_integrations_type_new.rb
+        connection.execute(<<~SQL)
+          WITH mapping(old_type, new_type) AS (VALUES
+            ('AsanaService',                   'Integrations::Asana'),
+            ('AssemblaService',                'Integrations::Assembla'),
+            ('BambooService',                  'Integrations::Bamboo'),
+            ('BugzillaService',                'Integrations::Bugzilla'),
+            ('BuildkiteService',               'Integrations::Buildkite'),
+            ('CampfireService',                'Integrations::Campfire'),
+            ('ConfluenceService',              'Integrations::Confluence'),
+            ('CustomIssueTrackerService',      'Integrations::CustomIssueTracker'),
+            ('DatadogService',                 'Integrations::Datadog'),
+            ('DiscordService',                 'Integrations::Discord'),
+            ('DroneCiService',                 'Integrations::DroneCi'),
+            ('EmailsOnPushService',            'Integrations::EmailsOnPush'),
+            ('EwmService',                     'Integrations::Ewm'),
+            ('ExternalWikiService',            'Integrations::ExternalWiki'),
+            ('FlowdockService',                'Integrations::Flowdock'),
+            ('HangoutsChatService',            'Integrations::HangoutsChat'),
+            ('IrkerService',                   'Integrations::Irker'),
+            ('JenkinsService',                 'Integrations::Jenkins'),
+            ('JiraService',                    'Integrations::Jira'),
+            ('MattermostService',              'Integrations::Mattermost'),
+            ('MattermostSlashCommandsService', 'Integrations::MattermostSlashCommands'),
+            ('MicrosoftTeamsService',          'Integrations::MicrosoftTeams'),
+            ('MockCiService',                  'Integrations::MockCi'),
+            ('MockMonitoringService',          'Integrations::MockMonitoring'),
+            ('PackagistService',               'Integrations::Packagist'),
+            ('PipelinesEmailService',          'Integrations::PipelinesEmail'),
+            ('PivotaltrackerService',          'Integrations::Pivotaltracker'),
+            ('PrometheusService',              'Integrations::Prometheus'),
+            ('PushoverService',                'Integrations::Pushover'),
+            ('RedmineService',                 'Integrations::Redmine'),
+            ('SlackService',                   'Integrations::Slack'),
+            ('SlackSlashCommandsService',      'Integrations::SlackSlashCommands'),
+            ('TeamcityService',                'Integrations::Teamcity'),
+            ('UnifyCircuitService',            'Integrations::UnifyCircuit'),
+            ('WebexTeamsService',              'Integrations::WebexTeams'),
+            ('YoutrackService',                'Integrations::Youtrack'),
+
+            -- EE-only integrations
+            ('GithubService',                  'Integrations::Github'),
+            ('GitlabSlackApplicationService',  'Integrations::GitlabSlackApplication')
+          )
+
+          UPDATE integrations SET type_new = mapping.new_type
+          FROM mapping
+          WHERE integrations.id BETWEEN #{sub_start_id} AND #{sub_stop_id}
+            AND integrations.type = mapping.old_type
+        SQL
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb
index bc113a1e33d..f5c8796bd18 100644
--- a/lib/gitlab/background_migration/backfill_project_repositories.rb
+++ b/lib/gitlab/background_migration/backfill_project_repositories.rb
@@ -189,7 +189,7 @@ module Gitlab
       end
 
       def perform(start_id, stop_id)
-        Gitlab::Database.bulk_insert(:project_repositories, project_repositories(start_id, stop_id)) # rubocop:disable Gitlab/BulkInsert
+        Gitlab::Database.main.bulk_insert(:project_repositories, project_repositories(start_id, stop_id)) # rubocop:disable Gitlab/BulkInsert
       end
 
       private
diff --git a/lib/gitlab/background_migration/backfill_snippet_repositories.rb b/lib/gitlab/background_migration/backfill_snippet_repositories.rb
index 6f37f1846d2..b58f0a3a3e0 100644
--- a/lib/gitlab/background_migration/backfill_snippet_repositories.rb
+++ b/lib/gitlab/background_migration/backfill_snippet_repositories.rb
@@ -105,7 +105,7 @@ module Gitlab
       end
 
       def commit_attrs
-        @commit_attrs ||= { branch_name: 'master', message: 'Initial commit' }
+        @commit_attrs ||= { branch_name: 'main', message: 'Initial commit' }
       end
 
       def create_commit(snippet)
diff --git a/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb b/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb
new file mode 100644
index 00000000000..107ac9b0c3b
--- /dev/null
+++ b/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class CopyCiBuildsColumnsToSecurityScans
+      extend ::Gitlab::Utils::Override
+
+      UPDATE_BATCH_SIZE = 500
+
+      def perform(start_id, stop_id)
+        (start_id..stop_id).step(UPDATE_BATCH_SIZE).each do |offset|
+          batch_start = offset
+          batch_stop = offset + UPDATE_BATCH_SIZE - 1
+
+          ActiveRecord::Base.connection.execute <<~SQL
+            UPDATE
+              security_scans
+            SET
+              project_id = ci_builds.project_id,
+              pipeline_id = ci_builds.commit_id
+            FROM ci_builds
+            WHERE ci_builds.type='Ci::Build' 
+              AND ci_builds.id=security_scans.build_id 
+              AND security_scans.id BETWEEN #{Integer(batch_start)} AND #{Integer(batch_stop)}
+          SQL
+        end
+
+        mark_job_as_succeeded(start_id, stop_id)
+      rescue StandardError => error
+        Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error)
+      end
+
+      private
+
+      def mark_job_as_succeeded(*arguments)
+        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
+          'CopyCiBuildsColumnsToSecurityScans',
+          arguments
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/create_security_setting.rb b/lib/gitlab/background_migration/create_security_setting.rb
new file mode 100644
index 00000000000..55b37bb03b5
--- /dev/null
+++ b/lib/gitlab/background_migration/create_security_setting.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This class doesn't create SecuritySetting
+    # as this feature exists only in EE
+    class CreateSecuritySetting
+      def perform(_from_id, _to_id)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::CreateSecuritySetting.prepend_mod_with('Gitlab::BackgroundMigration::CreateSecuritySetting')
diff --git a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb
index d2a9939b9ee..1c60473750d 100644
--- a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb
+++ b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb
@@ -34,7 +34,7 @@ module Gitlab
             end
           end
 
-          Gitlab::Database.bulk_insert(TEMP_TABLE, fingerprints) # rubocop:disable Gitlab/BulkInsert
+          Gitlab::Database.main.bulk_insert(TEMP_TABLE, fingerprints) # rubocop:disable Gitlab/BulkInsert
 
           execute("ANALYZE #{TEMP_TABLE}")
 
diff --git a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb
index 2bce5037d03..14c72bb4a72 100644
--- a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb
+++ b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb
@@ -65,7 +65,7 @@ module Gitlab
           next if service_ids.empty?
 
           migrated_ids += service_ids
-          Gitlab::Database.bulk_insert(table, data) # rubocop:disable Gitlab/BulkInsert
+          Gitlab::Database.main.bulk_insert(table, data) # rubocop:disable Gitlab/BulkInsert
         end
 
         return if migrated_ids.empty?
diff --git a/lib/gitlab/background_migration/populate_issue_email_participants.rb b/lib/gitlab/background_migration/populate_issue_email_participants.rb
index d6795296fb7..0a56ac1dae8 100644
--- a/lib/gitlab/background_migration/populate_issue_email_participants.rb
+++ b/lib/gitlab/background_migration/populate_issue_email_participants.rb
@@ -21,7 +21,7 @@ module Gitlab
           }
         end
 
-        Gitlab::Database.bulk_insert(:issue_email_participants, rows, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert
+        Gitlab::Database.main.bulk_insert(:issue_email_participants, rows, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert
       end
     end
   end
diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb
index a00d291245c..84ff7423254 100644
--- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb
+++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb
@@ -9,6 +9,8 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid
   end
 
   class VulnerabilitiesFinding < ActiveRecord::Base
+    include ShaAttribute
+
     self.table_name = "vulnerability_occurrences"
     belongs_to :primary_identifier, class_name: 'VulnerabilitiesIdentifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id'
     REPORT_TYPES = {
@@ -21,6 +23,9 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid
       api_fuzzing: 6
     }.with_indifferent_access.freeze
     enum report_type: REPORT_TYPES
+
+    sha_attribute :fingerprint
+    sha_attribute :location_fingerprint
   end
 
   class CalculateFindingUUID
diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb
index 44106897df8..28f1a10f9a7 100644
--- a/lib/gitlab/bare_repository_import/importer.rb
+++ b/lib/gitlab/bare_repository_import/importer.rb
@@ -73,7 +73,7 @@ module Gitlab
         if project.persisted? && mv_repositories(project)
           log " * Created #{project.name} (#{project_full_path})".color(:green)
 
-          project.write_repository_config
+          project.set_full_path
 
           ProjectCacheWorker.perform_async(project.id)
         else
diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb
index d29799f1029..2c60b2e36cb 100644
--- a/lib/gitlab/bitbucket_server_import/importer.rb
+++ b/lib/gitlab/bitbucket_server_import/importer.rb
@@ -7,7 +7,6 @@ module Gitlab
       attr_reader :project, :project_key, :repository_slug, :client, :errors, :users, :already_imported_cache_key
       attr_accessor :logger
 
-      REMOTE_NAME = 'bitbucket_server'
       BATCH_SIZE = 100
       # The base cache key to use for tracking already imported objects.
       ALREADY_IMPORTED_CACHE_KEY =
@@ -142,7 +141,7 @@ module Gitlab
         log_info(stage: 'import_repository', message: 'starting import')
 
         project.ensure_repository
-        project.repository.fetch_as_mirror(project.import_url, refmap: self.class.refmap, remote_name: REMOTE_NAME)
+        project.repository.fetch_as_mirror(project.import_url, refmap: self.class.refmap)
 
         log_info(stage: 'import_repository', message: 'finished import')
       rescue Gitlab::Shell::Error => e
diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb
index 4cbc0231bce..89c85cb50be 100644
--- a/lib/gitlab/cache/import/caching.rb
+++ b/lib/gitlab/cache/import/caching.rb
@@ -57,7 +57,7 @@ module Gitlab
 
         # Sets a cache key to the given value.
         #
-        # key - The cache key to write.
+        # raw_key - The cache key to write.
         # value - The value to set.
         # timeout - The time after which the cache key should expire.
         def self.write(raw_key, value, timeout: TIMEOUT)
@@ -73,7 +73,7 @@ module Gitlab
         # Increment the integer value of a key by one.
         # Sets the value to zero if missing before incrementing
         #
-        # key - The cache key to increment.
+        # raw_key - The cache key to increment.
         # timeout - The time after which the cache key should expire.
         # @return - the incremented value
         def self.increment(raw_key, timeout: TIMEOUT)
@@ -85,6 +85,22 @@ module Gitlab
           end
         end
 
+        # Increment the integer value of a key by the given value.
+        # Sets the value to zero if missing before incrementing
+        #
+        # raw_key - The cache key to increment.
+        # value - The value to increment the key
+        # timeout - The time after which the cache key should expire.
+        # @return - the incremented value
+        def self.increment_by(raw_key, value, timeout: TIMEOUT)
+          key = cache_key_for(raw_key)
+
+          Redis::Cache.with do |redis|
+            redis.incrby(key, value)
+            redis.expire(key, timeout)
+          end
+        end
+
         # Adds a value to a set.
         #
         # raw_key - The key of the set to add the value to.
diff --git a/lib/gitlab/chaos.rb b/lib/gitlab/chaos.rb
index 495f12882e5..1b4a647d16f 100644
--- a/lib/gitlab/chaos.rb
+++ b/lib/gitlab/chaos.rb
@@ -31,7 +31,7 @@ module Gitlab
       expected_end_time = Time.now + duration_s
 
       while Time.now < expected_end_time
-        ActiveRecord::Base.connection.execute("SELECT 1")
+        ApplicationRecord.connection.execute("SELECT 1")
 
         end_interval_time = Time.now + [duration_s, interval_s].min
         rand while Time.now < end_interval_time
diff --git a/lib/gitlab/chat/command.rb b/lib/gitlab/chat/command.rb
index 49b7dcf4bbe..0add53f8174 100644
--- a/lib/gitlab/chat/command.rb
+++ b/lib/gitlab/chat/command.rb
@@ -54,10 +54,12 @@ module Gitlab
           }
         )
 
-        service.execute(:chat) do |pipeline|
+        response = service.execute(:chat) do |pipeline|
           build_environment_variables(pipeline)
           build_chat_data(pipeline)
         end
+
+        response.payload
       end
 
       # pipeline - The `Ci::Pipeline` to create the environment variables for.
diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb
index a2d74d36b58..cfff6e919dc 100644
--- a/lib/gitlab/checks/branch_check.rb
+++ b/lib/gitlab/checks/branch_check.rb
@@ -122,7 +122,7 @@ module Gitlab
       def empty_project_push_message
         <<~MESSAGE
 
-        A default branch (e.g. master) does not yet exist for #{project.full_path}
+        A default branch (e.g. main) does not yet exist for #{project.full_path}
         Ask a project Owner or Maintainer to create a default branch:
 
           #{project_members_url}
diff --git a/lib/gitlab/checks/changes_access.rb b/lib/gitlab/checks/changes_access.rb
index 4e8b293a3e6..9ecc93f871b 100644
--- a/lib/gitlab/checks/changes_access.rb
+++ b/lib/gitlab/checks/changes_access.rb
@@ -29,11 +29,60 @@ module Gitlab
         true
       end
 
+      # All commits which have been newly introduced via any of the given
+      # changes. This set may also contain commits which are not referenced by
+      # any of the new revisions.
+      def commits
+        newrevs = @changes.map do |change|
+          newrev = change[:newrev]
+          newrev unless newrev.blank? || Gitlab::Git.blank_ref?(newrev)
+        end.compact
+
+        return [] if newrevs.empty?
+
+        @commits ||= project.repository.new_commits(newrevs, allow_quarantine: true)
+      end
+
+      # All commits which have been newly introduced via the given revision.
+      def commits_for(newrev)
+        commits_by_id = commits.index_by(&:id)
+
+        result = []
+        pending = Set[newrev]
+
+        # We go up the parent chain of our newrev and collect all commits which
+        # are new. In case a commit's ID cannot be found in the set of new
+        # commits, then it must already be a preexisting commit.
+        while pending.any?
+          rev = pending.first
+          pending.delete(rev)
+
+          # Remove the revision from commit candidates such that we don't walk
+          # it multiple times. If the hash doesn't contain the revision, then
+          # we have either already walked the commit or it's not new.
+          commit = commits_by_id.delete(rev)
+          next if commit.nil?
+
+          # Only add the parent ID to the pending set if we actually know its
+          # commit to guards us against readding an ID which we have already
+          # queued up before.
+          commit.parent_ids.each do |parent_id|
+            pending.add(parent_id) if commits_by_id.has_key?(parent_id)
+          end
+
+          result << commit
+        end
+
+        result
+      end
+
       protected
 
       def single_access_checks!
         # Iterate over all changes to find if user allowed all of them to be applied
         changes.each do |change|
+          commits = Gitlab::Lazy.new { commits_for(change[:newrev]) } if Feature.enabled?(:changes_batch_commits)
+
           # If user does not have access to make at least one change, cancel all
           # push by allowing the exception to bubble up
           Checks::SingleChangeAccess.new(
@@ -41,7 +90,8 @@ module Gitlab
             user_access: user_access,
             project: project,
             protocol: protocol,
-            logger: logger
+            logger: logger,
+            commits: commits
           ).validate!
         end
       end
diff --git a/lib/gitlab/checks/single_change_access.rb b/lib/gitlab/checks/single_change_access.rb
index 280b2dd25e2..2fd48dfbfe2 100644
--- a/lib/gitlab/checks/single_change_access.rb
+++ b/lib/gitlab/checks/single_change_access.rb
@@ -11,7 +11,7 @@ module Gitlab
 
       def initialize(
         change, user_access:, project:,
-        protocol:, logger:
+        protocol:, logger:, commits: nil
       )
         @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref)
         @branch_name = Gitlab::Git.branch_name(@ref)
@@ -19,6 +19,7 @@ module Gitlab
         @user_access = user_access
         @project = project
         @protocol = protocol
+        @commits = commits
 
         @logger = logger
         @logger.append_message("Running checks for ref: #{@branch_name || @tag_name}")
diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb
index 97988d8aa13..ef936581c10 100644
--- a/lib/gitlab/ci/ansi2html.rb
+++ b/lib/gitlab/ci/ansi2html.rb
@@ -33,6 +33,8 @@ module Gitlab
       Result = Struct.new(:html, :state, :append, :truncated, :offset, :size, :total, keyword_init: true) # rubocop:disable Lint/StructNewOverride
 
       class Converter
+        include EncodingHelper
+
         def on_0(_)
           reset
         end
@@ -256,6 +258,7 @@ module Gitlab
           start_offset = @offset
 
           stream.each_line do |line|
+            line = encode_utf8_no_detect(line)
             s = StringScanner.new(line)
 
             until s.eos?
diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb
index 8f2d47e7ccc..e48080993ab 100644
--- a/lib/gitlab/ci/ansi2json/line.rb
+++ b/lib/gitlab/ci/ansi2json/line.rb
@@ -9,6 +9,8 @@ module Gitlab
         # Line::Segment is a portion of a line that has its own style
         # and text. Multiple segments make the line content.
         class Segment
+          include EncodingHelper
+
           attr_accessor :text, :style
 
           def initialize(style:)
@@ -21,11 +23,12 @@ module Gitlab
           end
 
           def to_h
-            # Without force encoding to UTF-8 we could get an error
-            # when serializing the Hash to JSON.
+            # Without forcing the encoding to UTF-8 and then replacing
+            # invalid UTF-8 sequences we can get an error when serializing
+            # the Hash to JSON.
             # Encoding::UndefinedConversionError:
             #   "\xE2" from ASCII-8BIT to UTF-8
-            { text: text.force_encoding('UTF-8') }.tap do |result|
+            { text: encode_utf8_no_detect(text) }.tap do |result|
               result[:style] = style.to_s if style.set?
             end
           end
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 9c6428d701c..aceaf012f7e 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -17,13 +17,13 @@ module Gitlab
         Config::Yaml::Tags::TagError
       ].freeze
 
-      attr_reader :root, :context, :ref, :source
+      attr_reader :root, :context, :source_ref_path, :source
 
-      def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, ref: nil, source: nil)
-        @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline)
+      def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, source_ref_path: nil, source: nil)
+        @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline, ref: source_ref_path)
         @context.set_deadline(TIMEOUT_SECONDS)
 
-        @ref = ref
+        @source_ref_path = source_ref_path
         @source = source
 
         @config = expand_config(config)
@@ -108,13 +108,37 @@ module Gitlab
         end
       end
 
-      def build_context(project:, sha:, user:, parent_pipeline:)
+      def build_context(project:, sha:, user:, parent_pipeline:, ref:)
         Config::External::Context.new(
           project: project,
           sha: sha || find_sha(project),
           user: user,
           parent_pipeline: parent_pipeline,
-          variables: project&.predefined_variables&.to_runner_variables)
+          variables: build_variables(project: project, ref: ref))
+      end
+
+      def build_variables(project:, ref:)
+        Gitlab::Ci::Variables::Collection.new.tap do |variables|
+          break variables unless project
+
+          # The order of the following lines is important as priority of CI variables is
+          # defined globally within GitLab.
+          #
+          # See more detail in the docs: https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
+          variables.concat(project.predefined_variables)
+          variables.concat(pipeline_predefined_variables(ref: ref))
+          variables.concat(project.ci_instance_variables_for(ref: ref))
+          variables.concat(project.group.ci_variables_for(ref, project)) if project.group
+          variables.concat(project.ci_variables_for(ref: ref))
+        end
+      end
+
+      # https://gitlab.com/gitlab-org/gitlab/-/issues/337633 aims to add all predefined variables
+      # to this list, but only CI_COMMIT_REF_NAME is available right now to support compliance pipelines.
+      def pipeline_predefined_variables(ref:)
+        Gitlab::Ci::Variables::Collection.new.tap do |v|
+          v.append(key: 'CI_COMMIT_REF_NAME', value: ref)
+        end
       end
 
       def track_and_raise_for_dev_exception(error)
diff --git a/lib/gitlab/ci/config/entry/include.rb b/lib/gitlab/ci/config/entry/include.rb
index ad0ed00aa6f..368d8f07f8d 100644
--- a/lib/gitlab/ci/config/entry/include.rb
+++ b/lib/gitlab/ci/config/entry/include.rb
@@ -9,8 +9,10 @@ module Gitlab
         #
         class Include < ::Gitlab::Config::Entry::Node
           include ::Gitlab::Config::Entry::Validatable
+          include ::Gitlab::Config::Entry::Configurable
+          include ::Gitlab::Config::Entry::Attributable
 
-          ALLOWED_KEYS = %i[local file remote template artifact job project ref].freeze
+          ALLOWED_KEYS = %i[local file remote template artifact job project ref rules].freeze
 
           validations do
             validates :config, hash_or_string: true
@@ -27,6 +29,20 @@ module Gitlab
                 errors.add(:config, "must specify the file where to fetch the config from")
               end
             end
+
+            with_options allow_nil: true do
+              validates :rules, array_of_hashes: true
+            end
+          end
+
+          entry :rules, ::Gitlab::Ci::Config::Entry::Include::Rules,
+              description: 'List of evaluable Rules to determine file inclusion.',
+              inherit: false
+
+          attributes :rules
+
+          def skip_config_hash_validation?
+            true
           end
         end
       end
diff --git a/lib/gitlab/ci/config/entry/include/rules.rb b/lib/gitlab/ci/config/entry/include/rules.rb
new file mode 100644
index 00000000000..8eaf9e35aaf
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/include/rules.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        class Include
+          class Rules < ::Gitlab::Config::Entry::ComposableArray
+            include ::Gitlab::Config::Entry::Validatable
+
+            validations do
+              validates :config, presence: true
+              validates :config, type: Array
+            end
+
+            def value
+              @config
+            end
+
+            def composable_class
+              Entry::Include::Rules::Rule
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/include/rules/rule.rb b/lib/gitlab/ci/config/entry/include/rules/rule.rb
new file mode 100644
index 00000000000..d3d0f098814
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/include/rules/rule.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        class Include
+          class Rules::Rule < ::Gitlab::Config::Entry::Node
+            include ::Gitlab::Config::Entry::Validatable
+            include ::Gitlab::Config::Entry::Attributable
+
+            ALLOWED_KEYS = %i[if].freeze
+
+            attributes :if
+
+            validations do
+              validates :config, presence: true
+              validates :config, type: { with: Hash }
+              validates :config, allowed_keys: ALLOWED_KEYS
+
+              with_options allow_nil: true do
+                validates :if, expression: true
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/inherit/variables.rb b/lib/gitlab/ci/config/entry/inherit/variables.rb
index aa68833bdb8..adef4d1636a 100644
--- a/lib/gitlab/ci/config/entry/inherit/variables.rb
+++ b/lib/gitlab/ci/config/entry/inherit/variables.rb
@@ -13,9 +13,6 @@ module Gitlab
             strategy :ArrayStrategy, if: -> (config) { config.is_a?(Array) }
 
             class BooleanStrategy < ::Gitlab::Config::Entry::Boolean
-              def inherit?(_key)
-                value
-              end
             end
 
             class ArrayStrategy < ::Gitlab::Config::Entry::Node
@@ -25,20 +22,12 @@ module Gitlab
                 validates :config, type: Array
                 validates :config, array_of_strings: true
               end
-
-              def inherit?(key)
-                value.include?(key.to_s)
-              end
             end
 
             class UnknownStrategy < ::Gitlab::Config::Entry::Node
               def errors
                 ["#{location} should be a bool or array of strings"]
               end
-
-              def inherit?(key)
-                false
-              end
             end
           end
         end
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index e6d63969161..bd4d5f33689 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -16,11 +16,8 @@ module Gitlab
                             environment coverage retry parallel interruptible timeout
                             release dast_configuration secrets].freeze
 
-          REQUIRED_BY_NEEDS = %i[stage].freeze
-
           validations do
             validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS
-            validates :config, required_keys: REQUIRED_BY_NEEDS, if: :has_needs?
             validates :script, presence: true
 
             with_options allow_nil: true do
diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb
index 79dfb0eec1d..3543b5493bd 100644
--- a/lib/gitlab/ci/config/entry/processable.rb
+++ b/lib/gitlab/ci/config/entry/processable.rb
@@ -31,7 +31,7 @@ module Gitlab
 
               with_options allow_nil: true do
                 validates :extends, array_of_strings_or_string: true
-                validates :rules, array_of_hashes: true
+                validates :rules, nested_array_of_hashes: true
                 validates :resource_group, type: String
               end
             end
@@ -88,9 +88,6 @@ module Gitlab
                 validate_against_warnings
               end
 
-              # inherit root variables
-              @root_variables_value = deps&.variables_value # rubocop:disable Gitlab/ModuleWithInstanceVariables
-
               yield if block_given?
             end
           end
@@ -123,27 +120,13 @@ module Gitlab
               stage: stage_value,
               extends: extends,
               rules: rules_value,
-              variables: root_and_job_variables_value, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581
-              job_variables: job_variables,
+              job_variables: variables_value.to_h,
               root_variables_inheritance: root_variables_inheritance,
               only: only_value,
               except: except_value,
               resource_group: resource_group }.compact
           end
 
-          def root_and_job_variables_value
-            root_variables = @root_variables_value.to_h # rubocop:disable Gitlab/ModuleWithInstanceVariables
-            root_variables = root_variables.select do |key, _|
-              inherit_entry&.variables_entry&.inherit?(key)
-            end
-
-            root_variables.merge(variables_value.to_h)
-          end
-
-          def job_variables
-            variables_value.to_h
-          end
-
           def root_variables_inheritance
             inherit_entry&.variables_entry&.value
           end
diff --git a/lib/gitlab/ci/config/entry/rules.rb b/lib/gitlab/ci/config/entry/rules.rb
index bf74f995e80..53e52981471 100644
--- a/lib/gitlab/ci/config/entry/rules.rb
+++ b/lib/gitlab/ci/config/entry/rules.rb
@@ -13,7 +13,7 @@ module Gitlab
           end
 
           def value
-            @config
+            [@config].flatten
           end
 
           def composable_class
diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb
index 567a86c47e5..4bd8e250d7a 100644
--- a/lib/gitlab/ci/config/external/file/remote.rb
+++ b/lib/gitlab/ci/config/external/file/remote.rb
@@ -45,7 +45,7 @@ module Gitlab
                 errors.push("Remote file `#{location}` could not be fetched because of HTTP code `#{response.code}` error!")
               end
 
-              response.to_s if errors.none?
+              response.body if errors.none?
             end
           end
         end
diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb
index 3216d4eaac4..97e4922b2a1 100644
--- a/lib/gitlab/ci/config/external/mapper.rb
+++ b/lib/gitlab/ci/config/external/mapper.rb
@@ -33,6 +33,7 @@ module Gitlab
             locations
               .compact
               .map(&method(:normalize_location))
+              .filter_map(&method(:verify_rules))
               .flat_map(&method(:expand_project_files))
               .flat_map(&method(:expand_wildcard_paths))
               .map(&method(:expand_variables))
@@ -56,6 +57,15 @@ module Gitlab
             end
           end
 
+          def verify_rules(location)
+            # Behaves like there is no `rules`
+            return location unless ::Feature.enabled?(:ci_include_rules, context.project, default_enabled: :yaml)
+
+            return unless Rules.new(location[:rules]).evaluate(context).pass?
+
+            location
+          end
+
           def expand_project_files(location)
             return location unless location[:project]
 
@@ -65,8 +75,6 @@ module Gitlab
           end
 
           def expand_wildcard_paths(location)
-            return location unless ::Feature.enabled?(:ci_wildcard_file_paths, context.project, default_enabled: :yaml)
-
             # We only support local files for wildcard paths
             return location unless location[:local] && location[:local].include?('*')
 
diff --git a/lib/gitlab/ci/config/external/rules.rb b/lib/gitlab/ci/config/external/rules.rb
new file mode 100644
index 00000000000..5a788427172
--- /dev/null
+++ b/lib/gitlab/ci/config/external/rules.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module External
+        class Rules
+          def initialize(rule_hashes)
+            @rule_list = Build::Rules::Rule.fabricate_list(rule_hashes)
+          end
+
+          def evaluate(context)
+            Result.new(@rule_list.nil? || match_rule(context))
+          end
+
+          private
+
+          def match_rule(context)
+            @rule_list.find { |rule| rule.matches?(nil, context) }
+          end
+
+          Result = Struct.new(:result) do
+            def pass?
+              !!result
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb
index 5cabbc86d3e..312f98f850a 100644
--- a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb
+++ b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb
@@ -43,7 +43,6 @@ module Gitlab
             {
               name: name,
               instance: instance,
-              variables: variables, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581
               job_variables: variables,
               parallel: { total: total }
             }.compact
diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb
index d26a903c1f8..51051b0490f 100644
--- a/lib/gitlab/ci/features.rb
+++ b/lib/gitlab/ci/features.rb
@@ -3,7 +3,7 @@
 module Gitlab
   module Ci
     ##
-    # Ci::Features is a class that aggregates all CI/CD feature flags in one place.
+    # Deprecated: Ci::Features is a class that aggregates all CI/CD feature flags in one place.
     #
     module Features
       # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project`
diff --git a/lib/gitlab/ci/limit.rb b/lib/gitlab/ci/limit.rb
index c22a3c503d5..4f914388969 100644
--- a/lib/gitlab/ci/limit.rb
+++ b/lib/gitlab/ci/limit.rb
@@ -24,10 +24,13 @@ module Gitlab
       end
 
       def log_error!(extra_context = {})
-        error = LimitExceededError.new(message)
-        # TODO: change this to Gitlab::ErrorTracking.log_exception(error, extra_context)
-        # https://gitlab.com/gitlab-org/gitlab/issues/32906
-        ::Gitlab::ErrorTracking.track_exception(error, extra_context)
+        ::Gitlab::ErrorTracking.log_exception(limit_exceeded_error, extra_context)
+      end
+
+      protected
+
+      def limit_exceeded_error
+        LimitExceededError.new(message)
       end
     end
   end
diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb
index 4a7c11ee26e..cd2c135dd7e 100644
--- a/lib/gitlab/ci/lint.rb
+++ b/lib/gitlab/ci/lint.rb
@@ -38,6 +38,7 @@ module Gitlab
         pipeline = ::Ci::CreatePipelineService
           .new(@project, @current_user, ref: @project.default_branch)
           .execute(:push, dry_run: true, content: content)
+          .payload
 
         Result.new(
           jobs: dry_run_convert_to_jobs(pipeline.stages),
diff --git a/lib/gitlab/ci/model.rb b/lib/gitlab/ci/model.rb
deleted file mode 100644
index 1625cb841b6..00000000000
--- a/lib/gitlab/ci/model.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Ci
-    module Model
-      def table_name_prefix
-        "ci_"
-      end
-
-      def model_name
-        @model_name ||= ActiveModel::Name.new(self, nil, self.name.demodulize)
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb
index 3469537a2e2..1223d664214 100644
--- a/lib/gitlab/ci/parsers.rb
+++ b/lib/gitlab/ci/parsers.rb
@@ -11,7 +11,9 @@ module Gitlab
           cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura,
           terraform: ::Gitlab::Ci::Parsers::Terraform::Tfplan,
           accessibility: ::Gitlab::Ci::Parsers::Accessibility::Pa11y,
-          codequality: ::Gitlab::Ci::Parsers::Codequality::CodeClimate
+          codequality: ::Gitlab::Ci::Parsers::Codequality::CodeClimate,
+          sast: ::Gitlab::Ci::Parsers::Security::Sast,
+          secret_detection: ::Gitlab::Ci::Parsers::Security::SecretDetection
         }
       end
 
diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb
new file mode 100644
index 00000000000..41acb4d5040
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/common.rb
@@ -0,0 +1,266 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Parsers
+      module Security
+        class Common
+          SecurityReportParserError = Class.new(Gitlab::Ci::Parsers::ParserError)
+
+          def self.parse!(json_data, report, vulnerability_finding_signatures_enabled = false, validate: false)
+            new(json_data, report, vulnerability_finding_signatures_enabled, validate: validate).parse!
+          end
+
+          def initialize(json_data, report, vulnerability_finding_signatures_enabled = false, validate: false)
+            @json_data = json_data
+            @report = report
+            @validate = validate
+            @vulnerability_finding_signatures_enabled = vulnerability_finding_signatures_enabled
+          end
+
+          def parse!
+            return report_data unless valid?
+
+            raise SecurityReportParserError, "Invalid report format" unless report_data.is_a?(Hash)
+
+            create_scanner
+            create_scan
+            create_analyzer
+            set_report_version
+
+            create_findings
+
+            report_data
+          rescue JSON::ParserError
+            raise SecurityReportParserError, 'JSON parsing failed'
+          rescue StandardError => e
+            Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
+            raise SecurityReportParserError, "#{report.type} security report parsing failed"
+          end
+
+          private
+
+          attr_reader :json_data, :report, :validate
+
+          def valid?
+            return true if !validate || schema_validator.valid?
+
+            schema_validator.errors.each { |error| report.add_error('Schema', error) }
+
+            false
+          end
+
+          def schema_validator
+            @schema_validator ||= ::Gitlab::Ci::Parsers::Security::Validators::SchemaValidator.new(report.type, report_data)
+          end
+
+          def report_data
+            @report_data ||= Gitlab::Json.parse!(json_data)
+          end
+
+          def report_version
+            @report_version ||= report_data['version']
+          end
+
+          def top_level_scanner
+            @top_level_scanner ||= report_data.dig('scan', 'scanner')
+          end
+
+          def scan_data
+            @scan_data ||= report_data.dig('scan')
+          end
+
+          def analyzer_data
+            @analyzer_data ||= report_data.dig('scan', 'analyzer')
+          end
+
+          def tracking_data(data)
+            data['tracking']
+          end
+
+          def create_findings
+            if report_data["vulnerabilities"]
+              report_data["vulnerabilities"].each { |finding| create_finding(finding) }
+            end
+          end
+
+          def create_finding(data, remediations = [])
+            identifiers = create_identifiers(data['identifiers'])
+            links = create_links(data['links'])
+            location = create_location(data['location'] || {})
+            signatures = create_signatures(tracking_data(data))
+
+            if @vulnerability_finding_signatures_enabled && !signatures.empty?
+              # NOT the signature_sha - the compare key is hashed
+              # to create the project_fingerprint
+              highest_priority_signature = signatures.max_by(&:priority)
+              uuid = calculate_uuid_v5(identifiers.first, highest_priority_signature.signature_hex)
+            else
+              uuid = calculate_uuid_v5(identifiers.first, location&.fingerprint)
+            end
+
+            report.add_finding(
+              ::Gitlab::Ci::Reports::Security::Finding.new(
+                uuid: uuid,
+                report_type: report.type,
+                name: finding_name(data, identifiers, location),
+                compare_key: data['cve'] || '',
+                location: location,
+                severity: parse_severity_level(data['severity']),
+                confidence: parse_confidence_level(data['confidence']),
+                scanner: create_scanner(data['scanner']),
+                scan: report&.scan,
+                identifiers: identifiers,
+                links: links,
+                remediations: remediations,
+                raw_metadata: data.to_json,
+                metadata_version: report_version,
+                details: data['details'] || {},
+                signatures: signatures,
+                project_id: report.project_id,
+                vulnerability_finding_signatures_enabled: @vulnerability_finding_signatures_enabled))
+          end
+
+          def create_signatures(tracking)
+            tracking ||= { 'items' => [] }
+
+            signature_algorithms = Hash.new { |hash, key| hash[key] = [] }
+
+            tracking['items'].each do |item|
+              next unless item.key?('signatures')
+
+              item['signatures'].each do |signature|
+                alg = signature['algorithm']
+                signature_algorithms[alg] << signature['value']
+              end
+            end
+
+            signature_algorithms.map do |algorithm, values|
+              value = values.join('|')
+              signature = ::Gitlab::Ci::Reports::Security::FindingSignature.new(
+                algorithm_type: algorithm,
+                signature_value: value
+              )
+
+              if signature.valid?
+                signature
+              else
+                e = SecurityReportParserError.new("Vulnerability tracking signature is not valid: #{signature}")
+                Gitlab::ErrorTracking.track_exception(e)
+                nil
+              end
+            end.compact
+          end
+
+          def create_scan
+            return unless scan_data.is_a?(Hash)
+
+            report.scan = ::Gitlab::Ci::Reports::Security::Scan.new(scan_data)
+          end
+
+          def set_report_version
+            report.version = report_version
+          end
+
+          def create_analyzer
+            return unless analyzer_data.is_a?(Hash)
+
+            params = {
+              id: analyzer_data.dig('id'),
+              name: analyzer_data.dig('name'),
+              version: analyzer_data.dig('version'),
+              vendor: analyzer_data.dig('vendor', 'name')
+            }
+
+            return unless params.values.all?
+
+            report.analyzer = ::Gitlab::Ci::Reports::Security::Analyzer.new(**params)
+          end
+
+          def create_scanner(scanner_data = top_level_scanner)
+            return unless scanner_data.is_a?(Hash)
+
+            report.add_scanner(
+              ::Gitlab::Ci::Reports::Security::Scanner.new(
+                external_id: scanner_data['id'],
+                name: scanner_data['name'],
+                vendor: scanner_data.dig('vendor', 'name'),
+                version: scanner_data.dig('version')))
+          end
+
+          def create_identifiers(identifiers)
+            return [] unless identifiers.is_a?(Array)
+
+            identifiers.map { |identifier| create_identifier(identifier) }.compact
+          end
+
+          def create_identifier(identifier)
+            return unless identifier.is_a?(Hash)
+
+            report.add_identifier(
+              ::Gitlab::Ci::Reports::Security::Identifier.new(
+                external_type: identifier['type'],
+                external_id: identifier['value'],
+                name: identifier['name'],
+                url: identifier['url']))
+          end
+
+          def create_links(links)
+            return [] unless links.is_a?(Array)
+
+            links.map { |link| create_link(link) }.compact
+          end
+
+          def create_link(link)
+            return unless link.is_a?(Hash)
+
+            ::Gitlab::Ci::Reports::Security::Link.new(name: link['name'], url: link['url'])
+          end
+
+          def parse_severity_level(input)
+            input&.downcase.then { |value| ::Enums::Vulnerability.severity_levels.key?(value) ? value : 'unknown' }
+          end
+
+          def parse_confidence_level(input)
+            input&.downcase.then { |value| ::Enums::Vulnerability.confidence_levels.key?(value) ? value : 'unknown' }
+          end
+
+          def create_location(location_data)
+            raise NotImplementedError
+          end
+
+          def finding_name(data, identifiers, location)
+            return data['message'] if data['message'].present?
+            return data['name'] if data['name'].present?
+
+            identifier = identifiers.find(&:cve?) || identifiers.find(&:cwe?) || identifiers.first
+            "#{identifier.name} in #{location&.fingerprint_path}"
+          end
+
+          def calculate_uuid_v5(primary_identifier, location_fingerprint)
+            uuid_v5_name_components = {
+              report_type: report.type,
+              primary_identifier_fingerprint: primary_identifier&.fingerprint,
+              location_fingerprint: location_fingerprint,
+              project_id: report.project_id
+            }
+
+            if uuid_v5_name_components.values.any?(&:nil?)
+              Gitlab::AppLogger.warn(message: "One or more UUID name components are nil", components: uuid_v5_name_components)
+              return
+            end
+
+            ::Security::VulnerabilityUUID.generate(
+              report_type: uuid_v5_name_components[:report_type],
+              primary_identifier_fingerprint: uuid_v5_name_components[:primary_identifier_fingerprint],
+              location_fingerprint: uuid_v5_name_components[:location_fingerprint],
+              project_id: uuid_v5_name_components[:project_id]
+            )
+          end
+        end
+      end
+    end
+  end
+end
+
+Gitlab::Ci::Parsers::Security::Common.prepend_mod_with("Gitlab::Ci::Parsers::Security::Common")
diff --git a/lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb b/lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb
new file mode 100644
index 00000000000..24613a441be
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Parsers
+      module Security
+        module Concerns
+          module DeprecatedSyntax
+            extend ActiveSupport::Concern
+
+            included do
+              extend ::Gitlab::Utils::Override
+
+              override :parse_report
+            end
+
+            def report_data
+              @report_data ||= begin
+                data = super
+
+                if data.is_a?(Array)
+                  data = {
+                    "version" => self.class::DEPRECATED_REPORT_VERSION,
+                    "vulnerabilities" => data
+                  }
+                end
+
+                data
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/parsers/security/sast.rb b/lib/gitlab/ci/parsers/security/sast.rb
new file mode 100644
index 00000000000..e3c62614cd8
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/sast.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Parsers
+      module Security
+        class Sast < Common
+          include Security::Concerns::DeprecatedSyntax
+
+          DEPRECATED_REPORT_VERSION = "1.2"
+
+          private
+
+          def create_location(location_data)
+            ::Gitlab::Ci::Reports::Security::Locations::Sast.new(
+              file_path: location_data['file'],
+              start_line: location_data['start_line'],
+              end_line: location_data['end_line'],
+              class_name: location_data['class'],
+              method_name: location_data['method'])
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/parsers/security/secret_detection.rb b/lib/gitlab/ci/parsers/security/secret_detection.rb
new file mode 100644
index 00000000000..c6d95c1d391
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/secret_detection.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Parsers
+      module Security
+        class SecretDetection < Common
+          include Security::Concerns::DeprecatedSyntax
+
+          DEPRECATED_REPORT_VERSION = "1.2"
+
+          private
+
+          def create_location(location_data)
+            ::Gitlab::Ci::Reports::Security::Locations::SecretDetection.new(
+              file_path: location_data['file'],
+              start_line: location_data['start_line'],
+              end_line: location_data['end_line'],
+              class_name: location_data['class'],
+              method_name: location_data['method']
+            )
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
new file mode 100644
index 00000000000..3d92886cba8
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Parsers
+      module Security
+        module Validators
+          class SchemaValidator
+            class Schema
+              def root_path
+                File.join(__dir__, 'schemas')
+              end
+
+              def initialize(report_type)
+                @report_type = report_type
+              end
+
+              delegate :validate, to: :schemer
+
+              private
+
+              attr_reader :report_type
+
+              def schemer
+                JSONSchemer.schema(pathname)
+              end
+
+              def pathname
+                Pathname.new(schema_path)
+              end
+
+              def schema_path
+                File.join(root_path, file_name)
+              end
+
+              def file_name
+                "#{report_type}.json"
+              end
+            end
+
+            def initialize(report_type, report_data)
+              @report_type = report_type
+              @report_data = report_data
+            end
+
+            def valid?
+              errors.empty?
+            end
+
+            def errors
+              @errors ||= schema.validate(report_data).map { |error| JSONSchemer::Errors.pretty(error) }
+            end
+
+            private
+
+            attr_reader :report_type, :report_data
+
+            def schema
+              Schema.new(report_type)
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+Gitlab::Ci::Parsers::Security::Validators::SchemaValidator::Schema.prepend_mod_with("Gitlab::Ci::Parsers::Security::Validators::SchemaValidator::Schema")
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/sast.json b/lib/gitlab/ci/parsers/security/validators/schemas/sast.json
new file mode 100644
index 00000000000..a7159be0190
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/sast.json
@@ -0,0 +1,706 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Report format for GitLab SAST",
+  "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).",
+  "definitions": {
+    "detail_type": {
+      "oneOf": [
+        {
+          "$ref": "#/definitions/named_list"
+        },
+        {
+          "$ref": "#/definitions/list"
+        },
+        {
+          "$ref": "#/definitions/table"
+        },
+        {
+          "$ref": "#/definitions/text"
+        },
+        {
+          "$ref": "#/definitions/url"
+        },
+        {
+          "$ref": "#/definitions/code"
+        },
+        {
+          "$ref": "#/definitions/value"
+        },
+        {
+          "$ref": "#/definitions/diff"
+        },
+        {
+          "$ref": "#/definitions/markdown"
+        },
+        {
+          "$ref": "#/definitions/commit"
+        },
+        {
+          "$ref": "#/definitions/file_location"
+        },
+        {
+          "$ref": "#/definitions/module_location"
+        }
+      ]
+    },
+    "text_value": {
+      "type": "string"
+    },
+    "named_field": {
+      "type": "object",
+      "required": [
+        "name"
+      ],
+      "properties": {
+        "name": {
+          "$ref": "#/definitions/text_value",
+          "minLength": 1
+        },
+        "description": {
+          "$ref": "#/definitions/text_value"
+        }
+      }
+    },
+    "named_list": {
+      "type": "object",
+      "description": "An object with named and typed fields",
+      "required": [
+        "type",
+        "items"
+      ],
+      "properties": {
+        "type": {
+          "const": "named-list"
+        },
+        "items": {
+          "type": "object",
+          "patternProperties": {
+            "^.*$": {
+              "allOf": [
+                {
+                  "$ref": "#/definitions/named_field"
+                },
+                {
+                  "$ref": "#/definitions/detail_type"
+                }
+              ]
+            }
+          }
+        }
+      }
+    },
+    "list": {
+      "type": "object",
+      "description": "A list of typed fields",
+      "required": [
+        "type",
+        "items"
+      ],
+      "properties": {
+        "type": {
+          "const": "list"
+        },
+        "items": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/detail_type"
+          }
+        }
+      }
+    },
+    "table": {
+      "type": "object",
+      "description": "A table of typed fields",
+      "required": [
+        "type",
+        "rows"
+      ],
+      "properties": {
+        "type": {
+          "const": "table"
+        },
+        "header": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/detail_type"
+          }
+        },
+        "rows": {
+          "type": "array",
+          "items": {
+            "type": "array",
+            "items": {
+              "$ref": "#/definitions/detail_type"
+            }
+          }
+        }
+      }
+    },
+    "text": {
+      "type": "object",
+      "description": "Raw text",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "text"
+        },
+        "value": {
+          "$ref": "#/definitions/text_value"
+        }
+      }
+    },
+    "url": {
+      "type": "object",
+      "description": "A single URL",
+      "required": [
+        "type",
+        "href"
+      ],
+      "properties": {
+        "type": {
+          "const": "url"
+        },
+        "text": {
+          "$ref": "#/definitions/text_value"
+        },
+        "href": {
+          "type": "string",
+          "minLength": 1,
+          "examples": [
+            "http://mysite.com"
+          ]
+        }
+      }
+    },
+    "code": {
+      "type": "object",
+      "description": "A codeblock",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "code"
+        },
+        "value": {
+          "type": "string"
+        },
+        "lang": {
+          "type": "string",
+          "description": "A programming language"
+        }
+      }
+    },
+    "value": {
+      "type": "object",
+      "description": "A field that can store a range of types of value",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "value"
+        },
+        "value": {
+          "type": [
+            "number",
+            "string",
+            "boolean"
+          ]
+        }
+      }
+    },
+    "diff": {
+      "type": "object",
+      "description": "A diff",
+      "required": [
+        "type",
+        "before",
+        "after"
+      ],
+      "properties": {
+        "type": {
+          "const": "diff"
+        },
+        "before": {
+          "type": "string"
+        },
+        "after": {
+          "type": "string"
+        }
+      }
+    },
+    "markdown": {
+      "type": "object",
+      "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "markdown"
+        },
+        "value": {
+          "$ref": "#/definitions/text_value",
+          "examples": [
+            "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
+          ]
+        }
+      }
+    },
+    "commit": {
+      "type": "object",
+      "description": "A commit/tag/branch within the GitLab project",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "commit"
+        },
+        "value": {
+          "type": "string",
+          "description": "The commit SHA",
+          "minLength": 1
+        }
+      }
+    },
+    "file_location": {
+      "type": "object",
+      "description": "A location within a file in the project",
+      "required": [
+        "type",
+        "file_name",
+        "line_start"
+      ],
+      "properties": {
+        "type": {
+          "const": "file-location"
+        },
+        "file_name": {
+          "type": "string",
+          "minLength": 1
+        },
+        "line_start": {
+          "type": "integer"
+        },
+        "line_end": {
+          "type": "integer"
+        }
+      }
+    },
+    "module_location": {
+      "type": "object",
+      "description": "A location within a binary module of the form module+relative_offset",
+      "required": [
+        "type",
+        "module_name",
+        "offset"
+      ],
+      "properties": {
+        "type": {
+          "const": "module-location"
+        },
+        "module_name": {
+          "type": "string",
+          "minLength": 1,
+          "examples": [
+            "compiled_binary"
+          ]
+        },
+        "offset": {
+          "type": "integer",
+          "examples": [
+            100
+          ]
+        }
+      }
+    }
+  },
+  "self": {
+    "version": "14.0.0"
+  },
+  "required": [
+    "version",
+    "vulnerabilities"
+  ],
+  "additionalProperties": true,
+  "properties": {
+    "scan": {
+      "type": "object",
+      "required": [
+        "end_time",
+        "scanner",
+        "start_time",
+        "status",
+        "type"
+      ],
+      "properties": {
+        "end_time": {
+          "type": "string",
+          "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
+          "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+          "examples": [
+            "2020-01-28T03:26:02"
+          ]
+        },
+        "messages": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "description": "Communication intended for the initiator of a scan.",
+            "required": [
+              "level",
+              "value"
+            ],
+            "properties": {
+              "level": {
+                "type": "string",
+                "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
+                "enum": [
+                  "info",
+                  "warn",
+                  "fatal"
+                ],
+                "examples": [
+                  "info"
+                ]
+              },
+              "value": {
+                "type": "string",
+                "description": "The message to communicate.",
+                "minLength": 1,
+                "examples": [
+                  "Permission denied, scanning aborted"
+                ]
+              }
+            }
+          }
+        },
+        "scanner": {
+          "type": "object",
+          "description": "Object defining the scanner used to perform the scan.",
+          "required": [
+            "id",
+            "name",
+            "version",
+            "vendor"
+          ],
+          "properties": {
+            "id": {
+              "type": "string",
+              "description": "Unique id that identifies the scanner.",
+              "minLength": 1,
+              "examples": [
+                "my-sast-scanner"
+              ]
+            },
+            "name": {
+              "type": "string",
+              "description": "A human readable value that identifies the scanner, not required to be unique.",
+              "minLength": 1,
+              "examples": [
+                "My SAST Scanner"
+              ]
+            },
+            "url": {
+              "type": "string",
+              "description": "A link to more information about the scanner.",
+              "examples": [
+                "https://scanner.url"
+              ]
+            },
+            "version": {
+              "type": "string",
+              "description": "The version of the scanner.",
+              "minLength": 1,
+              "examples": [
+                "1.0.2"
+              ]
+            },
+            "vendor": {
+              "type": "object",
+              "description": "The vendor/maintainer of the scanner.",
+              "required": [
+                "name"
+              ],
+              "properties": {
+                "name": {
+                  "type": "string",
+                  "description": "The name of the vendor.",
+                  "minLength": 1,
+                  "examples": [
+                    "GitLab"
+                  ]
+                }
+              }
+            }
+          }
+        },
+        "start_time": {
+          "type": "string",
+          "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
+          "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+          "examples": [
+            "2020-02-14T16:01:59"
+          ]
+        },
+        "status": {
+          "type": "string",
+          "description": "Result of the scan.",
+          "enum": [
+            "success",
+            "failure"
+          ]
+        },
+        "type": {
+          "type": "string",
+          "description": "Type of the scan.",
+          "enum": [
+            "sast"
+          ]
+        }
+      }
+    },
+    "schema": {
+      "type": "string",
+      "description": "URI pointing to the validating security report schema.",
+      "format": "uri"
+    },
+    "version": {
+      "type": "string",
+      "description": "The version of the schema to which the JSON report conforms.",
+      "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
+    },
+    "vulnerabilities": {
+      "type": "array",
+      "description": "Array of vulnerability objects.",
+      "items": {
+        "type": "object",
+        "description": "Describes the vulnerability.",
+        "required": [
+          "category",
+          "cve",
+          "identifiers",
+          "location",
+          "scanner"
+        ],
+        "properties": {
+          "id": {
+            "type": "string",
+            "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+            "examples": [
+              "642735a5-1425-428d-8d4e-3c854885a3c9"
+            ]
+          },
+          "category": {
+            "type": "string",
+            "minLength": 1,
+            "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
+          },
+          "name": {
+            "type": "string",
+            "description": "The name of the vulnerability. This must not include the finding's specific information."
+          },
+          "message": {
+            "type": "string",
+            "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
+          },
+          "description": {
+            "type": "string",
+            "description": "A long text section describing the vulnerability more fully."
+          },
+          "cve": {
+            "type": "string",
+            "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
+          },
+          "severity": {
+            "type": "string",
+            "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
+            "enum": [
+              "Info",
+              "Unknown",
+              "Low",
+              "Medium",
+              "High",
+              "Critical"
+            ]
+          },
+          "confidence": {
+            "type": "string",
+            "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
+            "enum": [
+              "Ignore",
+              "Unknown",
+              "Experimental",
+              "Low",
+              "Medium",
+              "High",
+              "Confirmed"
+            ]
+          },
+          "solution": {
+            "type": "string",
+            "description": "Explanation of how to fix the vulnerability."
+          },
+          "scanner": {
+            "description": "Describes the scanner used to find this vulnerability.",
+            "type": "object",
+            "required": [
+              "id",
+              "name"
+            ],
+            "properties": {
+              "id": {
+                "type": "string",
+                "minLength": 1,
+                "description": "The scanner's ID, as a snake_case string."
+              },
+              "name": {
+                "type": "string",
+                "minLength": 1,
+                "description": "Human-readable name of the scanner."
+              }
+            }
+          },
+          "identifiers": {
+            "type": "array",
+            "minItems": 1,
+            "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
+            "items": {
+              "type": "object",
+              "required": [
+                "type",
+                "name",
+                "value"
+              ],
+              "properties": {
+                "type": {
+                  "type": "string",
+                  "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+                  "minLength": 1
+                },
+                "name": {
+                  "type": "string",
+                  "description": "Human-readable name of the identifier.",
+                  "minLength": 1
+                },
+                "url": {
+                  "type": "string",
+                  "description": "URL of the identifier's documentation.",
+                  "format": "uri"
+                },
+                "value": {
+                  "type": "string",
+                  "description": "Value of the identifier, for matching purpose.",
+                  "minLength": 1
+                }
+              }
+            }
+          },
+          "links": {
+            "type": "array",
+            "description": "An array of references to external documentation or articles that describe the vulnerability.",
+            "items": {
+              "type": "object",
+              "required": [
+                "url"
+              ],
+              "properties": {
+                "name": {
+                  "type": "string",
+                  "description": "Name of the vulnerability details link."
+                },
+                "url": {
+                  "type": "string",
+                  "description": "URL of the vulnerability details document.",
+                  "format": "uri"
+                }
+              }
+            }
+          },
+          "details": {
+            "$ref": "#/definitions/named_list/properties/items"
+          },
+          "location": {
+            "type": "object",
+            "description": "Identifies the vulnerability's location.",
+            "properties": {
+              "file": {
+                "type": "string",
+                "description": "Path to the file where the vulnerability is located."
+              },
+              "start_line": {
+                "type": "number",
+                "description": "The first line of the code affected by the vulnerability."
+              },
+              "end_line": {
+                "type": "number",
+                "description": "The last line of the code affected by the vulnerability."
+              },
+              "class": {
+                "type": "string",
+                "description": "Provides the name of the class where the vulnerability is located."
+              },
+              "method": {
+                "type": "string",
+                "description": "Provides the name of the method where the vulnerability is located."
+              }
+            }
+          },
+          "raw_source_code_extract": {
+            "type": "string",
+            "description": "Provides an unsanitized excerpt of the affected source code."
+          }
+        }
+      }
+    },
+    "remediations": {
+      "type": "array",
+      "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
+      "items": {
+        "type": "object",
+        "required": [
+          "fixes",
+          "summary",
+          "diff"
+        ],
+        "properties": {
+          "fixes": {
+            "type": "array",
+            "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
+            "items": {
+              "type": "object",
+              "required": [
+                "cve"
+              ],
+              "properties": {
+                "cve": {
+                  "type": "string",
+                  "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
+                }
+              }
+            }
+          },
+          "summary": {
+            "type": "string",
+            "minLength": 1,
+            "description": "An overview of how the vulnerabilities were fixed."
+          },
+          "diff": {
+            "type": "string",
+            "minLength": 1,
+            "description": "A base64-encoded remediation code diff, compatible with git apply."
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json b/lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json
new file mode 100644
index 00000000000..462e23a151c
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json
@@ -0,0 +1,729 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Report format for GitLab Secret Detection",
+  "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)",
+  "definitions": {
+    "detail_type": {
+      "oneOf": [
+        {
+          "$ref": "#/definitions/named_list"
+        },
+        {
+          "$ref": "#/definitions/list"
+        },
+        {
+          "$ref": "#/definitions/table"
+        },
+        {
+          "$ref": "#/definitions/text"
+        },
+        {
+          "$ref": "#/definitions/url"
+        },
+        {
+          "$ref": "#/definitions/code"
+        },
+        {
+          "$ref": "#/definitions/value"
+        },
+        {
+          "$ref": "#/definitions/diff"
+        },
+        {
+          "$ref": "#/definitions/markdown"
+        },
+        {
+          "$ref": "#/definitions/commit"
+        },
+        {
+          "$ref": "#/definitions/file_location"
+        },
+        {
+          "$ref": "#/definitions/module_location"
+        }
+      ]
+    },
+    "text_value": {
+      "type": "string"
+    },
+    "named_field": {
+      "type": "object",
+      "required": [
+        "name"
+      ],
+      "properties": {
+        "name": {
+          "$ref": "#/definitions/text_value",
+          "minLength": 1
+        },
+        "description": {
+          "$ref": "#/definitions/text_value"
+        }
+      }
+    },
+    "named_list": {
+      "type": "object",
+      "description": "An object with named and typed fields",
+      "required": [
+        "type",
+        "items"
+      ],
+      "properties": {
+        "type": {
+          "const": "named-list"
+        },
+        "items": {
+          "type": "object",
+          "patternProperties": {
+            "^.*$": {
+              "allOf": [
+                {
+                  "$ref": "#/definitions/named_field"
+                },
+                {
+                  "$ref": "#/definitions/detail_type"
+                }
+              ]
+            }
+          }
+        }
+      }
+    },
+    "list": {
+      "type": "object",
+      "description": "A list of typed fields",
+      "required": [
+        "type",
+        "items"
+      ],
+      "properties": {
+        "type": {
+          "const": "list"
+        },
+        "items": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/detail_type"
+          }
+        }
+      }
+    },
+    "table": {
+      "type": "object",
+      "description": "A table of typed fields",
+      "required": [
+        "type",
+        "rows"
+      ],
+      "properties": {
+        "type": {
+          "const": "table"
+        },
+        "header": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/detail_type"
+          }
+        },
+        "rows": {
+          "type": "array",
+          "items": {
+            "type": "array",
+            "items": {
+              "$ref": "#/definitions/detail_type"
+            }
+          }
+        }
+      }
+    },
+    "text": {
+      "type": "object",
+      "description": "Raw text",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "text"
+        },
+        "value": {
+          "$ref": "#/definitions/text_value"
+        }
+      }
+    },
+    "url": {
+      "type": "object",
+      "description": "A single URL",
+      "required": [
+        "type",
+        "href"
+      ],
+      "properties": {
+        "type": {
+          "const": "url"
+        },
+        "text": {
+          "$ref": "#/definitions/text_value"
+        },
+        "href": {
+          "type": "string",
+          "minLength": 1,
+          "examples": [
+            "http://mysite.com"
+          ]
+        }
+      }
+    },
+    "code": {
+      "type": "object",
+      "description": "A codeblock",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "code"
+        },
+        "value": {
+          "type": "string"
+        },
+        "lang": {
+          "type": "string",
+          "description": "A programming language"
+        }
+      }
+    },
+    "value": {
+      "type": "object",
+      "description": "A field that can store a range of types of value",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "value"
+        },
+        "value": {
+          "type": [
+            "number",
+            "string",
+            "boolean"
+          ]
+        }
+      }
+    },
+    "diff": {
+      "type": "object",
+      "description": "A diff",
+      "required": [
+        "type",
+        "before",
+        "after"
+      ],
+      "properties": {
+        "type": {
+          "const": "diff"
+        },
+        "before": {
+          "type": "string"
+        },
+        "after": {
+          "type": "string"
+        }
+      }
+    },
+    "markdown": {
+      "type": "object",
+      "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "markdown"
+        },
+        "value": {
+          "$ref": "#/definitions/text_value",
+          "examples": [
+            "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
+          ]
+        }
+      }
+    },
+    "commit": {
+      "type": "object",
+      "description": "A commit/tag/branch within the GitLab project",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "commit"
+        },
+        "value": {
+          "type": "string",
+          "description": "The commit SHA",
+          "minLength": 1
+        }
+      }
+    },
+    "file_location": {
+      "type": "object",
+      "description": "A location within a file in the project",
+      "required": [
+        "type",
+        "file_name",
+        "line_start"
+      ],
+      "properties": {
+        "type": {
+          "const": "file-location"
+        },
+        "file_name": {
+          "type": "string",
+          "minLength": 1
+        },
+        "line_start": {
+          "type": "integer"
+        },
+        "line_end": {
+          "type": "integer"
+        }
+      }
+    },
+    "module_location": {
+      "type": "object",
+      "description": "A location within a binary module of the form module+relative_offset",
+      "required": [
+        "type",
+        "module_name",
+        "offset"
+      ],
+      "properties": {
+        "type": {
+          "const": "module-location"
+        },
+        "module_name": {
+          "type": "string",
+          "minLength": 1,
+          "examples": [
+            "compiled_binary"
+          ]
+        },
+        "offset": {
+          "type": "integer",
+          "examples": [
+            100
+          ]
+        }
+      }
+    }
+  },
+  "self": {
+    "version": "14.0.0"
+  },
+  "required": [
+    "version",
+    "vulnerabilities"
+  ],
+  "additionalProperties": true,
+  "properties": {
+    "scan": {
+      "type": "object",
+      "required": [
+        "end_time",
+        "scanner",
+        "start_time",
+        "status",
+        "type"
+      ],
+      "properties": {
+        "end_time": {
+          "type": "string",
+          "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
+          "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+          "examples": [
+            "2020-01-28T03:26:02"
+          ]
+        },
+        "messages": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "description": "Communication intended for the initiator of a scan.",
+            "required": [
+              "level",
+              "value"
+            ],
+            "properties": {
+              "level": {
+                "type": "string",
+                "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
+                "enum": [
+                  "info",
+                  "warn",
+                  "fatal"
+                ],
+                "examples": [
+                  "info"
+                ]
+              },
+              "value": {
+                "type": "string",
+                "description": "The message to communicate.",
+                "minLength": 1,
+                "examples": [
+                  "Permission denied, scanning aborted"
+                ]
+              }
+            }
+          }
+        },
+        "scanner": {
+          "type": "object",
+          "description": "Object defining the scanner used to perform the scan.",
+          "required": [
+            "id",
+            "name",
+            "version",
+            "vendor"
+          ],
+          "properties": {
+            "id": {
+              "type": "string",
+              "description": "Unique id that identifies the scanner.",
+              "minLength": 1,
+              "examples": [
+                "my-sast-scanner"
+              ]
+            },
+            "name": {
+              "type": "string",
+              "description": "A human readable value that identifies the scanner, not required to be unique.",
+              "minLength": 1,
+              "examples": [
+                "My SAST Scanner"
+              ]
+            },
+            "url": {
+              "type": "string",
+              "description": "A link to more information about the scanner.",
+              "examples": [
+                "https://scanner.url"
+              ]
+            },
+            "version": {
+              "type": "string",
+              "description": "The version of the scanner.",
+              "minLength": 1,
+              "examples": [
+                "1.0.2"
+              ]
+            },
+            "vendor": {
+              "type": "object",
+              "description": "The vendor/maintainer of the scanner.",
+              "required": [
+                "name"
+              ],
+              "properties": {
+                "name": {
+                  "type": "string",
+                  "description": "The name of the vendor.",
+                  "minLength": 1,
+                  "examples": [
+                    "GitLab"
+                  ]
+                }
+              }
+            }
+          }
+        },
+        "start_time": {
+          "type": "string",
+          "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
+          "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+          "examples": [
+            "2020-02-14T16:01:59"
+          ]
+        },
+        "status": {
+          "type": "string",
+          "description": "Result of the scan.",
+          "enum": [
+            "success",
+            "failure"
+          ]
+        },
+        "type": {
+          "type": "string",
+          "description": "Type of the scan.",
+          "enum": [
+            "secret_detection"
+          ]
+        }
+      }
+    },
+    "schema": {
+      "type": "string",
+      "description": "URI pointing to the validating security report schema.",
+      "format": "uri"
+    },
+    "version": {
+      "type": "string",
+      "description": "The version of the schema to which the JSON report conforms.",
+      "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
+    },
+    "vulnerabilities": {
+      "type": "array",
+      "description": "Array of vulnerability objects.",
+      "items": {
+        "type": "object",
+        "description": "Describes the vulnerability.",
+        "required": [
+          "category",
+          "cve",
+          "identifiers",
+          "location",
+          "scanner"
+        ],
+        "properties": {
+          "id": {
+            "type": "string",
+            "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+            "examples": [
+              "642735a5-1425-428d-8d4e-3c854885a3c9"
+            ]
+          },
+          "category": {
+            "type": "string",
+            "minLength": 1,
+            "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
+          },
+          "name": {
+            "type": "string",
+            "description": "The name of the vulnerability. This must not include the finding's specific information."
+          },
+          "message": {
+            "type": "string",
+            "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
+          },
+          "description": {
+            "type": "string",
+            "description": "A long text section describing the vulnerability more fully."
+          },
+          "cve": {
+            "type": "string",
+            "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
+          },
+          "severity": {
+            "type": "string",
+            "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
+            "enum": [
+              "Info",
+              "Unknown",
+              "Low",
+              "Medium",
+              "High",
+              "Critical"
+            ]
+          },
+          "confidence": {
+            "type": "string",
+            "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
+            "enum": [
+              "Ignore",
+              "Unknown",
+              "Experimental",
+              "Low",
+              "Medium",
+              "High",
+              "Confirmed"
+            ]
+          },
+          "solution": {
+            "type": "string",
+            "description": "Explanation of how to fix the vulnerability."
+          },
+          "scanner": {
+            "description": "Describes the scanner used to find this vulnerability.",
+            "type": "object",
+            "required": [
+              "id",
+              "name"
+            ],
+            "properties": {
+              "id": {
+                "type": "string",
+                "minLength": 1,
+                "description": "The scanner's ID, as a snake_case string."
+              },
+              "name": {
+                "type": "string",
+                "minLength": 1,
+                "description": "Human-readable name of the scanner."
+              }
+            }
+          },
+          "identifiers": {
+            "type": "array",
+            "minItems": 1,
+            "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
+            "items": {
+              "type": "object",
+              "required": [
+                "type",
+                "name",
+                "value"
+              ],
+              "properties": {
+                "type": {
+                  "type": "string",
+                  "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+                  "minLength": 1
+                },
+                "name": {
+                  "type": "string",
+                  "description": "Human-readable name of the identifier.",
+                  "minLength": 1
+                },
+                "url": {
+                  "type": "string",
+                  "description": "URL of the identifier's documentation.",
+                  "format": "uri"
+                },
+                "value": {
+                  "type": "string",
+                  "description": "Value of the identifier, for matching purpose.",
+                  "minLength": 1
+                }
+              }
+            }
+          },
+          "links": {
+            "type": "array",
+            "description": "An array of references to external documentation or articles that describe the vulnerability.",
+            "items": {
+              "type": "object",
+              "required": [
+                "url"
+              ],
+              "properties": {
+                "name": {
+                  "type": "string",
+                  "description": "Name of the vulnerability details link."
+                },
+                "url": {
+                  "type": "string",
+                  "description": "URL of the vulnerability details document.",
+                  "format": "uri"
+                }
+              }
+            }
+          },
+          "details": {
+            "$ref": "#/definitions/named_list/properties/items"
+          },
+          "location": {
+            "required": [
+              "commit"
+            ],
+            "properties": {
+              "file": {
+                "type": "string",
+                "description": "Path to the file where the vulnerability is located"
+              },
+              "commit": {
+                "type": "object",
+                "description": "Represents the commit in which the vulnerability was detected",
+                "required": [
+                  "sha"
+                ],
+                "properties": {
+                  "author": {
+                    "type": "string"
+                  },
+                  "date": {
+                    "type": "string"
+                  },
+                  "message": {
+                    "type": "string"
+                  },
+                  "sha": {
+                    "type": "string",
+                    "minLength": 1
+                  }
+                }
+              },
+              "start_line": {
+                "type": "number",
+                "description": "The first line of the code affected by the vulnerability"
+              },
+              "end_line": {
+                "type": "number",
+                "description": "The last line of the code affected by the vulnerability"
+              },
+              "class": {
+                "type": "string",
+                "description": "Provides the name of the class where the vulnerability is located"
+              },
+              "method": {
+                "type": "string",
+                "description": "Provides the name of the method where the vulnerability is located"
+              }
+            }
+          },
+          "raw_source_code_extract": {
+            "type": "string",
+            "description": "Provides an unsanitized excerpt of the affected source code."
+          }
+        }
+      }
+    },
+    "remediations": {
+      "type": "array",
+      "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
+      "items": {
+        "type": "object",
+        "required": [
+          "fixes",
+          "summary",
+          "diff"
+        ],
+        "properties": {
+          "fixes": {
+            "type": "array",
+            "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
+            "items": {
+              "type": "object",
+              "required": [
+                "cve"
+              ],
+              "properties": {
+                "cve": {
+                  "type": "string",
+                  "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
+                }
+              }
+            }
+          },
+          "summary": {
+            "type": "string",
+            "minLength": 1,
+            "description": "An overview of how the vulnerabilities were fixed."
+          },
+          "diff": {
+            "type": "string",
+            "minLength": 1,
+            "description": "A base64-encoded remediation code diff, compatible with git apply."
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index 7564d0c3ed5..626eba97817 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -97,15 +97,16 @@ module Gitlab
               .observe({ source: pipeline.source.to_s }, pipeline.total_size)
           end
 
+          def observe_jobs_count_in_alive_pipelines
+            metrics.active_jobs_histogram
+              .observe({ plan: project.actual_plan_name }, project.all_pipelines.jobs_count_in_alive_pipelines)
+          end
+
           def increment_pipeline_failure_reason_counter(reason)
             metrics.pipeline_failure_reason_counter
               .increment(reason: (reason || :unknown_failure).to_s)
           end
 
-          def dangling_build?
-            %i[ondemand_dast_scan webide].include?(source)
-          end
-
           private
 
           # Verifies that origin_ref is a fully qualified tag reference (refs/tags/<tag-name>)
diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb
index 49ec1250a5f..5251dd3d40a 100644
--- a/lib/gitlab/ci/pipeline/chain/config/process.rb
+++ b/lib/gitlab/ci/pipeline/chain/config/process.rb
@@ -14,7 +14,7 @@ module Gitlab
               result = ::Gitlab::Ci::YamlProcessor.new(
                 @command.config_content, {
                   project: project,
-                  ref: @pipeline.ref,
+                  source_ref_path: @pipeline.source_ref_path,
                   sha: @pipeline.sha,
                   source: @pipeline.source,
                   user: current_user,
diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb
index dc648568129..bbfc6759b35 100644
--- a/lib/gitlab/ci/pipeline/chain/sequence.rb
+++ b/lib/gitlab/ci/pipeline/chain/sequence.rb
@@ -22,6 +22,7 @@ module Gitlab
 
             @command.observe_creation_duration(Time.now - @start)
             @command.observe_pipeline_size(@pipeline)
+            @command.observe_jobs_count_in_alive_pipelines
 
             @pipeline
           end
diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb
index e4e4f4f484a..76dfb4cbd87 100644
--- a/lib/gitlab/ci/pipeline/chain/skip.rb
+++ b/lib/gitlab/ci/pipeline/chain/skip.rb
@@ -22,16 +22,16 @@ module Gitlab
             end
           end
 
-          def skipped?
-            !@command.ignore_skip_ci && (commit_message_skips_ci? || push_option_skips_ci?)
-          end
-
           def break?
             skipped?
           end
 
           private
 
+          def skipped?
+            !@command.ignore_skip_ci && (commit_message_skips_ci? || push_option_skips_ci?)
+          end
+
           def commit_message_skips_ci?
             return false unless @pipeline.git_commit_message
 
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb
index 514241e8ae2..c7106f3ec39 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb
@@ -11,7 +11,7 @@ module Gitlab
             PATTERN = %r{^\/([^\/]|\\/)+[^\\]\/[ismU]*}.freeze
 
             def initialize(regexp)
-              super(regexp.gsub(/\\\//, '/'))
+              super(regexp.gsub(%r{\\/}, '/'))
 
               unless Gitlab::UntrustedRegexp::RubySyntax.valid?(@value)
                 raise Lexer::SyntaxError, 'Invalid regular expression!'
diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb
index 84b88374a7f..10de77afe74 100644
--- a/lib/gitlab/ci/pipeline/metrics.rb
+++ b/lib/gitlab/ci/pipeline/metrics.rb
@@ -24,7 +24,16 @@ module Gitlab
           name = :gitlab_ci_pipeline_size_builds
           comment = 'Pipeline size'
           labels = { source: nil }
-          buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000]
+          buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 3000]
+
+          ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
+        end
+
+        def self.active_jobs_histogram
+          name = :gitlab_ci_active_jobs
+          comment = 'Total amount of active jobs'
+          labels = { plan: nil }
+          buckets = [0, 200, 500, 1_000, 2_000, 5_000, 10_000]
 
           ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
         end
diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index 54d92745992..c393fed26de 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -39,7 +39,7 @@ module Gitlab
             @cache = Gitlab::Ci::Build::Cache
               .new(attributes.delete(:cache), @pipeline)
 
-            recalculate_yaml_variables!
+            calculate_yaml_variables!
           end
 
           def name
@@ -232,7 +232,7 @@ module Gitlab
             { options: { allow_failure_criteria: nil } }
           end
 
-          def recalculate_yaml_variables!
+          def calculate_yaml_variables!
             @seed_attributes[:yaml_variables] = Gitlab::Ci::Variables::Helpers.inherit_yaml_variables(
               from: @context.root_variables, to: @job_variables, inheritance: @root_variables_inheritance
             )
diff --git a/lib/gitlab/ci/reports/security/aggregated_report.rb b/lib/gitlab/ci/reports/security/aggregated_report.rb
new file mode 100644
index 00000000000..a8bb2196043
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/aggregated_report.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# Used to represent combined Security Reports. This is typically done for vulnerability deduplication purposes.
+
+module Gitlab
+  module Ci
+    module Reports
+      module Security
+        class AggregatedReport
+          attr_reader :findings
+
+          def initialize(reports, findings)
+            @reports = reports
+            @findings = findings
+          end
+
+          def created_at
+            @reports.map(&:created_at).compact.min
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb
new file mode 100644
index 00000000000..dc1c51b3ed0
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/finding.rb
@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Reports
+      module Security
+        class Finding
+          include ::VulnerabilityFindingHelpers
+
+          attr_reader :compare_key
+          attr_reader :confidence
+          attr_reader :identifiers
+          attr_reader :links
+          attr_reader :location
+          attr_reader :metadata_version
+          attr_reader :name
+          attr_reader :old_location
+          attr_reader :project_fingerprint
+          attr_reader :raw_metadata
+          attr_reader :report_type
+          attr_reader :scanner
+          attr_reader :scan
+          attr_reader :severity
+          attr_accessor :uuid
+          attr_accessor :overridden_uuid
+          attr_reader :remediations
+          attr_reader :details
+          attr_reader :signatures
+          attr_reader :project_id
+
+          delegate :file_path, :start_line, :end_line, to: :location
+
+          def initialize(compare_key:, identifiers:, links: [], remediations: [], location:, metadata_version:, name:, raw_metadata:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists
+            @compare_key = compare_key
+            @confidence = confidence
+            @identifiers = identifiers
+            @links = links
+            @location = location
+            @metadata_version = metadata_version
+            @name = name
+            @raw_metadata = raw_metadata
+            @report_type = report_type
+            @scanner = scanner
+            @scan = scan
+            @severity = severity
+            @uuid = uuid
+            @remediations = remediations
+            @details = details
+            @signatures = signatures
+            @project_id = project_id
+            @vulnerability_finding_signatures_enabled = vulnerability_finding_signatures_enabled
+
+            @project_fingerprint = generate_project_fingerprint
+          end
+
+          def to_hash
+            %i[
+              compare_key
+              confidence
+              identifiers
+              links
+              location
+              metadata_version
+              name
+              project_fingerprint
+              raw_metadata
+              report_type
+              scanner
+              scan
+              severity
+              uuid
+              details
+              signatures
+            ].each_with_object({}) do |key, hash|
+              hash[key] = public_send(key) # rubocop:disable GitlabSecurity/PublicSend
+            end
+          end
+
+          def primary_identifier
+            identifiers.first
+          end
+
+          def update_location(new_location)
+            @old_location = location
+            @location = new_location
+          end
+
+          def unsafe?(severity_levels)
+            severity.in?(severity_levels)
+          end
+
+          def eql?(other)
+            return false unless report_type == other.report_type && primary_identifier_fingerprint == other.primary_identifier_fingerprint
+
+            if @vulnerability_finding_signatures_enabled
+              matches_signatures(other.signatures, other.uuid)
+            else
+              location.fingerprint == other.location.fingerprint
+            end
+          end
+
+          def hash
+            if @vulnerability_finding_signatures_enabled && !signatures.empty?
+              highest_signature = signatures.max_by(&:priority)
+              report_type.hash ^ highest_signature.signature_hex.hash ^ primary_identifier_fingerprint.hash
+            else
+              report_type.hash ^ location.fingerprint.hash ^ primary_identifier_fingerprint.hash
+            end
+          end
+
+          def valid?
+            scanner.present? && primary_identifier.present? && location.present? && uuid.present?
+          end
+
+          def keys
+            @keys ||= identifiers.reject(&:type_identifier?).map do |identifier|
+              FindingKey.new(location_fingerprint: location&.fingerprint, identifier_fingerprint: identifier.fingerprint)
+            end
+          end
+
+          def primary_identifier_fingerprint
+            primary_identifier&.fingerprint
+          end
+
+          def <=>(other)
+            if severity == other.severity
+              compare_key <=> other.compare_key
+            else
+              ::Enums::Vulnerability.severity_levels[other.severity] <=>
+                ::Enums::Vulnerability.severity_levels[severity]
+            end
+          end
+
+          def scanner_order_to(other)
+            return 1 unless scanner
+            return -1 unless other&.scanner
+
+            scanner <=> other.scanner
+          end
+
+          private
+
+          def generate_project_fingerprint
+            Digest::SHA1.hexdigest(compare_key)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/reports/security/finding_key.rb b/lib/gitlab/ci/reports/security/finding_key.rb
new file mode 100644
index 00000000000..0acd923a60f
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/finding_key.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Reports
+      module Security
+        class FindingKey
+          def initialize(location_fingerprint:, identifier_fingerprint:)
+            @location_fingerprint = location_fingerprint
+            @identifier_fingerprint = identifier_fingerprint
+          end
+
+          def ==(other)
+            has_fingerprints? && other.has_fingerprints? &&
+              location_fingerprint == other.location_fingerprint &&
+                identifier_fingerprint == other.identifier_fingerprint
+          end
+
+          def hash
+            location_fingerprint.hash ^ identifier_fingerprint.hash
+          end
+
+          alias_method :eql?, :==
+
+          protected
+
+          attr_reader :location_fingerprint, :identifier_fingerprint
+
+          def has_fingerprints?
+            location_fingerprint.present? && identifier_fingerprint.present?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/reports/security/finding_signature.rb b/lib/gitlab/ci/reports/security/finding_signature.rb
new file mode 100644
index 00000000000..d1d7ef5c377
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/finding_signature.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Reports
+      module Security
+        class FindingSignature
+          include VulnerabilityFindingSignatureHelpers
+
+          attr_accessor :algorithm_type, :signature_value
+
+          def initialize(params = {})
+            @algorithm_type = params.dig(:algorithm_type)
+            @signature_value = params.dig(:signature_value)
+          end
+
+          def signature_sha
+            Digest::SHA1.digest(signature_value)
+          end
+
+          def signature_hex
+            signature_sha.unpack1("H*")
+          end
+
+          def to_hash
+            {
+              algorithm_type: algorithm_type,
+              signature_sha: signature_sha
+            }
+          end
+
+          def valid?
+            algorithm_types.key?(algorithm_type)
+          end
+
+          def eql?(other)
+            other.algorithm_type == algorithm_type &&
+              other.signature_sha == signature_sha
+          end
+
+          alias_method :==, :eql?
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/reports/security/locations/base.rb b/lib/gitlab/ci/reports/security/locations/base.rb
new file mode 100644
index 00000000000..9ad1d81287f
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/locations/base.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Reports
+      module Security
+        module Locations
+          class Base
+            include ::Gitlab::Utils::StrongMemoize
+
+            def ==(other)
+              other.fingerprint == fingerprint
+            end
+
+            def fingerprint
+              strong_memoize(:fingerprint) do
+                Digest::SHA1.hexdigest(fingerprint_data)
+              end
+            end
+
+            def as_json(options = nil)
+              fingerprint # side-effect call to initialize the ivar for serialization
+
+              super
+            end
+
+            def fingerprint_path
+              fingerprint_data
+            end
+
+            private
+
+            def fingerprint_data
+              raise NotImplementedError
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/reports/security/locations/sast.rb b/lib/gitlab/ci/reports/security/locations/sast.rb
new file mode 100644
index 00000000000..23ffa91e720
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/locations/sast.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Reports
+      module Security
+        module Locations
+          class Sast < Base
+            include Security::Concerns::FingerprintPathFromFile
+
+            attr_reader :class_name
+            attr_reader :end_line
+            attr_reader :file_path
+            attr_reader :method_name
+            attr_reader :start_line
+
+            def initialize(file_path:, start_line:, end_line: nil, class_name: nil, method_name: nil)
+              @class_name = class_name
+              @end_line = end_line
+              @file_path = file_path
+              @method_name = method_name
+              @start_line = start_line
+            end
+
+            def fingerprint_data
+              "#{file_path}:#{start_line}:#{end_line}"
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/reports/security/locations/secret_detection.rb b/lib/gitlab/ci/reports/security/locations/secret_detection.rb
new file mode 100644
index 00000000000..0fd5cc5af11
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/locations/secret_detection.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Reports
+      module Security
+        module Locations
+          class SecretDetection < Base
+            include Security::Concerns::FingerprintPathFromFile
+
+            attr_reader :class_name
+            attr_reader :end_line
+            attr_reader :file_path
+            attr_reader :method_name
+            attr_reader :start_line
+
+            def initialize(file_path:, start_line:, end_line: nil, class_name: nil, method_name: nil)
+              @class_name = class_name
+              @end_line = end_line
+              @file_path = file_path
+              @method_name = method_name
+              @start_line = start_line
+            end
+
+            def fingerprint_data
+              "#{file_path}:#{start_line}:#{end_line}"
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb
new file mode 100644
index 00000000000..1ba2d909d99
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/report.rb
@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Reports
+      module Security
+        class Report
+          attr_reader :created_at, :type, :pipeline, :findings, :scanners, :identifiers
+          attr_accessor :scan, :scanned_resources, :errors, :analyzer, :version
+
+          delegate :project_id, to: :pipeline
+
+          def initialize(type, pipeline, created_at)
+            @type = type
+            @pipeline = pipeline
+            @created_at = created_at
+            @findings = []
+            @scanners = {}
+            @identifiers = {}
+            @scanned_resources = []
+            @errors = []
+          end
+
+          def commit_sha
+            pipeline.sha
+          end
+
+          def add_error(type, message = 'An unexpected error happened!')
+            errors << { type: type, message: message }
+          end
+
+          def errored?
+            errors.present?
+          end
+
+          def add_scanner(scanner)
+            scanners[scanner.key] ||= scanner
+          end
+
+          def add_identifier(identifier)
+            identifiers[identifier.key] ||= identifier
+          end
+
+          def add_finding(finding)
+            findings << finding
+          end
+
+          def clone_as_blank
+            Report.new(type, pipeline, created_at)
+          end
+
+          def replace_with!(other)
+            instance_variables.each do |ivar|
+              instance_variable_set(ivar, other.public_send(ivar.to_s[1..-1])) # rubocop:disable GitlabSecurity/PublicSend
+            end
+          end
+
+          def merge!(other)
+            replace_with!(::Security::MergeReportsService.new(self, other).execute)
+          end
+
+          def primary_scanner
+            scanners.first&.second
+          end
+
+          def primary_scanner_order_to(other)
+            return 1 unless primary_scanner
+            return -1 unless other.primary_scanner
+
+            primary_scanner <=> other.primary_scanner
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/reports/security/reports.rb b/lib/gitlab/ci/reports/security/reports.rb
new file mode 100644
index 00000000000..b7a5e36b108
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/reports.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Reports
+      module Security
+        class Reports
+          attr_reader :reports, :pipeline
+
+          delegate :each, :empty?, to: :reports
+
+          def initialize(pipeline)
+            @reports = {}
+            @pipeline = pipeline
+          end
+
+          def get_report(report_type, report_artifact)
+            reports[report_type] ||= Report.new(report_type, pipeline, report_artifact.created_at)
+          end
+
+          def findings
+            reports.values.flat_map(&:findings)
+          end
+
+          def violates_default_policy_against?(target_reports, vulnerabilities_allowed, severity_levels)
+            unsafe_findings_count(target_reports, severity_levels) > vulnerabilities_allowed
+          end
+
+          private
+
+          def findings_diff(target_reports)
+            findings - target_reports&.findings.to_a
+          end
+
+          def unsafe_findings_count(target_reports, severity_levels)
+            findings_diff(target_reports).count {|finding| finding.unsafe?(severity_levels)}
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb b/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb
new file mode 100644
index 00000000000..6cb2e0ddb33
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb
@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Reports
+      module Security
+        class VulnerabilityReportsComparer
+          include Gitlab::Utils::StrongMemoize
+
+          attr_reader :base_report, :head_report
+
+          ACCEPTABLE_REPORT_AGE = 1.week
+
+          def initialize(project, base_report, head_report)
+            @base_report = base_report
+            @head_report = head_report
+
+            @signatures_enabled = project.licensed_feature_available?(:vulnerability_finding_signatures)
+
+            if @signatures_enabled
+              @added_findings = []
+              @fixed_findings = []
+              calculate_changes
+            end
+          end
+
+          def base_report_created_at
+            @base_report.created_at
+          end
+
+          def head_report_created_at
+            @head_report.created_at
+          end
+
+          def base_report_out_of_date
+            return false unless @base_report.created_at
+
+            ACCEPTABLE_REPORT_AGE.ago > @base_report.created_at
+          end
+
+          def added
+            strong_memoize(:added) do
+              if @signatures_enabled
+                @added_findings
+              else
+                head_report.findings - base_report.findings
+              end
+            end
+          end
+
+          def fixed
+            strong_memoize(:fixed) do
+              if @signatures_enabled
+                @fixed_findings
+              else
+                base_report.findings - head_report.findings
+              end
+            end
+          end
+
+          private
+
+          def calculate_changes
+            # This is a deconstructed version of the eql? method on
+            # Ci::Reports::Security::Finding. It:
+            #
+            # * precomputes for the head_findings (using FindingMatcher):
+            #   * sets of signature shas grouped by priority
+            #   * mappings of signature shas to the head finding object
+            #
+            # These are then used when iterating the base findings to perform
+            # fast(er) prioritized, signature-based comparisons between each base finding
+            # and the head findings.
+            #
+            # Both the head_findings and base_findings arrays are iterated once
+
+            base_findings = base_report.findings
+            head_findings = head_report.findings
+
+            matcher = FindingMatcher.new(head_findings)
+
+            base_findings.each do |base_finding|
+              matched_head_finding = matcher.find_and_remove_match!(base_finding)
+
+              @fixed_findings << base_finding if matched_head_finding.nil?
+            end
+
+            @added_findings = matcher.unmatched_head_findings.values
+          end
+        end
+
+        class FindingMatcher
+          attr_reader :unmatched_head_findings, :head_findings
+
+          include Gitlab::Utils::StrongMemoize
+
+          def initialize(head_findings)
+            @head_findings = head_findings
+            @unmatched_head_findings = @head_findings.index_by(&:object_id)
+          end
+
+          def find_and_remove_match!(base_finding)
+            matched_head_finding = find_matched_head_finding_for(base_finding)
+
+            # no signatures matched, so check the normal uuids of the base and head findings
+            # for a match
+            matched_head_finding = head_signatures_shas[base_finding.uuid] if matched_head_finding.nil?
+
+            @unmatched_head_findings.delete(matched_head_finding.object_id) unless matched_head_finding.nil?
+
+            matched_head_finding
+          end
+
+          private
+
+          def find_matched_head_finding_for(base_finding)
+            base_signature = sorted_signatures_for(base_finding).find do |signature|
+              # at this point a head_finding exists that has a signature with a
+              # matching priority, and a matching sha --> lookup the actual finding
+              # object from head_signatures_shas
+              head_signatures_shas[signature.signature_sha].eql?(base_finding)
+            end
+
+            base_signature.present? ? head_signatures_shas[base_signature.signature_sha] : nil
+          end
+
+          def sorted_signatures_for(base_finding)
+            base_finding.signatures.select { |signature| head_finding_signature?(signature) }
+                                   .sort_by { |sig| -sig.priority }
+          end
+
+          def head_finding_signature?(signature)
+            head_signatures_priorities[signature.priority].include?(signature.signature_sha)
+          end
+
+          def head_signatures_priorities
+            strong_memoize(:head_signatures_priorities) do
+              signatures_priorities = Hash.new { |hash, key| hash[key] = Set.new }
+
+              head_findings.each_with_object(signatures_priorities) do |head_finding, memo|
+                head_finding.signatures.each do |signature|
+                  memo[signature.priority].add(signature.signature_sha)
+                end
+              end
+            end
+          end
+
+          def head_signatures_shas
+            strong_memoize(:head_signatures_shas) do
+              head_findings.each_with_object({}) do |head_finding, memo|
+                head_finding.signatures.each do |signature|
+                  memo[signature.signature_sha] = head_finding
+                end
+                # for the final uuid check when no signatures have matched
+                memo[head_finding.uuid] = head_finding
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml
index ebb0b5948f1..71f38ededd9 100644
--- a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml
@@ -5,7 +5,7 @@
 
 # This template is on early stage of development.
 # Use it with caution. For usage instruction please read
-# https://gitlab.com/gitlab-org/5-minute-production-app/deploy-template/-/blob/v2.3.0/README.md
+# https://gitlab.com/gitlab-org/5-minute-production-app/deploy-template/-/blob/v3.0.0/README.md
 
 include:
   # workflow rules to prevent duplicate detached pipelines
diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml
index 1910913f2bd..f39a84bceec 100644
--- a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml
@@ -3,7 +3,7 @@
 # This specific template is located at:
 # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Bash.gitlab-ci.yml
 
-# See https://docs.gitlab.com/ee/ci/yaml/README.html for all available options
+# See https://docs.gitlab.com/ee/ci/yaml/index.html for all available options
 
 # you can delete this line if you're not using Docker
 image: busybox:latest
diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml
index d2d3b3ed61e..f147ad9332d 100644
--- a/lib/gitlab/ci/templates/Django.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml
@@ -18,7 +18,7 @@ variables:
   POSTGRES_DB: database_name
 
 # This folder is cached between builds
-# http://docs.gitlab.com/ee/ci/yaml/README.html#cache
+# https://docs.gitlab.com/ee/ci/yaml/index.html#cache
 cache:
   paths:
     - ~/.cache/pip/
diff --git a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml
index 38036c1f964..21a599fc78d 100644
--- a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml
@@ -10,7 +10,7 @@
 # A pipeline is composed of independent jobs that run scripts, grouped into stages.
 # Stages run in sequential order, but jobs within stages run in parallel.
 #
-# For more information, see: https://docs.gitlab.com/ee/ci/yaml/README.html#stages
+# For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages
 
 stages:          # List of stages for jobs, and their order of execution
   - build
diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
index 48e877684f6..43ecc4b96d5 100644
--- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
@@ -27,7 +27,7 @@ code_quality:
       }
     - docker pull --quiet "$CODE_QUALITY_IMAGE"
     - |
-      docker run \
+      docker run --rm \
         $(propagate_env_vars \
           SOURCE_CODE \
           TIMEOUT_SECONDS \
diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
index 00fcfa64a18..208951fa1a1 100644
--- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
 .dast-auto-deploy:
-  image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.6.0"
+  image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.12.0"
 
 dast_environment_deploy:
   extends: .dast-auto-deploy
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
index 530ab1d0f99..5c466f0984c 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
 .auto-deploy:
-  image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.6.0"
+  image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.12.0"
   dependencies: []
 
 review:
diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml
index 80125a9bc01..917a28bb1ee 100644
--- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml
@@ -252,6 +252,7 @@ semgrep-sast:
         - '**/*.jsx'
         - '**/*.ts'
         - '**/*.tsx'
+        - '**/*.c'
 
 sobelow-sast:
   extends: .sast-analyzer
diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml
index d0595491400..18f0f20203d 100644
--- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml
@@ -27,8 +27,8 @@ secret_detection:
       when: never
     - if: $CI_COMMIT_BRANCH
   script:
-    - if [[ $CI_COMMIT_TAG ]]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi
-    - if [[ $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH ]]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi
+    - if [ -n "$CI_COMMIT_TAG" ]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi
+    - if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi
     - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME
     - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt
     - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt
diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml
index 43e4ac02d41..ff7bac15017 100644
--- a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml
@@ -18,7 +18,7 @@ variables:
   MYSQL_ROOT_PASSWORD: secret
 
 # This folder is cached between builds
-# http://docs.gitlab.com/ee/ci/yaml/README.html#cache
+# https://docs.gitlab.com/ee/ci/yaml/index.html#cache
 cache:
   paths:
     - vendor/
diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml
index e48801b7970..16bc0026aa8 100644
--- a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml
@@ -16,7 +16,7 @@ services:
   - postgres:latest
 
 # This folder is cached between builds
-# http://docs.gitlab.com/ee/ci/yaml/README.html#cache
+# https://docs.gitlab.com/ee/ci/yaml/index.html#cache
 cache:
   paths:
     - node_modules/
diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml
index d3726fe34c5..9da50439be8 100644
--- a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml
@@ -6,7 +6,7 @@
 image: node:latest
 
 # This folder is cached between builds
-# http://docs.gitlab.com/ee/ci/yaml/README.html#cache
+# https://docs.gitlab.com/ee/ci/yaml/index.html#cache
 cache:
   paths:
     - node_modules/
diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml
index 490fc779e17..0c8b98dc1cf 100644
--- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml
@@ -29,7 +29,8 @@ before_script:
   - ruby -v  # Print out ruby version for debugging
   # Uncomment next line if your rails app needs a JS runtime:
   # - apt-get update -q && apt-get install nodejs -yqq
-  - bundle install -j $(nproc) --path vendor  # Install dependencies into ./vendor/ruby
+  - bundle config set path 'vendor' # Install dependencies into ./vendor/ruby
+  - bundle install -j $(nproc)
 
 # Optional - Delete if not using `rubocop`
 rubocop:
diff --git a/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml
index f4f066cc7c2..ed4876c2bcc 100644
--- a/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml
@@ -8,7 +8,7 @@
 # - A `test` stage to be present in the pipeline.
 # - You must define the `CIS_KUBECONFIG` variable to allow analyzer to connect to your Kubernetes cluster and fetch found vulnerabilities.
 #
-# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html).
+# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html).
 # List of available variables: https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/#available-variables
 
 variables:
diff --git a/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml
new file mode 100644
index 00000000000..d27a08db181
--- /dev/null
+++ b/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml
@@ -0,0 +1,23 @@
+# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# https://docs.gitlab.com/ee/development/cicd/templates.html
+# This specific template is located at:
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml
+
+stages:
+  - build
+  - test
+  - deploy
+  - dast
+
+variables:
+  DAST_RUNNER_VALIDATION_VERSION: 1
+
+validation:
+  stage: dast
+  image:
+    name: "registry.gitlab.com/security-products/dast-runner-validation:$DAST_RUNNER_VALIDATION_VERSION"
+  variables:
+    GIT_STRATEGY: none
+  allow_failure: false
+  script:
+    - ~/validate.sh
diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml
index e30777d8401..86b7d57d3cb 100644
--- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml
@@ -18,7 +18,7 @@ variables:
     bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec, semgrep,
     bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python,
     license-finder,
-    dast, api-fuzzing
+    dast, dast-runner-validation, api-fuzzing
 
   SECURE_BINARIES_DOWNLOAD_IMAGES: "true"
   SECURE_BINARIES_PUSH_IMAGES: "true"
@@ -230,6 +230,16 @@ dast:
       - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
           $SECURE_BINARIES_ANALYZERS =~ /\bdast\b/
 
+dast-runner-validation:
+  extends: .download_images
+  variables:
+    SECURE_BINARIES_ANALYZER_VERSION: "1"
+    SECURE_BINARIES_IMAGE: "registry.gitlab.com/security-products/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}"
+  only:
+    variables:
+      - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+          $SECURE_BINARIES_ANALYZERS =~ /\bdast-runner-validation\b/
+
 api-fuzzing:
   extends: .download_images
   variables:
diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
index 272b980b4b2..1a857ef3eb3 100644
--- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
@@ -4,7 +4,7 @@
 # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
 
 include:
-  - template: Terraform/Base.latest.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
+  - template: Terraform/Base.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
 
 stages:
   - init
diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
index d34a847f2d5..a9f6fd88d0b 100644
--- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
@@ -14,15 +14,22 @@ stages:
   - cleanup
 
 init:
-  extends: .init
+  extends: .terraform:init
 
 validate:
-  extends: .validate
+  extends: .terraform:validate
 
 build:
-  extends: .build
+  extends: .terraform:build
 
 deploy:
-  extends: .deploy
+  extends: .terraform:deploy
   dependencies:
     - build
+  environment:
+    name: $TF_STATE_NAME
+
+cleanup:
+  extends: .terraform:destroy
+  dependencies:
+    - deploy
diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml
new file mode 100644
index 00000000000..39c3374e534
--- /dev/null
+++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml
@@ -0,0 +1,64 @@
+# Terraform/Base.latest
+#
+# The purpose of this template is to provide flexibility to the user so
+# they are able to only include the jobs that they find interesting.
+#
+# Therefore, this template is not supposed to run any jobs. The idea is to only
+# create hidden jobs. See: https://docs.gitlab.com/ee/ci/yaml/#hide-jobs
+#
+# There is a more opinionated template which we suggest the users to abide,
+# which is the lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
+
+image:
+  name: registry.gitlab.com/gitlab-org/terraform-images/releases/terraform:1.0.3
+
+variables:
+  TF_ROOT: ${CI_PROJECT_DIR}  # The relative path to the root directory of the Terraform project
+  TF_STATE_NAME: ${TF_STATE_NAME:-default}  # The name of the state file used by the GitLab Managed Terraform state backend
+
+cache:
+  key: "${TF_ROOT}"
+  paths:
+    - ${TF_ROOT}/.terraform/
+    - ${TF_ROOT}/.terraform.lock.hcl
+
+.init: &init
+  stage: init
+  script:
+    - cd ${TF_ROOT}
+    - gitlab-terraform init
+
+.validate: &validate
+  stage: validate
+  script:
+    - cd ${TF_ROOT}
+    - gitlab-terraform validate
+
+.build: &build
+  stage: build
+  script:
+    - cd ${TF_ROOT}
+    - gitlab-terraform plan
+    - gitlab-terraform plan-json
+  artifacts:
+    paths:
+      - ${TF_ROOT}/plan.cache
+    reports:
+      terraform: ${TF_ROOT}/plan.json
+
+.deploy: &deploy
+  stage: deploy
+  script:
+    - cd ${TF_ROOT}
+    - gitlab-terraform apply
+  when: manual
+  only:
+    variables:
+      - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+
+.destroy: &destroy
+  stage: cleanup
+  script:
+    - cd ${TF_ROOT}
+    - gitlab-terraform destroy
+  when: manual
diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
index 200388a274c..c30860ad174 100644
--- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
@@ -13,7 +13,8 @@ image:
   name: registry.gitlab.com/gitlab-org/terraform-images/stable:latest
 
 variables:
-  TF_ROOT: ${CI_PROJECT_DIR}
+  TF_ROOT: ${CI_PROJECT_DIR}  # The relative path to the root directory of the Terraform project
+  TF_STATE_NAME: ${TF_STATE_NAME:-default}  # The name of the state file used by the GitLab Managed Terraform state backend
 
 cache:
   key: "${TF_ROOT}"
@@ -21,43 +22,46 @@ cache:
     - ${TF_ROOT}/.terraform/
     - ${TF_ROOT}/.terraform.lock.hcl
 
-.init: &init
+.terraform:init: &terraform_init
   stage: init
   script:
     - cd ${TF_ROOT}
     - gitlab-terraform init
 
-.validate: &validate
+.terraform:validate: &terraform_validate
   stage: validate
   script:
     - cd ${TF_ROOT}
     - gitlab-terraform validate
 
-.build: &build
+.terraform:build: &terraform_build
   stage: build
   script:
     - cd ${TF_ROOT}
     - gitlab-terraform plan
     - gitlab-terraform plan-json
+  resource_group: ${TF_STATE_NAME}
   artifacts:
     paths:
       - ${TF_ROOT}/plan.cache
     reports:
       terraform: ${TF_ROOT}/plan.json
 
-.deploy: &deploy
+.terraform:deploy: &terraform_deploy
   stage: deploy
   script:
     - cd ${TF_ROOT}
     - gitlab-terraform apply
+  resource_group: ${TF_STATE_NAME}
   when: manual
   only:
     variables:
       - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
 
-.destroy: &destroy
+.terraform:destroy: &terraform_destroy
   stage: cleanup
   script:
     - cd ${TF_ROOT}
     - gitlab-terraform destroy
+  resource_group: ${TF_STATE_NAME}
   when: manual
diff --git a/lib/gitlab/ci/yaml_processor/dag.rb b/lib/gitlab/ci/yaml_processor/dag.rb
index 0140218d9bc..8ab9573dd20 100644
--- a/lib/gitlab/ci/yaml_processor/dag.rb
+++ b/lib/gitlab/ci/yaml_processor/dag.rb
@@ -23,7 +23,7 @@ module Gitlab
 
           new(nodes).tsort
         rescue TSort::Cyclic
-          raise ValidationError, 'The pipeline has circular dependencies.'
+          raise ValidationError, 'The pipeline has circular dependencies'
         rescue MissingNodeError
         end
 
diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb
index dd5107bad9a..a97c7050fbb 100644
--- a/lib/gitlab/ci/yaml_processor/result.rb
+++ b/lib/gitlab/ci/yaml_processor/result.rb
@@ -69,7 +69,7 @@ module Gitlab
             when: job[:when] || 'on_success',
             environment: job[:environment_name],
             coverage_regex: job[:coverage],
-            yaml_variables: transform_to_yaml_variables(job[:variables]), # https://gitlab.com/gitlab-org/gitlab/-/issues/300581
+            # yaml_variables is calculated with using job_variables in Seed::Build
             job_variables: transform_to_yaml_variables(job[:job_variables]),
             root_variables_inheritance: job[:root_variables_inheritance],
             needs_attributes: job.dig(:needs, :job),
diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb
index 8120f2c1243..13c6eaf4993 100644
--- a/lib/gitlab/config/entry/validators.rb
+++ b/lib/gitlab/config/entry/validators.rb
@@ -78,10 +78,26 @@ module Gitlab
           include LegacyValidationHelpers
 
           def validate_each(record, attribute, value)
-            unless value.is_a?(Array) && value.map { |hsh| hsh.is_a?(Hash) }.all?
+            unless validate_array_of_hashes(value)
               record.errors.add(attribute, 'should be an array of hashes')
             end
           end
+
+          private
+
+          def validate_array_of_hashes(value)
+            value.is_a?(Array) && value.all? { |obj| obj.is_a?(Hash) }
+          end
+        end
+
+        class NestedArrayOfHashesValidator < ArrayOfHashesValidator
+          include NestedArrayHelpers
+
+          def validate_each(record, attribute, value)
+            unless validate_nested_array(value, 1, &method(:validate_array_of_hashes))
+              record.errors.add(attribute, 'should be an array containing hashes and arrays of hashes')
+            end
+          end
         end
 
         class ArrayOrStringValidator < ActiveModel::EachValidator
diff --git a/lib/gitlab/config_checker/external_database_checker.rb b/lib/gitlab/config_checker/external_database_checker.rb
index 606d45e0f0f..a56f2413615 100644
--- a/lib/gitlab/config_checker/external_database_checker.rb
+++ b/lib/gitlab/config_checker/external_database_checker.rb
@@ -6,7 +6,7 @@ module Gitlab
       extend self
 
       def check
-        return [] if Gitlab::Database.postgresql_minimum_supported_version?
+        return [] if Gitlab::Database.main.postgresql_minimum_supported_version?
 
         [
           {
@@ -15,7 +15,7 @@ module Gitlab
                        '%{pg_version_minimum} is required for this version of GitLab. ' \
                        'Please upgrade your environment to a supported PostgreSQL version, ' \
                        'see %{pg_requirements_url} for details.') % {
-                                                                      pg_version_current: Gitlab::Database.version,
+                                                                      pg_version_current: Gitlab::Database.main.version,
                                                                       pg_version_minimum: Gitlab::Database::MINIMUM_POSTGRES_VERSION,
                                                                       pg_requirements_url: '<a href="https://docs.gitlab.com/ee/install/requirements.html#database">database requirements</a>'
                                                                     }
diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb
index fbf021345ca..d40a6323d4f 100644
--- a/lib/gitlab/conflict/file.rb
+++ b/lib/gitlab/conflict/file.rb
@@ -23,7 +23,7 @@ module Gitlab
       # 'raw' holds the Gitlab::Git::Conflict::File that this instance wraps
       attr_reader :raw
 
-      delegate :type, :content, :their_path, :our_path, :our_mode, :our_blob, :repository, to: :raw
+      delegate :type, :content, :path, :ancestor_path, :their_path, :our_path, :our_mode, :our_blob, :repository, to: :raw
 
       def initialize(raw, merge_request:)
         @raw = raw
@@ -227,6 +227,26 @@ module Gitlab
                                                      new_path: our_path)
       end
 
+      def conflict_type(diff_file)
+        if ancestor_path.present?
+          if our_path.present? && their_path.present?
+            :both_modified
+          elsif their_path.blank?
+            :modified_source_removed_target
+          else
+            :modified_target_removed_source
+          end
+        else
+          if our_path.present? && their_path.present?
+            :both_added
+          elsif their_path.blank?
+            diff_file.renamed_file? ? :renamed_same_file : :removed_target_renamed_source
+          else
+            :removed_source_renamed_target
+          end
+        end
+      end
+
       private
 
       def map_raw_lines(raw_lines)
diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb
index 047600af267..c4ee1dafe20 100644
--- a/lib/gitlab/conflict/file_collection.rb
+++ b/lib/gitlab/conflict/file_collection.rb
@@ -7,14 +7,14 @@ module Gitlab
 
       attr_reader :merge_request, :resolver
 
-      def initialize(merge_request)
+      def initialize(merge_request, allow_tree_conflicts: false)
         our_commit = merge_request.source_branch_head.raw
         their_commit = merge_request.target_branch_head.raw
         @target_repo = merge_request.target_project.repository
         @source_repo = merge_request.source_project.repository.raw
         @our_commit_id = our_commit.id
         @their_commit_id = their_commit.id
-        @resolver = Gitlab::Git::Conflict::Resolver.new(@target_repo.raw, @our_commit_id, @their_commit_id)
+        @resolver = Gitlab::Git::Conflict::Resolver.new(@target_repo.raw, @our_commit_id, @their_commit_id, allow_tree_conflicts: allow_tree_conflicts)
         @merge_request = merge_request
       end
 
diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb
index 842920ba02e..bdcedd1896d 100644
--- a/lib/gitlab/content_security_policy/config_loader.rb
+++ b/lib/gitlab/content_security_policy/config_loader.rb
@@ -7,39 +7,40 @@ module Gitlab
                       form_action frame_ancestors frame_src img_src manifest_src
                       media_src object_src report_uri script_src style_src worker_src).freeze
 
-      def self.default_settings_hash
-        settings_hash = {
-          'enabled' => Rails.env.development? || Rails.env.test?,
-          'report_only' => false,
-          'directives' => {
-            'default_src' => "'self'",
-            'base_uri' => "'self'",
-            'connect_src' => "'self'",
-            'font_src' => "'self'",
-            'form_action' => "'self' https: http:",
-            'frame_ancestors' => "'self'",
-            'frame_src' => "'self' https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com",
-            'img_src' => "'self' data: blob: http: https:",
-            'manifest_src' => "'self'",
-            'media_src' => "'self'",
-            'script_src' => "'strict-dynamic' 'self' 'unsafe-inline' 'unsafe-eval' https://www.google.com/recaptcha/ https://www.recaptcha.net https://apis.google.com",
-            'style_src' => "'self' 'unsafe-inline'",
-            'worker_src' => "'self' blob: data:",
-            'object_src' => "'none'",
-            'report_uri' => nil
-          }
+      def self.default_enabled
+        Rails.env.development? || Rails.env.test?
+      end
+
+      def self.default_directives
+        directives = {
+          'default_src' => "'self'",
+          'base_uri' => "'self'",
+          'connect_src' => "'self'",
+          'font_src' => "'self'",
+          'form_action' => "'self' https: http:",
+          'frame_ancestors' => "'self'",
+          'frame_src' => "'self' https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com",
+          'img_src' => "'self' data: blob: http: https:",
+          'manifest_src' => "'self'",
+          'media_src' => "'self'",
+          'script_src' => "'strict-dynamic' 'self' 'unsafe-inline' 'unsafe-eval' https://www.google.com/recaptcha/ https://www.recaptcha.net https://apis.google.com",
+          'style_src' => "'self' 'unsafe-inline'",
+          'worker_src' => "'self' blob: data:",
+          'object_src' => "'none'",
+          'report_uri' => nil
         }
 
         # frame-src was deprecated in CSP level 2 in favor of child-src
         # CSP level 3 "undeprecated" frame-src and browsers fall back on child-src if it's missing
         # However Safari seems to read child-src first so we'll just keep both equal
-        settings_hash['directives']['child_src'] = settings_hash['directives']['frame_src']
+        directives['child_src'] = directives['frame_src']
 
-        allow_webpack_dev_server(settings_hash) if Rails.env.development?
-        allow_cdn(settings_hash) if ENV['GITLAB_CDN_HOST'].present?
-        allow_customersdot(settings_hash) if Rails.env.development? && ENV['CUSTOMER_PORTAL_URL'].present?
+        allow_webpack_dev_server(directives) if Rails.env.development?
+        allow_cdn(directives, Settings.gitlab.cdn_host) if Settings.gitlab.cdn_host.present?
+        allow_customersdot(directives) if Rails.env.development? && ENV['CUSTOMER_PORTAL_URL'].present?
+        allow_sentry(directives) if Gitlab.config.sentry&.enabled && Gitlab.config.sentry&.clientside_dsn
 
-        settings_hash
+        directives
       end
 
       def initialize(csp_directives)
@@ -66,31 +67,37 @@ module Gitlab
         arguments.strip.split(' ').map(&:strip)
       end
 
-      def self.allow_webpack_dev_server(settings_hash)
+      def self.allow_webpack_dev_server(directives)
         secure = Settings.webpack.dev_server['https']
         host_and_port = "#{Settings.webpack.dev_server['host']}:#{Settings.webpack.dev_server['port']}"
         http_url = "#{secure ? 'https' : 'http'}://#{host_and_port}"
         ws_url = "#{secure ? 'wss' : 'ws'}://#{host_and_port}"
 
-        append_to_directive(settings_hash, 'connect_src', "#{http_url} #{ws_url}")
+        append_to_directive(directives, 'connect_src', "#{http_url} #{ws_url}")
       end
 
-      def self.allow_cdn(settings_hash)
-        cdn_host = ENV['GITLAB_CDN_HOST']
-
-        append_to_directive(settings_hash, 'script_src', cdn_host)
-        append_to_directive(settings_hash, 'style_src', cdn_host)
-        append_to_directive(settings_hash, 'font_src', cdn_host)
+      def self.allow_cdn(directives, cdn_host)
+        append_to_directive(directives, 'script_src', cdn_host)
+        append_to_directive(directives, 'style_src', cdn_host)
+        append_to_directive(directives, 'font_src', cdn_host)
       end
 
-      def self.append_to_directive(settings_hash, directive, text)
-        settings_hash['directives'][directive] = "#{settings_hash['directives'][directive]} #{text}".strip
+      def self.append_to_directive(directives, directive, text)
+        directives[directive] = "#{directives[directive]} #{text}".strip
       end
 
-      def self.allow_customersdot(settings_hash)
+      def self.allow_customersdot(directives)
         customersdot_host = ENV['CUSTOMER_PORTAL_URL']
 
-        append_to_directive(settings_hash, 'frame_src', customersdot_host)
+        append_to_directive(directives, 'frame_src', customersdot_host)
+      end
+
+      def self.allow_sentry(directives)
+        sentry_dsn = Gitlab.config.sentry.clientside_dsn
+        sentry_uri = URI(sentry_dsn)
+        sentry_uri.user = nil
+
+        append_to_directive(directives, 'connect_src', sentry_uri.to_s)
       end
     end
   end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index e7ffeeb9849..bfe3f06a56b 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -85,7 +85,7 @@ module Gitlab
         active_db_connection = ActiveRecord::Base.connection.active? rescue false
 
         active_db_connection &&
-          Gitlab::Database.cached_table_exists?('application_settings')
+          Gitlab::Database.main.cached_table_exists?('application_settings')
       rescue ActiveRecord::NoDatabaseError
         false
       end
diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb
index f50ca5119b7..267c2d32ca9 100644
--- a/lib/gitlab/data_builder/deployment.rb
+++ b/lib/gitlab/data_builder/deployment.rb
@@ -16,6 +16,7 @@ module Gitlab
           object_kind: 'deployment',
           status: deployment.status,
           status_changed_at: status_changed_at,
+          deployment_id: deployment.id,
           deployable_id: deployment.deployable_id,
           deployable_url: deployable_url,
           environment: deployment.environment.name,
diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb
index 4d70e3949dd..385f1e57705 100644
--- a/lib/gitlab/data_builder/pipeline.rb
+++ b/lib/gitlab/data_builder/pipeline.rb
@@ -2,20 +2,56 @@
 
 module Gitlab
   module DataBuilder
-    module Pipeline
-      extend self
+    # Some callers want to include retried builds, so we wrap the payload hash
+    # in a SimpleDelegator with additional methods.
+    class Pipeline < SimpleDelegator
+      def self.build(pipeline)
+        new(pipeline)
+      end
 
-      def build(pipeline)
-        {
+      def initialize(pipeline)
+        @pipeline = pipeline
+
+        super(
           object_kind: 'pipeline',
           object_attributes: hook_attrs(pipeline),
           merge_request: pipeline.merge_request && merge_request_attrs(pipeline.merge_request),
           user: pipeline.user.try(:hook_attrs),
           project: pipeline.project.hook_attrs(backward: false),
           commit: pipeline.commit.try(:hook_attrs),
-          builds: pipeline.builds.latest.map(&method(:build_hook_attrs))
-        }
+          builds: Gitlab::Lazy.new do
+            preload_builds(pipeline, :latest_builds)
+            pipeline.latest_builds.map(&method(:build_hook_attrs))
+          end
+        )
+      end
+
+      def with_retried_builds
+        merge(
+          builds: Gitlab::Lazy.new do
+            preload_builds(@pipeline, :builds)
+            @pipeline.builds.map(&method(:build_hook_attrs))
+          end
+        )
+      end
+
+      private
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def preload_builds(pipeline, association)
+        ActiveRecord::Associations::Preloader.new.preload(pipeline,
+          {
+            association => {
+              **::Ci::Pipeline::PROJECT_ROUTE_AND_NAMESPACE_ROUTE,
+              runner: :tags,
+              job_artifacts_archive: [],
+              user: [],
+              metadata: []
+            }
+          }
+        )
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
       def hook_attrs(pipeline)
         {
@@ -91,7 +127,8 @@ module Gitlab
 
         {
           name: build.expanded_environment_name,
-          action: build.environment_action
+          action: build.environment_action,
+          deployment_tier: build.persisted_environment.try(:tier)
         }
       end
     end
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index a269b8d0366..acad19e096c 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -45,27 +45,18 @@ module Gitlab
     # It does not include the default public schema
     EXTRA_SCHEMAS = [DYNAMIC_PARTITIONS_SCHEMA, STATIC_PARTITIONS_SCHEMA].freeze
 
-    DEFAULT_POOL_HEADROOM = 10
-
-    # We configure the database connection pool size automatically based on the
-    # configured concurrency. We also add some headroom, to make sure we don't run
-    # out of connections when more threads besides the 'user-facing' ones are
-    # running.
-    #
-    # Read more about this in doc/development/database/client_side_connection_pool.md
-    def self.default_pool_size
-      headroom = (ENV["DB_POOL_HEADROOM"].presence || DEFAULT_POOL_HEADROOM).to_i
-
-      Gitlab::Runtime.max_threads + headroom
-    end
+    DATABASES = ActiveRecord::Base
+      .connection_handler
+      .connection_pools
+      .each_with_object({}) do |pool, hash|
+        hash[pool.db_config.name.to_sym] = Connection.new(pool.connection_klass)
+      end
+      .freeze
 
-    def self.config
-      default_config_hash = ActiveRecord::Base.configurations.find_db_config(Rails.env)&.configuration_hash || {}
+    PRIMARY_DATABASE_NAME = ActiveRecord::Base.connection_db_config.name.to_sym
 
-      default_config_hash.with_indifferent_access.tap do |hash|
-        # Match config/initializers/database_config.rb
-        hash[:pool] ||= default_pool_size
-      end
+    def self.main
+      DATABASES[PRIMARY_DATABASE_NAME]
     end
 
     def self.has_config?(database_name)
@@ -87,93 +78,34 @@ module Gitlab
       name.to_s == CI_DATABASE_NAME
     end
 
-    def self.username
-      config['username'] || ENV['USER']
-    end
-
-    def self.database_name
-      config['database']
-    end
-
-    def self.adapter_name
-      config['adapter']
-    end
-
-    def self.human_adapter_name
-      if postgresql?
-        'PostgreSQL'
-      else
-        'Unknown'
-      end
-    end
-
-    # Disables prepared statements for the current database connection.
-    def self.disable_prepared_statements
-      ActiveRecord::Base.establish_connection(config.merge(prepared_statements: false))
-    end
-
-    # @deprecated
-    def self.postgresql?
-      adapter_name.casecmp('postgresql') == 0
-    end
-
-    def self.read_only?
-      false
-    end
-
-    def self.read_write?
-      !self.read_only?
-    end
-
-    # Check whether the underlying database is in read-only mode
-    def self.db_read_only?
-      pg_is_in_recovery =
-        ActiveRecord::Base
-          .connection
-          .execute('SELECT pg_is_in_recovery()')
-          .first
-          .fetch('pg_is_in_recovery')
-
-      Gitlab::Utils.to_boolean(pg_is_in_recovery)
-    end
-
-    def self.db_read_write?
-      !self.db_read_only?
-    end
-
-    def self.version
-      @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1]
-    end
-
-    def self.postgresql_minimum_supported_version?
-      version.to_f >= MINIMUM_POSTGRES_VERSION
-    end
-
     def self.check_postgres_version_and_print_warning
-      return if Gitlab::Database.postgresql_minimum_supported_version?
       return if Gitlab::Runtime.rails_runner?
 
-      Kernel.warn ERB.new(Rainbow.new.wrap(<<~EOS).red).result
-
-                  ██     ██  █████  ██████  ███    ██ ██ ███    ██  ██████ 
-                  ██     ██ ██   ██ ██   ██ ████   ██ ██ ████   ██ ██      
-                  ██  █  ██ ███████ ██████  ██ ██  ██ ██ ██ ██  ██ ██   ███ 
-                  ██ ███ ██ ██   ██ ██   ██ ██  ██ ██ ██ ██  ██ ██ ██    ██ 
-                   ███ ███  ██   ██ ██   ██ ██   ████ ██ ██   ████  ██████  
-
-        ******************************************************************************
-          You are using PostgreSQL <%= Gitlab::Database.version %>, but PostgreSQL >= <%= Gitlab::Database::MINIMUM_POSTGRES_VERSION %>
-          is required for this version of GitLab.
-          <% if Rails.env.development? || Rails.env.test? %>
-          If using gitlab-development-kit, please find the relevant steps here:
-            https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md#upgrade-postgresql
-          <% end %>
-          Please upgrade your environment to a supported PostgreSQL version, see
-          https://docs.gitlab.com/ee/install/requirements.html#database for details.
-        ******************************************************************************
-      EOS
-    rescue ActiveRecord::ActiveRecordError, PG::Error
-      # ignore - happens when Rake tasks yet have to create a database, e.g. for testing
+      DATABASES.each do |name, connection|
+        next if connection.postgresql_minimum_supported_version?
+
+        Kernel.warn ERB.new(Rainbow.new.wrap(<<~EOS).red).result
+
+                    ██     ██  █████  ██████  ███    ██ ██ ███    ██  ██████ 
+                    ██     ██ ██   ██ ██   ██ ████   ██ ██ ████   ██ ██      
+                    ██  █  ██ ███████ ██████  ██ ██  ██ ██ ██ ██  ██ ██   ███ 
+                    ██ ███ ██ ██   ██ ██   ██ ██  ██ ██ ██ ██  ██ ██ ██    ██ 
+                     ███ ███  ██   ██ ██   ██ ██   ████ ██ ██   ████  ██████  
+
+          ******************************************************************************
+            You are using PostgreSQL <%= Gitlab::Database.main.version %> for the #{name} database, but PostgreSQL >= <%= Gitlab::Database::MINIMUM_POSTGRES_VERSION %>
+            is required for this version of GitLab.
+            <% if Rails.env.development? || Rails.env.test? %>
+            If using gitlab-development-kit, please find the relevant steps here:
+              https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md#upgrade-postgresql
+            <% end %>
+            Please upgrade your environment to a supported PostgreSQL version, see
+            https://docs.gitlab.com/ee/install/requirements.html#database for details.
+          ******************************************************************************
+        EOS
+      rescue ActiveRecord::ActiveRecordError, PG::Error
+        # ignore - happens when Rake tasks yet have to create a database, e.g. for testing
+      end
     end
 
     def self.nulls_order(field, direction = :asc, nulls_order = :nulls_last)
@@ -206,136 +138,20 @@ module Gitlab
       "'f'"
     end
 
-    def self.with_connection_pool(pool_size)
-      pool = create_connection_pool(pool_size)
-
-      begin
-        yield(pool)
-      ensure
-        pool.disconnect!
-      end
-    end
-
-    # Bulk inserts a number of rows into a table, optionally returning their
-    # IDs.
-    #
-    # table - The name of the table to insert the rows into.
-    # rows - An Array of Hash instances, each mapping the columns to their
-    #        values.
-    # return_ids - When set to true the return value will be an Array of IDs of
-    #              the inserted rows
-    # disable_quote - A key or an Array of keys to exclude from quoting (You
-    #                 become responsible for protection from SQL injection for
-    #                 these keys!)
-    # on_conflict - Defines an upsert. Values can be: :disabled (default) or
-    #               :do_nothing
-    def self.bulk_insert(table, rows, return_ids: false, disable_quote: [], on_conflict: nil)
-      return if rows.empty?
-
-      keys = rows.first.keys
-      columns = keys.map { |key| connection.quote_column_name(key) }
-
-      disable_quote = Array(disable_quote).to_set
-      tuples = rows.map do |row|
-        keys.map do |k|
-          disable_quote.include?(k) ? row[k] : connection.quote(row[k])
-        end
-      end
-
-      sql = <<-EOF
-        INSERT INTO #{table} (#{columns.join(', ')})
-        VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')}
-      EOF
-
-      sql = "#{sql} ON CONFLICT DO NOTHING" if on_conflict == :do_nothing
-
-      sql = "#{sql} RETURNING id" if return_ids
-
-      result = connection.execute(sql)
-
-      if return_ids
-        result.values.map { |tuple| tuple[0].to_i }
-      else
-        []
-      end
-    end
-
     def self.sanitize_timestamp(timestamp)
       MAX_TIMESTAMP_VALUE > timestamp ? timestamp : MAX_TIMESTAMP_VALUE.dup
     end
 
-    # pool_size - The size of the DB pool.
-    # host - An optional host name to use instead of the default one.
-    def self.create_connection_pool(pool_size, host = nil, port = nil)
-      original_config = Gitlab::Database.config
-
-      env_config = original_config.merge(pool: pool_size)
-      env_config[:host] = host if host
-      env_config[:port] = port if port
-
-      ActiveRecord::ConnectionAdapters::ConnectionHandler.new.establish_connection(env_config)
-    end
-
-    def self.connection
-      ActiveRecord::Base.connection
+    def self.allow_cross_joins_across_databases(url:)
+      # this method is implemented in:
+      # spec/support/database/prevent_cross_joins.rb
     end
-    private_class_method :connection
 
-    def self.cached_column_exists?(table_name, column_name)
-      connection.schema_cache.columns_hash(table_name).has_key?(column_name.to_s)
+    def self.allow_cross_database_modification_within_transaction(url:)
+      # this method is implemented in:
+      # spec/support/database/cross_database_modification_check.rb
     end
 
-    def self.cached_table_exists?(table_name)
-      exists? && connection.schema_cache.data_source_exists?(table_name)
-    end
-
-    def self.database_version
-      row = connection.execute("SELECT VERSION()").first
-
-      row['version']
-    end
-
-    def self.exists?
-      connection
-
-      true
-    rescue StandardError
-      false
-    end
-
-    def self.system_id
-      row = connection.execute('SELECT system_identifier FROM pg_control_system()').first
-
-      row['system_identifier']
-    end
-
-    # @param [ActiveRecord::Connection] ar_connection
-    # @return [String]
-    def self.get_write_location(ar_connection)
-      use_new_load_balancer_query = Gitlab::Utils.to_boolean(ENV['USE_NEW_LOAD_BALANCER_QUERY'], default: true)
-
-      sql = if use_new_load_balancer_query
-              <<~NEWSQL
-                SELECT CASE
-                    WHEN pg_is_in_recovery() = true AND EXISTS (SELECT 1 FROM pg_stat_get_wal_senders())
-                      THEN pg_last_wal_replay_lsn()::text
-                    WHEN pg_is_in_recovery() = false
-                      THEN pg_current_wal_insert_lsn()::text
-                      ELSE NULL
-                    END AS location;
-              NEWSQL
-            else
-              <<~SQL
-                SELECT pg_current_wal_insert_lsn()::text AS location
-              SQL
-            end
-
-      row = ar_connection.select_all(sql).first
-      row['location'] if row
-    end
-
-    private_class_method :database_version
-
     def self.add_post_migrate_path_to_rails(force: false)
       return if ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] && !force
 
@@ -352,47 +168,40 @@ module Gitlab
       end
     end
 
-    def self.dbname(ar_connection)
+    def self.db_config_names
+      ::ActiveRecord::Base.configurations.configs_for(env_name: Rails.env).map(&:name)
+    end
+
+    def self.db_config_name(ar_connection)
       if ar_connection.respond_to?(:pool) &&
           ar_connection.pool.respond_to?(:db_config) &&
-          ar_connection.pool.db_config.respond_to?(:database)
-        return ar_connection.pool.db_config.database
+          ar_connection.pool.db_config.respond_to?(:name)
+        return ar_connection.pool.db_config.name
       end
 
       'unknown'
     end
 
-    # inside_transaction? will return true if the caller is running within a transaction. Handles special cases
-    # when running inside a test environment, where tests may be wrapped in transactions
-    def self.inside_transaction?
-      if Rails.env.test?
-        ActiveRecord::Base.connection.open_transactions > open_transactions_baseline
-      else
-        ActiveRecord::Base.connection.open_transactions > 0
-      end
-    end
-
-    # These methods that access @open_transactions_baseline are not thread-safe.
-    # These are fine though because we only call these in RSpec's main thread. If we decide to run
-    # specs multi-threaded, we would need to use something like ThreadGroup to keep track of this value
-    def self.set_open_transactions_baseline
-      @open_transactions_baseline = ActiveRecord::Base.connection.open_transactions
-    end
-
-    def self.reset_open_transactions_baseline
-      @open_transactions_baseline = 0
+    def self.read_only?
+      false
     end
 
-    def self.open_transactions_baseline
-      @open_transactions_baseline ||= 0
+    def self.read_write?
+      !read_only?
     end
-    private_class_method :open_transactions_baseline
 
     # Monkeypatch rails with upgraded database observability
-    def self.install_monkey_patches
+    def self.install_transaction_metrics_patches!
       ActiveRecord::Base.prepend(ActiveRecordBaseTransactionMetrics)
     end
 
+    def self.install_transaction_context_patches!
+      ActiveRecord::ConnectionAdapters::TransactionManager
+        .prepend(TransactionManagerContext)
+      ActiveRecord::ConnectionAdapters::RealTransaction
+        .prepend(RealTransactionContext)
+    end
+
     # MonkeyPatch for ActiveRecord::Base for adding observability
     module ActiveRecordBaseTransactionMetrics
       extend ActiveSupport::Concern
@@ -407,6 +216,32 @@ module Gitlab
         end
       end
     end
+
+    # rubocop:disable Gitlab/ModuleWithInstanceVariables
+    module TransactionManagerContext
+      def transaction_context
+        @stack.first.try(:gitlab_transaction_context)
+      end
+    end
+
+    module RealTransactionContext
+      def gitlab_transaction_context
+        @gitlab_transaction_context ||= ::Gitlab::Database::Transaction::Context.new
+      end
+
+      def commit
+        gitlab_transaction_context.commit
+
+        super
+      end
+
+      def rollback
+        gitlab_transaction_context.rollback
+
+        super
+      end
+    end
+    # rubocop:enable Gitlab/ModuleWithInstanceVariables
   end
 end
 
diff --git a/lib/gitlab/database/as_with_materialized.rb b/lib/gitlab/database/as_with_materialized.rb
index eda991efbd5..07809c5b592 100644
--- a/lib/gitlab/database/as_with_materialized.rb
+++ b/lib/gitlab/database/as_with_materialized.rb
@@ -19,7 +19,7 @@ module Gitlab
       # Note: to be deleted after the minimum PG version is set to 12.0
       def self.materialized_supported?
         strong_memoize(:materialized_supported) do
-          Gitlab::Database.version.match?(/^1[2-9]\./) # version 12.x and above
+          Gitlab::Database.main.version.match?(/^1[2-9]\./) # version 12.x and above
         end
       end
 
diff --git a/lib/gitlab/database/async_indexes.rb b/lib/gitlab/database/async_indexes.rb
new file mode 100644
index 00000000000..d89d5238356
--- /dev/null
+++ b/lib/gitlab/database/async_indexes.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module AsyncIndexes
+      DEFAULT_INDEXES_PER_INVOCATION = 2
+
+      def self.create_pending_indexes!(how_many: DEFAULT_INDEXES_PER_INVOCATION)
+        PostgresAsyncIndex.order(:id).limit(how_many).each do |async_index|
+          IndexCreator.new(async_index).perform
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/async_indexes/index_creator.rb b/lib/gitlab/database/async_indexes/index_creator.rb
new file mode 100644
index 00000000000..00de79ec970
--- /dev/null
+++ b/lib/gitlab/database/async_indexes/index_creator.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module AsyncIndexes
+      class IndexCreator
+        include ExclusiveLeaseGuard
+
+        TIMEOUT_PER_ACTION = 1.day
+        STATEMENT_TIMEOUT = 9.hours
+
+        def initialize(async_index)
+          @async_index = async_index
+        end
+
+        def perform
+          try_obtain_lease do
+            if index_exists?
+              log_index_info('Skipping index creation as the index exists')
+            else
+              log_index_info('Creating async index')
+
+              set_statement_timeout do
+                connection.execute(async_index.definition)
+              end
+
+              log_index_info('Finished creating async index')
+            end
+
+            async_index.destroy
+          end
+        end
+
+        private
+
+        attr_reader :async_index
+
+        def index_exists?
+          connection.indexes(async_index.table_name).any? { |index| index.name == async_index.name }
+        end
+
+        def connection
+          @connection ||= ApplicationRecord.connection
+        end
+
+        def lease_timeout
+          TIMEOUT_PER_ACTION
+        end
+
+        def set_statement_timeout
+          connection.execute("SET statement_timeout TO '%ds'" % STATEMENT_TIMEOUT)
+          yield
+        ensure
+          connection.execute('RESET statement_timeout')
+        end
+
+        def log_index_info(message)
+          Gitlab::AppLogger.info(message: message, table_name: async_index.table_name, index_name: async_index.name)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/async_indexes/migration_helpers.rb b/lib/gitlab/database/async_indexes/migration_helpers.rb
new file mode 100644
index 00000000000..dff6376270a
--- /dev/null
+++ b/lib/gitlab/database/async_indexes/migration_helpers.rb
@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module AsyncIndexes
+      module MigrationHelpers
+        def unprepare_async_index(table_name, column_name, **options)
+          return unless async_index_creation_available?
+
+          index_name = options[:name] || index_name(table_name, column_name)
+
+          raise 'Specifying index name is mandatory - specify name: argument' unless index_name
+
+          unprepare_async_index_by_name(table_name, index_name)
+        end
+
+        def unprepare_async_index_by_name(table_name, index_name, **options)
+          return unless async_index_creation_available?
+
+          PostgresAsyncIndex.find_by(name: index_name).try do |async_index|
+            async_index.destroy
+          end
+        end
+
+        # Prepares an index for asynchronous creation.
+        #
+        # Stores the index information in the postgres_async_indexes table to be created later. The
+        # index will be always be created CONCURRENTLY, so that option does not need to be given.
+        # If an existing asynchronous definition exists with the same name, the existing entry will be
+        # updated with the new definition.
+        #
+        # If the requested index has already been created, it is not stored in the table for
+        # asynchronous creation.
+        def prepare_async_index(table_name, column_name, **options)
+          return unless async_index_creation_available?
+
+          index_name = options[:name] || index_name(table_name, column_name)
+
+          raise 'Specifying index name is mandatory - specify name: argument' unless index_name
+
+          options = options.merge({ algorithm: :concurrently })
+
+          if index_exists?(table_name, column_name, **options)
+            Gitlab::AppLogger.warn(
+              message: 'Index not prepared because it already exists',
+              table_name: table_name,
+              index_name: index_name)
+
+            return
+          end
+
+          index, algorithm, if_not_exists = add_index_options(table_name, column_name, **options)
+
+          create_index = ActiveRecord::ConnectionAdapters::CreateIndexDefinition.new(index, algorithm, if_not_exists)
+          schema_creation = ActiveRecord::ConnectionAdapters::PostgreSQL::SchemaCreation.new(ApplicationRecord.connection)
+          definition = schema_creation.accept(create_index)
+
+          async_index = PostgresAsyncIndex.safe_find_or_create_by!(name: index_name) do |rec|
+            rec.table_name = table_name
+            rec.definition = definition
+          end
+
+          Gitlab::AppLogger.info(
+            message: 'Prepared index for async creation',
+            table_name: async_index.table_name,
+            index_name: async_index.name)
+
+          async_index
+        end
+
+        private
+
+        def async_index_creation_available?
+          ApplicationRecord.connection.table_exists?(:postgres_async_indexes) &&
+            Feature.enabled?(:database_async_index_creation, type: :ops)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/async_indexes/postgres_async_index.rb b/lib/gitlab/database/async_indexes/postgres_async_index.rb
new file mode 100644
index 00000000000..236459e6216
--- /dev/null
+++ b/lib/gitlab/database/async_indexes/postgres_async_index.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module AsyncIndexes
+      class PostgresAsyncIndex < ApplicationRecord
+        self.table_name = 'postgres_async_indexes'
+
+        MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH
+        MAX_DEFINITION_LENGTH = 2048
+
+        validates :name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH }
+        validates :table_name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH }
+        validates :definition, presence: true, length: { maximum: MAX_DEFINITION_LENGTH }
+
+        def to_s
+          definition
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/batch_counter.rb b/lib/gitlab/database/batch_counter.rb
index 5f2e404c9da..7efa5b46ecb 100644
--- a/lib/gitlab/database/batch_counter.rb
+++ b/lib/gitlab/database/batch_counter.rb
@@ -31,7 +31,7 @@ module Gitlab
       end
 
       def count(batch_size: nil, mode: :itself, start: nil, finish: nil)
-        raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open?
+        raise 'BatchCount can not be run inside a transaction' if @relation.connection.transaction_open?
 
         check_mode!(mode)
 
diff --git a/lib/gitlab/database/connection.rb b/lib/gitlab/database/connection.rb
new file mode 100644
index 00000000000..21861e4fba8
--- /dev/null
+++ b/lib/gitlab/database/connection.rb
@@ -0,0 +1,249 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    # Configuration settings and methods for interacting with a PostgreSQL
+    # database, with support for multiple databases.
+    class Connection
+      DEFAULT_POOL_HEADROOM = 10
+
+      attr_reader :scope
+
+      # Initializes a new `Database`.
+      #
+      # The `scope` argument must be an object (such as `ActiveRecord::Base`)
+      # that supports retrieving connections and connection pools.
+      def initialize(scope = ActiveRecord::Base)
+        @config = nil
+        @scope = scope
+        @version = nil
+        @open_transactions_baseline = 0
+      end
+
+      # We configure the database connection pool size automatically based on
+      # the configured concurrency. We also add some headroom, to make sure we
+      # don't run out of connections when more threads besides the 'user-facing'
+      # ones are running.
+      #
+      # Read more about this in
+      # doc/development/database/client_side_connection_pool.md
+      def default_pool_size
+        headroom =
+          (ENV["DB_POOL_HEADROOM"].presence || DEFAULT_POOL_HEADROOM).to_i
+
+        Gitlab::Runtime.max_threads + headroom
+      end
+
+      def config
+        # The result of this method must not be cached, as other methods may use
+        # it after making configuration changes and expect those changes to be
+        # present. For example, `disable_prepared_statements` expects the
+        # configuration settings to always be up to date.
+        #
+        # See the following for more information:
+        #
+        # - https://gitlab.com/gitlab-org/release/retrospectives/-/issues/39
+        # - https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5238
+        scope.connection_db_config.configuration_hash.with_indifferent_access
+      end
+
+      def pool_size
+        config[:pool] || default_pool_size
+      end
+
+      def username
+        config[:username] || ENV['USER']
+      end
+
+      def database_name
+        config[:database]
+      end
+
+      def adapter_name
+        config[:adapter]
+      end
+
+      def human_adapter_name
+        if postgresql?
+          'PostgreSQL'
+        else
+          'Unknown'
+        end
+      end
+
+      def postgresql?
+        adapter_name.casecmp('postgresql') == 0
+      end
+
+      def db_config_with_default_pool_size
+        db_config_object = scope.connection_db_config
+        config = db_config_object.configuration_hash.merge(pool: default_pool_size)
+
+        ActiveRecord::DatabaseConfigurations::HashConfig.new(
+          db_config_object.env_name,
+          db_config_object.name,
+          config
+        )
+      end
+
+      # Disables prepared statements for the current database connection.
+      def disable_prepared_statements
+        scope.establish_connection(config.merge(prepared_statements: false))
+      end
+
+      # Check whether the underlying database is in read-only mode
+      def db_read_only?
+        pg_is_in_recovery =
+          scope
+            .connection
+            .execute('SELECT pg_is_in_recovery()')
+            .first
+            .fetch('pg_is_in_recovery')
+
+        Gitlab::Utils.to_boolean(pg_is_in_recovery)
+      end
+
+      def db_read_write?
+        !db_read_only?
+      end
+
+      def version
+        @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1]
+      end
+
+      def database_version
+        connection.execute("SELECT VERSION()").first['version']
+      end
+
+      def postgresql_minimum_supported_version?
+        version.to_f >= MINIMUM_POSTGRES_VERSION
+      end
+
+      # Bulk inserts a number of rows into a table, optionally returning their
+      # IDs.
+      #
+      # table - The name of the table to insert the rows into.
+      # rows - An Array of Hash instances, each mapping the columns to their
+      #        values.
+      # return_ids - When set to true the return value will be an Array of IDs of
+      #              the inserted rows
+      # disable_quote - A key or an Array of keys to exclude from quoting (You
+      #                 become responsible for protection from SQL injection for
+      #                 these keys!)
+      # on_conflict - Defines an upsert. Values can be: :disabled (default) or
+      #               :do_nothing
+      def bulk_insert(table, rows, return_ids: false, disable_quote: [], on_conflict: nil)
+        return if rows.empty?
+
+        keys = rows.first.keys
+        columns = keys.map { |key| connection.quote_column_name(key) }
+
+        disable_quote = Array(disable_quote).to_set
+        tuples = rows.map do |row|
+          keys.map do |k|
+            disable_quote.include?(k) ? row[k] : connection.quote(row[k])
+          end
+        end
+
+        sql = <<-EOF
+          INSERT INTO #{table} (#{columns.join(', ')})
+          VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')}
+        EOF
+
+        sql = "#{sql} ON CONFLICT DO NOTHING" if on_conflict == :do_nothing
+
+        sql = "#{sql} RETURNING id" if return_ids
+
+        result = connection.execute(sql)
+
+        if return_ids
+          result.values.map { |tuple| tuple[0].to_i }
+        else
+          []
+        end
+      end
+
+      def cached_column_exists?(table_name, column_name)
+        connection
+          .schema_cache.columns_hash(table_name)
+          .has_key?(column_name.to_s)
+      end
+
+      def cached_table_exists?(table_name)
+        exists? && connection.schema_cache.data_source_exists?(table_name)
+      end
+
+      def exists?
+        connection
+
+        true
+      rescue StandardError
+        false
+      end
+
+      def system_id
+        row = connection
+          .execute('SELECT system_identifier FROM pg_control_system()')
+          .first
+
+        row['system_identifier']
+      end
+
+      # @param [ActiveRecord::Connection] ar_connection
+      # @return [String]
+      def get_write_location(ar_connection)
+        use_new_load_balancer_query = Gitlab::Utils
+          .to_boolean(ENV['USE_NEW_LOAD_BALANCER_QUERY'], default: true)
+
+        sql =
+          if use_new_load_balancer_query
+            <<~NEWSQL
+              SELECT CASE
+                  WHEN pg_is_in_recovery() = true AND EXISTS (SELECT 1 FROM pg_stat_get_wal_senders())
+                    THEN pg_last_wal_replay_lsn()::text
+                  WHEN pg_is_in_recovery() = false
+                    THEN pg_current_wal_insert_lsn()::text
+                    ELSE NULL
+                  END AS location;
+            NEWSQL
+          else
+            <<~SQL
+              SELECT pg_current_wal_insert_lsn()::text AS location
+            SQL
+          end
+
+        row = ar_connection.select_all(sql).first
+        row['location'] if row
+      end
+
+      # inside_transaction? will return true if the caller is running within a
+      # transaction. Handles special cases when running inside a test
+      # environment, where tests may be wrapped in transactions
+      def inside_transaction?
+        base = Rails.env.test? ? @open_transactions_baseline : 0
+
+        scope.connection.open_transactions > base
+      end
+
+      # These methods that access @open_transactions_baseline are not
+      # thread-safe.  These are fine though because we only call these in
+      # RSpec's main thread. If we decide to run specs multi-threaded, we would
+      # need to use something like ThreadGroup to keep track of this value
+      def set_open_transactions_baseline
+        @open_transactions_baseline = scope.connection.open_transactions
+      end
+
+      def reset_open_transactions_baseline
+        @open_transactions_baseline = 0
+      end
+
+      private
+
+      def connection
+        scope.connection
+      end
+    end
+  end
+end
+
+Gitlab::Database::Connection.prepend_mod_with('Gitlab::Database::Connection')
diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb
index a7bfafe2815..870cf25984b 100644
--- a/lib/gitlab/database/count/reltuples_count_strategy.rb
+++ b/lib/gitlab/database/count/reltuples_count_strategy.rb
@@ -54,7 +54,7 @@ module Gitlab
 
           # Querying tuple stats only works on the primary. Due to load balancing, the
           # easiest way to do this is to start a transaction.
-          ActiveRecord::Base.transaction do
+          ActiveRecord::Base.transaction do # rubocop: disable Database/MultipleDatabases
             get_statistics(non_sti_table_names, check_statistics: check_statistics).each_with_object({}) do |row, data|
               model = table_to_model[row.table_name]
               data[model] = row.estimate
diff --git a/lib/gitlab/database/count/tablesample_count_strategy.rb b/lib/gitlab/database/count/tablesample_count_strategy.rb
index e9387a91a14..489bc0aacea 100644
--- a/lib/gitlab/database/count/tablesample_count_strategy.rb
+++ b/lib/gitlab/database/count/tablesample_count_strategy.rb
@@ -61,7 +61,7 @@ module Gitlab
             #{where_clause(model)}
           SQL
 
-          rows = ActiveRecord::Base.connection.select_all(query)
+          rows = ActiveRecord::Base.connection.select_all(query) # rubocop: disable Database/MultipleDatabases
 
           Integer(rows.first['count'])
         end
diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb
index 7774dd9fffe..c8a30c68bc6 100644
--- a/lib/gitlab/database/grant.rb
+++ b/lib/gitlab/database/grant.rb
@@ -10,7 +10,7 @@ module Gitlab
         # We _must not_ use quote_table_name as this will produce double
         # quotes on PostgreSQL and for "has_table_privilege" we need single
         # quotes.
-        connection = ActiveRecord::Base.connection
+        connection = ActiveRecord::Base.connection # rubocop: disable Database/MultipleDatabases
         quoted_table = connection.quote(table)
 
         begin
diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb
index 31d41a6d6c0..08f108eb8e4 100644
--- a/lib/gitlab/database/load_balancing.rb
+++ b/lib/gitlab/database/load_balancing.rb
@@ -23,7 +23,7 @@ module Gitlab
 
       # The connection proxy to use for load balancing (if enabled).
       def self.proxy
-        unless @proxy
+        unless load_balancing_proxy = ActiveRecord::Base.load_balancing_proxy
           Gitlab::ErrorTracking.track_exception(
             ProxyNotConfiguredError.new(
               "Attempting to access the database load balancing proxy, but it wasn't configured.\n" \
@@ -31,12 +31,12 @@ module Gitlab
             ))
         end
 
-        @proxy
+        load_balancing_proxy
       end
 
       # Returns a Hash containing the load balancing configuration.
       def self.configuration
-        Gitlab::Database.config[:load_balancing] || {}
+        Gitlab::Database.main.config[:load_balancing] || {}
       end
 
       # Returns the maximum replica lag size in bytes.
@@ -79,7 +79,7 @@ module Gitlab
       end
 
       def self.pool_size
-        Gitlab::Database.config[:pool]
+        Gitlab::Database.main.pool_size
       end
 
       # Returns true if load balancing is to be enabled.
@@ -107,12 +107,12 @@ module Gitlab
 
       # Configures proxying of requests.
       def self.configure_proxy(proxy = ConnectionProxy.new(hosts))
-        @proxy = proxy
+        ActiveRecord::Base.load_balancing_proxy = proxy
 
-        # This hijacks the "connection" method to ensure both
-        # `ActiveRecord::Base.connection` and all models use the same load
-        # balancing proxy.
-        ActiveRecord::Base.singleton_class.prepend(ActiveRecordProxy)
+        # Populate service discovery immediately if it is configured
+        if service_discovery_enabled?
+          ServiceDiscovery.new(service_discovery_configuration).perform_service_discovery
+        end
       end
 
       def self.active_record_models
@@ -132,9 +132,22 @@ module Gitlab
       # recognize the connection, this method returns the primary role
       # directly. In future, we may need to check for other sources.
       def self.db_role_for_connection(connection)
-        return ROLE_PRIMARY if !enable? || @proxy.blank?
+        return ROLE_UNKNOWN unless connection
+
+        # The connection proxy does not have a role assigned
+        # as this is dependent on a execution context
+        return ROLE_UNKNOWN if connection.is_a?(ConnectionProxy)
 
-        proxy.load_balancer.db_role_for_connection(connection)
+        # During application init we might receive `NullPool`
+        return ROLE_UNKNOWN unless connection.respond_to?(:pool) &&
+          connection.pool.respond_to?(:db_config) &&
+          connection.pool.db_config.respond_to?(:name)
+
+        if connection.pool.db_config.name.ends_with?(LoadBalancer::REPLICA_SUFFIX)
+          ROLE_REPLICA
+        else
+          ROLE_PRIMARY
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/load_balancing/active_record_proxy.rb b/lib/gitlab/database/load_balancing/active_record_proxy.rb
index 7763497e770..deaea62d774 100644
--- a/lib/gitlab/database/load_balancing/active_record_proxy.rb
+++ b/lib/gitlab/database/load_balancing/active_record_proxy.rb
@@ -7,7 +7,7 @@ module Gitlab
       # "connection" method.
       module ActiveRecordProxy
         def connection
-          LoadBalancing.proxy
+          ::Gitlab::Database::LoadBalancing.proxy
         end
       end
     end
diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb
index 3a09689a724..938f4951532 100644
--- a/lib/gitlab/database/load_balancing/connection_proxy.rb
+++ b/lib/gitlab/database/load_balancing/connection_proxy.rb
@@ -41,31 +41,31 @@ module Gitlab
         def select_all(arel, name = nil, binds = [], preparable: nil)
           if arel.respond_to?(:locked) && arel.locked
             # SELECT ... FOR UPDATE queries should be sent to the primary.
-            write_using_load_balancer(:select_all, [arel, name, binds],
+            write_using_load_balancer(:select_all, arel, name, binds,
                                       sticky: true)
           else
-            read_using_load_balancer(:select_all, [arel, name, binds])
+            read_using_load_balancer(:select_all, arel, name, binds)
           end
         end
 
         NON_STICKY_READS.each do |name|
-          define_method(name) do |*args, &block|
-            read_using_load_balancer(name, args, &block)
+          define_method(name) do |*args, **kwargs, &block|
+            read_using_load_balancer(name, *args, **kwargs, &block)
           end
         end
 
         STICKY_WRITES.each do |name|
-          define_method(name) do |*args, &block|
-            write_using_load_balancer(name, args, sticky: true, &block)
+          define_method(name) do |*args, **kwargs, &block|
+            write_using_load_balancer(name, *args, sticky: true, **kwargs, &block)
           end
         end
 
-        def transaction(*args, &block)
+        def transaction(*args, **kwargs, &block)
           if current_session.fallback_to_replicas_for_ambiguous_queries?
             track_read_only_transaction!
-            read_using_load_balancer(:transaction, args, &block)
+            read_using_load_balancer(:transaction, *args, **kwargs, &block)
           else
-            write_using_load_balancer(:transaction, args, sticky: true, &block)
+            write_using_load_balancer(:transaction, *args, sticky: true, **kwargs, &block)
           end
 
         ensure
@@ -73,26 +73,26 @@ module Gitlab
         end
 
         # Delegates all unknown messages to a read-write connection.
-        def method_missing(name, *args, &block)
+        def method_missing(...)
           if current_session.fallback_to_replicas_for_ambiguous_queries?
-            read_using_load_balancer(name, args, &block)
+            read_using_load_balancer(...)
           else
-            write_using_load_balancer(name, args, &block)
+            write_using_load_balancer(...)
           end
         end
 
         # Performs a read using the load balancer.
         #
         # name - The name of the method to call on a connection object.
-        def read_using_load_balancer(name, args, &block)
+        def read_using_load_balancer(...)
           if current_session.use_primary? &&
              !current_session.use_replicas_for_read_queries?
             @load_balancer.read_write do |connection|
-              connection.send(name, *args, &block)
+              connection.send(...)
             end
           else
             @load_balancer.read do |connection|
-              connection.send(name, *args, &block)
+              connection.send(...)
             end
           end
         end
@@ -102,7 +102,7 @@ module Gitlab
         # name - The name of the method to call on a connection object.
         # sticky - If set to true the session will stick to the master after
         #          the write.
-        def write_using_load_balancer(name, args, sticky: false, &block)
+        def write_using_load_balancer(name, *args, sticky: false, **kwargs, &block)
           if read_only_transaction?
             raise WriteInsideReadOnlyTransactionError, 'A write query is performed inside a read-only transaction'
           end
@@ -113,7 +113,7 @@ module Gitlab
             # secondary instead of on a primary (when necessary).
             current_session.write! if sticky
 
-            connection.send(name, *args, &block)
+            connection.send(name, *args, **kwargs, &block)
           end
         end
 
diff --git a/lib/gitlab/database/load_balancing/host.rb b/lib/gitlab/database/load_balancing/host.rb
index 3e74b5ea727..4c5357ae8e3 100644
--- a/lib/gitlab/database/load_balancing/host.rb
+++ b/lib/gitlab/database/load_balancing/host.rb
@@ -29,11 +29,11 @@ module Gitlab
           @host = host
           @port = port
           @load_balancer = load_balancer
-          @pool = Database.create_connection_pool(LoadBalancing.pool_size, host, port)
+          @pool = load_balancer.create_replica_connection_pool(::Gitlab::Database::LoadBalancing.pool_size, host, port)
           @online = true
           @last_checked_at = Time.zone.now
 
-          interval = LoadBalancing.replica_check_interval
+          interval = ::Gitlab::Database::LoadBalancing.replica_check_interval
           @intervals = (interval..(interval * 2)).step(0.5).to_a
         end
 
@@ -41,10 +41,10 @@ module Gitlab
         #
         # timeout - The time after which the pool should be forcefully
         #           disconnected.
-        def disconnect!(timeout = 120)
-          start_time = Metrics::System.monotonic_time
+        def disconnect!(timeout: 120)
+          start_time = ::Gitlab::Metrics::System.monotonic_time
 
-          while (Metrics::System.monotonic_time - start_time) <= timeout
+          while (::Gitlab::Metrics::System.monotonic_time - start_time) <= timeout
             break if pool.connections.none?(&:in_use?)
 
             sleep(2)
@@ -54,7 +54,7 @@ module Gitlab
         end
 
         def offline!
-          LoadBalancing::Logger.warn(
+          ::Gitlab::Database::LoadBalancing::Logger.warn(
             event: :host_offline,
             message: 'Marking host as offline',
             db_host: @host,
@@ -72,14 +72,14 @@ module Gitlab
           refresh_status
 
           if @online
-            LoadBalancing::Logger.info(
+            ::Gitlab::Database::LoadBalancing::Logger.info(
               event: :host_online,
               message: 'Host is online after replica status check',
               db_host: @host,
               db_port: @port
             )
           else
-            LoadBalancing::Logger.warn(
+            ::Gitlab::Database::LoadBalancing::Logger.warn(
               event: :host_offline,
               message: 'Host is offline after replica status check',
               db_host: @host,
@@ -108,7 +108,7 @@ module Gitlab
 
         def replication_lag_below_threshold?
           if (lag_time = replication_lag_time)
-            lag_time <= LoadBalancing.max_replication_lag_time
+            lag_time <= ::Gitlab::Database::LoadBalancing.max_replication_lag_time
           else
             false
           end
@@ -125,7 +125,7 @@ module Gitlab
           # only do this if we haven't replicated in a while so we only need
           # to connect to the primary when truly necessary.
           if (lag_size = replication_lag_size)
-            lag_size <= LoadBalancing.max_replication_difference
+            lag_size <= ::Gitlab::Database::LoadBalancing.max_replication_difference
           else
             false
           end
diff --git a/lib/gitlab/database/load_balancing/host_list.rb b/lib/gitlab/database/load_balancing/host_list.rb
index 24800012947..aa731521732 100644
--- a/lib/gitlab/database/load_balancing/host_list.rb
+++ b/lib/gitlab/database/load_balancing/host_list.rb
@@ -8,13 +8,11 @@ module Gitlab
         # hosts - The list of secondary hosts to add.
         def initialize(hosts = [])
           @hosts = hosts.shuffle
-          @pools = Set.new
           @index = 0
           @mutex = Mutex.new
           @hosts_gauge = Gitlab::Metrics.gauge(:db_load_balancing_hosts, 'Current number of load balancing hosts')
 
           set_metrics!
-          update_pools
         end
 
         def hosts
@@ -35,15 +33,16 @@ module Gitlab
           @mutex.synchronize { @hosts.map { |host| [host.host, host.port] } }
         end
 
-        def manage_pool?(pool)
-          @pools.include?(pool)
-        end
-
         def hosts=(hosts)
           @mutex.synchronize do
+            ::Gitlab::Database::LoadBalancing::Logger.info(
+              event: :host_list_update,
+              message: "Updating the host list for service discovery",
+              host_list_length: hosts.length,
+              old_host_list_length: @hosts.length
+            )
             @hosts = hosts
             unsafe_shuffle
-            update_pools
           end
 
           set_metrics!
@@ -89,10 +88,6 @@ module Gitlab
         def set_metrics!
           @hosts_gauge.set({}, @hosts.length)
         end
-
-        def update_pools
-          @pools = Set.new(@hosts.map(&:pool))
-        end
       end
     end
   end
diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb
index a5d67ebc050..e3f5d0ac470 100644
--- a/lib/gitlab/database/load_balancing/load_balancer.rb
+++ b/lib/gitlab/database/load_balancing/load_balancer.rb
@@ -7,20 +7,21 @@ module Gitlab
       #
       # Each host in the load balancer uses the same credentials as the primary
       # database.
-      #
-      # This class *requires* that `ActiveRecord::Base.retrieve_connection`
-      # always returns a connection to the primary.
       class LoadBalancer
         CACHE_KEY = :gitlab_load_balancer_host
-        VALID_HOSTS_CACHE_KEY = :gitlab_load_balancer_valid_hosts
+
+        REPLICA_SUFFIX = '_replica'
 
         attr_reader :host_list
 
         # hosts - The hostnames/addresses of the additional databases.
-        def initialize(hosts = [])
+        def initialize(hosts = [], model = ActiveRecord::Base)
+          @model = model
           @host_list = HostList.new(hosts.map { |addr| Host.new(addr, self) })
-          @connection_db_roles = {}.compare_by_identity
-          @connection_db_roles_count = {}.compare_by_identity
+        end
+
+        def disconnect!(timeout: 120)
+          host_list.hosts.each { |host| host.disconnect!(timeout: timeout) }
         end
 
         # Yields a connection that can be used for reads.
@@ -28,7 +29,6 @@ module Gitlab
         # If no secondaries were available this method will use the primary
         # instead.
         def read(&block)
-          connection = nil
           conflict_retried = 0
 
           while host
@@ -36,12 +36,8 @@ module Gitlab
 
             begin
               connection = host.connection
-              track_connection_role(connection, ROLE_REPLICA)
-
               return yield connection
             rescue StandardError => error
-              untrack_connection_role(connection)
-
               if serialization_failure?(error)
                 # This error can occur when a query conflicts. See
                 # https://www.postgresql.org/docs/current/static/hot-standby.html#HOT-STANDBY-CONFLICT
@@ -84,8 +80,6 @@ module Gitlab
           )
 
           read_write(&block)
-        ensure
-          untrack_connection_role(connection)
         end
 
         # Yields a connection that can be used for both reads and writes.
@@ -95,22 +89,9 @@ module Gitlab
           # Instead of immediately grinding to a halt we'll retry the operation
           # a few times.
           retry_with_backoff do
-            connection = ActiveRecord::Base.retrieve_connection
-            track_connection_role(connection, ROLE_PRIMARY)
-
+            connection = pool.connection
             yield connection
           end
-        ensure
-          untrack_connection_role(connection)
-        end
-
-        # Recognize the role (primary/replica) of the database this connection
-        # is connecting to. If the connection is not issued by this load
-        # balancer, return nil
-        def db_role_for_connection(connection)
-          return @connection_db_roles[connection] if @connection_db_roles[connection]
-          return ROLE_REPLICA if @host_list.manage_pool?(connection.pool)
-          return ROLE_PRIMARY if connection.pool == ActiveRecord::Base.connection_pool
         end
 
         # Returns a host to use for queries.
@@ -118,28 +99,27 @@ module Gitlab
         # Hosts are scoped per thread so that multiple threads don't
         # accidentally re-use the same host + connection.
         def host
-          RequestStore[CACHE_KEY] ||= current_host_list.next
+          request_cache[CACHE_KEY] ||= @host_list.next
         end
 
         # Releases the host and connection for the current thread.
         def release_host
-          if host = RequestStore[CACHE_KEY]
+          if host = request_cache[CACHE_KEY]
             host.disable_query_cache!
             host.release_connection
           end
 
-          RequestStore.delete(CACHE_KEY)
-          RequestStore.delete(VALID_HOSTS_CACHE_KEY)
+          request_cache.delete(CACHE_KEY)
         end
 
         def release_primary_connection
-          ActiveRecord::Base.connection_pool.release_connection
+          pool.release_connection
         end
 
         # Returns the transaction write location of the primary.
         def primary_write_location
           location = read_write do |connection|
-            ::Gitlab::Database.get_write_location(connection)
+            ::Gitlab::Database.main.get_write_location(connection)
           end
 
           return location if location
@@ -148,55 +128,17 @@ module Gitlab
         end
 
         # Returns true if there was at least one host that has caught up with the given transaction.
-        #
-        # In case of a retry, this method also stores the set of hosts that have caught up.
-        #
-        # UPD: `select_caught_up_hosts` seems to have redundant logic managing host list (`:gitlab_load_balancer_valid_hosts`),
-        # while we only need a single host: https://gitlab.com/gitlab-org/gitlab/-/issues/326125#note_615271604
-        # Also, shuffling the list afterwards doesn't seem to be necessary.
-        # This may be improved by merging this method with `select_up_to_date_host`.
-        # Could be removed when `:load_balancing_refine_load_balancer_methods` FF is rolled out
-        def select_caught_up_hosts(location)
-          all_hosts = @host_list.hosts
-          valid_hosts = all_hosts.select { |host| host.caught_up?(location) }
-
-          return false if valid_hosts.empty?
-
-          # Hosts can come online after the time when this scan was done,
-          # so we need to remember the ones that can be used. If the host went
-          # offline, we'll just rely on the retry mechanism to use the primary.
-          set_consistent_hosts_for_request(HostList.new(valid_hosts))
-
-          # Since we will be using a subset from the original list, let's just
-          # pick a random host and mix up the original list to ensure we don't
-          # only end up using one replica.
-          RequestStore[CACHE_KEY] = valid_hosts.sample
-          @host_list.shuffle
-
-          true
-        end
-
-        # Returns true if there was at least one host that has caught up with the given transaction.
-        # Similar to `#select_caught_up_hosts`, picks a random host, to rotate replicas we use.
-        # Unlike `#select_caught_up_hosts`, does not iterate over all hosts if finds any.
-        #
-        # It is going to be merged with `select_caught_up_hosts`, because they intend to do the same.
         def select_up_to_date_host(location)
           all_hosts = @host_list.hosts.shuffle
           host = all_hosts.find { |host| host.caught_up?(location) }
 
           return false unless host
 
-          RequestStore[CACHE_KEY] = host
+          request_cache[CACHE_KEY] = host
 
           true
         end
 
-        # Could be removed when `:load_balancing_refine_load_balancer_methods` FF is rolled out
-        def set_consistent_hosts_for_request(hosts)
-          RequestStore[VALID_HOSTS_CACHE_KEY] = hosts
-        end
-
         # Yields a block, retrying it upon error using an exponential backoff.
         def retry_with_backoff(retries = 3, time = 2)
           retried = 0
@@ -247,30 +189,50 @@ module Gitlab
           end
         end
 
-        private
+        # pool_size - The size of the DB pool.
+        # host - An optional host name to use instead of the default one.
+        # port - An optional port to connect to.
+        def create_replica_connection_pool(pool_size, host = nil, port = nil)
+          db_config = pool.db_config
 
-        def ensure_caching!
-          host.enable_query_cache! unless host.query_cache_enabled
-        end
+          env_config = db_config.configuration_hash.dup
+          env_config[:pool] = pool_size
+          env_config[:host] = host if host
+          env_config[:port] = port if port
+
+          replica_db_config = ActiveRecord::DatabaseConfigurations::HashConfig.new(
+            db_config.env_name,
+            db_config.name + REPLICA_SUFFIX,
+            env_config
+          )
 
-        def track_connection_role(connection, role)
-          @connection_db_roles[connection] = role
-          @connection_db_roles_count[connection] ||= 0
-          @connection_db_roles_count[connection] += 1
+          # We cannot use ActiveRecord::Base.connection_handler.establish_connection
+          # as it will rewrite ActiveRecord::Base.connection
+          ActiveRecord::ConnectionAdapters::ConnectionHandler
+            .new
+            .establish_connection(replica_db_config)
         end
 
-        def untrack_connection_role(connection)
-          return if connection.blank? || @connection_db_roles_count[connection].blank?
+        private
 
-          @connection_db_roles_count[connection] -= 1
-          if @connection_db_roles_count[connection] <= 0
-            @connection_db_roles.delete(connection)
-            @connection_db_roles_count.delete(connection)
-          end
+        # ActiveRecord::ConnectionAdapters::ConnectionHandler handles fetching,
+        # and caching for connections pools for each "connection", so we
+        # leverage that.
+        def pool
+          ActiveRecord::Base.connection_handler.retrieve_connection_pool(
+            @model.connection_specification_name,
+            role: ActiveRecord::Base.writing_role,
+            shard: ActiveRecord::Base.default_shard
+          )
+        end
+
+        def ensure_caching!
+          host.enable_query_cache! unless host.query_cache_enabled
         end
 
-        def current_host_list
-          RequestStore[VALID_HOSTS_CACHE_KEY] || @host_list
+        def request_cache
+          base = RequestStore[:gitlab_load_balancer] ||= {}
+          base[pool] ||= {}
         end
       end
     end
diff --git a/lib/gitlab/database/load_balancing/rack_middleware.rb b/lib/gitlab/database/load_balancing/rack_middleware.rb
index 8e7e6865402..f8a31622b7d 100644
--- a/lib/gitlab/database/load_balancing/rack_middleware.rb
+++ b/lib/gitlab/database/load_balancing/rack_middleware.rb
@@ -18,9 +18,9 @@ module Gitlab
         # namespace - The namespace to use for sticking.
         # id - The identifier to use for sticking.
         def self.stick_or_unstick(env, namespace, id)
-          return unless LoadBalancing.enable?
+          return unless ::Gitlab::Database::LoadBalancing.enable?
 
-          Sticking.unstick_or_continue_sticking(namespace, id)
+          ::Gitlab::Database::LoadBalancing::Sticking.unstick_or_continue_sticking(namespace, id)
 
           env[STICK_OBJECT] ||= Set.new
           env[STICK_OBJECT] << [namespace, id]
@@ -56,7 +56,7 @@ module Gitlab
           namespaces_and_ids = sticking_namespaces_and_ids(env)
 
           namespaces_and_ids.each do |namespace, id|
-            Sticking.unstick_or_continue_sticking(namespace, id)
+            ::Gitlab::Database::LoadBalancing::Sticking.unstick_or_continue_sticking(namespace, id)
           end
         end
 
@@ -65,17 +65,17 @@ module Gitlab
           namespaces_and_ids = sticking_namespaces_and_ids(env)
 
           namespaces_and_ids.each do |namespace, id|
-            Sticking.stick_if_necessary(namespace, id)
+            ::Gitlab::Database::LoadBalancing::Sticking.stick_if_necessary(namespace, id)
           end
         end
 
         def clear
           load_balancer.release_host
-          Session.clear_session
+          ::Gitlab::Database::LoadBalancing::Session.clear_session
         end
 
         def load_balancer
-          LoadBalancing.proxy.load_balancer
+          ::Gitlab::Database::LoadBalancing.proxy.load_balancer
         end
 
         # Determines the sticking namespace and identifier based on the Rack
diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb
index 9b42b25be1c..251961c8246 100644
--- a/lib/gitlab/database/load_balancing/service_discovery.rb
+++ b/lib/gitlab/database/load_balancing/service_discovery.rb
@@ -13,7 +13,8 @@ module Gitlab
       # balancer with said hosts. Requests may continue to use the old hosts
       # until they complete.
       class ServiceDiscovery
-        attr_reader :interval, :record, :record_type, :disconnect_timeout
+        attr_reader :interval, :record, :record_type, :disconnect_timeout,
+                    :load_balancer
 
         MAX_SLEEP_ADJUSTMENT = 10
 
@@ -40,7 +41,17 @@ module Gitlab
         # disconnect_timeout - The time after which an old host should be
         #                      forcefully disconnected.
         # use_tcp - Use TCP instaed of UDP to look up resources
-        def initialize(nameserver:, port:, record:, record_type: 'A', interval: 60, disconnect_timeout: 120, use_tcp: false)
+        # load_balancer - The load balancer instance to use
+        def initialize(
+          nameserver:,
+          port:,
+          record:,
+          record_type: 'A',
+          interval: 60,
+          disconnect_timeout: 120,
+          use_tcp: false,
+          load_balancer: LoadBalancing.proxy.load_balancer
+        )
           @nameserver = nameserver
           @port = port
           @record = record
@@ -48,34 +59,36 @@ module Gitlab
           @interval = interval
           @disconnect_timeout = disconnect_timeout
           @use_tcp = use_tcp
+          @load_balancer = load_balancer
         end
 
         def start
           Thread.new do
             loop do
-              interval =
-                begin
-                  refresh_if_necessary
-                rescue StandardError => error
-                  # Any exceptions that might occur should be reported to
-                  # Sentry, instead of silently terminating this thread.
-                  Gitlab::ErrorTracking.track_exception(error)
-
-                  Gitlab::AppLogger.error(
-                    "Service discovery encountered an error: #{error.message}"
-                  )
-
-                  self.interval
-                end
+              next_sleep_duration = perform_service_discovery
 
               # We slightly randomize the sleep() interval. This should reduce
               # the likelihood of _all_ processes refreshing at the same time,
               # possibly putting unnecessary pressure on the DNS server.
-              sleep(interval + rand(MAX_SLEEP_ADJUSTMENT))
+              sleep(next_sleep_duration + rand(MAX_SLEEP_ADJUSTMENT))
             end
           end
         end
 
+        def perform_service_discovery
+          refresh_if_necessary
+        rescue StandardError => error
+          # Any exceptions that might occur should be reported to
+          # Sentry, instead of silently terminating this thread.
+          Gitlab::ErrorTracking.track_exception(error)
+
+          Gitlab::AppLogger.error(
+            "Service discovery encountered an error: #{error.message}"
+          )
+
+          interval
+        end
+
         # Refreshes the hosts, but only if the DNS record returned a new list of
         # addresses.
         #
@@ -108,7 +121,7 @@ module Gitlab
           # host/connection. While this connection will be checked in and out,
           # it won't be explicitly disconnected.
           old_hosts.each do |host|
-            host.disconnect!(disconnect_timeout)
+            host.disconnect!(timeout: disconnect_timeout)
           end
         end
 
@@ -147,10 +160,6 @@ module Gitlab
           end.sort
         end
 
-        def load_balancer
-          LoadBalancing.proxy.load_balancer
-        end
-
         def resolver
           @resolver ||= Net::DNS::Resolver.new(
             nameservers: Resolver.new(@nameserver).resolve,
diff --git a/lib/gitlab/database/load_balancing/sticking.rb b/lib/gitlab/database/load_balancing/sticking.rb
index 8e1aa079216..20d42b9a694 100644
--- a/lib/gitlab/database/load_balancing/sticking.rb
+++ b/lib/gitlab/database/load_balancing/sticking.rb
@@ -53,14 +53,8 @@ module Gitlab
           # write location. If no such location exists, err on the side of caution.
           return false unless location
 
-          if ::Feature.enabled?(:load_balancing_refine_load_balancer_methods)
-            load_balancer.select_up_to_date_host(location).tap do |selected|
-              unstick(namespace, id) if selected
-            end
-          else
-            load_balancer.select_caught_up_hosts(location).tap do |selected|
-              unstick(namespace, id) if selected
-            end
+          load_balancer.select_up_to_date_host(location).tap do |selected|
+            unstick(namespace, id) if selected
           end
         end
 
@@ -109,7 +103,7 @@ module Gitlab
             if LoadBalancing.enable?
               load_balancer.primary_write_location
             else
-              Gitlab::Database.get_write_location(ActiveRecord::Base.connection)
+              Gitlab::Database.main.get_write_location(ActiveRecord::Base.connection)
             end
 
           return if location.blank?
diff --git a/lib/gitlab/database/metrics.rb b/lib/gitlab/database/metrics.rb
new file mode 100644
index 00000000000..5dabbc81b9c
--- /dev/null
+++ b/lib/gitlab/database/metrics.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    class Metrics
+      extend ::Gitlab::Utils::StrongMemoize
+
+      class << self
+        def subtransactions_increment(model_name)
+          subtransactions_counter.increment(model: model_name)
+        end
+
+        private
+
+        def subtransactions_counter
+          strong_memoize(:subtransactions_counter) do
+            name = :gitlab_active_record_subtransactions_total
+            comment = 'Total amount of subtransactions created by ActiveRecord'
+
+            ::Gitlab::Metrics.counter(name, comment)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 842ab4f7b80..23d9b16dc09 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -6,6 +6,7 @@ module Gitlab
       include Migrations::BackgroundMigrationHelpers
       include DynamicModelHelpers
       include RenameTableHelpers
+      include AsyncIndexes::MigrationHelpers
 
       # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
       MAX_IDENTIFIER_NAME_LENGTH = 63
@@ -152,6 +153,9 @@ module Gitlab
         disable_statement_timeout do
           add_index(table_name, column_name, **options)
         end
+
+        # We created this index. Now let's remove the queuing entry for async creation in case it's still there.
+        unprepare_async_index(table_name, column_name, **options)
       end
 
       # Removes an existed index, concurrently
@@ -178,6 +182,9 @@ module Gitlab
         disable_statement_timeout do
           remove_index(table_name, **options.merge({ column: column_name }))
         end
+
+        # We removed this index. Now let's make sure it's not queued for async creation.
+        unprepare_async_index(table_name, column_name, **options)
       end
 
       # Removes an existing index, concurrently
@@ -208,6 +215,9 @@ module Gitlab
         disable_statement_timeout do
           remove_index(table_name, **options.merge({ name: index_name }))
         end
+
+        # We removed this index. Now let's make sure it's not queued for async creation.
+        unprepare_async_index_by_name(table_name, index_name, **options)
       end
 
       # Adds a foreign key with only minimal locking on the tables involved.
@@ -221,8 +231,13 @@ module Gitlab
       # on_delete - The action to perform when associated data is removed,
       #             defaults to "CASCADE".
       # name - The name of the foreign key.
+      # validate - Flag that controls whether the new foreign key will be validated after creation.
+      #            If the flag is not set, the constraint will only be enforced for new data.
+      # reverse_lock_order - Flag that controls whether we should attempt to acquire locks in the reverse
+      #                      order of the ALTER TABLE. This can be useful in situations where the foreign
+      #                      key creation could deadlock with another process.
       #
-      def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, target_column: :id, name: nil, validate: true)
+      def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, target_column: :id, name: nil, validate: true, reverse_lock_order: false)
         # Transactions would result in ALTER TABLE locks being held for the
         # duration of the transaction, defeating the purpose of this method.
         if transaction_open?
@@ -250,6 +265,8 @@ module Gitlab
           # data.
 
           with_lock_retries do
+            execute("LOCK TABLE #{target}, #{source} IN SHARE ROW EXCLUSIVE MODE") if reverse_lock_order
+
             execute <<-EOF.strip_heredoc
             ALTER TABLE #{source}
             ADD CONSTRAINT #{options[:name]}
@@ -324,9 +341,9 @@ module Gitlab
       # - Per connection (requires a cleanup after the execution)
       #
       # When using a per connection disable statement, code must be inside
-      # a block so we can automatically execute `RESET ALL` after block finishes
+      # a block so we can automatically execute `RESET statement_timeout` after block finishes
       # otherwise the statement will still be disabled until connection is dropped
-      # or `RESET ALL` is executed
+      # or `RESET statement_timeout` is executed
       def disable_statement_timeout
         if block_given?
           if statement_timeout_disabled?
@@ -340,7 +357,7 @@ module Gitlab
 
               yield
             ensure
-              execute('RESET ALL')
+              execute('RESET statement_timeout')
             end
           end
         else
@@ -1248,8 +1265,8 @@ module Gitlab
 
       def check_trigger_permissions!(table)
         unless Grant.create_and_execute_trigger?(table)
-          dbname = Database.database_name
-          user = Database.username
+          dbname = Database.main.database_name
+          user = Database.main.username
 
           raise <<-EOF
 Your database user is not allowed to create, drop, or execute triggers on the
@@ -1569,8 +1586,8 @@ into similar problems in the future (e.g. when new tables are created).
       def create_extension(extension)
         execute('CREATE EXTENSION IF NOT EXISTS %s' % extension)
       rescue ActiveRecord::StatementInvalid => e
-        dbname = Database.database_name
-        user = Database.username
+        dbname = Database.main.database_name
+        user = Database.main.username
 
         warn(<<~MSG) if e.to_s =~ /permission denied/
           GitLab requires the PostgreSQL extension '#{extension}' installed in database '#{dbname}', but
@@ -1597,8 +1614,8 @@ into similar problems in the future (e.g. when new tables are created).
       def drop_extension(extension)
         execute('DROP EXTENSION IF EXISTS %s' % extension)
       rescue ActiveRecord::StatementInvalid => e
-        dbname = Database.database_name
-        user = Database.username
+        dbname = Database.main.database_name
+        user = Database.main.username
 
         warn(<<~MSG) if e.to_s =~ /permission denied/
           This migration attempts to drop the PostgreSQL extension '#{extension}'
diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb
index 28491a934a0..19d80ba1d64 100644
--- a/lib/gitlab/database/migrations/background_migration_helpers.rb
+++ b/lib/gitlab/database/migrations/background_migration_helpers.rb
@@ -264,6 +264,34 @@ module Gitlab
           migration
         end
 
+        # Force a background migration to complete.
+        #
+        # WARNING: This method will block the caller and move the background migration from an
+        # asynchronous migration to a synchronous migration.
+        #
+        # 1. Steal work from sidekiq and perform immediately (avoid duplicates generated by step 2).
+        # 2. Process any pending tracked jobs.
+        # 3. Steal work from sidekiq and perform immediately (clear anything left from step 2).
+        # 4. Optionally remove job tracking information.
+        #
+        # This method does not garauntee that all jobs completed successfully.
+        def finalize_background_migration(class_name, delete_tracking_jobs: ['succeeded'])
+          # Empty the sidekiq queue.
+          Gitlab::BackgroundMigration.steal(class_name)
+
+          # Process pending tracked jobs.
+          jobs = Gitlab::Database::BackgroundMigrationJob.pending.for_migration_class(class_name)
+          jobs.find_each do |job|
+            BackgroundMigrationWorker.new.perform(job.class_name, job.arguments)
+          end
+
+          # Empty the sidekiq queue.
+          Gitlab::BackgroundMigration.steal(class_name)
+
+          # Delete job tracking rows.
+          delete_job_tracking(class_name, status: delete_tracking_jobs) if delete_tracking_jobs
+        end
+
         def perform_background_migration_inline?
           Rails.env.test? || Rails.env.development?
         end
@@ -304,6 +332,12 @@ module Gitlab
           end
         end
 
+        def delete_job_tracking(class_name, status: 'succeeded')
+          status = Array(status).map { |s| Gitlab::Database::BackgroundMigrationJob.statuses[s] }
+          jobs = Gitlab::Database::BackgroundMigrationJob.where(status: status).for_migration_class(class_name)
+          jobs.each_batch { |batch| batch.delete_all }
+        end
+
         private
 
         def track_in_database(class_name, arguments)
diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb
index e9ef80d5198..d1e55eb825c 100644
--- a/lib/gitlab/database/migrations/instrumentation.rb
+++ b/lib/gitlab/database/migrations/instrumentation.rb
@@ -9,18 +9,20 @@ module Gitlab
 
         attr_reader :observations
 
-        def initialize(observers = ::Gitlab::Database::Migrations::Observers.all_observers)
-          @observers = observers
+        def initialize(observer_classes = ::Gitlab::Database::Migrations::Observers.all_observers)
+          @observer_classes = observer_classes
           @observations = []
         end
 
-        def observe(migration, &block)
-          observation = Observation.new(migration)
+        def observe(version:, name:, &block)
+          observation = Observation.new(version, name)
           observation.success = true
 
+          observers = observer_classes.map { |c| c.new(observation) }
+
           exception = nil
 
-          on_each_observer { |observer| observer.before }
+          on_each_observer(observers) { |observer| observer.before }
 
           observation.walltime = Benchmark.realtime do
             yield
@@ -29,8 +31,8 @@ module Gitlab
             observation.success = false
           end
 
-          on_each_observer { |observer| observer.after }
-          on_each_observer { |observer| observer.record(observation) }
+          on_each_observer(observers) { |observer| observer.after }
+          on_each_observer(observers) { |observer| observer.record }
 
           record_observation(observation)
 
@@ -41,13 +43,13 @@ module Gitlab
 
         private
 
-        attr_reader :observers
+        attr_reader :observer_classes
 
         def record_observation(observation)
           @observations << observation
         end
 
-        def on_each_observer(&block)
+        def on_each_observer(observers, &block)
           observers.each do |observer|
             yield observer
           rescue StandardError => e
diff --git a/lib/gitlab/database/migrations/observation.rb b/lib/gitlab/database/migrations/observation.rb
index 046843824a4..54eedec3c7b 100644
--- a/lib/gitlab/database/migrations/observation.rb
+++ b/lib/gitlab/database/migrations/observation.rb
@@ -4,7 +4,8 @@ module Gitlab
   module Database
     module Migrations
       Observation = Struct.new(
-        :migration,
+        :version,
+        :name,
         :walltime,
         :success,
         :total_database_size_change,
diff --git a/lib/gitlab/database/migrations/observers.rb b/lib/gitlab/database/migrations/observers.rb
index 979a098d699..140b3feed64 100644
--- a/lib/gitlab/database/migrations/observers.rb
+++ b/lib/gitlab/database/migrations/observers.rb
@@ -6,10 +6,10 @@ module Gitlab
       module Observers
         def self.all_observers
           [
-            TotalDatabaseSizeChange.new,
-            QueryStatistics.new,
-            QueryLog.new,
-            QueryDetails.new
+            TotalDatabaseSizeChange,
+            QueryStatistics,
+            QueryLog,
+            QueryDetails
           ]
         end
       end
diff --git a/lib/gitlab/database/migrations/observers/migration_observer.rb b/lib/gitlab/database/migrations/observers/migration_observer.rb
index 9bfbf35887d..85d18abb9ef 100644
--- a/lib/gitlab/database/migrations/observers/migration_observer.rb
+++ b/lib/gitlab/database/migrations/observers/migration_observer.rb
@@ -5,10 +5,11 @@ module Gitlab
     module Migrations
       module Observers
         class MigrationObserver
-          attr_reader :connection
+          attr_reader :connection, :observation
 
-          def initialize
+          def initialize(observation)
             @connection = ActiveRecord::Base.connection
+            @observation = observation
           end
 
           def before
@@ -19,7 +20,7 @@ module Gitlab
             # implement in subclass
           end
 
-          def record(observation)
+          def record
             raise NotImplementedError, 'implement in subclass'
           end
         end
diff --git a/lib/gitlab/database/migrations/observers/query_details.rb b/lib/gitlab/database/migrations/observers/query_details.rb
index 52b6464d449..dadacd2d2fc 100644
--- a/lib/gitlab/database/migrations/observers/query_details.rb
+++ b/lib/gitlab/database/migrations/observers/query_details.rb
@@ -6,8 +6,8 @@ module Gitlab
       module Observers
         class QueryDetails < MigrationObserver
           def before
-            @file_path = File.join(Instrumentation::RESULT_DIR, 'current-details.json')
-            @file = File.open(@file_path, 'wb')
+            file_path = File.join(Instrumentation::RESULT_DIR, "#{observation.version}_#{observation.name}-query-details.json")
+            @file = File.open(file_path, 'wb')
             @writer = Oj::StreamWriter.new(@file, {})
             @writer.push_array
             @subscriber = ActiveSupport::Notifications.subscribe('sql.active_record') do |*args|
@@ -22,8 +22,8 @@ module Gitlab
             @file.close
           end
 
-          def record(observation)
-            File.rename(@file_path, File.join(Instrumentation::RESULT_DIR, "#{observation.migration}-query-details.json"))
+          def record
+            # no-op
           end
 
           def record_sql_event(_name, started, finished, _unique_id, payload)
diff --git a/lib/gitlab/database/migrations/observers/query_log.rb b/lib/gitlab/database/migrations/observers/query_log.rb
index 45df07fe391..e15d733d2a2 100644
--- a/lib/gitlab/database/migrations/observers/query_log.rb
+++ b/lib/gitlab/database/migrations/observers/query_log.rb
@@ -7,8 +7,8 @@ module Gitlab
         class QueryLog < MigrationObserver
           def before
             @logger_was = ActiveRecord::Base.logger
-            @log_file_path = File.join(Instrumentation::RESULT_DIR, 'current.log')
-            @logger = Logger.new(@log_file_path)
+            file_path = File.join(Instrumentation::RESULT_DIR, "#{observation.version}_#{observation.name}.log")
+            @logger = Logger.new(file_path)
             ActiveRecord::Base.logger = @logger
           end
 
@@ -17,8 +17,8 @@ module Gitlab
             @logger.close
           end
 
-          def record(observation)
-            File.rename(@log_file_path, File.join(Instrumentation::RESULT_DIR, "#{observation.migration}.log"))
+          def record
+            # no-op
           end
         end
       end
diff --git a/lib/gitlab/database/migrations/observers/query_statistics.rb b/lib/gitlab/database/migrations/observers/query_statistics.rb
index 466f4724256..54504646a79 100644
--- a/lib/gitlab/database/migrations/observers/query_statistics.rb
+++ b/lib/gitlab/database/migrations/observers/query_statistics.rb
@@ -16,7 +16,7 @@ module Gitlab
             connection.execute('select pg_stat_statements_reset()')
           end
 
-          def record(observation)
+          def record
             return unless enabled?
 
             observation.query_statistics = connection.execute(<<~SQL)
diff --git a/lib/gitlab/database/migrations/observers/total_database_size_change.rb b/lib/gitlab/database/migrations/observers/total_database_size_change.rb
index 0b76b0bef5e..2e89498b79f 100644
--- a/lib/gitlab/database/migrations/observers/total_database_size_change.rb
+++ b/lib/gitlab/database/migrations/observers/total_database_size_change.rb
@@ -13,7 +13,7 @@ module Gitlab
             @size_after = get_total_database_size
           end
 
-          def record(observation)
+          def record
             return unless @size_after && @size_before
 
             observation.total_database_size_change = @size_after - @size_before
diff --git a/lib/gitlab/database/multi_threaded_migration.rb b/lib/gitlab/database/multi_threaded_migration.rb
deleted file mode 100644
index 65a6cb8e369..00000000000
--- a/lib/gitlab/database/multi_threaded_migration.rb
+++ /dev/null
@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Database
-    module MultiThreadedMigration
-      MULTI_THREAD_AR_CONNECTION = :thread_local_ar_connection
-
-      # This overwrites the default connection method so that every thread can
-      # use a thread-local connection, while still supporting all of Rails'
-      # migration methods.
-      def connection
-        Thread.current[MULTI_THREAD_AR_CONNECTION] ||
-          ActiveRecord::Base.connection
-      end
-
-      # Starts a thread-pool for N threads, along with N threads each using a
-      # single connection. The provided block is yielded from inside each
-      # thread.
-      #
-      # Example:
-      #
-      #     with_multiple_threads(4) do
-      #       execute('SELECT ...')
-      #     end
-      #
-      # thread_count - The number of threads to start.
-      #
-      # join - When set to true this method will join the threads, blocking the
-      #        caller until all threads have finished running.
-      #
-      # Returns an Array containing the started threads.
-      def with_multiple_threads(thread_count, join: true)
-        pool = Gitlab::Database.create_connection_pool(thread_count)
-
-        threads = Array.new(thread_count) do
-          Thread.new do
-            pool.with_connection do |connection|
-              Thread.current[MULTI_THREAD_AR_CONNECTION] = connection
-              yield
-            ensure
-              Thread.current[MULTI_THREAD_AR_CONNECTION] = nil
-            end
-          end
-        end
-
-        threads.each(&:join) if join
-
-        threads
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/database/partitioning/detached_partition_dropper.rb b/lib/gitlab/database/partitioning/detached_partition_dropper.rb
new file mode 100644
index 00000000000..dc63d93fd07
--- /dev/null
+++ b/lib/gitlab/database/partitioning/detached_partition_dropper.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+module Gitlab
+  module Database
+    module Partitioning
+      class DetachedPartitionDropper
+        def perform
+          return unless Feature.enabled?(:drop_detached_partitions, default_enabled: :yaml)
+
+          Gitlab::AppLogger.info(message: "Checking for previously detached partitions to drop")
+          Postgresql::DetachedPartition.ready_to_drop.find_each do |detached_partition|
+            conn.transaction do
+              # Another process may have already dropped the table and deleted this entry
+              next unless (detached_partition = Postgresql::DetachedPartition.lock.find_by(id: detached_partition.id))
+
+              unless check_partition_detached?(detached_partition)
+                Gitlab::AppLogger.error(message: "Attempt to drop attached database partition", partition_name: detached_partition.table_name)
+                detached_partition.destroy!
+                next
+              end
+
+              drop_one(detached_partition)
+            end
+          rescue StandardError => e
+            Gitlab::AppLogger.error(message: "Failed to drop previously detached partition",
+                                    partition_name: detached_partition.table_name,
+                                    exception_class: e.class,
+                                    exception_message: e.message)
+          end
+        end
+
+        private
+
+        def drop_one(detached_partition)
+          conn.transaction do
+            conn.execute(<<~SQL)
+              DROP TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{conn.quote_table_name(detached_partition.table_name)}
+            SQL
+
+            detached_partition.destroy!
+          end
+          Gitlab::AppLogger.info(message: "Dropped previously detached partition", partition_name: detached_partition.table_name)
+        end
+
+        def check_partition_detached?(detached_partition)
+          # PostgresPartition checks the pg_inherits view, so our partition will only show here if it's still attached
+          # and thus should not be dropped
+          !PostgresPartition.for_identifier("#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{detached_partition.table_name}").exists?
+        end
+
+        def conn
+          @conn ||= ApplicationRecord.connection
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/partitioning/monthly_strategy.rb b/lib/gitlab/database/partitioning/monthly_strategy.rb
index 4c68399cb68..7992c2fdaa7 100644
--- a/lib/gitlab/database/partitioning/monthly_strategy.rb
+++ b/lib/gitlab/database/partitioning/monthly_strategy.rb
@@ -86,7 +86,7 @@ module Gitlab
         end
 
         def pruning_old_partitions?
-          Feature.enabled?(:partition_pruning_dry_run) && retain_for.present?
+          retain_for.present?
         end
 
         def oldest_active_date
diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb
index c2a9422a42a..7e433ecdd39 100644
--- a/lib/gitlab/database/partitioning/partition_manager.rb
+++ b/lib/gitlab/database/partitioning/partition_manager.rb
@@ -4,6 +4,8 @@ module Gitlab
   module Database
     module Partitioning
       class PartitionManager
+        UnsafeToDetachPartitionError = Class.new(StandardError)
+
         def self.register(model)
           raise ArgumentError, "Only models with a #partitioning_strategy can be registered." unless model.respond_to?(:partitioning_strategy)
 
@@ -16,6 +18,7 @@ module Gitlab
 
         LEASE_TIMEOUT = 1.minute
         MANAGEMENT_LEASE_KEY = 'database_partition_management_%s'
+        RETAIN_DETACHED_PARTITIONS_FOR = 1.week
 
         attr_reader :models
 
@@ -35,13 +38,16 @@ module Gitlab
               partitions_to_create = missing_partitions(model)
               create(partitions_to_create) unless partitions_to_create.empty?
 
-              if Feature.enabled?(:partition_pruning_dry_run)
+              if Feature.enabled?(:partition_pruning, default_enabled: :yaml)
                 partitions_to_detach = extra_partitions(model)
                 detach(partitions_to_detach) unless partitions_to_detach.empty?
               end
             end
           rescue StandardError => e
-            Gitlab::AppLogger.error("Failed to create / detach partition(s) for #{model.table_name}: #{e.class}: #{e.message}")
+            Gitlab::AppLogger.error(message: "Failed to create / detach partition(s)",
+                                    table_name: model.table_name,
+                                    exception_class: e.class,
+                                    exception_message: e.message)
           end
         end
 
@@ -54,7 +60,6 @@ module Gitlab
         end
 
         def extra_partitions(model)
-          return [] unless Feature.enabled?(:partition_pruning_dry_run)
           return [] unless connection.table_exists?(model.table_name)
 
           model.partitioning_strategy.extra_partitions
@@ -74,7 +79,9 @@ module Gitlab
               partitions.each do |partition|
                 connection.execute partition.to_sql
 
-                Gitlab::AppLogger.info("Created partition #{partition.partition_name} for table #{partition.table}")
+                Gitlab::AppLogger.info(message: "Created partition",
+                                       partition_name: partition.partition_name,
+                                       table_name: partition.table)
               end
             end
           end
@@ -89,7 +96,24 @@ module Gitlab
         end
 
         def detach_one_partition(partition)
-          Gitlab::AppLogger.info("Planning to detach #{partition.partition_name} for table #{partition.table}")
+          assert_partition_detachable!(partition)
+
+          connection.execute partition.to_detach_sql
+
+          Postgresql::DetachedPartition.create!(table_name: partition.partition_name,
+                                                drop_after: RETAIN_DETACHED_PARTITIONS_FOR.from_now)
+
+          Gitlab::AppLogger.info(message: "Detached Partition",
+                                 partition_name: partition.partition_name,
+                                 table_name: partition.table)
+        end
+
+        def assert_partition_detachable!(partition)
+          parent_table_identifier = "#{connection.current_schema}.#{partition.table}"
+
+          if (example_fk = PostgresForeignKey.by_referenced_table_identifier(parent_table_identifier).first)
+            raise UnsafeToDetachPartitionError, "Cannot detach #{partition.partition_name}, it would block while checking foreign key #{example_fk.name} on #{example_fk.constrained_table_identifier}"
+          end
         end
 
         def with_lock_retries(&block)
diff --git a/lib/gitlab/database/partitioning/partition_monitoring.rb b/lib/gitlab/database/partitioning/partition_monitoring.rb
index ad122fd47fe..6963ecd2cc1 100644
--- a/lib/gitlab/database/partitioning/partition_monitoring.rb
+++ b/lib/gitlab/database/partitioning/partition_monitoring.rb
@@ -16,6 +16,7 @@ module Gitlab
 
             gauge_present.set({ table: model.table_name }, strategy.current_partitions.size)
             gauge_missing.set({ table: model.table_name }, strategy.missing_partitions.size)
+            gauge_extra.set({ table: model.table_name }, strategy.extra_partitions.size)
           end
         end
 
@@ -28,6 +29,10 @@ module Gitlab
         def gauge_missing
           @gauge_missing ||= Gitlab::Metrics.gauge(:db_partitions_missing, 'Number of database partitions currently expected, but not present')
         end
+
+        def gauge_extra
+          @gauge_extra ||= Gitlab::Metrics.gauge(:db_partitions_extra, 'Number of database partitions currently attached to tables, but outside of their retention window and scheduled to be dropped')
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/partitioning/time_partition.rb b/lib/gitlab/database/partitioning/time_partition.rb
index 7dca60c0854..1221f042530 100644
--- a/lib/gitlab/database/partitioning/time_partition.rb
+++ b/lib/gitlab/database/partitioning/time_partition.rb
@@ -47,6 +47,13 @@ module Gitlab
           SQL
         end
 
+        def to_detach_sql
+          <<~SQL
+            ALTER TABLE #{conn.quote_table_name(table)}
+            DETACH PARTITION #{fully_qualified_partition}
+          SQL
+        end
+
         def ==(other)
           table == other.table && partition_name == other.partition_name && from == other.from && to == other.to
         end
diff --git a/lib/gitlab/database/postgres_foreign_key.rb b/lib/gitlab/database/postgres_foreign_key.rb
new file mode 100644
index 00000000000..94f74724295
--- /dev/null
+++ b/lib/gitlab/database/postgres_foreign_key.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    class PostgresForeignKey < ApplicationRecord
+      self.primary_key = :oid
+
+      scope :by_referenced_table_identifier, ->(identifier) do
+        raise ArgumentError, "Referenced table name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/
+
+        where(referenced_table_identifier: identifier)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb
index 580cab5622d..2e3f674cf82 100644
--- a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb
+++ b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb
@@ -57,7 +57,7 @@ module Gitlab
         # @param finish final pkey range
         # @return [Gitlab::Database::PostgresHll::Buckets] HyperLogLog data structure instance that can estimate number of unique elements
         def execute(batch_size: nil, start: nil, finish: nil)
-          raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open?
+          raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open? # rubocop: disable Database/MultipleDatabases
 
           batch_size ||= DEFAULT_BATCH_SIZE
           start = actual_start(start)
diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb
index 58e4e7e7924..1079bfdeda3 100644
--- a/lib/gitlab/database/postgres_index.rb
+++ b/lib/gitlab/database/postgres_index.rb
@@ -19,7 +19,12 @@ module Gitlab
       end
 
       # Indexes with reindexing support
-      scope :reindexing_support, -> { where(partitioned: false, exclusion: false, expression: false, type: Gitlab::Database::Reindexing::SUPPORTED_TYPES) }
+      scope :reindexing_support, -> do
+        where(partitioned: false, exclusion: false, expression: false, type: Gitlab::Database::Reindexing::SUPPORTED_TYPES)
+          .not_match("#{Gitlab::Database::Reindexing::ReindexConcurrently::TEMPORARY_INDEX_PATTERN}$")
+      end
+
+      scope :reindexing_leftovers, -> { match("#{Gitlab::Database::Reindexing::ReindexConcurrently::TEMPORARY_INDEX_PATTERN}$") }
 
       scope :not_match, ->(regex) { where("name !~ ?", regex) }
 
diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb
index 0986372586b..7da60d8375d 100644
--- a/lib/gitlab/database/postgres_partition.rb
+++ b/lib/gitlab/database/postgres_partition.rb
@@ -7,10 +7,14 @@ module Gitlab
 
       belongs_to :postgres_partitioned_table, foreign_key: 'parent_identifier', primary_key: 'identifier'
 
-      scope :by_identifier, ->(identifier) do
+      scope :for_identifier, ->(identifier) do
         raise ArgumentError, "Partition name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/
 
-        find(identifier)
+        where(primary_key => identifier)
+      end
+
+      scope :by_identifier, ->(identifier) do
+        for_identifier(identifier).first!
       end
 
       scope :for_parent_table, ->(name) { where("parent_identifier = concat(current_schema(), '.', ?)", name).order(:name) }
diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb
index 841e04ccbd1..04b409a9306 100644
--- a/lib/gitlab/database/reindexing.rb
+++ b/lib/gitlab/database/reindexing.rb
@@ -8,6 +8,13 @@ module Gitlab
 
       SUPPORTED_TYPES = %w(btree gist).freeze
 
+      # When dropping an index, we acquire a SHARE UPDATE EXCLUSIVE lock,
+      # which only conflicts with DDL and vacuum. We therefore execute this with a rather
+      # high lock timeout and a long pause in between retries. This is an alternative to
+      # setting a high statement timeout, which would lead to a long running query with effects
+      # on e.g. vacuum.
+      REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30
+
       # candidate_indexes: Array of Gitlab::Database::PostgresIndex
       def self.perform(candidate_indexes, how_many: DEFAULT_INDEXES_PER_INVOCATION)
         IndexSelection.new(candidate_indexes).take(how_many).each do |index|
@@ -15,10 +22,22 @@ module Gitlab
         end
       end
 
-      def self.candidate_indexes
-        Gitlab::Database::PostgresIndex
-          .not_match("#{ReindexConcurrently::TEMPORARY_INDEX_PATTERN}$")
-          .reindexing_support
+      def self.cleanup_leftovers!
+        PostgresIndex.reindexing_leftovers.each do |index|
+          Gitlab::AppLogger.info("Removing index #{index.identifier} which is a leftover, temporary index from previous reindexing activity")
+
+          retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new(
+            timing_configuration: REMOVE_INDEX_RETRY_CONFIG,
+            klass: self.class,
+            logger: Gitlab::AppLogger
+          )
+
+          retries.run(raise_on_exhaustion: false) do
+            ApplicationRecord.connection.tap do |conn|
+              conn.execute("DROP INDEX CONCURRENTLY IF EXISTS #{conn.quote_table_name(index.schema)}.#{conn.quote_table_name(index.name)}")
+            end
+          end
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/reindexing/reindex_concurrently.rb b/lib/gitlab/database/reindexing/reindex_concurrently.rb
index 8d9f9f5abdd..7a720f7c539 100644
--- a/lib/gitlab/database/reindexing/reindex_concurrently.rb
+++ b/lib/gitlab/database/reindexing/reindex_concurrently.rb
@@ -11,13 +11,6 @@ module Gitlab
         STATEMENT_TIMEOUT = 9.hours
         PG_MAX_INDEX_NAME_LENGTH = 63
 
-        # When dropping an index, we acquire a SHARE UPDATE EXCLUSIVE lock,
-        # which only conflicts with DDL and vacuum. We therefore execute this with a rather
-        # high lock timeout and a long pause in between retries. This is an alternative to
-        # setting a high statement timeout, which would lead to a long running query with effects
-        # on e.g. vacuum.
-        REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30
-
         attr_reader :index, :logger
 
         def initialize(index, logger: Gitlab::AppLogger)
diff --git a/lib/gitlab/database/schema_migrations/context.rb b/lib/gitlab/database/schema_migrations/context.rb
index bd8b9bed2c1..35105121bbd 100644
--- a/lib/gitlab/database/schema_migrations/context.rb
+++ b/lib/gitlab/database/schema_migrations/context.rb
@@ -6,17 +6,14 @@ module Gitlab
       class Context
         attr_reader :connection
 
+        DEFAULT_SCHEMA_MIGRATIONS_PATH = "db/schema_migrations"
+
         def initialize(connection)
           @connection = connection
         end
 
         def schema_directory
-          @schema_directory ||=
-            if ActiveRecord::Base.configurations.primary?(database_name)
-              File.join(db_dir, 'schema_migrations')
-            else
-              File.join(db_dir, "#{database_name}_schema_migrations")
-            end
+          @schema_directory ||= Rails.root.join(database_schema_migrations_path).to_s
         end
 
         def versions_to_create
@@ -32,8 +29,8 @@ module Gitlab
           @database_name ||= @connection.pool.db_config.name
         end
 
-        def db_dir
-          @db_dir ||= Rails.application.config.paths["db"].first
+        def database_schema_migrations_path
+          @connection.pool.db_config.configuration_hash[:schema_migrations_path] || DEFAULT_SCHEMA_MIGRATIONS_PATH
         end
       end
     end
diff --git a/lib/gitlab/database/similarity_score.rb b/lib/gitlab/database/similarity_score.rb
index 20bf6fa4d30..bb8b9f333fe 100644
--- a/lib/gitlab/database/similarity_score.rb
+++ b/lib/gitlab/database/similarity_score.rb
@@ -67,7 +67,7 @@ module Gitlab
       def self.build_expression(search:, rules:)
         return EXPRESSION_ON_INVALID_INPUT if search.blank? || rules.empty?
 
-        quoted_search = ActiveRecord::Base.connection.quote(search.to_s)
+        quoted_search = ApplicationRecord.connection.quote(search.to_s)
 
         first_expression, *expressions = rules.map do |rule|
           rule_to_arel(quoted_search, rule)
@@ -110,7 +110,7 @@ module Gitlab
 
       # CAST(multiplier AS numeric)
       def self.multiplier_expression(rule)
-        quoted_multiplier = ActiveRecord::Base.connection.quote(rule.fetch(:multiplier, DEFAULT_MULTIPLIER).to_s)
+        quoted_multiplier = ApplicationRecord.connection.quote(rule.fetch(:multiplier, DEFAULT_MULTIPLIER).to_s)
 
         Arel::Nodes::NamedFunction.new('CAST', [Arel.sql(quoted_multiplier).as('numeric')])
       end
diff --git a/lib/gitlab/database/transaction/context.rb b/lib/gitlab/database/transaction/context.rb
new file mode 100644
index 00000000000..a50dd30b75b
--- /dev/null
+++ b/lib/gitlab/database/transaction/context.rb
@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module Transaction
+      class Context
+        attr_reader :context
+
+        LOG_DEPTH_THRESHOLD = 8
+        LOG_SAVEPOINTS_THRESHOLD = 32
+        LOG_DURATION_S_THRESHOLD = 300
+        LOG_THROTTLE_DURATION = 1
+
+        def initialize
+          @context = {}
+        end
+
+        def set_start_time
+          @context[:start_time] = current_timestamp
+        end
+
+        def increment_savepoints
+          @context[:savepoints] = @context[:savepoints].to_i + 1
+        end
+
+        def increment_rollbacks
+          @context[:rollbacks] = @context[:rollbacks].to_i + 1
+        end
+
+        def increment_releases
+          @context[:releases] = @context[:releases].to_i + 1
+        end
+
+        def set_depth(depth)
+          @context[:depth] = [@context[:depth].to_i, depth].max
+        end
+
+        def track_sql(sql)
+          (@context[:queries] ||= []).push(sql)
+        end
+
+        def duration
+          return unless @context[:start_time].present?
+
+          current_timestamp - @context[:start_time]
+        end
+
+        def depth_threshold_exceeded?
+          @context[:depth].to_i > LOG_DEPTH_THRESHOLD
+        end
+
+        def savepoints_threshold_exceeded?
+          @context[:savepoints].to_i > LOG_SAVEPOINTS_THRESHOLD
+        end
+
+        def duration_threshold_exceeded?
+          duration.to_i > LOG_DURATION_S_THRESHOLD
+        end
+
+        def log_savepoints?
+          depth_threshold_exceeded? || savepoints_threshold_exceeded?
+        end
+
+        def log_duration?
+          duration_threshold_exceeded?
+        end
+
+        def should_log?
+          !logged_already? && (log_savepoints? || log_duration?)
+        end
+
+        def commit
+          log(:commit)
+        end
+
+        def rollback
+          log(:rollback)
+        end
+
+        private
+
+        def queries
+          @context[:queries].to_a.join("\n")
+        end
+
+        def current_timestamp
+          ::Gitlab::Metrics::System.monotonic_time
+        end
+
+        def logged_already?
+          return false if @context[:last_log_timestamp].nil?
+
+          (current_timestamp - @context[:last_log_timestamp].to_i) < LOG_THROTTLE_DURATION
+        end
+
+        def set_last_log_timestamp
+          @context[:last_log_timestamp] = current_timestamp
+        end
+
+        def log(operation)
+          return unless should_log?
+
+          set_last_log_timestamp
+
+          attributes = {
+            class: self.class.name,
+            result: operation,
+            duration_s: duration,
+            depth: @context[:depth].to_i,
+            savepoints_count: @context[:savepoints].to_i,
+            rollbacks_count: @context[:rollbacks].to_i,
+            releases_count: @context[:releases].to_i,
+            sql: queries
+          }
+
+          application_info(attributes)
+        end
+
+        def application_info(attributes)
+          Gitlab::AppJsonLogger.info(attributes)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/transaction/observer.rb b/lib/gitlab/database/transaction/observer.rb
new file mode 100644
index 00000000000..7888f0916e3
--- /dev/null
+++ b/lib/gitlab/database/transaction/observer.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module Transaction
+      class Observer
+        INSTRUMENTED_STATEMENTS = %w[BEGIN SAVEPOINT ROLLBACK RELEASE].freeze
+        LONGEST_COMMAND_LENGTH = 'ROLLBACK TO SAVEPOINT'.length
+        START_COMMENT = '/*'
+        END_COMMENT = '*/'
+
+        def self.instrument_transactions(cmd, event)
+          connection = event.payload[:connection]
+          manager = connection&.transaction_manager
+          return unless manager.respond_to?(:transaction_context)
+
+          context = manager.transaction_context
+          return if context.nil?
+
+          if cmd.start_with?('BEGIN')
+            context.set_start_time
+            context.set_depth(0)
+            context.track_sql(event.payload[:sql])
+          elsif cmd.start_with?('SAVEPOINT ')
+            context.set_depth(manager.open_transactions)
+            context.increment_savepoints
+          elsif cmd.start_with?('ROLLBACK TO SAVEPOINT')
+            context.increment_rollbacks
+          elsif cmd.start_with?('RELEASE SAVEPOINT ')
+            context.increment_releases
+          end
+        end
+
+        def self.register!
+          ActiveSupport::Notifications.subscribe('sql.active_record') do |event|
+            sql = event.payload.dig(:sql).to_s
+            cmd = extract_sql_command(sql)
+
+            if cmd.start_with?(*INSTRUMENTED_STATEMENTS)
+              self.instrument_transactions(cmd, event)
+            end
+          end
+        end
+
+        def self.extract_sql_command(sql)
+          return sql unless sql.start_with?(START_COMMENT)
+
+          index = sql.index(END_COMMENT)
+
+          return sql unless index
+
+          # /* comment */ SELECT
+          #
+          # We offset using a position of the end comment + 1 character to
+          # accomodate a space between Marginalia comment and a SQL statement.
+          offset = index + END_COMMENT.length + 1
+
+          # Avoid duplicating the entire string. This isn't optimized to
+          # strip extra spaces, but we assume that this doesn't happen
+          # for performance reasons.
+          sql[offset..offset + LONGEST_COMMAND_LENGTH]
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/deprecation_json_logger.rb b/lib/gitlab/deprecation_json_logger.rb
new file mode 100644
index 00000000000..9796b24868b
--- /dev/null
+++ b/lib/gitlab/deprecation_json_logger.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class DeprecationJsonLogger < Gitlab::JsonLogger
+    def self.file_name_noext
+      'deprecation_json'
+    end
+  end
+end
diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb
index 6d04c4874c7..f73e060be7f 100644
--- a/lib/gitlab/diff/file_collection/base.rb
+++ b/lib/gitlab/diff/file_collection/base.rb
@@ -43,10 +43,16 @@ module Gitlab
           end
         end
 
+        # This is either the new path, otherwise the old path for the diff_file
         def diff_file_paths
           diff_files.map(&:file_path)
         end
 
+        # This is both the new and old paths for the diff_file
+        def diff_paths
+          diff_files.map(&:paths).flatten.uniq
+        end
+
         def pagination_data
           @pagination_data || empty_pagination_data
         end
diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb
index 28200643296..4fa2fe1724e 100644
--- a/lib/gitlab/email/handler/create_note_handler.rb
+++ b/lib/gitlab/email/handler/create_note_handler.rb
@@ -5,6 +5,7 @@ require 'gitlab/email/handler/reply_processing'
 
 # handles note/reply creation emails with these formats:
 #   incoming+1234567890abcdef1234567890abcdef@incoming.gitlab.com
+# Quoted material is _not_ stripped but appended as a `details` section
 module Gitlab
   module Email
     module Handler
@@ -24,7 +25,7 @@ module Gitlab
           validate_permission!(:create_note)
 
           raise NoteableNotFoundError unless noteable
-          raise EmptyEmailError if message.blank?
+          raise EmptyEmailError if note_message.blank?
 
           verify_record!(
             record: create_note,
@@ -47,7 +48,13 @@ module Gitlab
         end
 
         def create_note
-          sent_notification.create_reply(message)
+          sent_notification.create_reply(note_message)
+        end
+
+        def note_message
+          return message unless sent_notification.noteable_type == "Issue"
+
+          message_with_appended_reply
         end
       end
     end
diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb
index d508cf9360e..a717509e24d 100644
--- a/lib/gitlab/email/handler/reply_processing.rb
+++ b/lib/gitlab/email/handler/reply_processing.rb
@@ -35,13 +35,20 @@ module Gitlab
           @message_with_reply ||= process_message(trim_reply: false)
         end
 
+        def message_with_appended_reply
+          @message_with_appended_reply ||= process_message(append_reply: true)
+        end
+
         def process_message(**kwargs)
-          message = ReplyParser.new(mail, **kwargs).execute.strip
+          message, stripped_text = ReplyParser.new(mail, **kwargs).execute
+          message = message.strip
+
           message_with_attachments = add_attachments(message)
+          # Support bot is specifically forbidden from using slash commands.
+          message = strip_quick_actions(message_with_attachments)
+          return message unless kwargs[:append_reply]
 
-          # Support bot is specifically forbidden
-          # from using slash commands.
-          strip_quick_actions(message_with_attachments)
+          append_reply(message, stripped_text)
         end
 
         def add_attachments(reply)
@@ -92,10 +99,22 @@ module Gitlab
         def strip_quick_actions(content)
           return content unless author.support_bot?
 
+          quick_actions_extractor.redact_commands(content)
+        end
+
+        def quick_actions_extractor
           command_definitions = ::QuickActions::InterpretService.command_definitions
-          extractor = ::Gitlab::QuickActions::Extractor.new(command_definitions)
+          ::Gitlab::QuickActions::Extractor.new(command_definitions)
+        end
+
+        def append_reply(message, reply)
+          return message if message.blank? || reply.blank?
+
+          # Do not append if message only contains slash commands
+          body, _commands = quick_actions_extractor.extract_commands(message)
+          return message if body.empty?
 
-          extractor.redact_commands(content)
+          message + "\n\n<details><summary>...</summary>\n\n#{reply}\n\n</details>"
         end
       end
     end
diff --git a/lib/gitlab/email/message/in_product_marketing/admin_verify.rb b/lib/gitlab/email/message/in_product_marketing/admin_verify.rb
new file mode 100644
index 00000000000..234b93594b5
--- /dev/null
+++ b/lib/gitlab/email/message/in_product_marketing/admin_verify.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Email
+    module Message
+      module InProductMarketing
+        class AdminVerify < Base
+          def subject_line
+            s_('InProductMarketing|Create a custom CI runner with just a few clicks')
+          end
+
+          def tagline
+            nil
+          end
+
+          def title
+            s_('InProductMarketing|Spin up an autoscaling runner in GitLab')
+          end
+
+          def subtitle
+            s_('InProductMarketing|Use our AWS cloudformation template to spin up your runners in just a few clicks!')
+          end
+
+          def body_line1
+            ''
+          end
+
+          def body_line2
+            ''
+          end
+
+          def cta_text
+            s_('InProductMarketing|Create a custom runner')
+          end
+
+          def progress
+            super(track_name: 'Admin')
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/message/in_product_marketing/base.rb b/lib/gitlab/email/message/in_product_marketing/base.rb
index 89acc058a46..96551c89837 100644
--- a/lib/gitlab/email/message/in_product_marketing/base.rb
+++ b/lib/gitlab/email/message/in_product_marketing/base.rb
@@ -67,11 +67,11 @@ module Gitlab
             end
           end
 
-          def progress
+          def progress(current: series + 1, total: total_series, track_name: track.to_s.humanize)
             if Gitlab.com?
-              s_('InProductMarketing|This is email %{current_series} of %{total_series} in the %{track} series.') % { current_series: series + 1, total_series: total_series, track: track.to_s.humanize }
+              s_('InProductMarketing|This is email %{current_series} of %{total_series} in the %{track} series.') % { current_series: current, total_series: total, track: track_name }
             else
-              s_('InProductMarketing|This is email %{current_series} of %{total_series} in the %{track} series. To disable notification emails sent by your local GitLab instance, either contact your administrator or %{unsubscribe_link}.') % { current_series: series + 1, total_series: total_series, track: track.to_s.humanize, unsubscribe_link: unsubscribe_link }
+              s_('InProductMarketing|This is email %{current_series} of %{total_series} in the %{track} series. To disable notification emails sent by your local GitLab instance, either contact your administrator or %{unsubscribe_link}.') % { current_series: current, total_series: total, track: track_name, unsubscribe_link: unsubscribe_link }
             end
           end
 
@@ -109,7 +109,7 @@ module Gitlab
           private
 
           def track
-            self.class.name.demodulize.downcase.to_sym
+            self.class.name.demodulize.underscore.to_sym
           end
 
           def total_series
diff --git a/lib/gitlab/email/message/in_product_marketing/create.rb b/lib/gitlab/email/message/in_product_marketing/create.rb
index 5d3cac0a121..4b0c4af4911 100644
--- a/lib/gitlab/email/message/in_product_marketing/create.rb
+++ b/lib/gitlab/email/message/in_product_marketing/create.rb
@@ -84,7 +84,7 @@ module Gitlab
           end
 
           def basics_link
-            link(s_('InProductMarketing|Git basics'), help_page_url('gitlab-basics/README'))
+            link(s_('InProductMarketing|Git basics'), help_page_url('gitlab-basics/index'))
           end
 
           def import_link
diff --git a/lib/gitlab/email/message/in_product_marketing/team.rb b/lib/gitlab/email/message/in_product_marketing/team.rb
index 46c2797e534..cf723ad5efd 100644
--- a/lib/gitlab/email/message/in_product_marketing/team.rb
+++ b/lib/gitlab/email/message/in_product_marketing/team.rb
@@ -73,6 +73,10 @@ module Gitlab
               s_('InProductMarketing|Invite your team now')
             ][series]
           end
+
+          def progress
+            super(current: series + 2, total: 4)
+          end
         end
       end
     end
diff --git a/lib/gitlab/email/message/in_product_marketing/team_short.rb b/lib/gitlab/email/message/in_product_marketing/team_short.rb
new file mode 100644
index 00000000000..1d60a5fe4e5
--- /dev/null
+++ b/lib/gitlab/email/message/in_product_marketing/team_short.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Email
+    module Message
+      module InProductMarketing
+        class TeamShort < Base
+          def subject_line
+            s_('InProductMarketing|Team up in GitLab for greater efficiency')
+          end
+
+          def tagline
+            nil
+          end
+
+          def title
+            s_('InProductMarketing|Turn coworkers into collaborators')
+          end
+
+          def subtitle
+            s_('InProductMarketing|Invite your team today to build better code (and processes) together')
+          end
+
+          def body_line1
+            ''
+          end
+
+          def body_line2
+            ''
+          end
+
+          def cta_text
+            s_('InProductMarketing|Invite your colleagues today')
+          end
+
+          def progress
+            super(total: 4, track_name: 'Team')
+          end
+
+          def logo_path
+            'mailers/in_product_marketing/team-0.png'
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/message/in_product_marketing/trial.rb b/lib/gitlab/email/message/in_product_marketing/trial.rb
index d87dc5c1b81..222046a3966 100644
--- a/lib/gitlab/email/message/in_product_marketing/trial.rb
+++ b/lib/gitlab/email/message/in_product_marketing/trial.rb
@@ -68,6 +68,10 @@ module Gitlab
               s_('InProductMarketing|Start your trial now!')
             ][series]
           end
+
+          def progress
+            super(current: series + 2, total: 4)
+          end
         end
       end
     end
diff --git a/lib/gitlab/email/message/in_product_marketing/trial_short.rb b/lib/gitlab/email/message/in_product_marketing/trial_short.rb
new file mode 100644
index 00000000000..0fcd3fde4a6
--- /dev/null
+++ b/lib/gitlab/email/message/in_product_marketing/trial_short.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Email
+    module Message
+      module InProductMarketing
+        class TrialShort < Base
+          def subject_line
+            s_('InProductMarketing|Be a DevOps hero')
+          end
+
+          def tagline
+            nil
+          end
+
+          def title
+            s_('InProductMarketing|Expand your DevOps journey with a free GitLab trial')
+          end
+
+          def subtitle
+            s_('InProductMarketing|Start your trial today to experience single application success and discover all the features of GitLab Ultimate for free!')
+          end
+
+          def body_line1
+            ''
+          end
+
+          def body_line2
+            ''
+          end
+
+          def cta_text
+            s_('InProductMarketing|Start a trial')
+          end
+
+          def progress
+            super(total: 4, track_name: 'Trial')
+          end
+
+          def logo_path
+            'mailers/in_product_marketing/trial-0.png'
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/message/in_product_marketing/verify.rb b/lib/gitlab/email/message/in_product_marketing/verify.rb
index 88140c67804..e731c65121e 100644
--- a/lib/gitlab/email/message/in_product_marketing/verify.rb
+++ b/lib/gitlab/email/message/in_product_marketing/verify.rb
@@ -72,7 +72,7 @@ module Gitlab
           end
 
           def quick_start_link
-            link(s_('InProductMarketing|quick start guide'), help_page_url('ci/quick_start/README'))
+            link(s_('InProductMarketing|quick start guide'), help_page_url('ci/quick_start/index'))
           end
 
           def performance_link
diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb
index 7579f3d8680..0f0f4800062 100644
--- a/lib/gitlab/email/reply_parser.rb
+++ b/lib/gitlab/email/reply_parser.rb
@@ -6,20 +6,17 @@ module Gitlab
     class ReplyParser
       attr_accessor :message
 
-      def initialize(message, trim_reply: true)
+      def initialize(message, trim_reply: true, append_reply: false)
         @message = message
         @trim_reply = trim_reply
+        @append_reply = append_reply
       end
 
       def execute
         body = select_body(message)
 
         encoding = body.encoding
-
-        if @trim_reply
-          body = EmailReplyTrimmer.trim(body)
-        end
-
+        body, stripped_text = EmailReplyTrimmer.trim(body, @append_reply) if @trim_reply
         return '' unless body
 
         # not using /\s+$/ here because that deletes empty lines
@@ -30,7 +27,10 @@ module Gitlab
         # so we detect it manually here.
         return "" if body.lines.all? { |l| l.strip.empty? || l.start_with?('>') }
 
-        body.force_encoding(encoding).encode("UTF-8")
+        encoded_body = body.force_encoding(encoding).encode("UTF-8")
+        return encoded_body unless @append_reply
+
+        [encoded_body, stripped_text.force_encoding(encoding).encode("UTF-8")]
       end
 
       private
diff --git a/lib/gitlab/email/smtp_config.rb b/lib/gitlab/email/smtp_config.rb
new file mode 100644
index 00000000000..c9deb3fe324
--- /dev/null
+++ b/lib/gitlab/email/smtp_config.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Email
+    class SmtpConfig
+      def self.encrypted_secrets
+        Settings.encrypted(Gitlab.config.gitlab.email_smtp_secret_file)
+      end
+
+      def self.secrets
+        self.new
+      end
+
+      def initialize
+        @secrets ||= self.class.encrypted_secrets.config
+      rescue StandardError => e
+        Gitlab::AppLogger.error "SMTP encrypted secrets are invalid: #{e.inspect}"
+      end
+
+      def username
+        @secrets&.fetch(:user_name, nil)&.chomp
+      end
+
+      def password
+        @secrets&.fetch(:password, nil)&.chomp
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb
index 8ee53d0de28..50e7631d983 100644
--- a/lib/gitlab/encoding_helper.rb
+++ b/lib/gitlab/encoding_helper.rb
@@ -64,6 +64,15 @@ module Gitlab
       detect && detect[:type] == :binary
     end
 
+    # This is like encode_utf8 except we skip autodetection of the encoding. We
+    # assume the data must be interpreted as UTF-8.
+    def encode_utf8_no_detect(message)
+      message = force_encode_utf8(message)
+      return message if message.valid_encoding?
+
+      message.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+    end
+
     def encode_utf8(message, replace: "")
       message = force_encode_utf8(message)
       return message if message.valid_encoding?
diff --git a/lib/gitlab/encrypted_command_base.rb b/lib/gitlab/encrypted_command_base.rb
new file mode 100644
index 00000000000..b35c28b85cd
--- /dev/null
+++ b/lib/gitlab/encrypted_command_base.rb
@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+# rubocop:disable Rails/Output
+module Gitlab
+  class EncryptedCommandBase
+    DISPLAY_NAME = "Base"
+    EDIT_COMMAND_NAME = "base"
+
+    class << self
+      def encrypted_secrets
+        raise NotImplementedError
+      end
+
+      def write(contents)
+        encrypted = encrypted_secrets
+        return unless validate_config(encrypted)
+
+        validate_contents(contents)
+        encrypted.write(contents)
+
+        puts "File encrypted and saved."
+      rescue Interrupt
+        warn "Aborted changing file: nothing saved."
+      rescue ActiveSupport::MessageEncryptor::InvalidMessage
+        warn "Couldn't decrypt #{encrypted.content_path}. Perhaps you passed the wrong key?"
+      end
+
+      def edit
+        encrypted = encrypted_secrets
+        return unless validate_config(encrypted)
+
+        if ENV["EDITOR"].blank?
+          warn 'No $EDITOR specified to open file. Please provide one when running the command:'
+          warn "gitlab-rake #{self::EDIT_COMMAND_NAME} EDITOR=vim"
+          return
+        end
+
+        temp_file = Tempfile.new(File.basename(encrypted.content_path), File.dirname(encrypted.content_path))
+        contents_changed = false
+
+        encrypted.change do |contents|
+          contents = encrypted_file_template unless File.exist?(encrypted.content_path)
+          File.write(temp_file.path, contents)
+          system(ENV['EDITOR'], temp_file.path)
+          changes = File.read(temp_file.path)
+          contents_changed = contents != changes
+          validate_contents(changes)
+          changes
+        end
+
+        puts "Contents were unchanged." unless contents_changed
+        puts "File encrypted and saved."
+      rescue Interrupt
+        warn "Aborted changing file: nothing saved."
+      rescue ActiveSupport::MessageEncryptor::InvalidMessage
+        warn "Couldn't decrypt #{encrypted.content_path}. Perhaps you passed the wrong key?"
+      ensure
+        temp_file&.unlink
+      end
+
+      def show
+        encrypted = encrypted_secrets
+        return unless validate_config(encrypted)
+
+        puts encrypted.read.presence || "File '#{encrypted.content_path}' does not exist. Use `gitlab-rake #{self::EDIT_COMMAND_NAME}` to change that."
+      rescue ActiveSupport::MessageEncryptor::InvalidMessage
+        warn "Couldn't decrypt #{encrypted.content_path}. Perhaps you passed the wrong key?"
+      end
+
+      def validate_config(encrypted)
+        dir_path = File.dirname(encrypted.content_path)
+
+        unless File.exist?(dir_path)
+          warn "Directory #{dir_path} does not exist. Create the directory and try again."
+          return false
+        end
+
+        if encrypted.key.nil?
+          warn "Missing encryption key encrypted_settings_key_base."
+          return false
+        end
+
+        true
+      end
+
+      def validate_contents(contents)
+        begin
+          config = YAML.safe_load(contents, permitted_classes: [Symbol])
+          error_contents = "Did not include any key-value pairs" unless config.is_a?(Hash)
+        rescue Psych::Exception => e
+          error_contents = e.message
+        end
+
+        puts "WARNING: Content was not a valid #{self::DISPLAY_NAME} secret yml file. #{error_contents}" if error_contents
+
+        contents
+      end
+
+      def encrypted_file_template
+        raise NotImplementedError
+      end
+    end
+  end
+end
+# rubocop:enable Rails/Output
diff --git a/lib/gitlab/encrypted_ldap_command.rb b/lib/gitlab/encrypted_ldap_command.rb
index cdb3e268b51..3675646185e 100644
--- a/lib/gitlab/encrypted_ldap_command.rb
+++ b/lib/gitlab/encrypted_ldap_command.rb
@@ -2,93 +2,13 @@
 
 # rubocop:disable Rails/Output
 module Gitlab
-  class EncryptedLdapCommand
-    class << self
-      def write(contents)
-        encrypted = Gitlab::Auth::Ldap::Config.encrypted_secrets
-        return unless validate_config(encrypted)
-
-        validate_contents(contents)
-        encrypted.write(contents)
-
-        puts "File encrypted and saved."
-      rescue Interrupt
-        puts "Aborted changing file: nothing saved."
-      rescue ActiveSupport::MessageEncryptor::InvalidMessage
-        puts "Couldn't decrypt #{encrypted.content_path}. Perhaps you passed the wrong key?"
-      end
-
-      def edit
-        encrypted = Gitlab::Auth::Ldap::Config.encrypted_secrets
-        return unless validate_config(encrypted)
-
-        if ENV["EDITOR"].blank?
-          puts 'No $EDITOR specified to open file. Please provide one when running the command:'
-          puts 'gitlab-rake gitlab:ldap:secret:edit EDITOR=vim'
-          return
-        end
-
-        temp_file = Tempfile.new(File.basename(encrypted.content_path), File.dirname(encrypted.content_path))
-        contents_changed = false
-
-        encrypted.change do |contents|
-          contents = encrypted_file_template unless File.exist?(encrypted.content_path)
-          File.write(temp_file.path, contents)
-          system(ENV['EDITOR'], temp_file.path)
-          changes = File.read(temp_file.path)
-          contents_changed = contents != changes
-          validate_contents(changes)
-          changes
-        end
-
-        puts "Contents were unchanged." unless contents_changed
-        puts "File encrypted and saved."
-      rescue Interrupt
-        puts "Aborted changing file: nothing saved."
-      rescue ActiveSupport::MessageEncryptor::InvalidMessage
-        puts "Couldn't decrypt #{encrypted.content_path}. Perhaps you passed the wrong key?"
-      ensure
-        temp_file&.unlink
-      end
-
-      def show
-        encrypted = Gitlab::Auth::Ldap::Config.encrypted_secrets
-        return unless validate_config(encrypted)
+  class EncryptedLdapCommand < EncryptedCommandBase
+    DISPLAY_NAME = "LDAP"
+    EDIT_COMMAND_NAME = "gitlab:ldap:secret:edit"
 
-        puts encrypted.read.presence || "File '#{encrypted.content_path}' does not exist. Use `gitlab-rake gitlab:ldap:secret:edit` to change that."
-      rescue ActiveSupport::MessageEncryptor::InvalidMessage
-        puts "Couldn't decrypt #{encrypted.content_path}. Perhaps you passed the wrong key?"
-      end
-
-      private
-
-      def validate_config(encrypted)
-        dir_path = File.dirname(encrypted.content_path)
-
-        unless File.exist?(dir_path)
-          puts "Directory #{dir_path} does not exist. Create the directory and try again."
-          return false
-        end
-
-        if encrypted.key.nil?
-          puts "Missing encryption key encrypted_settings_key_base."
-          return false
-        end
-
-        true
-      end
-
-      def validate_contents(contents)
-        begin
-          config = YAML.safe_load(contents, permitted_classes: [Symbol])
-          error_contents = "Did not include any key-value pairs" unless config.is_a?(Hash)
-        rescue Psych::Exception => e
-          error_contents = e.message
-        end
-
-        puts "WARNING: Content was not a valid LDAP secret yml file. #{error_contents}" if error_contents
-
-        contents
+    class << self
+      def encrypted_secrets
+        Gitlab::Auth::Ldap::Config.encrypted_secrets
       end
 
       def encrypted_file_template
diff --git a/lib/gitlab/encrypted_smtp_command.rb b/lib/gitlab/encrypted_smtp_command.rb
new file mode 100644
index 00000000000..51a476b143d
--- /dev/null
+++ b/lib/gitlab/encrypted_smtp_command.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# rubocop:disable Rails/Output
+module Gitlab
+  class EncryptedSmtpCommand < EncryptedCommandBase
+    DISPLAY_NAME = "SMTP"
+    EDIT_COMMAND_NAME = "gitlab:smtp:secret:edit"
+
+    class << self
+      def encrypted_secrets
+        Gitlab::Email::SmtpConfig.encrypted_secrets
+      end
+
+      def encrypted_file_template
+        <<~YAML
+          # password: '123'
+          # user_name: 'gitlab-inst'
+        YAML
+      end
+    end
+  end
+end
+# rubocop:enable Rails/Output
diff --git a/lib/gitlab/etag_caching/router/restful.rb b/lib/gitlab/etag_caching/router/restful.rb
index fba4b9e433a..408a901f69d 100644
--- a/lib/gitlab/etag_caching/router/restful.rb
+++ b/lib/gitlab/etag_caching/router/restful.rb
@@ -71,7 +71,7 @@ module Gitlab
             'continuous_delivery'
           ],
           [
-            %r(#{RESERVED_WORDS_PREFIX}/environments\.json\z),
+            %r(#{RESERVED_WORDS_PREFIX}/-/environments\.json\z),
             'environments',
             'continuous_delivery'
           ],
diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb
index 0cf3969b490..2f78e4e5c0a 100644
--- a/lib/gitlab/experimentation.rb
+++ b/lib/gitlab/experimentation.rb
@@ -34,16 +34,13 @@
 module Gitlab
   module Experimentation
     EXPERIMENTS = {
-      invite_members_empty_group_version_a: {
-        tracking_category: 'Growth::Expansion::Experiment::InviteMembersEmptyGroupVersionA',
-        use_backwards_compatible_subject_index: true
-      },
-      contact_sales_btn_in_app: {
-        tracking_category: 'Growth::Conversion::Experiment::ContactSalesInApp',
-        use_backwards_compatible_subject_index: true
+      remove_known_trial_form_fields_welcoming: {
+        tracking_category: 'Growth::Conversion::Experiment::RemoveKnownTrialFormFieldsWelcoming',
+        rollout_strategy: :user
       },
-      remove_known_trial_form_fields: {
-        tracking_category: 'Growth::Conversion::Experiment::RemoveKnownTrialFormFields'
+      remove_known_trial_form_fields_noneditable: {
+        tracking_category: 'Growth::Conversion::Experiment::RemoveKnownTrialFormFieldsNoneditable',
+        rollout_strategy: :user
       },
       invite_members_new_dropdown: {
         tracking_category: 'Growth::Expansion::Experiment::InviteMembersNewDropdown'
@@ -52,9 +49,6 @@ module Gitlab
         tracking_category: 'Growth::Conversion::Experiment::ShowTrialStatusInSidebar',
         rollout_strategy: :group
       },
-      trial_onboarding_issues: {
-        tracking_category: 'Growth::Conversion::Experiment::TrialOnboardingIssues'
-      },
       learn_gitlab_a: {
         tracking_category: 'Growth::Conversion::Experiment::LearnGitLabA',
         rollout_strategy: :user
diff --git a/lib/gitlab/fake_application_settings.rb b/lib/gitlab/fake_application_settings.rb
index 211c0967f89..0b9a4c161ae 100644
--- a/lib/gitlab/fake_application_settings.rb
+++ b/lib/gitlab/fake_application_settings.rb
@@ -1,35 +1,51 @@
 # frozen_string_literal: true
 
-# This class extends an OpenStruct object by adding predicate methods to mimic
+# Fakes ActiveRecord attribute storage by adding predicate methods to mimic
 # ActiveRecord access. We rely on the initial values being true or false to
 # determine whether to define a predicate method because for a newly-added
 # column that has not been migrated yet, there is no way to determine the
 # column type without parsing db/structure.sql.
 module Gitlab
-  class FakeApplicationSettings < OpenStruct
-    include ApplicationSettingImplementation
-
-    # Mimic ActiveRecord predicate methods for boolean values
-    def self.define_predicate_methods(options)
-      options.each do |key, value|
-        next if key.to_s.end_with?('?')
-        next unless [true, false].include?(value)
-
-        define_method "#{key}?" do
-          actual_key = key.to_s.chomp('?')
-          self[actual_key]
+  class FakeApplicationSettings
+    prepend ApplicationSettingImplementation
+
+    def self.define_properties(settings)
+      settings.each do |key, value|
+        define_method key do
+          read_attribute(key)
+        end
+
+        if [true, false].include?(value)
+          define_method "#{key}?" do
+            read_attribute(key)
+          end
+        end
+
+        define_method "#{key}=" do |v|
+          @table[key.to_sym] = v
         end
       end
     end
 
-    def initialize(options = {})
-      super
+    def initialize(settings = {})
+      @table = settings.dup
 
-      FakeApplicationSettings.define_predicate_methods(options)
+      FakeApplicationSettings.define_properties(settings)
     end
 
-    alias_method :read_attribute, :[]
-    alias_method :has_attribute?, :[]
+    def read_attribute(key)
+      @table[key.to_sym]
+    end
+
+    def has_attribute?(key)
+      @table.key?(key.to_sym)
+    end
+
+    # Mimic behavior of OpenStruct, which absorbs any calls into undefined
+    # properties to return `nil`.
+    def method_missing(*)
+      nil
+    end
   end
 end
 
diff --git a/lib/gitlab/form_builders/gitlab_ui_form_builder.rb b/lib/gitlab/form_builders/gitlab_ui_form_builder.rb
new file mode 100644
index 00000000000..a5290508e42
--- /dev/null
+++ b/lib/gitlab/form_builders/gitlab_ui_form_builder.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module FormBuilders
+    class GitlabUiFormBuilder < ActionView::Helpers::FormBuilder
+      def gitlab_ui_checkbox_component(
+        method,
+        label,
+        help_text: nil,
+        checkbox_options: {},
+        checked_value: '1',
+        unchecked_value: '0',
+        label_options: {}
+      )
+        @template.content_tag(
+          :div,
+          class: 'gl-form-checkbox custom-control custom-checkbox'
+        ) do
+          @template.check_box(
+            @object_name,
+            method,
+            format_options(checkbox_options, ['custom-control-input']),
+            checked_value,
+            unchecked_value
+          ) +
+          @template.label(
+            @object_name, method, format_options(label_options, ['custom-control-label'])
+          ) do
+            if help_text
+              @template.content_tag(
+                :span,
+                label
+              ) +
+              @template.content_tag(
+                :p,
+                help_text,
+                class: 'help-text'
+              )
+            else
+              label
+            end
+          end
+        end
+      end
+
+      private
+
+      def format_options(options, classes)
+        classes << options[:class]
+
+        objectify_options(options.merge({ class: classes.flatten.compact }))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index 1c8e55ecf50..f72217dedde 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -77,8 +77,8 @@ module Gitlab
           end
         end
 
-        def raw(repository, sha)
-          repository.gitaly_blob_client.get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE)
+        def raw(repository, sha, limit: MAX_DATA_DISPLAY_SIZE)
+          repository.gitaly_blob_client.get_blob(oid: sha, limit: limit)
         end
 
         # Returns an array of Blob instances, specified in blob_references as
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index a863b952390..7fd4acb4179 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -111,8 +111,18 @@ module Gitlab
         #   Commit.between(repo, '29eda46b', 'master')
         #
         def between(repo, base, head)
+          # In either of these cases, we are guaranteed to return no commits, so
+          # shortcut the RPC call
+          return [] if Gitlab::Git.blank_ref?(base) || Gitlab::Git.blank_ref?(head)
+
           wrapped_gitaly_errors do
-            repo.gitaly_commit_client.between(base, head)
+            if Feature.enabled?(:between_uses_list_commits, default_enabled: :yaml)
+              revisions = [head, "^#{base}"] # base..head
+
+              repo.gitaly_commit_client.list_commits(revisions, reverse: true)
+            else
+              repo.gitaly_commit_client.between(base, head)
+            end
           end
         end
 
diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb
index 8815088d23c..6a7a7032665 100644
--- a/lib/gitlab/git/commit_stats.rb
+++ b/lib/gitlab/git/commit_stats.rb
@@ -14,21 +14,23 @@ module Gitlab
       # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/323
       def initialize(repo, commit)
         @id = commit.id
-        @additions = 0
-        @deletions = 0
-        @total = 0
 
-        wrapped_gitaly_errors do
-          gitaly_stats(repo, commit)
-        end
-      end
+        additions, deletions = fetch_stats(repo, commit)
 
-      def gitaly_stats(repo, commit)
-        stats = repo.gitaly_commit_client.commit_stats(@id)
-        @additions = stats.additions
-        @deletions = stats.deletions
+        @additions = additions.to_i
+        @deletions = deletions.to_i
         @total = @additions + @deletions
       end
+
+      def fetch_stats(repo, commit)
+        Rails.cache.fetch("commit_stats:#{repo.gl_project_path}:#{@id}") do
+          stats = wrapped_gitaly_errors do
+            repo.gitaly_commit_client.commit_stats(@id)
+          end
+
+          [stats.additions, stats.deletions]
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/git/conflict/file.rb b/lib/gitlab/git/conflict/file.rb
index 7ffe4a7ae81..049ca5a54b3 100644
--- a/lib/gitlab/git/conflict/file.rb
+++ b/lib/gitlab/git/conflict/file.rb
@@ -6,13 +6,14 @@ module Gitlab
       class File
         UnsupportedEncoding = Class.new(StandardError)
 
-        attr_reader :their_path, :our_path, :our_mode, :repository, :commit_oid
+        attr_reader :ancestor_path, :their_path, :our_path, :our_mode, :repository, :commit_oid
 
         attr_accessor :raw_content
 
         def initialize(repository, commit_oid, conflict, raw_content)
           @repository = repository
           @commit_oid = commit_oid
+          @ancestor_path = conflict[:ancestor][:path]
           @their_path = conflict[:theirs][:path]
           @our_path = conflict[:ours][:path]
           @our_mode = conflict[:ours][:mode]
@@ -94,6 +95,15 @@ module Gitlab
 
           resolution
         end
+
+        def path
+          # There are conflict scenarios (e.g. file is removed on source) wherein
+          # our_path will be blank/nil. Since we are indexing them by path in
+          # `#conflicts` helper and we want to match the diff file to a conflict
+          # in `DiffFileEntity#highlighted_diff_lines`, we need to fallback to
+          # their_path (this is the path on target).
+          our_path.presence || their_path
+        end
       end
     end
   end
diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb
index aa5d50d1fb1..2069d26400e 100644
--- a/lib/gitlab/git/conflict/resolver.rb
+++ b/lib/gitlab/git/conflict/resolver.rb
@@ -9,15 +9,16 @@ module Gitlab
         ConflictSideMissing = Class.new(StandardError)
         ResolutionError = Class.new(StandardError)
 
-        def initialize(target_repository, our_commit_oid, their_commit_oid)
+        def initialize(target_repository, our_commit_oid, their_commit_oid, allow_tree_conflicts: false)
           @target_repository = target_repository
           @our_commit_oid = our_commit_oid
           @their_commit_oid = their_commit_oid
+          @allow_tree_conflicts = allow_tree_conflicts
         end
 
         def conflicts
           @conflicts ||= wrapped_gitaly_errors do
-            gitaly_conflicts_client(@target_repository).list_conflict_files.to_a
+            gitaly_conflicts_client(@target_repository).list_conflict_files(allow_tree_conflicts: @allow_tree_conflicts).to_a
           rescue GRPC::FailedPrecondition => e
             raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing, e.message
           end
diff --git a/lib/gitlab/git/remote_mirror.rb b/lib/gitlab/git/remote_mirror.rb
index eb368af199d..2f618294e8e 100644
--- a/lib/gitlab/git/remote_mirror.rb
+++ b/lib/gitlab/git/remote_mirror.rb
@@ -5,11 +5,10 @@ module Gitlab
     class RemoteMirror
       include Gitlab::Git::WrapsGitalyErrors
 
-      attr_reader :repository, :ref_name, :remote_url, :only_branches_matching, :ssh_key, :known_hosts, :keep_divergent_refs
+      attr_reader :repository, :remote_url, :only_branches_matching, :ssh_key, :known_hosts, :keep_divergent_refs
 
-      def initialize(repository, ref_name, remote_url, only_branches_matching: [], ssh_key: nil, known_hosts: nil, keep_divergent_refs: false)
+      def initialize(repository, remote_url, only_branches_matching: [], ssh_key: nil, known_hosts: nil, keep_divergent_refs: false)
         @repository = repository
-        @ref_name = ref_name
         @remote_url = remote_url
         @only_branches_matching = only_branches_matching
         @ssh_key = ssh_key
@@ -20,7 +19,6 @@ module Gitlab
       def update
         wrapped_gitaly_errors do
           repository.gitaly_remote_client.update_remote_mirror(
-            ref_name,
             remote_url,
             only_branches_matching,
             ssh_key: ssh_key,
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 70d072e8082..1ab80fe2454 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -354,13 +354,9 @@ module Gitlab
         end
       end
 
-      def new_commits(newrevs)
+      def new_commits(newrevs, allow_quarantine: false)
         wrapped_gitaly_errors do
-          if Feature.enabled?(:list_commits)
-            gitaly_commit_client.list_commits(Array.wrap(newrevs) + %w[--not --all])
-          else
-            Array.wrap(newrevs).flat_map { |newrev| gitaly_ref_client.list_new_commits(newrev) }
-          end
+          gitaly_commit_client.list_new_commits(Array.wrap(newrevs), allow_quarantine: allow_quarantine)
         end
       end
 
@@ -703,24 +699,11 @@ module Gitlab
         write_ref(ref, start_point)
       end
 
-      # If `mirror_refmap` is present the remote is set as mirror with that mapping
-      def add_remote(remote_name, url, mirror_refmap: nil)
-        wrapped_gitaly_errors do
-          gitaly_remote_client.add_remote(remote_name, url, mirror_refmap)
-        end
-      end
-
-      def remove_remote(remote_name)
-        wrapped_gitaly_errors do
-          gitaly_remote_client.remove_remote(remote_name)
-        end
-      end
-
-      def find_remote_root_ref(remote_name, remote_url, authorization = nil)
-        return unless remote_name.present? && remote_url.present?
+      def find_remote_root_ref(remote_url, authorization = nil)
+        return unless remote_url.present?
 
         wrapped_gitaly_errors do
-          gitaly_remote_client.find_remote_root_ref(remote_name, remote_url, authorization)
+          gitaly_remote_client.find_remote_root_ref(remote_url, authorization)
         end
       end
 
@@ -820,18 +803,18 @@ module Gitlab
       # no_tags - should we use --no-tags flag?
       # prune - should we use --prune flag?
       # check_tags_changed - should we ask gitaly to calculate whether any tags changed?
-      def fetch_remote(remote, url: nil, refmap: nil, ssh_auth: nil, forced: false, no_tags: false, prune: true, check_tags_changed: false)
+      def fetch_remote(url, refmap: nil, ssh_auth: nil, forced: false, no_tags: false, prune: true, check_tags_changed: false, http_authorization_header: "")
         wrapped_gitaly_errors do
           gitaly_repository_client.fetch_remote(
-            remote,
-            url: url,
+            url,
             refmap: refmap,
             ssh_auth: ssh_auth,
             forced: forced,
             no_tags: no_tags,
             prune: prune,
             check_tags_changed: check_tags_changed,
-            timeout: GITLAB_PROJECTS_TIMEOUT
+            timeout: GITLAB_PROJECTS_TIMEOUT,
+            http_authorization_header: http_authorization_header
           )
         end
       end
@@ -844,8 +827,8 @@ module Gitlab
         end
       end
 
-      def blob_at(sha, path)
-        Gitlab::Git::Blob.find(self, sha, path) unless Gitlab::Git.blank_ref?(sha)
+      def blob_at(sha, path, limit: Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE)
+        Gitlab::Git::Blob.find(self, sha, path, limit: limit) unless Gitlab::Git.blank_ref?(sha)
       end
 
       # Items should be of format [[commit_id, path], [commit_id1, path1]]
@@ -922,13 +905,17 @@ module Gitlab
       end
       # rubocop:enable Metrics/ParameterLists
 
-      def write_config(full_path:)
+      def set_full_path(full_path:)
         return unless full_path.present?
 
         # This guard avoids Gitaly log/error spam
         raise NoRepository, 'repository does not exist' unless exists?
 
-        set_config('gitlab.fullpath' => full_path)
+        if Feature.enabled?(:set_full_path)
+          gitaly_repository_client.set_full_path(full_path)
+        else
+          set_config('gitlab.fullpath' => full_path)
+        end
       end
 
       def set_config(entries)
@@ -937,12 +924,6 @@ module Gitlab
         end
       end
 
-      def delete_config(*keys)
-        wrapped_gitaly_errors do
-          gitaly_repository_client.delete_config(keys)
-        end
-      end
-
       def disconnect_alternates
         wrapped_gitaly_errors do
           gitaly_repository_client.disconnect_alternates
diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb
index 389c9d32ccb..5993c8888d3 100644
--- a/lib/gitlab/git/rugged_impl/tree.rb
+++ b/lib/gitlab/git/rugged_impl/tree.rb
@@ -14,9 +14,12 @@ module Gitlab
           include Gitlab::Git::RuggedImpl::UseRugged
 
           override :tree_entries
-          def tree_entries(repository, sha, path, recursive)
+          def tree_entries(repository, sha, path, recursive, pagination_params = nil)
             if use_rugged?(repository, :rugged_tree_entries)
-              execute_rugged_call(:tree_entries_with_flat_path_from_rugged, repository, sha, path, recursive)
+              [
+                execute_rugged_call(:tree_entries_with_flat_path_from_rugged, repository, sha, path, recursive),
+                nil
+              ]
             else
               super
             end
diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb
index 568e894a02f..25895dc6728 100644
--- a/lib/gitlab/git/tag.rb
+++ b/lib/gitlab/git/tag.rb
@@ -5,6 +5,8 @@ module Gitlab
     class Tag < Ref
       extend Gitlab::EncodingHelper
 
+      delegate :id, to: :@raw_tag
+
       attr_reader :object_sha, :repository
 
       MAX_TAG_MESSAGE_DISPLAY_SIZE = 10.megabytes
@@ -24,6 +26,18 @@ module Gitlab
         def get_messages(repository, tag_ids)
           repository.gitaly_ref_client.get_tag_messages(tag_ids)
         end
+
+        def extract_signature_lazily(repository, tag_id)
+          BatchLoader.for(tag_id).batch(key: repository) do |tag_ids, loader, args|
+            batch_signature_extraction(args[:key], tag_ids).each do |tag_id, signature_data|
+              loader.call(tag_id, signature_data)
+            end
+          end
+        end
+
+        def batch_signature_extraction(repository, tag_ids)
+          repository.gitaly_ref_client.get_tag_signatures(tag_ids)
+        end
       end
 
       def initialize(repository, raw_tag)
@@ -81,7 +95,7 @@ module Gitlab
         when :PGP
           nil # not implemented, see https://gitlab.com/gitlab-org/gitlab/issues/19260
         when :X509
-          X509::Tag.new(@raw_tag).signature
+          X509::Tag.new(@repository, self).signature
         else
           nil
         end
diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb
index ed02f2e92ec..eb008507397 100644
--- a/lib/gitlab/git/tree.rb
+++ b/lib/gitlab/git/tree.rb
@@ -15,15 +15,15 @@ module Gitlab
         # Uses rugged for raw objects
         #
         # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/320
-        def where(repository, sha, path = nil, recursive = false)
+        def where(repository, sha, path = nil, recursive = false, pagination_params = nil)
           path = nil if path == '' || path == '/'
 
-          tree_entries(repository, sha, path, recursive)
+          tree_entries(repository, sha, path, recursive, pagination_params)
         end
 
-        def tree_entries(repository, sha, path, recursive)
+        def tree_entries(repository, sha, path, recursive, pagination_params = nil)
           wrapped_gitaly_errors do
-            repository.gitaly_commit_client.tree_entries(repository, sha, path, recursive)
+            repository.gitaly_commit_client.tree_entries(repository, sha, path, recursive, pagination_params)
           end
         end
 
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index 642a77ced11..759c6b93d9a 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -498,21 +498,10 @@ module Gitlab
     end
 
     def check_changes_size
-      changes_size =
-        if Feature.enabled?(:git_access_batched_changes_size, project, default_enabled: :yaml)
-          revs = ['--not', '--all', '--not']
-          revs += changes_list.map { |change| change[:newrev] }
+      revs = ['--not', '--all', '--not']
+      revs += changes_list.map { |change| change[:newrev] }
 
-          repository.blobs(revs).sum(&:size)
-        else
-          changes_size = 0
-
-          changes_list.each do |change|
-            changes_size += repository.new_blobs(change[:newrev]).sum(&:size)
-          end
-
-          changes_size
-        end
+      changes_size = repository.blobs(revs).sum(&:size)
 
       check_size_against_limit(changes_size)
     end
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index b894207f0aa..fa616a252e4 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -111,17 +111,22 @@ module Gitlab
         nil
       end
 
-      def tree_entries(repository, revision, path, recursive)
+      def tree_entries(repository, revision, path, recursive, pagination_params)
         request = Gitaly::GetTreeEntriesRequest.new(
           repository: @gitaly_repo,
           revision: encode_binary(revision),
           path: path.present? ? encode_binary(path) : '.',
-          recursive: recursive
+          recursive: recursive,
+          pagination_params: pagination_params
         )
+        request.sort = Gitaly::GetTreeEntriesRequest::SortBy::TREES_FIRST if pagination_params
 
         response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout)
 
-        response.flat_map do |message|
+        cursor = nil
+
+        entries = response.flat_map do |message|
+          cursor = message.pagination_cursor if message.pagination_cursor
           message.entries.map do |gitaly_tree_entry|
             Gitlab::Git::Tree.new(
               id: gitaly_tree_entry.oid,
@@ -135,6 +140,8 @@ module Gitlab
             )
           end
         end
+
+        [entries, cursor]
       end
 
       def commit_count(ref, options = {})
@@ -248,16 +255,42 @@ module Gitlab
         consume_commits_response(response)
       end
 
-      def list_commits(revisions)
+      def list_commits(revisions, reverse: false)
         request = Gitaly::ListCommitsRequest.new(
           repository: @gitaly_repo,
-          revisions: Array.wrap(revisions)
+          revisions: Array.wrap(revisions),
+          reverse: reverse
         )
 
         response = GitalyClient.call(@repository.storage, :commit_service, :list_commits, request, timeout: GitalyClient.medium_timeout)
         consume_commits_response(response)
       end
 
+      # List all commits which are new in the repository. If commits have been pushed into the repo
+      def list_new_commits(revisions, allow_quarantine: false)
+        git_env = Gitlab::Git::HookEnv.all(@gitaly_repo.gl_repository)
+        if allow_quarantine && git_env['GIT_OBJECT_DIRECTORY_RELATIVE'].present?
+          # If we have a quarantine environment, then we can optimize the check
+          # by doing a ListAllCommitsRequest. Instead of walking through
+          # references, we just walk through all quarantined objects, which is
+          # a lot more efficient. To do so, we throw away any alternate object
+          # directories, which point to the main object directory of the
+          # repository, and only keep the object directory which points into
+          # the quarantine object directory.
+          quarantined_repo = @gitaly_repo.dup
+          quarantined_repo.git_alternate_object_directories = Google::Protobuf::RepeatedField.new(:string)
+
+          request = Gitaly::ListAllCommitsRequest.new(
+            repository: quarantined_repo
+          )
+
+          response = GitalyClient.call(@repository.storage, :commit_service, :list_all_commits, request, timeout: GitalyClient.medium_timeout)
+          consume_commits_response(response)
+        else
+          list_commits(Array.wrap(revisions) + %w[--not --all])
+        end
+      end
+
       def list_commits_by_oid(oids)
         return [] if oids.empty?
 
diff --git a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb
index 38ec910111c..b1278e3bfac 100644
--- a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb
+++ b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb
@@ -43,6 +43,7 @@ module Gitlab
 
       def conflict_from_gitaly_file_header(header)
         {
+          ancestor: { path: header.ancestor_path },
           ours: { path: header.our_path, mode: header.our_mode },
           theirs: { path: header.their_path }
         }
diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb
index 300800189f1..982454b117e 100644
--- a/lib/gitlab/gitaly_client/conflicts_service.rb
+++ b/lib/gitlab/gitaly_client/conflicts_service.rb
@@ -14,11 +14,12 @@ module Gitlab
         @their_commit_oid = their_commit_oid
       end
 
-      def list_conflict_files
+      def list_conflict_files(allow_tree_conflicts: false)
         request = Gitaly::ListConflictFilesRequest.new(
           repository: @gitaly_repo,
           our_commit_oid: @our_commit_oid,
-          their_commit_oid: @their_commit_oid
+          their_commit_oid: @their_commit_oid,
+          allow_tree_conflicts: allow_tree_conflicts
         )
         response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request, timeout: GitalyClient.long_timeout)
         GitalyClient::ConflictFilesStitcher.new(response, @gitaly_repo)
diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb
index ac2db99ee01..7097d5bd181 100644
--- a/lib/gitlab/gitaly_client/ref_service.rb
+++ b/lib/gitlab/gitaly_client/ref_service.rb
@@ -62,24 +62,6 @@ module Gitlab
         encode!(response.name.dup)
       end
 
-      def list_new_commits(newrev)
-        request = Gitaly::ListNewCommitsRequest.new(
-          repository: @gitaly_repo,
-          commit_id: newrev
-        )
-
-        commits = []
-
-        response = GitalyClient.call(@storage, :ref_service, :list_new_commits, request, timeout: GitalyClient.medium_timeout)
-        response.each do |msg|
-          msg.commits.each do |c|
-            commits << Gitlab::Git::Commit.new(@repository, c)
-          end
-        end
-
-        commits
-      end
-
       def list_new_blobs(newrev, limit = 0, dynamic_timeout: nil)
         request = Gitaly::ListNewBlobsRequest.new(
           repository: @gitaly_repo,
@@ -196,6 +178,27 @@ module Gitlab
         messages
       end
 
+      def get_tag_signatures(tag_ids)
+        request = Gitaly::GetTagSignaturesRequest.new(repository: @gitaly_repo, tag_revisions: tag_ids)
+        response = GitalyClient.call(@repository.storage, :ref_service, :get_tag_signatures, request, timeout: GitalyClient.fast_timeout)
+
+        signatures = Hash.new { |h, k| h[k] = [+''.b, +''.b] }
+        current_tag_id = nil
+
+        response.each do |message|
+          message.signatures.each do |tag_signature|
+            current_tag_id = tag_signature.tag_id if tag_signature.tag_id.present?
+
+            signatures[current_tag_id].first << tag_signature.signature
+            signatures[current_tag_id].last << tag_signature.content
+          end
+        end
+
+        signatures
+      rescue GRPC::InvalidArgument => ex
+        raise ArgumentError, ex
+      end
+
       def pack_refs
         request = Gitaly::PackRefsRequest.new(repository: @gitaly_repo)
 
diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb
index 487127b7b74..535b987f91c 100644
--- a/lib/gitlab/gitaly_client/remote_service.rb
+++ b/lib/gitlab/gitaly_client/remote_service.rb
@@ -26,25 +26,7 @@ module Gitlab
         @storage = repository.storage
       end
 
-      def add_remote(name, url, mirror_refmaps)
-        request = Gitaly::AddRemoteRequest.new(
-          repository: @gitaly_repo,
-          name: name,
-          url: url,
-          mirror_refmaps: Array.wrap(mirror_refmaps).map(&:to_s)
-        )
-
-        GitalyClient.call(@storage, :remote_service, :add_remote, request, timeout: GitalyClient.fast_timeout)
-      end
-
-      def remove_remote(name)
-        request = Gitaly::RemoveRemoteRequest.new(repository: @gitaly_repo, name: name)
-
-        GitalyClient.call(@storage, :remote_service, :remove_remote, request, timeout: GitalyClient.long_timeout).result
-      end
-
-      # The remote_name parameter is deprecated and will be removed soon.
-      def find_remote_root_ref(remote_name, remote_url, authorization)
+      def find_remote_root_ref(remote_url, authorization)
         request = Gitaly::FindRemoteRootRefRequest.new(repository: @gitaly_repo,
                                                        remote_url: remote_url,
                                                        http_authorization_header: authorization)
@@ -55,18 +37,13 @@ module Gitlab
         encode_utf8(response.ref)
       end
 
-      def update_remote_mirror(ref_name, remote_url, only_branches_matching, ssh_key: nil, known_hosts: nil, keep_divergent_refs: false)
+      def update_remote_mirror(remote_url, only_branches_matching, ssh_key: nil, known_hosts: nil, keep_divergent_refs: false)
         req_enum = Enumerator.new do |y|
           first_request = Gitaly::UpdateRemoteMirrorRequest.new(
             repository: @gitaly_repo
           )
 
-          if remote_url
-            first_request.remote = Gitaly::UpdateRemoteMirrorRequest::Remote.new(url: remote_url)
-          else
-            first_request.ref_name = ref_name
-          end
-
+          first_request.remote = Gitaly::UpdateRemoteMirrorRequest::Remote.new(url: remote_url)
           first_request.ssh_key = ssh_key if ssh_key.present?
           first_request.known_hosts = known_hosts if known_hosts.present?
           first_request.keep_divergent_refs = keep_divergent_refs
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index 009aeaf868a..2e26b3341a2 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -73,18 +73,21 @@ module Gitlab
       # rubocop: disable Metrics/ParameterLists
       # The `remote` parameter is going away soonish anyway, at which point the
       # Rubocop warning can be enabled again.
-      def fetch_remote(remote, url:, refmap:, ssh_auth:, forced:, no_tags:, timeout:, prune: true, check_tags_changed: false)
+      def fetch_remote(url, refmap:, ssh_auth:, forced:, no_tags:, timeout:, prune: true, check_tags_changed: false, http_authorization_header: "")
         request = Gitaly::FetchRemoteRequest.new(
-          repository: @gitaly_repo, remote: remote, force: forced,
-          no_tags: no_tags, timeout: timeout, no_prune: !prune,
-          check_tags_changed: check_tags_changed
+          repository: @gitaly_repo,
+          force: forced,
+          no_tags: no_tags,
+          timeout: timeout,
+          no_prune: !prune,
+          check_tags_changed: check_tags_changed,
+          remote_params: Gitaly::Remote.new(
+            url: url,
+            mirror_refmaps: Array.wrap(refmap).map(&:to_s),
+            http_authorization_header: http_authorization_header
+          )
         )
 
-        if url
-          request.remote_params = Gitaly::Remote.new(url: url,
-                                                     mirror_refmaps: Array.wrap(refmap).map(&:to_s))
-        end
-
         if ssh_auth&.ssh_mirror_url?
           if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?
             request.ssh_key = ssh_auth.ssh_private_key
@@ -263,34 +266,33 @@ module Gitlab
         GitalyClient.call(@storage, :repository_service, :write_ref, request, timeout: GitalyClient.fast_timeout)
       end
 
-      def set_config(entries)
-        return if entries.empty?
-
-        request = Gitaly::SetConfigRequest.new(repository: @gitaly_repo)
-        entries.each do |key, value|
-          request.entries << build_set_config_entry(key, value)
-        end
-
+      def set_full_path(path)
         GitalyClient.call(
           @storage,
           :repository_service,
-          :set_config,
-          request,
+          :set_full_path,
+          Gitaly::SetFullPathRequest.new(
+            repository: @gitaly_repo,
+            path: path
+          ),
           timeout: GitalyClient.fast_timeout
         )
 
         nil
       end
 
-      def delete_config(keys)
-        return if keys.empty?
+      def set_config(entries)
+        return if entries.empty?
 
-        request = Gitaly::DeleteConfigRequest.new(repository: @gitaly_repo, keys: keys)
+        request = Gitaly::SetConfigRequest.new(repository: @gitaly_repo)
+        entries.each do |key, value|
+          request.entries << build_set_config_entry(key, value)
+        end
 
         GitalyClient.call(
           @storage,
           :repository_service,
-          :delete_config,
+          :set_config,
           request,
           timeout: GitalyClient.fast_timeout
         )
diff --git a/lib/gitlab/github_import/bulk_importing.rb b/lib/gitlab/github_import/bulk_importing.rb
index 0d448b55104..80f8f8bfbe2 100644
--- a/lib/gitlab/github_import/bulk_importing.rb
+++ b/lib/gitlab/github_import/bulk_importing.rb
@@ -3,23 +3,60 @@
 module Gitlab
   module GithubImport
     module BulkImporting
+      attr_reader :project, :client
+
+      # project - An instance of `Project`.
+      # client - An instance of `Gitlab::GithubImport::Client`.
+      def initialize(project, client)
+        @project = project
+        @client = client
+      end
+
       # Builds and returns an Array of objects to bulk insert into the
       # database.
       #
       # enum - An Enumerable that returns the objects to turn into database
       #        rows.
       def build_database_rows(enum)
-        enum.each_with_object([]) do |(object, _), rows|
-          rows << build(object) unless already_imported?(object)
+        rows = enum.each_with_object([]) do |(object, _), result|
+          result << build(object) unless already_imported?(object)
         end
+
+        log_and_increment_counter(rows.size, :fetched)
+
+        rows
       end
 
       # Bulk inserts the given rows into the database.
       def bulk_insert(model, rows, batch_size: 100)
         rows.each_slice(batch_size) do |slice|
-          Gitlab::Database.bulk_insert(model.table_name, slice) # rubocop:disable Gitlab/BulkInsert
+          Gitlab::Database.main.bulk_insert(model.table_name, slice) # rubocop:disable Gitlab/BulkInsert
+
+          log_and_increment_counter(slice.size, :imported)
         end
       end
+
+      def object_type
+        raise NotImplementedError
+      end
+
+      private
+
+      def log_and_increment_counter(value, operation)
+        Gitlab::Import::Logger.info(
+          import_type: :github,
+          project_id: project.id,
+          importer: self.class.name,
+          message: "#{value} #{object_type.to_s.pluralize} #{operation}"
+        )
+
+        Gitlab::GithubImport::ObjectCounter.increment(
+          project,
+          object_type,
+          operation,
+          value: value
+        )
+      end
     end
   end
 end
diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb
index d2f5af63621..9bda066efcc 100644
--- a/lib/gitlab/github_import/importer/diff_note_importer.rb
+++ b/lib/gitlab/github_import/importer/diff_note_importer.rb
@@ -46,7 +46,7 @@ module Gitlab
           # To work around this we're using bulk_insert with a single row. This
           # allows us to efficiently insert data (even if it's just 1 row)
           # without having to use all sorts of hacks to disable callbacks.
-          Gitlab::Database.bulk_insert(LegacyDiffNote.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert
+          Gitlab::Database.main.bulk_insert(LegacyDiffNote.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert
         rescue ActiveRecord::InvalidForeignKey
           # It's possible the project and the issue have been deleted since
           # scheduling this job. In this case we'll just skip creating the note.
diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb
index 13061d2c9df..f8665676ccf 100644
--- a/lib/gitlab/github_import/importer/issue_importer.rb
+++ b/lib/gitlab/github_import/importer/issue_importer.rb
@@ -75,7 +75,7 @@ module Gitlab
             end
           end
 
-          Gitlab::Database.bulk_insert(IssueAssignee.table_name, assignees) # rubocop:disable Gitlab/BulkInsert
+          Gitlab::Database.main.bulk_insert(IssueAssignee.table_name, assignees) # rubocop:disable Gitlab/BulkInsert
         end
       end
     end
diff --git a/lib/gitlab/github_import/importer/label_links_importer.rb b/lib/gitlab/github_import/importer/label_links_importer.rb
index 77eb4542195..b608bb48e38 100644
--- a/lib/gitlab/github_import/importer/label_links_importer.rb
+++ b/lib/gitlab/github_import/importer/label_links_importer.rb
@@ -40,7 +40,7 @@ module Gitlab
             }
           end
 
-          Gitlab::Database.bulk_insert(LabelLink.table_name, rows) # rubocop:disable Gitlab/BulkInsert
+          Gitlab::Database.main.bulk_insert(LabelLink.table_name, rows) # rubocop:disable Gitlab/BulkInsert
         end
 
         def find_target_id
diff --git a/lib/gitlab/github_import/importer/labels_importer.rb b/lib/gitlab/github_import/importer/labels_importer.rb
index 80246fa1b77..7293de56a9a 100644
--- a/lib/gitlab/github_import/importer/labels_importer.rb
+++ b/lib/gitlab/github_import/importer/labels_importer.rb
@@ -6,15 +6,9 @@ module Gitlab
       class LabelsImporter
         include BulkImporting
 
-        attr_reader :project, :client, :existing_labels
-
-        # project - An instance of `Project`.
-        # client - An instance of `Gitlab::GithubImport::Client`.
         # rubocop: disable CodeReuse/ActiveRecord
-        def initialize(project, client)
-          @project = project
-          @client = client
-          @existing_labels = project.labels.pluck(:title).to_set
+        def existing_labels
+          @existing_labels ||= project.labels.pluck(:title).to_set
         end
         # rubocop: enable CodeReuse/ActiveRecord
 
@@ -51,6 +45,10 @@ module Gitlab
         def each_label
           client.labels(project.import_source)
         end
+
+        def object_type
+          :label
+        end
       end
     end
   end
diff --git a/lib/gitlab/github_import/importer/lfs_objects_importer.rb b/lib/gitlab/github_import/importer/lfs_objects_importer.rb
index 40248ecbd31..775afd5f53a 100644
--- a/lib/gitlab/github_import/importer/lfs_objects_importer.rb
+++ b/lib/gitlab/github_import/importer/lfs_objects_importer.rb
@@ -35,7 +35,11 @@ module Gitlab
             yield object
           end
         rescue StandardError => e
-          error(project.id, e)
+          Gitlab::Import::ImportFailureService.track(
+            project_id: project.id,
+            error_source: importer_class.name,
+            exception: e
+          )
         end
       end
     end
diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb
index 71ff7465d9b..d11b151bbe2 100644
--- a/lib/gitlab/github_import/importer/milestones_importer.rb
+++ b/lib/gitlab/github_import/importer/milestones_importer.rb
@@ -6,15 +6,9 @@ module Gitlab
       class MilestonesImporter
         include BulkImporting
 
-        attr_reader :project, :client, :existing_milestones
-
-        # project - An instance of `Project`
-        # client - An instance of `Gitlab::GithubImport::Client`
         # rubocop: disable CodeReuse/ActiveRecord
-        def initialize(project, client)
-          @project = project
-          @client = client
-          @existing_milestones = project.milestones.pluck(:iid).to_set
+        def existing_milestones
+          @existing_milestones ||= project.milestones.pluck(:iid).to_set
         end
         # rubocop: enable CodeReuse/ActiveRecord
 
@@ -55,6 +49,10 @@ module Gitlab
         def each_milestone
           client.milestones(project.import_source, state: 'all')
         end
+
+        def object_type
+          :milestone
+        end
       end
     end
   end
diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb
index ae9996d81ef..1fd42a69fac 100644
--- a/lib/gitlab/github_import/importer/note_importer.rb
+++ b/lib/gitlab/github_import/importer/note_importer.rb
@@ -37,7 +37,7 @@ module Gitlab
           # We're using bulk_insert here so we can bypass any validations and
           # callbacks. Running these would result in a lot of unnecessary SQL
           # queries being executed when importing large projects.
-          Gitlab::Database.bulk_insert(Note.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert
+          Gitlab::Database.main.bulk_insert(Note.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert
         rescue ActiveRecord::InvalidForeignKey
           # It's possible the project and the issue have been deleted since
           # scheduling this job. In this case we'll just skip creating the note.
diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb
index b2f099761b1..2812fbd3dfe 100644
--- a/lib/gitlab/github_import/importer/pull_requests_importer.rb
+++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb
@@ -40,11 +40,7 @@ module Gitlab
           # updating the timestamp.
           project.update_column(:last_repository_updated_at, Time.zone.now)
 
-          if Feature.enabled?(:fetch_remote_params, project, default_enabled: :yaml)
-            project.repository.fetch_remote('github', url: project.import_url, refmap: Gitlab::GithubImport.refmap, forced: false)
-          else
-            project.repository.fetch_remote('github', forced: false)
-          end
+          project.repository.fetch_remote(project.import_url, refmap: Gitlab::GithubImport.refmap, forced: false)
 
           pname = project.path_with_namespace
 
diff --git a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb
index e389acbf877..bd65eb5899c 100644
--- a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb
+++ b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb
@@ -37,43 +37,6 @@ module Gitlab
           review.id
         end
 
-        def each_object_to_import(&block)
-          if use_github_review_importer_query_only_unimported_merge_requests?
-            each_merge_request_to_import(&block)
-          else
-            each_merge_request_skipping_imported(&block)
-          end
-        end
-
-        private
-
-        attr_reader :merge_requests_already_imported_cache_key
-
-        # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62036#note_587181108
-        def use_github_review_importer_query_only_unimported_merge_requests?
-          Feature.enabled?(
-            :github_review_importer_query_only_unimported_merge_requests,
-            default_enabled: :yaml
-          )
-        end
-
-        def each_merge_request_skipping_imported
-          project.merge_requests.find_each do |merge_request|
-            next if already_imported?(merge_request)
-
-            Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched)
-
-            client
-              .pull_request_reviews(project.import_source, merge_request.iid)
-              .each do |review|
-                review.merge_request_id = merge_request.id
-                yield(review)
-              end
-
-            mark_as_imported(merge_request)
-          end
-        end
-
         # The worker can be interrupted, by rate limit for instance,
         # in different situations. To avoid requesting already imported data,
         # if the worker is interrupted:
@@ -82,7 +45,7 @@ module Gitlab
         # - before importing all merge requests reviews
         #   Merge requests that had all the reviews imported are cached with
         #   `mark_merge_request_reviews_imported`
-        def each_merge_request_to_import
+        def each_object_to_import(&block)
           each_review_page do |page, merge_request|
             page.objects.each do |review|
               next if already_imported?(review)
@@ -97,6 +60,10 @@ module Gitlab
           end
         end
 
+        private
+
+        attr_reader :merge_requests_already_imported_cache_key
+
         def each_review_page
           merge_requests_to_import.find_each do |merge_request|
             # The page counter needs to be scoped by merge request to avoid skipping
diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb
index a3734ccf069..c1fbd868800 100644
--- a/lib/gitlab/github_import/importer/releases_importer.rb
+++ b/lib/gitlab/github_import/importer/releases_importer.rb
@@ -6,15 +6,9 @@ module Gitlab
       class ReleasesImporter
         include BulkImporting
 
-        attr_reader :project, :client, :existing_tags
-
-        # project - An instance of `Project`
-        # client - An instance of `Gitlab::GithubImport::Client`
         # rubocop: disable CodeReuse/ActiveRecord
-        def initialize(project, client)
-          @project = project
-          @client = client
-          @existing_tags = project.releases.pluck(:tag).to_set
+        def existing_tags
+          @existing_tags ||= project.releases.pluck(:tag).to_set
         end
         # rubocop: enable CodeReuse/ActiveRecord
 
@@ -50,6 +44,10 @@ module Gitlab
         def description_for(release)
           release.body.presence || "Release for tag #{release.tag_name}"
         end
+
+        def object_type
+          :release
+        end
       end
     end
   end
diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb
index 1401c92a44e..20068a33019 100644
--- a/lib/gitlab/github_import/importer/repository_importer.rb
+++ b/lib/gitlab/github_import/importer/repository_importer.rb
@@ -50,7 +50,7 @@ module Gitlab
           project.ensure_repository
 
           refmap = Gitlab::GithubImport.refmap
-          project.repository.fetch_as_mirror(project.import_url, refmap: refmap, forced: true, remote_name: 'github')
+          project.repository.fetch_as_mirror(project.import_url, refmap: refmap, forced: true)
 
           project.change_head(default_branch) if default_branch
 
@@ -59,8 +59,6 @@ module Gitlab
           Repositories::HousekeepingService.new(project, :gc).execute
 
           true
-        rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e
-          fail_import("Failed to import the repository: #{e.message}")
         end
 
         def import_wiki_repository
@@ -70,7 +68,8 @@ module Gitlab
         rescue ::Gitlab::Git::CommandError => e
           if e.message !~ /repository not exported/
             project.create_wiki
-            fail_import("Failed to import the wiki: #{e.message}")
+
+            raise e
           else
             true
           end
@@ -84,11 +83,6 @@ module Gitlab
           project.update_column(:last_repository_updated_at, Time.zone.now)
         end
 
-        def fail_import(message)
-          project.import_state.mark_as_failed(message)
-          false
-        end
-
         private
 
         def default_branch
diff --git a/lib/gitlab/github_import/logger.rb b/lib/gitlab/github_import/logger.rb
new file mode 100644
index 00000000000..980aa0a7982
--- /dev/null
+++ b/lib/gitlab/github_import/logger.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GithubImport
+    class Logger < ::Gitlab::Import::Logger
+      def default_attributes
+        super.merge(import_type: :github)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/object_counter.rb b/lib/gitlab/github_import/object_counter.rb
index e4835504c2d..4c9a8da601f 100644
--- a/lib/gitlab/github_import/object_counter.rb
+++ b/lib/gitlab/github_import/object_counter.rb
@@ -14,11 +14,16 @@ module Gitlab
       CACHING = Gitlab::Cache::Import::Caching
 
       class << self
-        def increment(project, object_type, operation)
+        # Increments the project and the global counters if the given value is >= 1
+        def increment(project, object_type, operation, value: 1)
+          integer = value.to_i
+
+          return if integer <= 0
+
           validate_operation!(operation)
 
-          increment_project_counter(project, object_type, operation)
-          increment_global_counter(object_type, operation)
+          increment_project_counter(project, object_type, operation, integer)
+          increment_global_counter(object_type, operation, integer)
         end
 
         def summary(project)
@@ -41,7 +46,7 @@ module Gitlab
         # and it's used to report the health of the Github Importer
         # in the Grafana Dashboard
         # https://dashboards.gitlab.net/d/2zgM_rImz/github-importer?orgId=1
-        def increment_global_counter(object_type, operation)
+        def increment_global_counter(object_type, operation, value)
           key = GLOBAL_COUNTER_KEY % {
             operation: operation,
             object_type: object_type
@@ -51,18 +56,26 @@ module Gitlab
             object_type: object_type.to_s.humanize
           }
 
-          Gitlab::Metrics.counter(key.to_sym, description).increment
+          Gitlab::Metrics.counter(key.to_sym, description).increment(by: value)
         end
 
         # Project counters are short lived, in Redis,
         # and it's used to report how successful a project
         # import was with the #summary method.
-        def increment_project_counter(project, object_type, operation)
-          counter_key = PROJECT_COUNTER_KEY % { project: project.id, operation: operation, object_type: object_type }
+        def increment_project_counter(project, object_type, operation, value)
+          counter_key = PROJECT_COUNTER_KEY % {
+            project: project.id,
+            operation: operation,
+            object_type: object_type
+          }
 
           add_counter_to_list(project, operation, counter_key)
 
-          CACHING.increment(counter_key)
+          if Feature.disabled?(:import_redis_increment_by, default_enabled: :yaml)
+            CACHING.increment(counter_key)
+          else
+            CACHING.increment_by(counter_key, value)
+          end
         end
 
         def add_counter_to_list(project, operation, key)
@@ -75,7 +88,7 @@ module Gitlab
 
         def validate_operation!(operation)
           unless operation.to_s.presence_in(OPERATIONS)
-            raise ArgumentError, "Operation must be #{OPERATIONS.join(' or ')}"
+            raise ArgumentError, "operation must be #{OPERATIONS.join(' or ')}"
           end
         end
       end
diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb
index 4598429d568..8c76f5a9d94 100644
--- a/lib/gitlab/github_import/parallel_scheduling.rb
+++ b/lib/gitlab/github_import/parallel_scheduling.rb
@@ -49,9 +49,14 @@ module Gitlab
 
         retval
       rescue StandardError => e
-        error(project.id, e)
+        Gitlab::Import::ImportFailureService.track(
+          project_id: project.id,
+          error_source: self.class.name,
+          exception: e,
+          fail_import: abort_on_failure
+        )
 
-        raise e
+        raise(e)
       end
 
       # Imports all the objects in sequence in the current thread.
@@ -165,6 +170,10 @@ module Gitlab
         raise NotImplementedError
       end
 
+      def abort_on_failure
+        false
+      end
+
       # Any options to be passed to the method used for retrieving the data to
       # import.
       def collection_options
@@ -174,36 +183,16 @@ module Gitlab
       private
 
       def info(project_id, extra = {})
-        logger.info(log_attributes(project_id, extra))
-      end
-
-      def error(project_id, exception)
-        logger.error(
-          log_attributes(
-            project_id,
-            message: 'importer failed',
-            'error.message': exception.message
-          )
-        )
-
-        Gitlab::ErrorTracking.track_exception(
-          exception,
-          log_attributes(project_id)
-        )
+        Logger.info(log_attributes(project_id, extra))
       end
 
       def log_attributes(project_id, extra = {})
         extra.merge(
-          import_source: :github,
           project_id: project_id,
           importer: importer_class.name,
           parallel: parallel?
         )
       end
-
-      def logger
-        @logger ||= Gitlab::Import::Logger.build
-      end
     end
   end
 end
diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb
index 058cd1ebd57..f583ef39d13 100644
--- a/lib/gitlab/github_import/user_finder.rb
+++ b/lib/gitlab/github_import/user_finder.rb
@@ -120,10 +120,18 @@ module Gitlab
         read_id_from_cache(ID_FOR_EMAIL_CACHE_KEY % email)
       end
 
-      # Queries and caches the GitLab user ID for a GitHub user ID, if one was
-      # found.
+      # If importing from github.com, queries and caches the GitLab user ID for
+      # a GitHub user ID, if one was found.
+      #
+      # When importing from Github Enterprise, do not query user by Github ID
+      # since we only have users' Github ID from github.com.
       def id_for_github_id(id)
-        gitlab_id = query_id_for_github_id(id) || nil
+        gitlab_id =
+          if project.github_enterprise_import?
+            nil
+          else
+            query_id_for_github_id(id)
+          end
 
         Gitlab::Cache::Import::Caching.write(ID_CACHE_KEY % id, gitlab_id)
       end
diff --git a/lib/gitlab/graphql/copy_field_description.rb b/lib/gitlab/graphql/copy_field_description.rb
index edd73083ff2..ed2273bc91a 100644
--- a/lib/gitlab/graphql/copy_field_description.rb
+++ b/lib/gitlab/graphql/copy_field_description.rb
@@ -11,7 +11,7 @@ module Gitlab
         # are always identical to the corresponding query field descriptions.
         #
         # E.g.:
-        #   argument :name, GraphQL::STRING_TYPE, description: copy_field_description(Types::UserType, :name)
+        #   argument :name, GraphQL::Types::String, description: copy_field_description(Types::UserType, :name)
         def copy_field_description(type, field_name)
           type.fields[field_name.to_s.camelize(:lower)].description
         end
diff --git a/lib/gitlab/graphql/markdown_field.rb b/lib/gitlab/graphql/markdown_field.rb
index 0b5bde8d8d9..6188d860aba 100644
--- a/lib/gitlab/graphql/markdown_field.rb
+++ b/lib/gitlab/graphql/markdown_field.rb
@@ -19,7 +19,7 @@ module Gitlab
           # Adding complexity to rendered notes since that could cause queries.
           kwargs[:complexity] ||= 5
 
-          field name, GraphQL::STRING_TYPE, **kwargs
+          field name, GraphQL::Types::String, **kwargs
 
           define_method resolver_method do
             # We need to `dup` the context so the MarkdownHelper doesn't modify it
diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb
index afe1554aec1..f830af68e07 100644
--- a/lib/gitlab/highlight.rb
+++ b/lib/gitlab/highlight.rb
@@ -50,11 +50,7 @@ module Gitlab
     attr_reader :context
 
     def self.file_size_limit
-      if Feature.enabled?(:one_megabyte_file_size_limit)
-        1024.kilobytes
-      else
-        Gitlab.config.extra['maximum_text_highlight_size_kilobytes']
-      end
+      Gitlab.config.extra['maximum_text_highlight_size_kilobytes']
     end
 
     private_class_method :file_size_limit
diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb
index 7e45cd216f5..8a19f208adf 100644
--- a/lib/gitlab/http.rb
+++ b/lib/gitlab/http.rb
@@ -14,7 +14,7 @@ module Gitlab
       Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout, Gitlab::HTTP::ReadTotalTimeout
     ].freeze
     HTTP_ERRORS = HTTP_TIMEOUT_ERRORS + [
-      SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError,
+      EOFError, SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError,
       Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH,
       Gitlab::HTTP::BlockedUrlError, Gitlab::HTTP::RedirectionTooDeep
     ].freeze
@@ -43,16 +43,29 @@ module Gitlab
           options
         end
 
-      unless options.has_key?(:use_read_total_timeout)
+      options[:skip_read_total_timeout] = true if options[:skip_read_total_timeout].nil? && options[:stream_body]
+
+      if options[:skip_read_total_timeout]
         return httparty_perform_request(http_method, path, options_with_timeouts, &block)
       end
 
       start_time = Gitlab::Metrics::System.monotonic_time
       read_total_timeout = options.fetch(:timeout, DEFAULT_READ_TOTAL_TIMEOUT)
+      tracked_timeout_error = false
 
       httparty_perform_request(http_method, path, options_with_timeouts) do |fragment|
         elapsed = Gitlab::Metrics::System.monotonic_time - start_time
-        raise ReadTotalTimeout, "Request timed out after #{elapsed} seconds" if elapsed > read_total_timeout
+
+        if elapsed > read_total_timeout
+          error = ReadTotalTimeout.new("Request timed out after #{elapsed} seconds")
+
+          raise error if options[:use_read_total_timeout]
+
+          unless tracked_timeout_error
+            Gitlab::ErrorTracking.track_exception(error)
+            tracked_timeout_error = true
+          end
+        end
 
         block.call fragment if block
       end
diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb
index 7ebbe9f1c14..5f1b9873fee 100644
--- a/lib/gitlab/i18n.rb
+++ b/lib/gitlab/i18n.rb
@@ -38,26 +38,26 @@ module Gitlab
     # Currently monthly updated manually by ~group::import PM.
     # https://gitlab.com/gitlab-org/gitlab/-/issues/18923
     TRANSLATION_LEVELS = {
-      'bg' => 1,
+      'bg' => 0,
       'cs_CZ' => 1,
-      'de' => 17,
+      'de' => 16,
       'en' => 100,
-      'eo' => 1,
-      'es' => 38,
-      'fil_PH' => 1,
+      'eo' => 0,
+      'es' => 36,
+      'fil_PH' => 0,
       'fr' => 12,
-      'gl_ES' => 1,
+      'gl_ES' => 0,
       'id_ID' => 0,
       'it' => 2,
-      'ja' => 42,
-      'ko' => 13,
-      'nl_NL' => 1,
-      'pl_PL' => 5,
-      'pt_BR' => 21,
-      'ru' => 29,
+      'ja' => 39,
+      'ko' => 12,
+      'nl_NL' => 0,
+      'pl_PL' => 6,
+      'pt_BR' => 36,
+      'ru' => 28,
       'tr_TR' => 16,
-      'uk' => 41,
-      'zh_CN' => 67,
+      'uk' => 40,
+      'zh_CN' => 74,
       'zh_HK' => 2,
       'zh_TW' => 3
     }.freeze
diff --git a/lib/gitlab/import/database_helpers.rb b/lib/gitlab/import/database_helpers.rb
index f8ea7a7adcd..e73c3afe9bd 100644
--- a/lib/gitlab/import/database_helpers.rb
+++ b/lib/gitlab/import/database_helpers.rb
@@ -11,7 +11,7 @@ module Gitlab
         # We use bulk_insert here so we can bypass any queries executed by
         # callbacks or validation rules, as doing this wouldn't scale when
         # importing very large projects.
-        result = Gitlab::Database # rubocop:disable Gitlab/BulkInsert
+        result = Gitlab::Database.main # rubocop:disable Gitlab/BulkInsert
                  .bulk_insert(relation.table_name, [attributes], return_ids: true)
 
         result.first
diff --git a/lib/gitlab/import/import_failure_service.rb b/lib/gitlab/import/import_failure_service.rb
new file mode 100644
index 00000000000..f808ed1b6e2
--- /dev/null
+++ b/lib/gitlab/import/import_failure_service.rb
@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Import
+    class ImportFailureService
+      def self.track(
+        exception:,
+        import_state: nil,
+        project_id: nil,
+        error_source: nil,
+        fail_import: false
+      )
+        new(
+          exception: exception,
+          import_state: import_state,
+          project_id: project_id,
+          error_source: error_source
+        ).execute(fail_import: fail_import)
+      end
+
+      def initialize(exception:, import_state: nil, project_id: nil, error_source: nil)
+        if import_state.blank? && project_id.blank?
+          raise ArgumentError, 'import_state OR project_id must be provided'
+        end
+
+        if project_id.blank?
+          @import_state = import_state
+          @project = import_state.project
+        else
+          @project = Project.find(project_id)
+          @import_state = @project.import_state
+        end
+
+        @exception = exception
+        @error_source = error_source
+      end
+
+      def execute(fail_import:)
+        track_exception
+        persist_failure
+
+        import_state.mark_as_failed(exception.message) if fail_import
+      end
+
+      private
+
+      attr_reader :exception, :import_state, :project, :error_source
+
+      def track_exception
+        attributes = {
+          import_type: project.import_type,
+          project_id: project.id,
+          source: error_source
+        }
+
+        Gitlab::Import::Logger.error(
+          attributes.merge(
+            message: 'importer failed',
+            'error.message': exception.message
+          )
+        )
+
+        Gitlab::ErrorTracking.track_exception(exception, attributes)
+      end
+
+      def persist_failure
+        project.import_failures.create(
+          source: error_source,
+          exception_class: exception.class.to_s,
+          exception_message: exception.message.truncate(255),
+          correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import/logger.rb b/lib/gitlab/import/logger.rb
index ab3e822a4e9..bd34aff734a 100644
--- a/lib/gitlab/import/logger.rb
+++ b/lib/gitlab/import/logger.rb
@@ -6,6 +6,10 @@ module Gitlab
       def self.file_name_noext
         'importer'
       end
+
+      def default_attributes
+        super.merge(feature_category: :importers)
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb
index 97b34088e3e..dc80c92f507 100644
--- a/lib/gitlab/import_export/json/legacy_reader.rb
+++ b/lib/gitlab/import_export/json/legacy_reader.rb
@@ -27,7 +27,7 @@ module Gitlab
           end
 
           def read_hash
-            ActiveSupport::JSON.decode(IO.read(@path))
+            Gitlab::Json.parse(IO.read(@path))
           rescue StandardError => e
             Gitlab::ErrorTracking.log_exception(e)
             raise Gitlab::ImportExport::Error, 'Incorrect JSON format'
diff --git a/lib/gitlab/import_export/json/ndjson_reader.rb b/lib/gitlab/import_export/json/ndjson_reader.rb
index 4899bd3b0ee..510da61d3ab 100644
--- a/lib/gitlab/import_export/json/ndjson_reader.rb
+++ b/lib/gitlab/import_export/json/ndjson_reader.rb
@@ -47,8 +47,8 @@ module Gitlab
         private
 
         def json_decode(string)
-          ActiveSupport::JSON.decode(string)
-        rescue ActiveSupport::JSON.parse_error => e
+          Gitlab::Json.parse(string)
+        rescue JSON::ParserError => e
           Gitlab::ErrorTracking.log_exception(e)
           raise Gitlab::ImportExport::Error, 'Incorrect JSON format'
         end
diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb
index d1e013a151c..9d28e1abeab 100644
--- a/lib/gitlab/import_export/json/streaming_serializer.rb
+++ b/lib/gitlab/import_export/json/streaming_serializer.rb
@@ -31,10 +31,12 @@ module Gitlab
         end
 
         def execute
-          serialize_root
+          read_from_replica_if_available do
+            serialize_root
 
-          includes.each do |relation_definition|
-            serialize_relation(relation_definition)
+            includes.each do |relation_definition|
+              serialize_relation(relation_definition)
+            end
           end
         end
 
@@ -166,6 +168,13 @@ module Gitlab
             )
           ])
         end
+
+        def read_from_replica_if_available(&block)
+          return yield unless ::Feature.enabled?(:load_balancing_for_export_workers, type: :development, default_enabled: :yaml)
+          return yield unless ::Gitlab::Database::LoadBalancing.enable?
+
+          ::Gitlab::Database::LoadBalancing::Session.current.use_replicas_for_read_queries(&block)
+        end
       end
     end
   end
diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb
index d73ae1410a3..9931b09e9ca 100644
--- a/lib/gitlab/import_export/lfs_restorer.rb
+++ b/lib/gitlab/import_export/lfs_restorer.rb
@@ -72,7 +72,7 @@ module Gitlab
         @lfs_json ||=
           begin
             json = IO.read(lfs_json_path)
-            ActiveSupport::JSON.decode(json)
+            Gitlab::Json.parse(json)
           rescue StandardError
             raise Gitlab::ImportExport::Error, 'Incorrect JSON format'
           end
diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml
index a84978a2a80..5633194a8f8 100644
--- a/lib/gitlab/import_export/project/import_export.yml
+++ b/lib/gitlab/import_export/project/import_export.yml
@@ -230,6 +230,7 @@ excluded_attributes:
     - :blocking_issues_count
     - :service_desk_reply_to
     - :upvotes_count
+    - :work_item_type_id
   merge_request:
     - :milestone_id
     - :sprint_id
@@ -293,6 +294,7 @@ excluded_attributes:
     - :encrypted_token
     - :encrypted_token_iv
     - :enabled
+    - :integrated
   service_desk_setting:
     - :outgoing_name
   priorities:
@@ -328,6 +330,7 @@ excluded_attributes:
     - :release_id
   project_members:
     - :source_id
+    - :invite_email_success
   metrics:
     - :merge_request_id
     - :pipeline_id
diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb
index 0753625b978..b03dceba303 100644
--- a/lib/gitlab/import_export/project/object_builder.rb
+++ b/lib/gitlab/import_export/project/object_builder.rb
@@ -106,7 +106,7 @@ module Gitlab
         end
 
         def design?
-          klass == DesignManagement::Design
+          klass == ::DesignManagement::Design
         end
 
         def diff_commit_user?
diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb
index 8a64abb9f62..0f21a16793d 100644
--- a/lib/gitlab/instrumentation/redis_interceptor.rb
+++ b/lib/gitlab/instrumentation/redis_interceptor.rb
@@ -5,8 +5,21 @@ module Gitlab
     module RedisInterceptor
       APDEX_EXCLUDE = %w[brpop blpop brpoplpush bzpopmin bzpopmax xread xreadgroup].freeze
 
+      # These are temporary to help with investigating
+      # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1183
+      DURATION_ERROR_THRESHOLD = 1.25.seconds
+
+      class MysteryRedisDurationError < StandardError
+        attr_reader :backtrace
+
+        def initialize(backtrace)
+          @backtrace = backtrace
+        end
+      end
+
       def call(*args, &block)
         start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined
+        start_real_time = Time.now
         instrumentation_class.instance_count_request
         instrumentation_class.redis_cluster_validate!(args.first)
 
@@ -27,6 +40,13 @@ module Gitlab
           instrumentation_class.add_duration(duration)
           instrumentation_class.add_call_details(duration, args)
         end
+
+        if duration > DURATION_ERROR_THRESHOLD && Feature.enabled?(:report_on_long_redis_durations, default_enabled: :yaml)
+          Gitlab::ErrorTracking.track_exception(MysteryRedisDurationError.new(caller),
+                                                command: command_from_args(args),
+                                                duration: duration,
+                                                timestamp: start_real_time.iso8601(5))
+        end
       end
 
       def write(command)
diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb
index c10c4a7bedf..23acf1e8e86 100644
--- a/lib/gitlab/instrumentation_helper.rb
+++ b/lib/gitlab/instrumentation_helper.rb
@@ -30,6 +30,7 @@ module Gitlab
       instrument_cpu(payload)
       instrument_thread_memory_allocations(payload)
       instrument_load_balancing(payload)
+      instrument_pid(payload)
     end
 
     def instrument_gitaly(payload)
@@ -99,6 +100,10 @@ module Gitlab
       payload[:cpu_s] = cpu_s.round(DURATION_PRECISION) if cpu_s
     end
 
+    def instrument_pid(payload)
+      payload[:pid] = Process.pid
+    end
+
     def instrument_thread_memory_allocations(payload)
       counters = ::Gitlab::Memory::Instrumentation.measure_thread_memory_allocations(
         ::Gitlab::RequestContext.instance.thread_memory_allocations)
diff --git a/lib/gitlab/integrations/sti_type.rb b/lib/gitlab/integrations/sti_type.rb
index b87c9936570..0fa9f435b5c 100644
--- a/lib/gitlab/integrations/sti_type.rb
+++ b/lib/gitlab/integrations/sti_type.rb
@@ -7,9 +7,13 @@ module Gitlab
         Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog
         Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Irker Jenkins Jira Mattermost
         MattermostSlashCommands MicrosoftTeams MockCi MockMonitoring Packagist PipelinesEmail Pivotaltracker
-        Prometheus Pushover Redmine Slack SlackSlashCommands Teamcity UnifyCircuit Youtrack WebexTeams
+        Prometheus Pushover Redmine Slack SlackSlashCommands Teamcity UnifyCircuit WebexTeams Youtrack
       )).freeze
 
+      def self.namespaced_integrations
+        NAMESPACED_INTEGRATIONS
+      end
+
       def cast(value)
         new_cast(value) || super
       end
@@ -32,16 +36,12 @@ module Gitlab
 
       private
 
-      def namespaced_integrations
-        NAMESPACED_INTEGRATIONS
-      end
-
       def new_cast(value)
         value = prepare_value(value)
         return unless value
 
         stripped_name = value.delete_suffix('Service')
-        return unless namespaced_integrations.include?(stripped_name)
+        return unless self.class.namespaced_integrations.include?(stripped_name)
 
         "Integrations::#{stripped_name}"
       end
diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb
index 3e7659db240..13d3bb2b8dc 100644
--- a/lib/gitlab/jira/http_client.rb
+++ b/lib/gitlab/jira/http_client.rb
@@ -40,6 +40,14 @@ module Gitlab
         @authenticated = result.response.is_a?(Net::HTTPOK)
         store_cookies(result) if options[:use_cookies]
 
+        # This is needed to make response.to_s work. HTTParty::Response internal uses a Net::HTTPResponse as @response.
+        # When a block is used, Net::HTTPResponse#body will be a Net::ReadAdapter instead of a String.
+        # In this case HTTParty::Response.to_s will default to inspecting the Net::HTTPResponse class instead
+        # of returning the content of body.
+        # See https://github.com/jnunemaker/httparty/blob/v0.18.1/lib/httparty/response.rb#L86-L92
+        # See https://github.com/ruby/net-http/blob/v0.1.1/lib/net/http/response.rb#L346-L350
+        result.response.body = result.body
+
         result
       end
 
diff --git a/lib/gitlab/jira_import/issue_serializer.rb b/lib/gitlab/jira_import/issue_serializer.rb
index 43280606bb6..ab748d67fbf 100644
--- a/lib/gitlab/jira_import/issue_serializer.rb
+++ b/lib/gitlab/jira_import/issue_serializer.rb
@@ -52,9 +52,10 @@ module Gitlab
       end
 
       def map_user_id(jira_user)
-        return unless jira_user&.dig('accountId')
+        jira_user_identifier = jira_user&.dig('accountId') || jira_user&.dig('key')
+        return unless jira_user_identifier
 
-        Gitlab::JiraImport.get_user_mapping(project.id, jira_user['accountId'])
+        Gitlab::JiraImport.get_user_mapping(project.id, jira_user_identifier)
       end
 
       def reporter
diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb
index 4314c131ada..41c18f82a4b 100644
--- a/lib/gitlab/json_cache.rb
+++ b/lib/gitlab/json_cache.rb
@@ -58,7 +58,7 @@ module Gitlab
     private
 
     def parse_value(raw, klass)
-      value = ActiveSupport::JSON.decode(raw.to_s)
+      value = Gitlab::Json.parse(raw.to_s)
 
       case value
       when Hash then parse_entry(value, klass)
@@ -66,7 +66,7 @@ module Gitlab
       else
         value
       end
-    rescue ActiveSupport::JSON.parse_error
+    rescue JSON::ParserError
       nil
     end
 
diff --git a/lib/gitlab/json_logger.rb b/lib/gitlab/json_logger.rb
index 3a74df8dc8f..d0dcd232ecc 100644
--- a/lib/gitlab/json_logger.rb
+++ b/lib/gitlab/json_logger.rb
@@ -7,7 +7,7 @@ module Gitlab
     end
 
     def format_message(severity, timestamp, progname, message)
-      data = {}
+      data = default_attributes
       data[:severity] = severity
       data[:time] = timestamp.utc.iso8601(3)
       data[Labkit::Correlation::CorrelationId::LOG_KEY] = Labkit::Correlation::CorrelationId.current_id
@@ -21,5 +21,11 @@ module Gitlab
 
       Gitlab::Json.dump(data) + "\n"
     end
+
+    protected
+
+    def default_attributes
+      {}
+    end
   end
 end
diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb
index 86c0aa2b48d..45582f19214 100644
--- a/lib/gitlab/kas.rb
+++ b/lib/gitlab/kas.rb
@@ -5,6 +5,7 @@ module Gitlab
     INTERNAL_API_REQUEST_HEADER = 'Gitlab-Kas-Api-Request'
     VERSION_FILE = 'GITLAB_KAS_VERSION'
     JWT_ISSUER = 'gitlab-kas'
+    K8S_PROXY_PATH = 'k8s-proxy'
 
     include JwtAuthenticatable
 
@@ -39,6 +40,12 @@ module Gitlab
         Gitlab.config.gitlab_kas.external_url
       end
 
+      def tunnel_url
+        uri = URI.join(external_url, K8S_PROXY_PATH)
+        uri.scheme = uri.scheme.in?(%w(grpcs wss)) ? 'https' : 'http'
+        uri.to_s
+      end
+
       # Return GitLab KAS internal_url
       #
       # @return [String] internal_url
diff --git a/lib/gitlab/kubernetes/default_namespace.rb b/lib/gitlab/kubernetes/default_namespace.rb
index c95362b024b..c22c2fe394d 100644
--- a/lib/gitlab/kubernetes/default_namespace.rb
+++ b/lib/gitlab/kubernetes/default_namespace.rb
@@ -36,14 +36,17 @@ module Gitlab
         end
       end
 
-      def default_project_namespace(slug)
-        namespace_slug = "#{project.path}-#{project.id}".downcase
-
-        if cluster.namespace_per_environment?
-          namespace_slug += "-#{slug}"
-        end
+      def default_project_namespace(environment_slug)
+        maybe_environment_suffix = cluster.namespace_per_environment? ? "-#{environment_slug}" : ''
+        suffix = "-#{project.id}#{maybe_environment_suffix}"
+        namespace = project_path_slug(63 - suffix.length) + suffix
+        Gitlab::NamespaceSanitizer.sanitize(namespace)
+      end
 
-        Gitlab::NamespaceSanitizer.sanitize(namespace_slug)
+      def project_path_slug(max_length)
+        Gitlab::NamespaceSanitizer
+          .sanitize(project.path.downcase)
+          .first(max_length)
       end
 
       ##
diff --git a/lib/gitlab/kubernetes/kubeconfig/entry/cluster.rb b/lib/gitlab/kubernetes/kubeconfig/entry/cluster.rb
new file mode 100644
index 00000000000..836517d4e1f
--- /dev/null
+++ b/lib/gitlab/kubernetes/kubeconfig/entry/cluster.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    module Kubeconfig
+      module Entry
+        class Cluster
+          attr_reader :name
+
+          def initialize(name:, url:, ca_pem: nil)
+            @name = name
+            @url = url
+            @ca_pem = ca_pem
+          end
+
+          def to_h
+            {
+              name: name,
+              cluster: cluster
+            }
+          end
+
+          private
+
+          attr_reader :url, :ca_pem
+
+          def cluster
+            {
+              server: url,
+              'certificate-authority-data': certificate_authority_data
+            }.compact
+          end
+
+          def certificate_authority_data
+            return unless ca_pem.present?
+
+            Base64.strict_encode64(ca_pem)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/kubernetes/kubeconfig/entry/context.rb b/lib/gitlab/kubernetes/kubeconfig/entry/context.rb
new file mode 100644
index 00000000000..8ff17ab9cff
--- /dev/null
+++ b/lib/gitlab/kubernetes/kubeconfig/entry/context.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    module Kubeconfig
+      module Entry
+        class Context
+          attr_reader :name
+
+          def initialize(name:, cluster:, user:, namespace: nil)
+            @name = name
+            @cluster = cluster
+            @user = user
+            @namespace = namespace
+          end
+
+          def to_h
+            {
+              name: name,
+              context: context
+            }
+          end
+
+          private
+
+          attr_reader :cluster, :user, :namespace
+
+          def context
+            {
+              cluster: cluster,
+              namespace: namespace,
+              user: user
+            }.compact
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/kubernetes/kubeconfig/entry/user.rb b/lib/gitlab/kubernetes/kubeconfig/entry/user.rb
new file mode 100644
index 00000000000..784f6d67802
--- /dev/null
+++ b/lib/gitlab/kubernetes/kubeconfig/entry/user.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    module Kubeconfig
+      module Entry
+        class User
+          attr_reader :name
+
+          def initialize(name:, token:)
+            @name = name
+            @token = token
+          end
+
+          def to_h
+            {
+              name: name,
+              user: { token: token }
+            }
+          end
+
+          private
+
+          attr_reader :token
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/kubernetes/kubeconfig/template.rb b/lib/gitlab/kubernetes/kubeconfig/template.rb
new file mode 100644
index 00000000000..da0861ee86a
--- /dev/null
+++ b/lib/gitlab/kubernetes/kubeconfig/template.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    module Kubeconfig
+      class Template
+        ENTRIES = {
+          cluster: Gitlab::Kubernetes::Kubeconfig::Entry::Cluster,
+          user: Gitlab::Kubernetes::Kubeconfig::Entry::User,
+          context: Gitlab::Kubernetes::Kubeconfig::Entry::Context
+        }.freeze
+
+        def initialize
+          @clusters = []
+          @users = []
+          @contexts = []
+        end
+
+        def valid?
+          contexts.present?
+        end
+
+        def add_cluster(**args)
+          clusters << new_entry(:cluster, **args)
+        end
+
+        def add_user(**args)
+          users << new_entry(:user, **args)
+        end
+
+        def add_context(**args)
+          contexts << new_entry(:context, **args)
+        end
+
+        def to_h
+          {
+            apiVersion: 'v1',
+            kind: 'Config',
+            clusters: clusters.map(&:to_h),
+            users: users.map(&:to_h),
+            contexts: contexts.map(&:to_h)
+          }
+        end
+
+        def to_yaml
+          YAML.dump(to_h.deep_stringify_keys)
+        end
+
+        private
+
+        attr_reader :clusters, :users, :contexts
+
+        def new_entry(entry, **args)
+          ENTRIES.fetch(entry).new(**args)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/language_detection.rb b/lib/gitlab/language_detection.rb
index 1e5edb79f10..fc9fb5caa09 100644
--- a/lib/gitlab/language_detection.rb
+++ b/lib/gitlab/language_detection.rb
@@ -18,7 +18,7 @@ module Gitlab
     end
 
     # Newly detected languages, returned in a structure accepted by
-    # Gitlab::Database.bulk_insert
+    # Gitlab::Database.main.bulk_insert
     def insertions(programming_languages)
       lang_to_id = programming_languages.to_h { |p| [p.name, p.id] }
 
diff --git a/lib/gitlab/markdown_cache.rb b/lib/gitlab/markdown_cache.rb
index d0702190ac0..db39904ea23 100644
--- a/lib/gitlab/markdown_cache.rb
+++ b/lib/gitlab/markdown_cache.rb
@@ -2,10 +2,14 @@
 
 module Gitlab
   module MarkdownCache
-    # Increment this number every time the renderer changes its output.
+    # Increment this number to invalidate cached HTML from Markdown documents.
+    # Even when reverting an MR, we should increment this because we only
+    # persist the cache when the new version is higher.
+    #
     # Changing this value puts strain on the database, as every row with
-    # cached markdown needs to be updated. As a result, this line should
-    # not be changed.
+    # cached markdown needs to be updated. As a result, avoid changing
+    # this if the change to the renderer output is a new feature or a
+    # minor bug fix.
     # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330313
     CACHE_COMMONMARK_VERSION        = 28
     CACHE_COMMONMARK_VERSION_START  = 10
diff --git a/lib/gitlab/markdown_cache/active_record/extension.rb b/lib/gitlab/markdown_cache/active_record/extension.rb
index 1de890c84f9..e21268c5fff 100644
--- a/lib/gitlab/markdown_cache/active_record/extension.rb
+++ b/lib/gitlab/markdown_cache/active_record/extension.rb
@@ -10,7 +10,9 @@ module Gitlab
           # Using before_update here conflicts with elasticsearch-model somehow
           before_create :refresh_markdown_cache, if: :invalidated_markdown_cache?
           before_update :refresh_markdown_cache, if: :invalidated_markdown_cache?
-          after_save :store_mentions!, if: :mentionable_attributes_changed?
+          # The import case needs to be fixed to avoid large number of
+          # SQL queries: https://gitlab.com/gitlab-org/gitlab/-/issues/21801
+          after_save :store_mentions!, if: :mentionable_attributes_changed?, unless: ->(obj) { obj.is_a?(Importable) && obj.importing? }
         end
 
         # Always exclude _html fields from attributes (including serialization).
@@ -37,6 +39,7 @@ module Gitlab
 
         def save_markdown(updates)
           return unless persisted? && Gitlab::Database.read_write?
+          return if cached_markdown_version.to_i < cached_markdown_version_in_database.to_i
 
           update_columns(updates)
         end
diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb
index b99261b5c4d..6ba336d37cd 100644
--- a/lib/gitlab/metrics/requests_rack_middleware.rb
+++ b/lib/gitlab/metrics/requests_rack_middleware.rb
@@ -13,7 +13,7 @@ module Gitlab
         "put" => %w(200 202 204 400 401 403 404 405 406 409 410 422 500)
       }.freeze
 
-      HEALTH_ENDPOINT = /^\/-\/(liveness|readiness|health|metrics)\/?$/.freeze
+      HEALTH_ENDPOINT = %r{^/-/(liveness|readiness|health|metrics)/?$}.freeze
 
       FEATURE_CATEGORY_DEFAULT = 'unknown'
 
@@ -66,28 +66,28 @@ module Gitlab
       def call(env)
         method = env['REQUEST_METHOD'].downcase
         method = 'INVALID' unless HTTP_METHODS.key?(method)
-        started = Gitlab::Metrics::System.monotonic_time
+        started = ::Gitlab::Metrics::System.monotonic_time
         health_endpoint = health_endpoint?(env['PATH_INFO'])
         status = 'undefined'
 
         begin
           status, headers, body = @app.call(env)
 
-          elapsed = Gitlab::Metrics::System.monotonic_time - started
+          elapsed = ::Gitlab::Metrics::System.monotonic_time - started
 
-          if !health_endpoint && Gitlab::Metrics.record_duration_for_status?(status)
-            RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method }, elapsed)
+          if !health_endpoint && ::Gitlab::Metrics.record_duration_for_status?(status)
+            self.class.http_request_duration_seconds.observe({ method: method }, elapsed)
           end
 
           [status, headers, body]
         rescue StandardError
-          RequestsRackMiddleware.rack_uncaught_errors_count.increment
+          self.class.rack_uncaught_errors_count.increment
           raise
         ensure
           if health_endpoint
-            RequestsRackMiddleware.http_health_requests_total.increment(status: status.to_s, method: method)
+            self.class.http_health_requests_total.increment(status: status.to_s, method: method)
           else
-            RequestsRackMiddleware.http_requests_total.increment(
+            self.class.http_requests_total.increment(
               status: status.to_s,
               method: method,
               feature_category: feature_category.presence || FEATURE_CATEGORY_DEFAULT
diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb
index 258aa93be38..52d80c3c27e 100644
--- a/lib/gitlab/metrics/samplers/base_sampler.rb
+++ b/lib/gitlab/metrics/samplers/base_sampler.rb
@@ -23,7 +23,7 @@ module Gitlab
         def safe_sample
           sample
         rescue StandardError => e
-          Gitlab::AppLogger.warn("#{self.class}: #{e}, stopping")
+          ::Gitlab::AppLogger.warn("#{self.class}: #{e}, stopping")
           stop
         end
 
diff --git a/lib/gitlab/metrics/subscribers/action_cable.rb b/lib/gitlab/metrics/subscribers/action_cable.rb
index 631b9209f14..9f955dfe79f 100644
--- a/lib/gitlab/metrics/subscribers/action_cable.rb
+++ b/lib/gitlab/metrics/subscribers/action_cable.rb
@@ -28,7 +28,7 @@ module Gitlab
           if event.payload.present?
             channel = event.payload[:channel_class]
             operation = operation_name_from(event.payload)
-            data_size = ::ActiveSupport::JSON.encode(event.payload[:data]).bytesize
+            data_size = Gitlab::Json.generate(event.payload[:data]).bytesize
 
             transmitted_bytes_histogram.observe({ channel: channel, operation: operation }, data_size)
           end
diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb
index e1f1f37c905..fa129025bfe 100644
--- a/lib/gitlab/metrics/subscribers/action_view.rb
+++ b/lib/gitlab/metrics/subscribers/action_view.rb
@@ -40,7 +40,7 @@ module Gitlab
         end
 
         def current_transaction
-          Transaction.current
+          ::Gitlab::Metrics::Transaction.current
         end
       end
     end
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
index a8fcad9ff9f..59b2f88041f 100644
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -8,17 +8,15 @@ module Gitlab
         attach_to :active_record
 
         IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze
-        DB_COUNTERS = %i{db_count db_write_count db_cached_count}.freeze
-        SQL_COMMANDS_WITH_COMMENTS_REGEX = /\A(\/\*.*\*\/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$/i.freeze
+        DB_COUNTERS = %i{count write_count cached_count}.freeze
+        SQL_COMMANDS_WITH_COMMENTS_REGEX = %r{\A(/\*.*\*/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$}i.freeze
 
         SQL_DURATION_BUCKET = [0.05, 0.1, 0.25].freeze
         TRANSACTION_DURATION_BUCKET = [0.1, 0.25, 1].freeze
 
-        DB_LOAD_BALANCING_COUNTERS = %i{
-          db_replica_count db_replica_cached_count db_replica_wal_count db_replica_wal_cached_count
-          db_primary_count db_primary_cached_count db_primary_wal_count db_primary_wal_cached_count
-        }.freeze
-        DB_LOAD_BALANCING_DURATIONS = %i{db_primary_duration_s db_replica_duration_s}.freeze
+        DB_LOAD_BALANCING_ROLES = %i{replica primary}.freeze
+        DB_LOAD_BALANCING_COUNTERS = %i{count cached_count wal_count wal_cached_count}.freeze
+        DB_LOAD_BALANCING_DURATIONS = %i{duration_s}.freeze
 
         SQL_WAL_LOCATION_REGEX = /(pg_current_wal_insert_lsn\(\)::text|pg_last_wal_replay_lsn\(\)::text)/.freeze
 
@@ -40,9 +38,10 @@ module Gitlab
           payload = event.payload
           return if ignored_query?(payload)
 
-          increment(:db_count)
-          increment(:db_cached_count) if cached_query?(payload)
-          increment(:db_write_count) unless select_sql_command?(payload)
+          db_config_name = db_config_name(event.payload)
+          increment(:count, db_config_name: db_config_name)
+          increment(:cached_count, db_config_name: db_config_name) if cached_query?(payload)
+          increment(:write_count, db_config_name: db_config_name) unless select_sql_command?(payload)
 
           observe(:gitlab_sql_duration_seconds, event) do
             buckets SQL_DURATION_BUCKET
@@ -61,24 +60,17 @@ module Gitlab
           return {} unless Gitlab::SafeRequestStore.active?
 
           {}.tap do |payload|
-            DB_COUNTERS.each do |counter|
-              payload[counter] = Gitlab::SafeRequestStore[counter].to_i
+            db_counter_keys.each do |key|
+              payload[key] = Gitlab::SafeRequestStore[key].to_i
             end
 
             if ::Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable?
-              DB_LOAD_BALANCING_COUNTERS.each do |counter|
+              load_balancing_metric_counter_keys.each do |counter|
                 payload[counter] = ::Gitlab::SafeRequestStore[counter].to_i
               end
-              DB_LOAD_BALANCING_DURATIONS.each do |duration|
-                payload[duration] = ::Gitlab::SafeRequestStore[duration].to_f.round(3)
-              end
 
-              if Feature.enabled?(:multiple_database_metrics, default_enabled: :yaml)
-                ::Gitlab::SafeRequestStore[:duration_by_database]&.each do |dbname, duration_by_role|
-                  duration_by_role.each do |db_role, duration|
-                    payload[:"db_#{db_role}_#{dbname}_duration_s"] = duration.to_f.round(3)
-                  end
-                end
+              load_balancing_metric_duration_keys.each do |duration|
+                payload[duration] = ::Gitlab::SafeRequestStore[duration].to_f.round(3)
               end
             end
           end
@@ -92,12 +84,15 @@ module Gitlab
 
         def increment_db_role_counters(db_role, payload)
           cached = cached_query?(payload)
-          increment("db_#{db_role}_count".to_sym)
-          increment("db_#{db_role}_cached_count".to_sym) if cached
+
+          db_config_name = db_config_name(payload)
+
+          increment(:count, db_role: db_role, db_config_name: db_config_name)
+          increment(:cached_count, db_role: db_role, db_config_name: db_config_name) if cached
 
           if wal_command?(payload)
-            increment("db_#{db_role}_wal_count".to_sym)
-            increment("db_#{db_role}_wal_cached_count".to_sym) if cached
+            increment(:wal_count, db_role: db_role, db_config_name: db_config_name)
+            increment(:wal_cached_count, db_role: db_role, db_config_name: db_config_name) if cached
           end
         end
 
@@ -109,15 +104,13 @@ module Gitlab
           return unless ::Gitlab::SafeRequestStore.active?
 
           duration = event.duration / 1000.0
-          duration_key = "db_#{db_role}_duration_s".to_sym
+          duration_key = compose_metric_key(:duration_s, db_role)
           ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration
 
           # Per database metrics
-          dbname = ::Gitlab::Database.dbname(event.payload[:connection])
-          ::Gitlab::SafeRequestStore[:duration_by_database] ||= {}
-          ::Gitlab::SafeRequestStore[:duration_by_database][dbname] ||= {}
-          ::Gitlab::SafeRequestStore[:duration_by_database][dbname][db_role] ||= 0
-          ::Gitlab::SafeRequestStore[:duration_by_database][dbname][db_role] += duration
+          db_config_name = db_config_name(event.payload)
+          duration_key = compose_metric_key(:duration_s, db_role, db_config_name)
+          ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration
         end
 
         def ignored_query?(payload)
@@ -132,19 +125,84 @@ module Gitlab
           payload[:sql].match(SQL_COMMANDS_WITH_COMMENTS_REGEX)
         end
 
-        def increment(counter)
-          current_transaction&.increment("gitlab_transaction_#{counter}_total".to_sym, 1)
+        def increment(counter, db_config_name:, db_role: nil)
+          log_key = compose_metric_key(counter, db_role)
 
-          Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1
+          prometheus_key = if db_role
+                             :"gitlab_transaction_db_#{db_role}_#{counter}_total"
+                           else
+                             :"gitlab_transaction_db_#{counter}_total"
+                           end
+
+          if ENV['GITLAB_MULTIPLE_DATABASE_METRICS']
+            current_transaction&.increment(prometheus_key, 1, { db_config_name: db_config_name })
+          else
+            current_transaction&.increment(prometheus_key, 1)
+          end
+
+          Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1
+
+          # To avoid confusing log keys we only log the db_config_name metrics
+          # when we are also logging the db_role. Otherwise it will be hard to
+          # tell if the log key is referring to a db_role OR a db_config_name.
+          if db_role.present? && db_config_name.present?
+            log_key = compose_metric_key(counter, db_role, db_config_name)
+            Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1
+          end
         end
 
         def observe(histogram, event, &block)
-          current_transaction&.observe(histogram, event.duration / 1000.0, &block)
+          db_config_name = db_config_name(event.payload)
+
+          if ENV['GITLAB_MULTIPLE_DATABASE_METRICS']
+            current_transaction&.observe(histogram, event.duration / 1000.0, { db_config_name: db_config_name }, &block)
+          else
+            current_transaction&.observe(histogram, event.duration / 1000.0, &block)
+          end
         end
 
         def current_transaction
           ::Gitlab::Metrics::WebTransaction.current || ::Gitlab::Metrics::BackgroundTransaction.current
         end
+
+        def db_config_name(payload)
+          ::Gitlab::Database.db_config_name(payload[:connection])
+        end
+
+        def self.db_counter_keys
+          DB_COUNTERS.map { |c| compose_metric_key(c) }
+        end
+
+        def self.load_balancing_metric_counter_keys
+          load_balancing_metric_keys(DB_LOAD_BALANCING_COUNTERS)
+        end
+
+        def self.load_balancing_metric_duration_keys
+          load_balancing_metric_keys(DB_LOAD_BALANCING_DURATIONS)
+        end
+
+        def self.load_balancing_metric_keys(metrics)
+          [].tap do |counters|
+            DB_LOAD_BALANCING_ROLES.each do |role|
+              metrics.each do |metric|
+                counters << compose_metric_key(metric, role)
+                next unless ENV['GITLAB_MULTIPLE_DATABASE_METRICS']
+
+                ::Gitlab::Database.db_config_names.each do |config_name|
+                  counters << compose_metric_key(metric, role, config_name)
+                end
+              end
+            end
+          end
+        end
+
+        def compose_metric_key(metric, db_role = nil, db_config_name = nil)
+          self.class.compose_metric_key(metric, db_role, db_config_name)
+        end
+
+        def self.compose_metric_key(metric, db_role = nil, db_config_name = nil)
+          [:db, db_role, db_config_name, metric].compact.join("_").to_sym
+        end
       end
     end
   end
diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb
index b274d2b1079..45344e79796 100644
--- a/lib/gitlab/metrics/subscribers/rails_cache.rb
+++ b/lib/gitlab/metrics/subscribers/rails_cache.rb
@@ -65,7 +65,7 @@ module Gitlab
         private
 
         def current_transaction
-          Transaction.current
+          ::Gitlab::Metrics::Transaction.current
         end
 
         def metric_cache_operation_duration_seconds
diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb
index 4b65bbcc791..a1a0356ff58 100644
--- a/lib/gitlab/middleware/go.rb
+++ b/lib/gitlab/middleware/go.rb
@@ -127,23 +127,25 @@ module Gitlab
 
       def project_for_paths(paths, request)
         project = Project.where_full_path_in(paths).first
-        return unless Ability.allowed?(current_user(request, project), :read_project, project)
+
+        return unless authentication_result(request, project).can_perform_action_on_project?(:read_project, project)
 
         project
       end
 
-      def current_user(request, project)
-        return unless has_basic_credentials?(request)
+      def authentication_result(request, project)
+        empty_result = Gitlab::Auth::Result::EMPTY
+        return empty_result unless has_basic_credentials?(request)
 
         login, password = user_name_and_password(request)
         auth_result = Gitlab::Auth.find_for_git_client(login, password, project: project, ip: request.ip)
-        return unless auth_result.success?
+        return empty_result unless auth_result.success?
 
-        return unless auth_result.actor&.can?(:access_git)
+        return empty_result unless auth_result.can?(:access_git)
 
-        return unless auth_result.authentication_abilities.include?(:read_project)
+        return empty_result unless auth_result.authentication_abilities_include?(:read_project)
 
-        auth_result.actor
+        auth_result
       end
     end
   end
diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb
index 329041e3ba2..30b3fe3d893 100644
--- a/lib/gitlab/middleware/multipart.rb
+++ b/lib/gitlab/middleware/multipart.rb
@@ -177,7 +177,7 @@ module Gitlab
           @app.call(env)
         end
       rescue UploadedFile::InvalidPathError => e
-        [400, { 'Content-Type' => 'text/plain' }, e.message]
+        [400, { 'Content-Type' => 'text/plain' }, [e.message]]
       end
     end
   end
diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb
index b29240985f1..b5e304599ab 100644
--- a/lib/gitlab/optimistic_locking.rb
+++ b/lib/gitlab/optimistic_locking.rb
@@ -11,7 +11,7 @@ module Gitlab
       retry_attempts = 0
 
       begin
-        ActiveRecord::Base.transaction do
+        ActiveRecord::Base.transaction do # rubocop: disable Database/MultipleDatabases
           yield(subject)
         end
       rescue ActiveRecord::StaleObjectError
diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb
index b65c8613d00..38618a0ac06 100644
--- a/lib/gitlab/otp_key_rotator.rb
+++ b/lib/gitlab/otp_key_rotator.rb
@@ -36,7 +36,7 @@ module Gitlab
       raise ArgumentError, "New key is too short! Must be 256 bits" if new_key.size < 64
 
       write_csv do |csv|
-        ActiveRecord::Base.transaction do
+        User.transaction do
           User.with_two_factor.in_batches do |relation| # rubocop: disable Cop/InBatches
             rows = relation.pluck(:id, :encrypted_otp_secret, :encrypted_otp_secret_iv, :encrypted_otp_secret_salt)
             rows.each do |row|
@@ -54,7 +54,7 @@ module Gitlab
 
     # rubocop: disable CodeReuse/ActiveRecord
     def rollback!
-      ActiveRecord::Base.transaction do
+      User.transaction do
         CSV.foreach(filename, headers: HEADERS, return_headers: false) do |row|
           User.where(id: row['user_id']).update_all(encrypted_otp_secret: row['old_value'])
         end
diff --git a/lib/gitlab/pagination/keyset/column_condition_builder.rb b/lib/gitlab/pagination/keyset/column_condition_builder.rb
new file mode 100644
index 00000000000..ca436000abe
--- /dev/null
+++ b/lib/gitlab/pagination/keyset/column_condition_builder.rb
@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Pagination
+    module Keyset
+      class ColumnConditionBuilder
+        # This class builds the WHERE conditions for the keyset pagination library.
+        # It produces WHERE conditions for one column at a time.
+        #
+        # Requisite 1: Only the last column (columns.last) is non-nullable and distinct.
+        # Requisite 2: Only one column is distinct and non-nullable.
+        #
+        # Scenario: We want to order by columns named X, Y and Z and build the conditions
+        #          used in the WHERE clause of a pagination query using a set of cursor values.
+        #   X is the column definition for a nullable column
+        #   Y is the column definition for a non-nullable but not distinct column
+        #   Z is the column definition for a distinct, non-nullable column used as a tie breaker.
+        #
+        #   Then the method is initially invoked with these arguments:
+        #     columns = [ColumnDefinition for X, ColumnDefinition for Y, ColumnDefinition for Z]
+        #     values = { X: x, Y: y, Z: z } => these represent cursor values for pagination
+        #                                      (x could be nil since X is nullable)
+        #     current_conditions is initialized to [] to store the result during the iteration calls
+        #     invoked within the Order#build_where_values method.
+        #
+        #   The elements of current_conditions are instances of Arel::Nodes and -
+        #    will be concatenated using OR or UNION to be used in the WHERE clause.
+        #
+        #   Example: Let's say we want to build WHERE clause conditions for
+        #     ORDER BY X DESC NULLS LAST, Y ASC, Z DESC
+        #
+        #     Iteration 1:
+        #       columns = [X, Y, Z]
+        #       At the end, current_conditions should be:
+        #         [(Z < z)]
+        #
+        #     Iteration 2:
+        #       columns = [X, Y]
+        #       At the end, current_conditions should be:
+        #         [(Y > y) OR (Y = y AND Z < z)]
+        #
+        #     Iteration 3:
+        #       columns = [X]
+        #       At the end, current_conditions should be:
+        #         [((X IS NOT NULL AND Y > y) OR (X IS NOT NULL AND Y = y AND Z < z))
+        #           OR
+        #          ((x IS NULL) OR (X IS NULL))]
+        #
+        # Parameters:
+        #
+        #  - columns: instance of ColumnOrderDefinition
+        #  - value: cursor value for the column
+        def initialize(column, value)
+          @column = column
+          @value = value
+        end
+
+        def where_conditions(current_conditions)
+          return not_nullable_conditions(current_conditions) if column.not_nullable?
+          return nulls_first_conditions(current_conditions) if column.nulls_first?
+
+          # Here we are dealing with the case of column_definition.nulls_last?
+          # Suppose ORDER BY X DESC NULLS FIRST, Y ASC, Z DESC is the ordering clause
+          # and we already have built the conditions for columns Y and Z.
+          #
+          # We first need a set of conditions to use when x (the value for X) is NULL:
+          #   null_conds = [
+          #     (x IS NULL AND X IS NULL AND Y<y),
+          #     (x IS NULL AND X IS NULL AND Y=y AND Z<z),
+          null_conds = current_conditions.map do |conditional|
+            Arel::Nodes::And.new([value_is_null, column_is_null, conditional])
+          end
+
+          # We then need a set of conditions to use when m has an actual value:
+          #   non_null_conds = [
+          #     (x IS NOT NULL AND X IS NULL),
+          #     (x IS NOT NULL AND X < x)
+          #     (x IS NOT NULL AND X = x AND Y > y),
+          #     (x IS NOT NULL AND X = x AND Y = y AND Z < z),
+          tie_breaking_conds = current_conditions.map do |conditional|
+            Arel::Nodes::And.new([column_equals_to_value, conditional])
+          end
+
+          non_null_conds = [column_is_null, compare_column_with_value, *tie_breaking_conds].map do |conditional|
+            Arel::Nodes::And.new([value_is_not_null, conditional])
+          end
+
+          [*null_conds, *non_null_conds]
+        end
+
+        private
+
+        # WHEN THE COLUMN IS NON-NULLABLE AND DISTINCT
+        #   Per Assumption 1, only the last column can be non-nullable and distinct
+        #   (column Z is non-nullable/distinct and comes last in the example).
+        #   So the Order#build_where_conditions is being called for the first time with current_conditions = [].
+        #
+        #   At the end of the call, we should expect:
+        #     current_conditions should be [(Z < z)]
+        #
+        # WHEN THE COLUMN IS NON-NULLABLE BUT NOT DISTINCT
+        #   Let's say Z has been processed and we are about to process the column Y next.
+        #   (per requisite 1, if a non-nullable but not distinct column is being processed,
+        #    at the least, the conditional for the non-nullable/distinct column exists)
+        #
+        #   At the start of the method call:
+        #     current_conditions = [(Z < z)]
+        #     comparison_node = (Y < y)
+        #     eqaulity_node = (Y = y)
+        #
+        #   We should add a comparison node for the next column Y, (Y < y)
+        #    then break a tie using the previous conditionals, (Y = y AND Z < z)
+        #
+        #   At the end of the call, we should expect:
+        #     current_conditions = [(Y < y), (Y = y AND Z < z)]
+        def not_nullable_conditions(current_conditions)
+          tie_break_conds = current_conditions.map do |conditional|
+            Arel::Nodes::And.new([column_equals_to_value, conditional])
+          end
+
+          [compare_column_with_value, *tie_break_conds]
+        end
+
+        def nulls_first_conditions(current_conditions)
+          # Using the same scenario described earlier,
+          # suppose the ordering clause is ORDER BY X DESC NULLS FIRST, Y ASC, Z DESC
+          # and we have built the conditions for columns Y and Z in previous iterations:
+          #
+          #   current_conditions = [(Y > y), (Y = y AND Z < z)]
+          #
+          # In this branch of the iteration,
+          # we first need a set of conditions to use when m (the value for M) is NULL:
+          #   null_conds = [
+          #     (x IS NULL AND X IS NULL AND Y > y),
+          #     (x IS NULL AND X IS NULL AND Y = y AND Z < z),
+          #     (x IS NULL AND X IS NOT NULL)]
+          #
+          # Note that when x has an actual value, say x = 3, null_conds evalutes to FALSE.
+          tie_breaking_conds = current_conditions.map do |conditional|
+            Arel::Nodes::And.new([column_is_null, conditional])
+          end
+
+          null_conds = [*tie_breaking_conds, column_is_not_null].map do |conditional|
+            Arel::Nodes::And.new([value_is_null, conditional])
+          end
+
+          # We then need a set of conditions to use when m has an actual value:
+          #   non_null_conds = [
+          #     (x IS NOT NULL AND X < x),
+          #     (x IS NOT NULL AND X = x AND Y > y),
+          #     (x IS NOT NULL AND X = x AND Y = y AND Z < z)]
+          #
+          # Note again that when x IS NULL, non_null_conds evaluates to FALSE.
+          tie_breaking_conds = current_conditions.map do |conditional|
+            Arel::Nodes::And.new([column_equals_to_value, conditional])
+          end
+
+          # The combined OR condition (null_where_cond OR non_null_where_cond) will return a correct result -
+          # without having to account for whether x is nil or an actual value at the application level.
+          non_null_conds = [compare_column_with_value, *tie_breaking_conds].map do |conditional|
+            Arel::Nodes::And.new([value_is_not_null, conditional])
+          end
+
+          [*null_conds, *non_null_conds]
+        end
+
+        def column_equals_to_value
+          @equality_node ||= column.column_expression.eq(value)
+        end
+
+        def column_is_null
+          @column_is_null ||= column.column_expression.eq(nil)
+        end
+
+        def column_is_not_null
+          @column_is_not_null ||= column.column_expression.not_eq(nil)
+        end
+
+        def value_is_null
+          @value_is_null ||= build_quoted_value.eq(nil)
+        end
+
+        def value_is_not_null
+          @value_is_not_null ||= build_quoted_value.not_eq(nil)
+        end
+
+        def compare_column_with_value
+          if column.descending_order?
+            column.column_expression.lt(value)
+          else
+            column.column_expression.gt(value)
+          end
+        end
+
+        # Turns the given value to an SQL literal by casting it to the proper format.
+        def build_quoted_value
+          return value if value.instance_of?(Arel::Nodes::SqlLiteral)
+
+          Arel::Nodes.build_quoted(value, column.column_expression)
+        end
+
+        attr_reader :column, :value
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb
index 19d44ee69dd..ccfa9334a12 100644
--- a/lib/gitlab/pagination/keyset/order.rb
+++ b/lib/gitlab/pagination/keyset/order.rb
@@ -141,24 +141,10 @@ module Gitlab
 
           return use_composite_row_comparison(values) if composite_row_comparison_possible?
 
-          where_values = []
-
-          reversed_column_definitions = column_definitions.reverse
-          reversed_column_definitions.each_with_index do |column_definition, i|
-            value = values[column_definition.attribute_name]
-
-            conditions_for_column(column_definition, value).each do |condition|
-              column_definitions_after_index = reversed_column_definitions.last(column_definitions.reverse.size - i - 1)
-
-              equal_conditon_for_rest = column_definitions_after_index.map do |definition|
-                definition.column_expression.eq(values[definition.attribute_name])
-              end
-
-              where_values << Arel::Nodes::Grouping.new(Arel::Nodes::And.new([condition, *equal_conditon_for_rest].compact))
-            end
-          end
-
-          where_values
+          column_definitions
+            .map { ColumnConditionBuilder.new(_1, values[_1.attribute_name]) }
+            .reverse
+            .reduce([]) { |where_conditions, column| column.where_conditions(where_conditions) }
         end
 
         def where_values_with_or_query(values)
@@ -222,32 +208,6 @@ module Gitlab
           scope
         end
 
-        def conditions_for_column(column_definition, value)
-          conditions = []
-          # Depending on the order, build a query condition fragment for taking the next rows
-          if column_definition.distinct? || (!column_definition.distinct? && value.present?)
-            conditions << compare_column_with_value(column_definition, value)
-          end
-
-          # When the column is nullable, additional conditions for NULL a NOT NULL values are necessary.
-          # This depends on the position of the nulls (top or bottom of the resultset).
-          if column_definition.nulls_first? && value.blank?
-            conditions << column_definition.column_expression.not_eq(nil)
-          elsif column_definition.nulls_last? && value.present?
-            conditions << column_definition.column_expression.eq(nil)
-          end
-
-          conditions
-        end
-
-        def compare_column_with_value(column_definition, value)
-          if column_definition.descending_order?
-            column_definition.column_expression.lt(value)
-          else
-            column_definition.column_expression.gt(value)
-          end
-        end
-
         def build_or_query(expressions)
           return [] if expressions.blank?
 
diff --git a/lib/gitlab/pagination/keyset/simple_order_builder.rb b/lib/gitlab/pagination/keyset/simple_order_builder.rb
index 76d6bbadaa4..5e79910a3e9 100644
--- a/lib/gitlab/pagination/keyset/simple_order_builder.rb
+++ b/lib/gitlab/pagination/keyset/simple_order_builder.rb
@@ -122,6 +122,7 @@ module Gitlab
 
           return unless attribute
           return unless tie_breaker_attribute
+          return unless attribute.respond_to?(:name)
 
           model_class.column_names.include?(attribute.name.to_s) &&
             arel_table[primary_key].to_s == tie_breaker_attribute.to_s
diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb
index 5c9b029a107..b2179d80a18 100644
--- a/lib/gitlab/profiler.rb
+++ b/lib/gitlab/profiler.rb
@@ -119,18 +119,18 @@ module Gitlab
 
     def self.with_custom_logger(logger)
       original_colorize_logging = ActiveSupport::LogSubscriber.colorize_logging
-      original_activerecord_logger = ActiveRecord::Base.logger
+      original_activerecord_logger = ApplicationRecord.logger
       original_actioncontroller_logger = ActionController::Base.logger
 
       if logger
         ActiveSupport::LogSubscriber.colorize_logging = false
-        ActiveRecord::Base.logger = logger
+        ApplicationRecord.logger = logger
         ActionController::Base.logger = logger
       end
 
       yield.tap do
         ActiveSupport::LogSubscriber.colorize_logging = original_colorize_logging
-        ActiveRecord::Base.logger = original_activerecord_logger
+        ApplicationRecord.logger = original_activerecord_logger
         ActionController::Base.logger = original_actioncontroller_logger
       end
     end
diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index e52023c4612..fb9447f9665 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -8,7 +8,8 @@ module Gitlab
       @project = project
       @repository_ref = repository_ref.presence
 
-      super(current_user, query, [project], order_by: order_by, sort: sort, filters: filters)
+      # use the default filter for project searches since we are already limiting by a single project
+      super(current_user, query, [project], order_by: order_by, sort: sort, filters: filters, default_project_filter: true)
     end
 
     def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, preload_method: nil)
@@ -24,7 +25,7 @@ module Gitlab
       when 'users'
         users.page(page).per(per_page)
       else
-        super(scope, page: page, per_page: per_page, without_count: false)
+        super(scope, page: page, per_page: per_page, without_count: true)
       end
     end
 
diff --git a/lib/gitlab/query_limiting/active_support_subscriber.rb b/lib/gitlab/query_limiting/active_support_subscriber.rb
index 138fae7b641..4bfd526914b 100644
--- a/lib/gitlab/query_limiting/active_support_subscriber.rb
+++ b/lib/gitlab/query_limiting/active_support_subscriber.rb
@@ -6,10 +6,10 @@ module Gitlab
       attach_to :active_record
 
       def sql(event)
-        return if !Transaction.current || event.payload.fetch(:cached, event.payload[:name] == 'CACHE')
+        return if !::Gitlab::QueryLimiting::Transaction.current || event.payload.fetch(:cached, event.payload[:name] == 'CACHE')
 
-        Transaction.current.increment
-        Transaction.current.executed_sql(event.payload[:sql])
+        ::Gitlab::QueryLimiting::Transaction.current.increment
+        ::Gitlab::QueryLimiting::Transaction.current.executed_sql(event.payload[:sql])
       end
     end
   end
diff --git a/lib/gitlab/query_limiting/middleware.rb b/lib/gitlab/query_limiting/middleware.rb
index 714fe42884a..76de547b14f 100644
--- a/lib/gitlab/query_limiting/middleware.rb
+++ b/lib/gitlab/query_limiting/middleware.rb
@@ -13,7 +13,7 @@ module Gitlab
       end
 
       def call(env)
-        transaction, retval = Transaction.run do
+        transaction, retval = ::Gitlab::QueryLimiting::Transaction.run do
           @app.call(env)
         end
 
diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb
index efe07aa8ab2..cf5c9296d8c 100644
--- a/lib/gitlab/quick_actions/issuable_actions.rb
+++ b/lib/gitlab/quick_actions/issuable_actions.rb
@@ -241,8 +241,49 @@ module Gitlab
           "#{comment} #{TABLEFLIP}"
         end
 
+        desc _('Set severity')
+        explanation _('Sets the severity')
+        params '1 / S1 / Critical'
+        types Issue
+        condition do
+          !quick_action_target.persisted? || quick_action_target.supports_severity?
+        end
+        parse_params do |severity|
+          find_severity(severity)
+        end
+        command :severity do |severity|
+          next unless quick_action_target.supports_severity?
+
+          if severity
+            if quick_action_target.persisted?
+              ::Issues::UpdateService.new(project: quick_action_target.project, current_user: current_user, params: { severity: severity }).execute(quick_action_target)
+            else
+              quick_action_target.build_issuable_severity(severity: severity)
+            end
+
+            @execution_message[:severity] = _("Severity updated to %{severity}.") % { severity: severity.capitalize }
+          else
+            @execution_message[:severity] = _('No severity matches the provided parameter')
+          end
+        end
+
         private
 
+        def find_severity(severity_param)
+          return unless severity_param
+
+          severity_param = severity_param.downcase
+          severities = IssuableSeverity::SEVERITY_QUICK_ACTION_PARAMS.values.map { |vals| vals.map(&:downcase) }
+
+          matched_severity = severities.find do |severity_values|
+            severity_values.include?(severity_param)
+          end
+
+          return unless matched_severity
+
+          matched_severity[0]
+        end
+
         def run_label_command(labels:, command:, updates_key:)
           return if labels.empty?
 
diff --git a/lib/gitlab/reactive_cache_set_cache.rb b/lib/gitlab/reactive_cache_set_cache.rb
index 7fee1d0727f..e4a92ed5122 100644
--- a/lib/gitlab/reactive_cache_set_cache.rb
+++ b/lib/gitlab/reactive_cache_set_cache.rb
@@ -10,11 +10,6 @@ module Gitlab
       @expires_in = expires_in
     end
 
-    # NOTE Remove as part of #331319
-    def old_cache_key(key)
-      "#{cache_namespace}:#{key}:set"
-    end
-
     def cache_key(key)
       super(key)
     end
diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb
index bbcc2732e89..3c8ac07215d 100644
--- a/lib/gitlab/redis/wrapper.rb
+++ b/lib/gitlab/redis/wrapper.rb
@@ -8,6 +8,10 @@ require 'active_support/core_ext/hash/keys'
 require 'active_support/core_ext/module/delegation'
 require 'active_support/core_ext/string/inflections'
 
+# Explicitly load Redis::Store::Factory so we can read Redis configuration in
+# TestEnv
+require 'redis/store/factory'
+
 module Gitlab
   module Redis
     class Wrapper
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 0bd2ac180c3..698a417283e 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -184,19 +184,19 @@ module Gitlab
         #   - Must not have a scheme, such as http:// or https://
         #   - Must not have a port number, such as :8080 or :8443
 
-        @go_package_regex ||= /
+        @go_package_regex ||= %r{
           \b (?# word boundary)
           (?<domain>
             [0-9a-z](?:(?:-|[0-9a-z]){0,61}[0-9a-z])? (?# first domain)
             (?:\.[0-9a-z](?:(?:-|[0-9a-z]){0,61}[0-9a-z])?)* (?# inner domains)
             \.[a-z]{2,} (?# top-level domain)
           )
-          (?<path>\/(?:
-            [-\/$_.+!*'(),0-9a-z] (?# plain URL character)
+          (?<path>/(?:
+            [-/$_.+!*'(),0-9a-z] (?# plain URL character)
             | %[0-9a-f]{2})* (?# URL encoded character)
           )? (?# path)
           \b (?# word boundary)
-        /ix.freeze
+        }ix.freeze
       end
 
       def generic_package_version_regex
@@ -416,7 +416,7 @@ module Gitlab
     end
 
     def base64_regex
-      @base64_regex ||= /(?:[A-Za-z0-9+\/]{4})*(?:[A-Za-z0-9+\/]{2}==|[A-Za-z0-9+\/]{3}=)?/.freeze
+      @base64_regex ||= %r{(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?}.freeze
     end
 
     def feature_flag_regex
diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb
index 7de53c4b3ff..3061fb96190 100644
--- a/lib/gitlab/repository_set_cache.rb
+++ b/lib/gitlab/repository_set_cache.rb
@@ -13,11 +13,6 @@ module Gitlab
       @expires_in = expires_in
     end
 
-    # NOTE Remove as part of #331319
-    def old_cache_key(type)
-      "#{type}:#{namespace}:set"
-    end
-
     def cache_key(type)
       super("#{type}:#{namespace}")
     end
diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb
index e6851af8264..90513e346f2 100644
--- a/lib/gitlab/search_results.rb
+++ b/lib/gitlab/search_results.rb
@@ -168,7 +168,7 @@ module Gitlab
       issues = IssuesFinder.new(current_user, issuable_params.merge(finder_params)).execute
 
       unless default_project_filter
-        issues = issues.where(project_id: project_ids_relation) # rubocop: disable CodeReuse/ActiveRecord
+        issues = issues.in_projects(project_ids_relation)
       end
 
       apply_sort(issues, scope: 'issues')
diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb
index 9fc7a44ec99..feb2c3c1d7d 100644
--- a/lib/gitlab/set_cache.rb
+++ b/lib/gitlab/set_cache.rb
@@ -10,11 +10,6 @@ module Gitlab
       @expires_in = expires_in
     end
 
-    # NOTE Remove as part of https://gitlab.com/gitlab-org/gitlab/-/issues/331319
-    def old_cache_key(key)
-      "#{key}:set"
-    end
-
     def cache_key(key)
       "#{cache_namespace}:#{key}:set"
     end
@@ -25,7 +20,6 @@ module Gitlab
 
       with do |redis|
         keys_to_expire = keys.map { |key| cache_key(key) }
-        keys_to_expire += keys.map { |key| old_cache_key(key) } # NOTE Remove as part of #331319
 
         Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
           redis.unlink(*keys_to_expire)
diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb
index 7ed1958a8d0..751405f1045 100644
--- a/lib/gitlab/setup_helper.rb
+++ b/lib/gitlab/setup_helper.rb
@@ -21,7 +21,7 @@ module Gitlab
       end
     rescue Errno::EEXIST
       puts 'Skipping config.toml generation:'
-      puts 'A configuration file already exists.'
+      puts "A configuration file for #{config_path} already exists."
     rescue ArgumentError => e
       puts 'Skipping config.toml generation:'
       puts e.message
@@ -32,7 +32,7 @@ module Gitlab
       extend Gitlab::SetupHelper
       class << self
         def configuration_toml(dir, _, _)
-          config = { redis: { URL: redis_url } }
+          config = { redis: { URL: redis_url, DB: redis_db } }
 
           TomlRB.dump(config)
         end
@@ -41,6 +41,10 @@ module Gitlab
           Gitlab::Redis::SharedState.url
         end
 
+        def redis_db
+          Gitlab::Redis::SharedState.params.fetch(:db, 0)
+        end
+
         def get_config_path(dir, _)
           File.join(dir, 'config_path')
         end
diff --git a/lib/gitlab/sidekiq_cluster/cli.rb b/lib/gitlab/sidekiq_cluster/cli.rb
index e20834fa912..05319ba17a2 100644
--- a/lib/gitlab/sidekiq_cluster/cli.rb
+++ b/lib/gitlab/sidekiq_cluster/cli.rb
@@ -37,6 +37,7 @@ module Gitlab
         @logger.formatter = ::Gitlab::SidekiqLogging::JSONFormatter.new
         @rails_path = Dir.pwd
         @dryrun = false
+        @list_queues = false
       end
 
       def run(argv = ARGV)
@@ -47,6 +48,11 @@ module Gitlab
 
         option_parser.parse!(argv)
 
+        if @dryrun && @list_queues
+          raise CommandError,
+            'The --dryrun and --list-queues options are mutually exclusive'
+        end
+
         worker_metadatas = SidekiqConfig::CliMethods.worker_metadatas(@rails_path)
         worker_queues = SidekiqConfig::CliMethods.worker_queues(@rails_path)
 
@@ -73,6 +79,12 @@ module Gitlab
             'No queues found, you must select at least one queue'
         end
 
+        if @list_queues
+          puts queue_groups.map(&:sort) # rubocop:disable Rails/Output
+
+          return
+        end
+
         unless @dryrun
           @logger.info("Starting cluster with #{queue_groups.length} processes")
         end
@@ -202,6 +214,10 @@ module Gitlab
           opt.on('-d', '--dryrun', 'Print commands that would be run without this flag, and quit') do |int|
             @dryrun = true
           end
+
+          opt.on('--list-queues', 'List matching queues, and quit') do |int|
+            @list_queues = true
+          end
         end
       end
     end
diff --git a/lib/gitlab/sidekiq_config/dummy_worker.rb b/lib/gitlab/sidekiq_config/dummy_worker.rb
index ef0dce0cf84..b7f53da8e00 100644
--- a/lib/gitlab/sidekiq_config/dummy_worker.rb
+++ b/lib/gitlab/sidekiq_config/dummy_worker.rb
@@ -5,7 +5,6 @@ module Gitlab
     # For queues that don't have explicit workers - default and mailers
     class DummyWorker
       ATTRIBUTE_METHODS = {
-        queue: :queue,
         name: :name,
         feature_category: :get_feature_category,
         has_external_dependencies: :worker_has_external_dependencies?,
@@ -20,6 +19,10 @@ module Gitlab
         @attributes = attributes
       end
 
+      def generated_queue_name
+        @attributes[:queue]
+      end
+
       def queue_namespace
         nil
       end
diff --git a/lib/gitlab/sidekiq_config/worker.rb b/lib/gitlab/sidekiq_config/worker.rb
index aea4209f631..a343573440f 100644
--- a/lib/gitlab/sidekiq_config/worker.rb
+++ b/lib/gitlab/sidekiq_config/worker.rb
@@ -6,9 +6,11 @@ module Gitlab
       include Comparable
 
       attr_reader :klass
-      delegate :feature_category_not_owned?, :get_feature_category, :get_sidekiq_options,
-               :get_tags, :get_urgency, :get_weight, :get_worker_resource_boundary,
-               :idempotent?, :queue, :queue_namespace, :worker_has_external_dependencies?,
+
+      delegate :feature_category_not_owned?, :generated_queue_name, :get_feature_category,
+               :get_sidekiq_options, :get_tags, :get_urgency, :get_weight,
+               :get_worker_resource_boundary, :idempotent?, :queue_namespace,
+               :worker_has_external_dependencies?,
                to: :klass
 
       def initialize(klass, ee:)
@@ -35,7 +37,7 @@ module Gitlab
 
       # Put namespaced queues first
       def to_sort
-        [queue_namespace ? 0 : 1, queue]
+        [queue_namespace ? 0 : 1, generated_queue_name]
       end
 
       # YAML representation
@@ -45,7 +47,7 @@ module Gitlab
 
       def to_yaml
         {
-          name: queue,
+          name: generated_queue_name,
           worker_name: klass.name,
           feature_category: get_feature_category,
           has_external_dependencies: worker_has_external_dependencies?,
@@ -62,7 +64,7 @@ module Gitlab
       end
 
       def queue_and_weight
-        [queue, get_weight]
+        [generated_queue_name, get_weight]
       end
 
       def retries
diff --git a/lib/gitlab/signed_tag.rb b/lib/gitlab/signed_tag.rb
new file mode 100644
index 00000000000..3b22cb7622d
--- /dev/null
+++ b/lib/gitlab/signed_tag.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class SignedTag
+    include Gitlab::Utils::StrongMemoize
+
+    def initialize(repository, tag)
+      @repository = repository
+      @tag = tag
+
+      if Feature.enabled?(:get_tag_signatures)
+        @signature_data = Gitlab::Git::Tag.extract_signature_lazily(repository, tag.id) if repository
+      else
+        @signature_data = [signature_text_of_message.b, signed_text_of_message.b]
+      end
+    end
+
+    def signature
+      return unless @tag.has_signature?
+    end
+
+    def signature_text
+      @signature_data&.fetch(0)
+    end
+
+    def signed_text
+      @signature_data&.fetch(1)
+    end
+
+    private
+
+    def signature_text_of_message
+      @tag.message.slice(@tag.message.index("-----BEGIN SIGNED MESSAGE-----")..-1)
+    rescue StandardError
+      nil
+    end
+
+    def signed_text_of_message
+      %{object #{@tag.target_commit.id}
+type commit
+tag #{@tag.name}
+tagger #{@tag.tagger.name} <#{@tag.tagger.email}> #{@tag.tagger.date.seconds} #{@tag.tagger.timezone}
+
+#{@tag.message.gsub(/-----BEGIN SIGNED MESSAGE-----(.*)-----END SIGNED MESSAGE-----/m, "")}}
+    end
+  end
+end
diff --git a/lib/gitlab/slash_commands/presenters/help.rb b/lib/gitlab/slash_commands/presenters/help.rb
index 714ca77c3e5..71bc0dc0123 100644
--- a/lib/gitlab/slash_commands/presenters/help.rb
+++ b/lib/gitlab/slash_commands/presenters/help.rb
@@ -48,7 +48,7 @@ module Gitlab
              *Documentation*
 
              For more information about GitLab chatops, refer to its
-             documentation: https://docs.gitlab.com/ee/ci/chatops/README.html.
+             documentation: https://docs.gitlab.com/ee/ci/chatops/index.html.
           MESSAGE
 
           message
diff --git a/lib/gitlab/sql/glob.rb b/lib/gitlab/sql/glob.rb
index f3421bd95d2..adc6ccbd1b9 100644
--- a/lib/gitlab/sql/glob.rb
+++ b/lib/gitlab/sql/glob.rb
@@ -17,7 +17,7 @@ module Gitlab
       end
 
       def q(string)
-        ActiveRecord::Base.connection.quote(string)
+        ApplicationRecord.connection.quote(string)
       end
     end
   end
diff --git a/lib/gitlab/sql/set_operator.rb b/lib/gitlab/sql/set_operator.rb
index 59a808eafa9..18275da3ef0 100644
--- a/lib/gitlab/sql/set_operator.rb
+++ b/lib/gitlab/sql/set_operator.rb
@@ -19,6 +19,7 @@ module Gitlab
     #     Project.where("id IN (#{sql})")
     class SetOperator
       def initialize(relations, remove_duplicates: true, remove_order: true)
+        verify_select_values!(relations) if Rails.env.test? || Rails.env.development?
         @relations = relations
         @remove_duplicates = remove_duplicates
         @remove_order = remove_order
@@ -33,7 +34,7 @@ module Gitlab
         # aren't incremented properly when joining relations together this way.
         # By using "unprepared_statements" we remove the usage of placeholders
         # (thus fixing this problem), at a slight performance cost.
-        fragments = ActiveRecord::Base.connection.unprepared_statement do
+        fragments = ApplicationRecord.connection.unprepared_statement do
           relations.map do |rel|
             remove_order ? rel.reorder(nil).to_sql : rel.to_sql
           end.reject(&:blank?)
@@ -54,6 +55,34 @@ module Gitlab
       private
 
       attr_reader :relations, :remove_duplicates, :remove_order
+
+      def verify_select_values!(relations)
+        all_select_values = relations.map do |relation|
+          if relation.respond_to?(:select_values)
+            relation.select_values
+          else
+            relation.projections # Handle Arel based subqueries
+          end
+        end
+
+        unless all_select_values.map(&:size).uniq.one?
+          relation_select_sizes = all_select_values.map.with_index do |select_values, index|
+            if select_values.empty?
+              "Relation ##{index}: *, all columns"
+            else
+              "Relation ##{index}: #{select_values.size} select values"
+            end
+          end
+
+          exception_text = <<~EOF
+          Relations with uneven select values were passed. The UNION query could break when the underlying table changes (add or remove columns).
+
+          #{relation_select_sizes.join("\n")}
+          EOF
+
+          raise(exception_text)
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/tracking/docs/helper.rb b/lib/gitlab/tracking/docs/helper.rb
index 81874aac9a5..4e03858b771 100644
--- a/lib/gitlab/tracking/docs/helper.rb
+++ b/lib/gitlab/tracking/docs/helper.rb
@@ -16,7 +16,7 @@ module Gitlab
             <!---
               This documentation is auto generated by a script.
 
-              Please do not edit this file directly, check generate_metrics_dictionary task on lib/tasks/gitlab/usage_data.rake.
+              Please do not edit this file directly, check generate_event_dictionary task on lib/tasks/gitlab/snowplow.rake.
             --->
 
             <!-- vale gitlab.Spelling = NO -->
diff --git a/lib/gitlab/usage/docs/helper.rb b/lib/gitlab/usage/docs/helper.rb
deleted file mode 100644
index bfe674b945e..00000000000
--- a/lib/gitlab/usage/docs/helper.rb
+++ /dev/null
@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Usage
-    module Docs
-      # Helper with functions to be used by HAML templates
-      module Helper
-        def auto_generated_comment
-          <<-MARKDOWN.strip_heredoc
-            ---
-            stage: Growth
-            group: Product Intelligence
-            info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers
-            ---
-
-            <!---
-              This documentation is auto generated by a script.
-
-              Please do not edit this file directly, check generate_metrics_dictionary task on lib/tasks/gitlab/usage_data.rake.
-            --->
-          MARKDOWN
-        end
-
-        def render_name(name)
-          "### `#{name}`"
-        end
-
-        def render_description(object)
-          return 'Missing description' unless object[:description].present?
-
-          object[:description]
-        end
-
-        def render_object_schema(object)
-          "[Object JSON schema](#{object.json_schema_path})"
-        end
-
-        def render_yaml_link(yaml_path)
-          "[YAML definition](#{yaml_path})"
-        end
-
-        def render_status(object)
-          "Status: #{format(:status, object[:status])}"
-        end
-
-        def render_owner(object)
-          "Group: `#{object[:product_group]}`"
-        end
-
-        def render_tiers(object)
-          "Tiers:#{format(:tier, object[:tier])}"
-        end
-
-        def render_data_category(object)
-          "Data Category: `#{object[:data_category]}`"
-        end
-
-        def format(key, value)
-          Gitlab::Usage::Docs::ValueFormatter.format(key, value)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/usage/docs/renderer.rb b/lib/gitlab/usage/docs/renderer.rb
deleted file mode 100644
index 7a7c58005bb..00000000000
--- a/lib/gitlab/usage/docs/renderer.rb
+++ /dev/null
@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Usage
-    module Docs
-      class Renderer
-        include Gitlab::Usage::Docs::Helper
-        DICTIONARY_PATH = Rails.root.join('doc', 'development', 'usage_ping')
-        TEMPLATE_PATH = Rails.root.join('lib', 'gitlab', 'usage', 'docs', 'templates', 'default.md.haml')
-
-        def initialize(metrics_definitions)
-          @layout = Haml::Engine.new(File.read(TEMPLATE_PATH))
-          @metrics_definitions = metrics_definitions.sort
-        end
-
-        def contents
-          # Render and remove an extra trailing new line
-          @contents ||= @layout.render(self, metrics_definitions: @metrics_definitions).sub!(/\n(?=\Z)/, '')
-        end
-
-        def write
-          filename = DICTIONARY_PATH.join('dictionary.md').to_s
-
-          FileUtils.mkdir_p(DICTIONARY_PATH)
-          File.write(filename, contents)
-
-          filename
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/usage/docs/templates/default.md.haml b/lib/gitlab/usage/docs/templates/default.md.haml
deleted file mode 100644
index 83a3a5b6698..00000000000
--- a/lib/gitlab/usage/docs/templates/default.md.haml
+++ /dev/null
@@ -1,48 +0,0 @@
-= auto_generated_comment
-
-:plain
-  # Metrics Dictionary
-
-  This file is autogenerated, please do not edit directly.
-
-  To generate these files from the GitLab repository, run:
-
-  ```shell
-  bundle exec rake gitlab:usage_data:generate_metrics_dictionary
-  ```
-
-  The Metrics Dictionary is based on the following metrics definition YAML files:
-
-  - [`config/metrics`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/config/metrics)
-  - [`ee/config/metrics`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/config/metrics)
-
-  Each table includes a `milestone`, which corresponds to the GitLab version when the metric
-  was released.
-
-  <!-- vale off -->
-  <!-- Docs linting disabled after this line. -->
-  <!-- See https://docs.gitlab.com/ee/development/documentation/testing.html#disable-vale-tests -->
-
-  ## Metrics Definitions
-
-\
-- metrics_definitions.each do |name, object|
-
-  = render_name(name)
-  \
-  = render_description(object.attributes)
-  - if object.has_json_schema?
-    \
-    = render_object_schema(object)
-  \
-  = render_yaml_link(object.yaml_path)
-  \
-  = render_owner(object.attributes)
-  - if object.attributes[:data_category].present?
-    \
-    = render_data_category(object.attributes)
-  \
-  = render_status(object.attributes)
-  \
-  = render_tiers(object.attributes)
-  \
diff --git a/lib/gitlab/usage/docs/value_formatter.rb b/lib/gitlab/usage/docs/value_formatter.rb
deleted file mode 100644
index 379e5df4d52..00000000000
--- a/lib/gitlab/usage/docs/value_formatter.rb
+++ /dev/null
@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Usage
-    module Docs
-      class ValueFormatter
-        def self.format(key, value)
-          return '' unless value.present?
-
-          case key
-          when :key_path
-            "**`#{value}`**"
-          when :data_source
-            value.to_s.capitalize
-          when :product_group, :product_category, :status
-            "`#{value}`"
-          when :introduced_by_url
-            "[Introduced by](#{value})"
-          when :distribution, :tier
-            Array(value).map { |tier| " `#{tier}`" }.join(',')
-          else
-            value
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/usage/metric.rb b/lib/gitlab/usage/metric.rb
index f3469209f48..5b1ac189c13 100644
--- a/lib/gitlab/usage/metric.rb
+++ b/lib/gitlab/usage/metric.rb
@@ -3,40 +3,43 @@
 module Gitlab
   module Usage
     class Metric
-      include ActiveModel::Model
+      attr_reader :definition
 
-      InvalidMetricError = Class.new(RuntimeError)
-
-      attr_accessor :key_path, :value
+      def initialize(definition)
+        @definition = definition
+      end
 
-      validates :key_path, presence: true
+      class << self
+        def all
+          @all ||= Gitlab::Usage::MetricDefinition.with_instrumentation_class.map do |definition|
+            self.new(definition)
+          end
+        end
+      end
 
-      def definition
-        self.class.definitions[key_path]
+      def with_value
+        unflatten_key_path(intrumentation_object.value)
       end
 
-      def unflatten_key_path
-        unflatten(key_path.split('.'), value)
+      def with_instrumentation
+        unflatten_key_path(intrumentation_object.instrumentation)
       end
 
-      class << self
-        def definitions
-          @definitions ||= Gitlab::Usage::MetricDefinition.definitions
-        end
+      private
 
-        def dictionary
-          definitions.map { |key, definition| definition.to_dictionary }
-        end
+      def unflatten_key_path(value)
+        ::Gitlab::Usage::Metrics::KeyPathProcessor.process(definition.key_path, value)
       end
 
-      private
+      def instrumentation_class
+        "Gitlab::Usage::Metrics::Instrumentations::#{definition.instrumentation_class}"
+      end
 
-      def unflatten(keys, value)
-        loop do
-          value = { keys.pop.to_sym => value }
-          break if keys.blank?
-        end
-        value
+      def intrumentation_object
+        instrumentation_class.constantize.new(
+          time_frame: definition.time_frame,
+          options: definition.attributes[:options]
+        )
       end
     end
   end
diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb
index ccd2c69e2e7..db0cb4c6326 100644
--- a/lib/gitlab/usage/metric_definition.rb
+++ b/lib/gitlab/usage/metric_definition.rb
@@ -7,6 +7,8 @@ module Gitlab
       BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master'
       SKIP_VALIDATION_STATUSES = %w[deprecated removed].to_set.freeze
 
+      InvalidError = Class.new(RuntimeError)
+
       attr_reader :path
       attr_reader :attributes
 
@@ -48,11 +50,15 @@ module Gitlab
               Metric file: #{path}
             ERROR_MSG
 
-            Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new(error_message))
+            Gitlab::ErrorTracking.track_and_raise_for_dev_exception(InvalidError.new(error_message))
           end
         end
       end
 
+      def category_to_lowercase
+        attributes[:data_category]&.downcase!
+      end
+
       alias_method :to_dictionary, :to_h
 
       class << self
@@ -69,6 +75,10 @@ module Gitlab
           @all ||= definitions.map { |_key_path, definition| definition }
         end
 
+        def with_instrumentation_class
+          all.select { |definition| definition.attributes[:instrumentation_class].present? }
+        end
+
         def schemer
           @schemer ||= ::JSONSchemer.schema(Pathname.new(METRIC_SCHEMA_PATH))
         end
@@ -90,9 +100,9 @@ module Gitlab
           definition = YAML.safe_load(definition)
           definition.deep_symbolize_keys!
 
-          self.new(path, definition).tap(&:validate!)
+          self.new(path, definition).tap(&:validate!).tap(&:category_to_lowercase)
         rescue StandardError => e
-          Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new(e.message))
+          Gitlab::ErrorTracking.track_and_raise_for_dev_exception(InvalidError.new(e.message))
         end
 
         def load_all_from_path!(definitions, glob_path)
@@ -100,7 +110,7 @@ module Gitlab
             definition = load_from_file(path)
 
             if previous = definitions[definition.key]
-              Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new("Metric '#{definition.key}' is already defined in '#{previous.path}'"))
+              Gitlab::ErrorTracking.track_and_raise_for_dev_exception(InvalidError.new("Metric '#{definition.key}' is already defined in '#{previous.path}'"))
             end
 
             definitions[definition.key] = definition
@@ -114,6 +124,10 @@ module Gitlab
         attributes[method] || super
       end
 
+      def respond_to_missing?(method, *args)
+        attributes[method].present? || super
+      end
+
       def skip_validation?
         !!attributes[:skip_validation] || @skip_validation || SKIP_VALIDATION_STATUSES.include?(attributes[:status])
       end
diff --git a/lib/gitlab/usage/metrics/aggregates.rb b/lib/gitlab/usage/metrics/aggregates.rb
new file mode 100644
index 00000000000..a32c413dba8
--- /dev/null
+++ b/lib/gitlab/usage/metrics/aggregates.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Usage
+    module Metrics
+      module Aggregates
+        UNION_OF_AGGREGATED_METRICS = 'OR'
+        INTERSECTION_OF_AGGREGATED_METRICS = 'AND'
+        ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze
+        AGGREGATED_METRICS_PATH = Rails.root.join('config/metrics/aggregates/*.yml')
+        AggregatedMetricError = Class.new(StandardError)
+        UnknownAggregationOperator = Class.new(AggregatedMetricError)
+        UnknownAggregationSource = Class.new(AggregatedMetricError)
+        DisallowedAggregationTimeFrame = Class.new(AggregatedMetricError)
+
+        DATABASE_SOURCE = 'database'
+        REDIS_SOURCE = 'redis'
+
+        SOURCES = {
+          DATABASE_SOURCE => Sources::PostgresHll,
+          REDIS_SOURCE => Sources::RedisHll
+        }.freeze
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb
index 3ec06fba5d1..2545a505984 100644
--- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb
+++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb
@@ -4,23 +4,6 @@ module Gitlab
   module Usage
     module Metrics
       module Aggregates
-        UNION_OF_AGGREGATED_METRICS = 'OR'
-        INTERSECTION_OF_AGGREGATED_METRICS = 'AND'
-        ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze
-        AGGREGATED_METRICS_PATH = Rails.root.join('config/metrics/aggregates/*.yml')
-        AggregatedMetricError = Class.new(StandardError)
-        UnknownAggregationOperator = Class.new(AggregatedMetricError)
-        UnknownAggregationSource = Class.new(AggregatedMetricError)
-        DisallowedAggregationTimeFrame = Class.new(AggregatedMetricError)
-
-        DATABASE_SOURCE = 'database'
-        REDIS_SOURCE = 'redis'
-
-        SOURCES = {
-          DATABASE_SOURCE => Sources::PostgresHll,
-          REDIS_SOURCE => Sources::RedisHll
-        }.freeze
-
         class Aggregate
           include Gitlab::Usage::TimeFrame
 
diff --git a/lib/gitlab/usage/metrics/aggregates/sources.rb b/lib/gitlab/usage/metrics/aggregates/sources.rb
new file mode 100644
index 00000000000..f782a64f3b5
--- /dev/null
+++ b/lib/gitlab/usage/metrics/aggregates/sources.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Usage
+    module Metrics
+      module Aggregates
+        module Sources
+          UnionNotAvailable = Class.new(AggregatedMetricError)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb b/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb
index 009b8e62543..1bdf3a7f9d8 100644
--- a/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb
+++ b/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb
@@ -5,8 +5,6 @@ module Gitlab
     module Metrics
       module Aggregates
         module Sources
-          UnionNotAvailable = Class.new(AggregatedMetricError)
-
           class RedisHll
             extend Calculations::Intersection
             def self.calculate_metrics_union(metric_names:, start_date:, end_date:, recorded_at: nil)
diff --git a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb
index 7b5bee3f8bd..a264f9484f3 100644
--- a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb
@@ -15,6 +15,10 @@ module Gitlab
             @time_frame = time_frame
             @options = options
           end
+
+          def instrumentation
+            value
+          end
         end
       end
     end
diff --git a/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb
index dd1f9948815..ee51180973c 100644
--- a/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb
@@ -5,8 +5,8 @@ module Gitlab
     module Metrics
       module Instrumentations
         class CollectedDataCategoriesMetric < GenericMetric
-          def value
-            ::ServicePing::PermitDataCategoriesService.new.execute
+          value do
+            ::ServicePing::PermitDataCategoriesService.new.execute.to_a
           end
         end
       end
diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb
index 7b3a545185b..d7fc798ebe2 100644
--- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb
@@ -33,16 +33,17 @@ module Gitlab
               @metric_relation = block
             end
 
-            def operation(symbol, column: nil)
+            def operation(symbol, column: nil, &block)
               @metric_operation = symbol
               @column = column
+              @metric_operation_block = block if block_given?
             end
 
             def cache_start_and_finish_as(cache_key)
               @cache_key = cache_key
             end
 
-            attr_reader :metric_operation, :metric_relation, :metric_start, :metric_finish, :column, :cache_key
+            attr_reader :metric_operation, :metric_relation, :metric_start, :metric_finish, :metric_operation_block, :column, :cache_key
           end
 
           def value
@@ -52,13 +53,18 @@ module Gitlab
               .call(relation,
                     self.class.column,
                     start: start,
-                    finish: finish)
+                    finish: finish,
+                    &self.class.metric_operation_block)
           end
 
           def to_sql
             Gitlab::Usage::Metrics::Query.for(self.class.metric_operation, relation, self.class.column)
           end
 
+          def instrumentation
+            to_sql
+          end
+
           def suggested_name
             Gitlab::Usage::Metrics::NameSuggestion.for(
               self.class.metric_operation,
diff --git a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb
index 1849773e33d..0f4b903b99c 100644
--- a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb
@@ -12,27 +12,35 @@ module Gitlab
           #     Gitlab::CurrentSettings.uuid
           #   end
           # end
+          FALLBACK = -1
+
           class << self
-            attr_reader :metric_operation
-            @metric_operation = :alt
+            attr_reader :metric_value
+
+            def fallback(custom_fallback = FALLBACK)
+              return @metric_fallback if defined?(@metric_fallback)
+
+              @metric_fallback = custom_fallback
+            end
 
             def value(&block)
               @metric_value = block
             end
+          end
 
-            attr_reader :metric_value
+          def initialize(time_frame: 'none', options: {})
+            @time_frame = time_frame
+            @options = options
           end
 
           def value
-            alt_usage_data do
+            alt_usage_data(fallback: self.class.fallback) do
               self.class.metric_value.call
             end
           end
 
           def suggested_name
-            Gitlab::Usage::Metrics::NameSuggestion.for(
-              self.class.metric_operation
-            )
+            Gitlab::Usage::Metrics::NameSuggestion.for(:alt)
           end
         end
       end
diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb
index a36e612a1cb..bb27cca1bb9 100644
--- a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb
@@ -12,11 +12,6 @@ module Gitlab
           #   events:
           #     - g_analytics_valuestream
           # end
-          class << self
-            attr_reader :metric_operation
-            @metric_operation = :redis
-          end
-
           def initialize(time_frame:, options: {})
             super
 
@@ -36,9 +31,7 @@ module Gitlab
           end
 
           def suggested_name
-            Gitlab::Usage::Metrics::NameSuggestion.for(
-              self.class.metric_operation
-            )
+            Gitlab::Usage::Metrics::NameSuggestion.for(:redis)
           end
 
           private
diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb
new file mode 100644
index 00000000000..a25bad2436b
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Usage
+    module Metrics
+      module Instrumentations
+        # Usage example
+        #
+        # In metric YAML definition:
+        #
+        # instrumentation_class: RedisMetric
+        # options:
+        #   event: pushes
+        #   counter_class: SourceCodeCounter
+        #
+        class RedisMetric < BaseMetric
+          def initialize(time_frame:, options: {})
+            super
+
+            raise ArgumentError, "'event' option is required" unless metric_event.present?
+            raise ArgumentError, "'counter class' option is required" unless counter_class.present?
+          end
+
+          def metric_event
+            options[:event]
+          end
+
+          def counter_class_name
+            options[:counter_class]
+          end
+
+          def counter_class
+            "Gitlab::UsageDataCounters::#{counter_class_name}".constantize
+          end
+
+          def value
+            redis_usage_data do
+              counter_class.read(metric_event)
+            end
+          end
+
+          def suggested_name
+            Gitlab::Usage::Metrics::NameSuggestion.for(:redis)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage/metrics/names_suggestions/generator.rb b/lib/gitlab/usage/metrics/names_suggestions/generator.rb
index a669b43f395..b47dc5689d4 100644
--- a/lib/gitlab/usage/metrics/names_suggestions/generator.rb
+++ b/lib/gitlab/usage/metrics/names_suggestions/generator.rb
@@ -10,6 +10,12 @@ module Gitlab
               uncached_data.deep_stringify_keys.dig(*key_path.split('.'))
             end
 
+            def add_metric(metric, time_frame: 'none')
+              metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize
+
+              metric_class.new(time_frame: time_frame).suggested_name
+            end
+
             private
 
             def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index aabc706901e..910c8397f20 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -72,8 +72,8 @@ module Gitlab
       def license_usage_data
         {
           recorded_at: recorded_at,
-          uuid: alt_usage_data { Gitlab::CurrentSettings.uuid },
-          hostname: alt_usage_data { Gitlab.config.gitlab.host },
+          uuid: add_metric('UuidMetric'),
+          hostname: add_metric('HostnameMetric'),
           version: alt_usage_data { Gitlab::VERSION },
           installation_type: alt_usage_data { installation_type },
           active_user_count: count(User.active),
@@ -93,7 +93,7 @@ module Gitlab
         {
           counts: {
             assignee_lists: count(List.assignee),
-            boards: count(Board),
+            boards: add_metric('CountBoardsMetric', time_frame: 'all'),
             ci_builds: count(::Ci::Build),
             ci_internal_pipelines: count(::Ci::Pipeline.internal),
             ci_external_pipelines: count(::Ci::Pipeline.external),
@@ -108,6 +108,7 @@ module Gitlab
             deployments: deployment_count(Deployment),
             successful_deployments: deployment_count(Deployment.success),
             failed_deployments: deployment_count(Deployment.failed),
+            feature_flags: count(Operations::FeatureFlag),
             # rubocop: enable UsageData/LargeTable:
             environments: count(::Environment),
             clusters: count(::Clusters::Cluster),
@@ -138,7 +139,7 @@ module Gitlab
             in_review_folder: count(::Environment.in_review_folder),
             grafana_integrated_projects: count(GrafanaIntegration.enabled),
             groups: count(Group),
-            issues: count(Issue, start: minimum_id(Issue), finish: maximum_id(Issue)),
+            issues: add_metric('CountIssuesMetric', time_frame: 'all'),
             issues_created_from_gitlab_error_tracking_ui: count(SentryIssue),
             issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue),
             issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
@@ -255,9 +256,10 @@ module Gitlab
         {
           settings: {
             ldap_encrypted_secrets_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth::Ldap::Config.encrypted_secrets.active? },
+            smtp_encrypted_secrets_enabled: alt_usage_data(fallback: nil) { Gitlab::Email::SmtpConfig.encrypted_secrets.active? },
             operating_system: alt_usage_data(fallback: nil) { operating_system },
             gitaly_apdex: alt_usage_data { gitaly_apdex },
-            collected_data_categories: alt_usage_data(fallback: []) { Gitlab::Usage::Metrics::Instrumentations::CollectedDataCategoriesMetric.new(time_frame: 'none').value }
+            collected_data_categories: add_metric('CollectedDataCategoriesMetric', time_frame: 'none')
           }
         }
       end
@@ -328,9 +330,9 @@ module Gitlab
             version: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.container_registry_version }
           },
           database: {
-            adapter: alt_usage_data { Gitlab::Database.adapter_name },
-            version: alt_usage_data { Gitlab::Database.version },
-            pg_system_id: alt_usage_data { Gitlab::Database.system_id }
+            adapter: alt_usage_data { Gitlab::Database.main.adapter_name },
+            version: alt_usage_data { Gitlab::Database.main.version },
+            pg_system_id: alt_usage_data { Gitlab::Database.main.system_id }
           },
           mail: {
             smtp_server: alt_usage_data { ActionMailer::Base.smtp_settings[:address] }
@@ -644,16 +646,17 @@ module Gitlab
       # Omitted because of encrypted properties: `projects_jira_cloud_active`, `projects_jira_server_active`
       # rubocop: disable CodeReuse/ActiveRecord
       def usage_activity_by_stage_plan(time_period)
+        time_frame = time_period.present? ? '28d' : 'none'
         {
-          issues: distinct_count(::Issue.where(time_period), :author_id),
+          issues: add_metric('CountUsersCreatingIssuesMetric', time_frame: time_frame),
           notes: distinct_count(::Note.where(time_period), :author_id),
           projects: distinct_count(::Project.where(time_period), :creator_id),
           todos: distinct_count(::Todo.where(time_period), :author_id),
           service_desk_enabled_projects: distinct_count_service_desk_enabled_projects(time_period),
           service_desk_issues: count(::Issue.service_desk.where(time_period)),
-          projects_jira_active: distinct_count(::Project.with_active_jira_integrations.where(time_period), :creator_id),
-          projects_jira_dvcs_cloud_active: distinct_count(::Project.with_active_jira_integrations.with_jira_dvcs_cloud.where(time_period), :creator_id),
-          projects_jira_dvcs_server_active: distinct_count(::Project.with_active_jira_integrations.with_jira_dvcs_server.where(time_period), :creator_id)
+          projects_jira_active: distinct_count(::Project.with_active_integration(::Integrations::Jira) .where(time_period), :creator_id),
+          projects_jira_dvcs_cloud_active: distinct_count(::Project.with_active_integration(::Integrations::Jira) .with_jira_dvcs_cloud.where(time_period), :creator_id),
+          projects_jira_dvcs_server_active: distinct_count(::Project.with_active_integration(::Integrations::Jira) .with_jira_dvcs_server.where(time_period), :creator_id)
         }
       end
       # rubocop: enable CodeReuse/ActiveRecord
diff --git a/lib/gitlab/usage_data_counters.rb b/lib/gitlab/usage_data_counters.rb
index ed9dad37f3e..796cbfdb3d6 100644
--- a/lib/gitlab/usage_data_counters.rb
+++ b/lib/gitlab/usage_data_counters.rb
@@ -15,7 +15,8 @@ module Gitlab
       MergeRequestCounter,
       DesignsCounter,
       KubernetesAgentCounter,
-      StaticSiteEditorCounter
+      StaticSiteEditorCounter,
+      DiffsCounter
     ].freeze
 
     UsageDataCounterError = Class.new(StandardError)
diff --git a/lib/gitlab/usage_data_counters/diffs_counter.rb b/lib/gitlab/usage_data_counters/diffs_counter.rb
new file mode 100644
index 00000000000..b06c7e26f54
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/diffs_counter.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module UsageDataCounters
+    class DiffsCounter < BaseCounter
+      KNOWN_EVENTS = %w[searches].freeze
+      PREFIX = 'diff'
+    end
+  end
+end
diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
index 597df9936ea..96562a44391 100644
--- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb
+++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
@@ -117,7 +117,7 @@ module Gitlab
         private
 
         def track(values, event_name, context: '', time: Time.zone.now)
-          return unless usage_ping_enabled?
+          return unless ::ServicePing::ServicePingSettings.enabled?
 
           event = event_for(event_name)
           Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new("Unknown event #{event_name}")) unless event.present?
@@ -131,10 +131,6 @@ module Gitlab
           Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
         end
 
-        def usage_ping_enabled?
-          Gitlab::CurrentSettings.usage_ping_enabled?
-        end
-
         # The array of valid context on which we allow tracking
         def valid_context_list
           Plan.all_plans
@@ -193,6 +189,7 @@ module Gitlab
 
         def events_in_same_slot?(events)
           # if we check one event then redis_slot is only one to check
+          return false if events.empty?
           return true if events.size == 1
 
           slot = events.first[:redis_slot]
diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml
index 5023161a9dd..7ad51bfe832 100644
--- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml
+++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml
@@ -232,3 +232,8 @@
   redis_slot: code_review
   category: code_review
   aggregation: weekly
+- name: i_code_review_user_searches_diff
+  redis_slot: code_review
+  category: code_review
+  aggregation: weekly
+  feature_flag: diff_searching_usage_data
diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml
index fe1eb090fa4..3db0482d38e 100644
--- a/lib/gitlab/usage_data_counters/known_events/common.yml
+++ b/lib/gitlab/usage_data_counters/known_events/common.yml
@@ -170,7 +170,6 @@
   category: testing
   redis_slot: testing
   aggregation: weekly
-  feature_flag: usage_data_i_testing_full_code_quality_report_total
 - name: i_testing_web_performance_widget_total
   category: testing
   redis_slot: testing
diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml
index c1eabb352f7..7df351859fb 100644
--- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml
+++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml
@@ -179,6 +179,10 @@
   category: quickactions
   redis_slot: quickactions
   aggregation: weekly
+- name: i_quickactions_severity
+  category: quickactions
+  redis_slot: quickactions
+  aggregation: weekly
 - name: i_quickactions_shrug
   category: quickactions
   redis_slot: quickactions
diff --git a/lib/gitlab/usage_data_counters/redis_counter.rb b/lib/gitlab/usage_data_counters/redis_counter.rb
index 2406f771fd8..591e431c871 100644
--- a/lib/gitlab/usage_data_counters/redis_counter.rb
+++ b/lib/gitlab/usage_data_counters/redis_counter.rb
@@ -4,13 +4,13 @@ module Gitlab
   module UsageDataCounters
     module RedisCounter
       def increment(redis_counter_key)
-        return unless Gitlab::CurrentSettings.usage_ping_enabled
+        return unless ::ServicePing::ServicePingSettings.enabled?
 
         Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) }
       end
 
       def increment_by(redis_counter_key, incr)
-        return unless Gitlab::CurrentSettings.usage_ping_enabled
+        return unless ::ServicePing::ServicePingSettings.enabled?
 
         Gitlab::Redis::SharedState.with { |redis| redis.incrby(redis_counter_key, incr) }
       end
diff --git a/lib/gitlab/usage_data_metrics.rb b/lib/gitlab/usage_data_metrics.rb
index dde5dde19e0..1ef201121d9 100644
--- a/lib/gitlab/usage_data_metrics.rb
+++ b/lib/gitlab/usage_data_metrics.rb
@@ -5,26 +5,7 @@ module Gitlab
     class << self
       # Build the Usage Ping JSON payload from metrics YAML definitions which have instrumentation class set
       def uncached_data
-        ::Gitlab::Usage::MetricDefinition.all.map do |definition|
-          instrumentation_class = definition.attributes[:instrumentation_class]
-          options = definition.attributes[:options]
-
-          if instrumentation_class.present?
-            metric_value = "Gitlab::Usage::Metrics::Instrumentations::#{instrumentation_class}".constantize.new(
-              time_frame: definition.attributes[:time_frame],
-              options: options).value
-
-            metric_payload(definition.key_path, metric_value)
-          else
-            {}
-          end
-        end.reduce({}, :deep_merge)
-      end
-
-      private
-
-      def metric_payload(key_path, value)
-        ::Gitlab::Usage::Metrics::KeyPathProcessor.process(key_path, value)
+        ::Gitlab::Usage::Metric.all.map(&:with_value).reduce({}, :deep_merge)
       end
     end
   end
diff --git a/lib/gitlab/usage_data_non_sql_metrics.rb b/lib/gitlab/usage_data_non_sql_metrics.rb
index 44d5baa42f6..1ff4588d091 100644
--- a/lib/gitlab/usage_data_non_sql_metrics.rb
+++ b/lib/gitlab/usage_data_non_sql_metrics.rb
@@ -5,6 +5,16 @@ module Gitlab
     SQL_METRIC_DEFAULT = -3
 
     class << self
+      def uncached_data
+        super.with_indifferent_access.deep_merge(instrumentation_metrics_queries.with_indifferent_access)
+      end
+
+      def add_metric(metric, time_frame: 'none')
+        metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize
+
+        metric_class.new(time_frame: time_frame).instrumentation
+      end
+
       def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
         SQL_METRIC_DEFAULT
       end
@@ -37,6 +47,12 @@ module Gitlab
           projects_jira_cloud_active: 0
         }
       end
+
+      private
+
+      def instrumentation_metrics_queries
+        ::Gitlab::Usage::Metric.all.map(&:with_instrumentation).reduce({}, :deep_merge)
+      end
     end
   end
 end
diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb
index 63e6cf03d1f..f64da2fbf13 100644
--- a/lib/gitlab/usage_data_queries.rb
+++ b/lib/gitlab/usage_data_queries.rb
@@ -5,6 +5,16 @@ module Gitlab
   # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41091
   class UsageDataQueries < UsageData
     class << self
+      def uncached_data
+        super.with_indifferent_access.deep_merge(instrumentation_metrics_queries.with_indifferent_access)
+      end
+
+      def add_metric(metric, time_frame: 'none')
+        metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize
+
+        metric_class.new(time_frame: time_frame).instrumentation
+      end
+
       def count(relation, column = nil, *args, **kwargs)
         Gitlab::Usage::Metrics::Query.for(:count, relation, column)
       end
@@ -58,6 +68,12 @@ module Gitlab
       def epics_deepest_relationship_level
         { epics_deepest_relationship_level: 0 }
       end
+
+      private
+
+      def instrumentation_metrics_queries
+        ::Gitlab::Usage::Metric.all.map(&:with_instrumentation).reduce({}, :deep_merge)
+      end
     end
   end
 end
diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb
index 0b1acaf7dd8..cb34ed69a9c 100644
--- a/lib/gitlab/utils.rb
+++ b/lib/gitlab/utils.rb
@@ -13,7 +13,7 @@ module Gitlab
       return unless path.is_a?(String)
 
       path = decode_path(path)
-      path_regex = /(\A(\.{1,2})\z|\A\.\.[\/\\]|[\/\\]\.\.\z|[\/\\]\.\.[\/\\]|\n)/
+      path_regex = %r{(\A(\.{1,2})\z|\A\.\.[/\\]|[/\\]\.\.\z|[/\\]\.\.[/\\]|\n)}
 
       if path.match?(path_regex)
         raise PathTraversalAttackError, 'Invalid path'
diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb
index faa524d171c..d46210f6efe 100644
--- a/lib/gitlab/utils/usage_data.rb
+++ b/lib/gitlab/utils/usage_data.rb
@@ -44,6 +44,12 @@ module Gitlab
       DISTRIBUTED_HLL_FALLBACK = -2
       MAX_BUCKET_SIZE = 100
 
+      def add_metric(metric, time_frame: 'none')
+        metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize
+
+        metric_class.new(time_frame: time_frame).value
+      end
+
       def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
         if batch
           Gitlab::Database::BatchCount.batch_count(relation, column, batch_size: batch_size, start: start, finish: finish)
diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb
index abfb7e2310e..64029d4d3fe 100644
--- a/lib/gitlab/visibility_level.rb
+++ b/lib/gitlab/visibility_level.rb
@@ -12,6 +12,7 @@ module Gitlab
     included do
       scope :public_only,               -> { where(visibility_level: PUBLIC) }
       scope :public_and_internal_only,  -> { where(visibility_level: [PUBLIC, INTERNAL] ) }
+      scope :private_only,              -> { where(visibility_level: PRIVATE) }
       scope :non_public_only,           -> { where.not(visibility_level: PUBLIC) }
 
       scope :public_to_user, -> (user = nil) do
diff --git a/lib/gitlab/web_ide/config/entry/terminal.rb b/lib/gitlab/web_ide/config/entry/terminal.rb
index ec07023461f..3da2c3b2ced 100644
--- a/lib/gitlab/web_ide/config/entry/terminal.rb
+++ b/lib/gitlab/web_ide/config/entry/terminal.rb
@@ -54,7 +54,6 @@ module Gitlab
           def to_hash
             {
               tag_list: tags || [],
-              yaml_variables: yaml_variables, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581
               job_variables: yaml_variables,
               options: {
                 image: image_value,
diff --git a/lib/gitlab/x509/tag.rb b/lib/gitlab/x509/tag.rb
index ad85b200130..cf24e6f62bd 100644
--- a/lib/gitlab/x509/tag.rb
+++ b/lib/gitlab/x509/tag.rb
@@ -4,37 +4,16 @@ require 'digest'
 
 module Gitlab
   module X509
-    class Tag
+    class Tag < Gitlab::SignedTag
       include Gitlab::Utils::StrongMemoize
 
-      def initialize(raw_tag)
-        @raw_tag = raw_tag
-      end
-
       def signature
-        signature = X509::Signature.new(signature_text, signed_text, @raw_tag.tagger.email, Time.at(@raw_tag.tagger.date.seconds))
-
-        return if signature.verified_signature.nil?
-
-        signature
-      end
-
-      private
-
-      def signature_text
-        @raw_tag.message.slice(@raw_tag.message.index("-----BEGIN SIGNED MESSAGE-----")..-1)
-      rescue StandardError
-        nil
-      end
-
-      def signed_text
-        # signed text is reconstructed as long as there is no specific gitaly function
-        %{object #{@raw_tag.target_commit.id}
-type commit
-tag #{@raw_tag.name}
-tagger #{@raw_tag.tagger.name} <#{@raw_tag.tagger.email}> #{@raw_tag.tagger.date.seconds} #{@raw_tag.tagger.timezone}
+        strong_memoize(:signature) do
+          super
 
-#{@raw_tag.message.gsub(/-----BEGIN SIGNED MESSAGE-----(.*)-----END SIGNED MESSAGE-----/m, "")}}
+          signature = X509::Signature.new(signature_text, signed_text, @tag.tagger.email, Time.at(@tag.tagger.date.seconds))
+          signature unless signature.verified_signature.nil?
+        end
       end
     end
   end
diff --git a/lib/peek/views/active_record.rb b/lib/peek/views/active_record.rb
index 8e1200338c2..a3fe206c86f 100644
--- a/lib/peek/views/active_record.rb
+++ b/lib/peek/views/active_record.rb
@@ -66,7 +66,8 @@ module Peek
           backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller),
           cached: data[:cached] ? 'Cached' : '',
           transaction: data[:connection].transaction_open? ? 'In a transaction' : '',
-          db_role: db_role(data)
+          db_role: db_role(data),
+          db_config_name: "Config name: #{::Gitlab::Database.db_config_name(data[:connection])}"
         }
       end
 
@@ -76,7 +77,15 @@ module Peek
         role = ::Gitlab::Database::LoadBalancing.db_role_for_connection(data[:connection]) ||
           ::Gitlab::Database::LoadBalancing::ROLE_UNKNOWN
 
-        role.to_s.capitalize
+        "Role: #{role.to_s.capitalize}"
+      end
+
+      def format_call_details(call)
+        if ENV['GITLAB_MULTIPLE_DATABASE_METRICS']
+          super
+        else
+          super.except(:db_config_name)
+        end
       end
     end
   end
diff --git a/lib/product_analytics/tracker.rb b/lib/product_analytics/tracker.rb
index d4a88b879f0..e5550c2e6ef 100644
--- a/lib/product_analytics/tracker.rb
+++ b/lib/product_analytics/tracker.rb
@@ -6,6 +6,6 @@ module ProductAnalytics
     URL = Gitlab.config.gitlab.url + '/-/sp.js'
 
     # The collector URL minus protocol and /i
-    COLLECTOR_URL = Gitlab.config.gitlab.url.sub(/\Ahttps?\:\/\//, '') + '/-/collector'
+    COLLECTOR_URL = Gitlab.config.gitlab.url.sub(%r{\Ahttps?\://}, '') + '/-/collector'
   end
 end
diff --git a/lib/sidebars/concerns/has_partial.rb b/lib/sidebars/concerns/has_partial.rb
new file mode 100644
index 00000000000..e89bc66bc40
--- /dev/null
+++ b/lib/sidebars/concerns/has_partial.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# This module has the necessary methods to render
+# menus using a custom partial
+module Sidebars
+  module Concerns
+    module HasPartial
+      def menu_partial
+        nil
+      end
+
+      def menu_partial_options
+        {}
+      end
+
+      def menu_with_partial?
+        menu_partial.present?
+      end
+    end
+  end
+end
diff --git a/lib/sidebars/concerns/has_pill.rb b/lib/sidebars/concerns/has_pill.rb
index 5082ed477e6..4bbf69bf16b 100644
--- a/lib/sidebars/concerns/has_pill.rb
+++ b/lib/sidebars/concerns/has_pill.rb
@@ -5,6 +5,8 @@
 module Sidebars
   module Concerns
     module HasPill
+      include ActionView::Helpers::NumberHelper
+
       def has_pill?
         false
       end
@@ -18,6 +20,17 @@ module Sidebars
       def pill_html_options
         {}
       end
+
+      def format_cached_count(count_service, count)
+        if count > count_service::CACHED_COUNT_THRESHOLD
+          number_to_human(
+            count,
+            units: { thousand: 'k', million: 'm' }, precision: 1, significant: false, format: '%n%u'
+          )
+        else
+          number_with_delimiter(count)
+        end
+      end
     end
   end
 end
diff --git a/lib/sidebars/groups/menus/ci_cd_menu.rb b/lib/sidebars/groups/menus/ci_cd_menu.rb
new file mode 100644
index 00000000000..e870bbf5ebc
--- /dev/null
+++ b/lib/sidebars/groups/menus/ci_cd_menu.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Sidebars
+  module Groups
+    module Menus
+      class CiCdMenu < ::Sidebars::Menu
+        override :configure_menu_items
+        def configure_menu_items
+          add_item(runners_menu_item)
+
+          true
+        end
+
+        override :link
+        def link
+          renderable_items.first.link
+        end
+
+        override :title
+        def title
+          _('CI/CD')
+        end
+
+        override :sprite_icon
+        def sprite_icon
+          'rocket'
+        end
+
+        private
+
+        def runners_menu_item
+          return ::Sidebars::NilMenuItem.new(item_id: :runners) unless show_runners?
+
+          ::Sidebars::MenuItem.new(
+            title: _('Runners'),
+            link: group_runners_path(context.group),
+            active_routes: { path: 'groups/runners#index' },
+            item_id: :runners
+          )
+        end
+
+        # TODO Proper policies, such as `read_group_runners`, should be implemented per
+        # See https://gitlab.com/gitlab-org/gitlab/-/issues/334802
+        def show_runners?
+          can?(context.current_user, :admin_group, context.group) &&
+            Feature.enabled?(:runner_list_group_view_vue_ui, context.group, default_enabled: :yaml)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/sidebars/groups/menus/group_information_menu.rb b/lib/sidebars/groups/menus/group_information_menu.rb
new file mode 100644
index 00000000000..b28cb927ad2
--- /dev/null
+++ b/lib/sidebars/groups/menus/group_information_menu.rb
@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Sidebars
+  module Groups
+    module Menus
+      class GroupInformationMenu < ::Sidebars::Menu
+        override :configure_menu_items
+        def configure_menu_items
+          add_item(activity_menu_item)
+          add_item(labels_menu_item)
+          add_item(members_menu_item)
+
+          true
+        end
+
+        override :link
+        def link
+          renderable_items.first.link
+        end
+
+        override :title
+        def title
+          context.group.subgroup? ? _('Subgroup information') : _('Group information')
+        end
+
+        override :sprite_icon
+        def sprite_icon
+          'group'
+        end
+
+        override :active_routes
+        def active_routes
+          { path: 'groups#subgroups' }
+        end
+
+        private
+
+        def activity_menu_item
+          unless can?(context.current_user, :read_group_activity, context.group)
+            return ::Sidebars::NilMenuItem.new(item_id: :activity)
+          end
+
+          ::Sidebars::MenuItem.new(
+            title: _('Activity'),
+            link: activity_group_path(context.group),
+            active_routes: { path: 'groups#activity' },
+            item_id: :activity
+          )
+        end
+
+        def labels_menu_item
+          unless can?(context.current_user, :read_group_labels, context.group)
+            return ::Sidebars::NilMenuItem.new(item_id: :labels)
+          end
+
+          ::Sidebars::MenuItem.new(
+            title: _('Labels'),
+            link: group_labels_path(context.group),
+            active_routes: { controller: :labels },
+            item_id: :labels
+          )
+        end
+
+        def members_menu_item
+          unless can?(context.current_user, :read_group_member, context.group)
+            return ::Sidebars::NilMenuItem.new(item_id: :members)
+          end
+
+          ::Sidebars::MenuItem.new(
+            title: _('Members'),
+            link: group_group_members_path(context.group),
+            active_routes: { path: 'group_members#index' },
+            item_id: :members
+          )
+        end
+      end
+    end
+  end
+end
diff --git a/lib/sidebars/groups/menus/issues_menu.rb b/lib/sidebars/groups/menus/issues_menu.rb
new file mode 100644
index 00000000000..95641c09076
--- /dev/null
+++ b/lib/sidebars/groups/menus/issues_menu.rb
@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Sidebars
+  module Groups
+    module Menus
+      class IssuesMenu < ::Sidebars::Menu
+        include Gitlab::Utils::StrongMemoize
+
+        override :configure_menu_items
+        def configure_menu_items
+          return unless can?(context.current_user, :read_group_issues, context.group)
+
+          add_item(list_menu_item)
+          add_item(boards_menu_item)
+          add_item(milestones_menu_item)
+
+          true
+        end
+
+        override :link
+        def link
+          issues_group_path(context.group)
+        end
+
+        override :title
+        def title
+          _('Issues')
+        end
+
+        override :sprite_icon
+        def sprite_icon
+          'issues'
+        end
+
+        override :has_pill?
+        def has_pill?
+          true
+        end
+
+        override :pill_count
+        def pill_count
+          strong_memoize(:pill_count) do
+            count_service = ::Groups::OpenIssuesCountService
+            count = count_service.new(context.group, context.current_user).count
+
+            format_cached_count(count_service, count)
+          end
+        end
+
+        override :pill_html_options
+        def pill_html_options
+          {
+            class: 'issue_counter'
+          }
+        end
+
+        private
+
+        def list_menu_item
+          ::Sidebars::MenuItem.new(
+            title: _('List'),
+            link: issues_group_path(context.group),
+            active_routes: { path: 'groups#issues' },
+            container_html_options: { aria: { label: _('Issues') } },
+            item_id: :issue_list
+          )
+        end
+
+        def boards_menu_item
+          unless can?(context.current_user, :read_group_boards, context.group)
+            return ::Sidebars::NilMenuItem.new(item_id: :boards)
+          end
+
+          title = context.group.multiple_issue_boards_available? ? s_('IssueBoards|Boards') : s_('IssueBoards|Board')
+
+          ::Sidebars::MenuItem.new(
+            title: title,
+            link: group_boards_path(context.group),
+            active_routes: { path: %w[boards#index boards#show] },
+            item_id: :boards
+          )
+        end
+
+        def milestones_menu_item
+          unless can?(context.current_user, :read_group_milestones, context.group)
+            return ::Sidebars::NilMenuItem.new(item_id: :milestones)
+          end
+
+          ::Sidebars::MenuItem.new(
+            title: _('Milestones'),
+            link: group_milestones_path(context.group),
+            active_routes: { path: 'milestones#index' },
+            item_id: :milestones
+          )
+        end
+      end
+    end
+  end
+end
+
+Sidebars::Groups::Menus::IssuesMenu.prepend_mod_with('Sidebars::Groups::Menus::IssuesMenu')
diff --git a/lib/sidebars/groups/menus/kubernetes_menu.rb b/lib/sidebars/groups/menus/kubernetes_menu.rb
new file mode 100644
index 00000000000..4ea294a4837
--- /dev/null
+++ b/lib/sidebars/groups/menus/kubernetes_menu.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Sidebars
+  module Groups
+    module Menus
+      class KubernetesMenu < ::Sidebars::Menu
+        override :link
+        def link
+          group_clusters_path(context.group)
+        end
+
+        override :title
+        def title
+          _('Kubernetes')
+        end
+
+        override :sprite_icon
+        def sprite_icon
+          'cloud-gear'
+        end
+
+        override :render?
+        def render?
+          can?(context.current_user, :read_cluster, context.group)
+        end
+
+        override :extra_container_html_options
+        def extra_container_html_options
+          {
+            class: 'shortcuts-kubernetes'
+          }
+        end
+
+        override :active_routes
+        def active_routes
+          { controller: :clusters }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/sidebars/groups/menus/merge_requests_menu.rb b/lib/sidebars/groups/menus/merge_requests_menu.rb
new file mode 100644
index 00000000000..7faf50305c6
--- /dev/null
+++ b/lib/sidebars/groups/menus/merge_requests_menu.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Sidebars
+  module Groups
+    module Menus
+      class MergeRequestsMenu < ::Sidebars::Menu
+        include Gitlab::Utils::StrongMemoize
+
+        override :link
+        def link
+          merge_requests_group_path(context.group)
+        end
+
+        override :title
+        def title
+          _('Merge requests')
+        end
+
+        override :sprite_icon
+        def sprite_icon
+          'git-merge'
+        end
+
+        override :render?
+        def render?
+          can?(context.current_user, :read_group_merge_requests, context.group)
+        end
+
+        override :has_pill?
+        def has_pill?
+          true
+        end
+
+        override :pill_count
+        def pill_count
+          strong_memoize(:pill_count) do
+            count_service = ::Groups::MergeRequestsCountService
+            count = count_service.new(context.group, context.current_user).count
+
+            format_cached_count(count_service, count)
+          end
+        end
+
+        override :pill_html_options
+        def pill_html_options
+          {
+            class: 'merge_counter js-merge-counter'
+          }
+        end
+
+        override :active_routes
+        def active_routes
+          { path: 'groups#merge_requests' }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/sidebars/groups/menus/packages_registries_menu.rb b/lib/sidebars/groups/menus/packages_registries_menu.rb
new file mode 100644
index 00000000000..e46e2820c04
--- /dev/null
+++ b/lib/sidebars/groups/menus/packages_registries_menu.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Sidebars
+  module Groups
+    module Menus
+      class PackagesRegistriesMenu < ::Sidebars::Menu
+        override :configure_menu_items
+        def configure_menu_items
+          add_item(packages_registry_menu_item)
+          add_item(container_registry_menu_item)
+          add_item(dependency_proxy_menu_item)
+
+          true
+        end
+
+        override :link
+        def link
+          renderable_items.first.link
+        end
+
+        override :title
+        def title
+          _('Packages & Registries')
+        end
+
+        override :sprite_icon
+        def sprite_icon
+          'package'
+        end
+
+        private
+
+        def packages_registry_menu_item
+          unless context.group.packages_feature_enabled?
+            return ::Sidebars::NilMenuItem.new(item_id: :packages_registry)
+          end
+
+          ::Sidebars::MenuItem.new(
+            title: _('Package Registry'),
+            link: group_packages_path(context.group),
+            active_routes: { controller: 'groups/packages' },
+            item_id: :packages_registry
+          )
+        end
+
+        def container_registry_menu_item
+          if !::Gitlab.config.registry.enabled || !can?(context.current_user, :read_container_image, context.group)
+            return ::Sidebars::NilMenuItem.new(item_id: :container_registry)
+          end
+
+          ::Sidebars::MenuItem.new(
+            title: _('Container Registry'),
+            link: group_container_registries_path(context.group),
+            active_routes: { controller: 'groups/registry/repositories' },
+            item_id: :container_registry
+          )
+        end
+
+        def dependency_proxy_menu_item
+          unless context.group.dependency_proxy_feature_available?
+            return ::Sidebars::NilMenuItem.new(item_id: :dependency_proxy)
+          end
+
+          ::Sidebars::MenuItem.new(
+            title: _('Dependency Proxy'),
+            link: group_dependency_proxy_path(context.group),
+            active_routes: { controller: 'groups/dependency_proxies' },
+            item_id: :dependency_proxy
+          )
+        end
+      end
+    end
+  end
+end
diff --git a/lib/sidebars/groups/menus/settings_menu.rb b/lib/sidebars/groups/menus/settings_menu.rb
new file mode 100644
index 00000000000..8bc6077d302
--- /dev/null
+++ b/lib/sidebars/groups/menus/settings_menu.rb
@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Sidebars
+  module Groups
+    module Menus
+      class SettingsMenu < ::Sidebars::Menu
+        override :configure_menu_items
+        def configure_menu_items
+          return false unless can?(context.current_user, :admin_group, context.group)
+
+          add_item(general_menu_item)
+          add_item(integrations_menu_item)
+          add_item(group_projects_menu_item)
+          add_item(repository_menu_item)
+          add_item(ci_cd_menu_item)
+          add_item(applications_menu_item)
+          add_item(packages_and_registries_menu_item)
+
+          true
+        end
+
+        override :link
+        def link
+          edit_group_path(context.group)
+        end
+
+        override :title
+        def title
+          _('Settings')
+        end
+
+        override :sprite_icon
+        def sprite_icon
+          'settings'
+        end
+
+        override :extra_nav_link_html_options
+        def extra_nav_link_html_options
+          {
+            class: 'shortcuts-settings'
+          }
+        end
+
+        private
+
+        def general_menu_item
+          ::Sidebars::MenuItem.new(
+            title: _('General'),
+            link: edit_group_path(context.group),
+            active_routes: { path: 'groups#edit' },
+            item_id: :general
+          )
+        end
+
+        def integrations_menu_item
+          ::Sidebars::MenuItem.new(
+            title: _('Integrations'),
+            link: group_settings_integrations_path(context.group),
+            active_routes: { controller: :integrations },
+            item_id: :integrations
+          )
+        end
+
+        def group_projects_menu_item
+          ::Sidebars::MenuItem.new(
+            title: _('Projects'),
+            link: projects_group_path(context.group),
+            active_routes: { path: 'groups#projects' },
+            item_id: :group_projects
+          )
+        end
+
+        def repository_menu_item
+          ::Sidebars::MenuItem.new(
+            title: _('Repository'),
+            link: group_settings_repository_path(context.group),
+            active_routes: { controller: :repository },
+            item_id: :repository
+          )
+        end
+
+        def ci_cd_menu_item
+          ::Sidebars::MenuItem.new(
+            title: _('CI/CD'),
+            link: group_settings_ci_cd_path(context.group),
+            active_routes: { path: %w[ci_cd#show groups/runners#show groups/runners#edit] },
+            item_id: :ci_cd
+          )
+        end
+
+        def applications_menu_item
+          ::Sidebars::MenuItem.new(
+            title: _('Applications'),
+            link: group_settings_applications_path(context.group),
+            active_routes: { controller: :applications },
+            item_id: :applications
+          )
+        end
+
+        def packages_and_registries_menu_item
+          unless context.group.packages_feature_enabled?
+            return ::Sidebars::NilMenuItem.new(item_id: :packages_and_registries)
+          end
+
+          ::Sidebars::MenuItem.new(
+            title: _('Packages & Registries'),
+            link: group_settings_packages_and_registries_path(context.group),
+            active_routes: { controller: :packages_and_registries },
+            item_id: :packages_and_registries
+          )
+        end
+      end
+    end
+  end
+end
+
+Sidebars::Groups::Menus::SettingsMenu.prepend_mod_with('Sidebars::Groups::Menus::SettingsMenu')
diff --git a/lib/sidebars/groups/panel.rb b/lib/sidebars/groups/panel.rb
index fe669bf0b29..73b943c5655 100644
--- a/lib/sidebars/groups/panel.rb
+++ b/lib/sidebars/groups/panel.rb
@@ -6,6 +6,14 @@ module Sidebars
       override :configure_menus
       def configure_menus
         set_scope_menu(Sidebars::Groups::Menus::ScopeMenu.new(context))
+
+        add_menu(Sidebars::Groups::Menus::GroupInformationMenu.new(context))
+        add_menu(Sidebars::Groups::Menus::IssuesMenu.new(context))
+        add_menu(Sidebars::Groups::Menus::MergeRequestsMenu.new(context))
+        add_menu(Sidebars::Groups::Menus::CiCdMenu.new(context))
+        add_menu(Sidebars::Groups::Menus::KubernetesMenu.new(context))
+        add_menu(Sidebars::Groups::Menus::PackagesRegistriesMenu.new(context))
+        add_menu(Sidebars::Groups::Menus::SettingsMenu.new(context))
       end
 
       override :render_raw_menus_partial
@@ -20,3 +28,5 @@ module Sidebars
     end
   end
 end
+
+Sidebars::Groups::Panel.prepend_mod_with('Sidebars::Groups::Panel')
diff --git a/lib/sidebars/menu.rb b/lib/sidebars/menu.rb
index dcdc130b0d7..3b8872fd572 100644
--- a/lib/sidebars/menu.rb
+++ b/lib/sidebars/menu.rb
@@ -12,6 +12,7 @@ module Sidebars
     include ::Sidebars::Concerns::Renderable
     include ::Sidebars::Concerns::ContainerWithHtmlOptions
     include ::Sidebars::Concerns::HasActiveRoutes
+    include ::Sidebars::Concerns::HasPartial
 
     attr_reader :context
     delegate :current_user, :container, to: :@context
@@ -29,7 +30,7 @@ module Sidebars
 
     override :render?
     def render?
-      has_renderable_items?
+      has_renderable_items? || menu_with_partial?
     end
 
     # Menus might have or not a link
diff --git a/lib/sidebars/projects/menus/packages_registries_menu.rb b/lib/sidebars/projects/menus/packages_registries_menu.rb
index 27e318d73c5..d49bb680853 100644
--- a/lib/sidebars/projects/menus/packages_registries_menu.rb
+++ b/lib/sidebars/projects/menus/packages_registries_menu.rb
@@ -31,7 +31,7 @@ module Sidebars
         private
 
         def packages_registry_menu_item
-          if !::Gitlab.config.packages.enabled || !can?(context.current_user, :read_package, context.project)
+          if packages_registry_disabled?
             return ::Sidebars::NilMenuItem.new(item_id: :packages_registry)
           end
 
@@ -58,7 +58,7 @@ module Sidebars
         end
 
         def infrastructure_registry_menu_item
-          if Feature.disabled?(:infrastructure_registry_page, context.current_user, default_enabled: :yaml)
+          if packages_registry_disabled?
             return ::Sidebars::NilMenuItem.new(item_id: :infrastructure_registry)
           end
 
@@ -69,6 +69,10 @@ module Sidebars
             item_id: :infrastructure_registry
           )
         end
+
+        def packages_registry_disabled?
+          !::Gitlab.config.packages.enabled || !can?(context.current_user, :read_package, context.project)
+        end
       end
     end
   end
diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab
index ac47c5be1e8..96e3a015115 100755
--- a/lib/support/init.d/gitlab
+++ b/lib/support/init.d/gitlab
@@ -44,7 +44,7 @@ gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log"
 gitlab_pages_enabled=false
 gitlab_pages_dir=$(cd $app_root/../gitlab-pages 2> /dev/null && pwd)
 gitlab_pages_pid_path="$pid_path/gitlab-pages.pid"
-gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090"
+gitlab_pages_options="-config $app_root/gitlab-pages/gitlab-pages.conf"
 gitlab_pages_log="$app_root/log/gitlab-pages.log"
 shell_path="/bin/bash"
 gitaly_enabled=true
diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example
index 53bebe55fa3..0233c26cecc 100644
--- a/lib/support/init.d/gitlab.default.example
+++ b/lib/support/init.d/gitlab.default.example
@@ -68,7 +68,7 @@ gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log"
 # The -pages-domain must be specified the same as in `gitlab.yml > pages > host`.
 # Set `gitlab_pages_enabled=true` if you want to enable the Pages feature.
 gitlab_pages_enabled=false
-gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090"
+gitlab_pages_options="-config $app_root/gitlab-pages/gitlab-pages.conf"
 gitlab_pages_log="$app_root/log/gitlab-pages.log"
 
 # mail_room_enabled specifies whether mail_room, which is used to process incoming email, is enabled.
diff --git a/lib/support/nginx/gitlab-pages-ssl b/lib/support/nginx/gitlab-pages-ssl
index 62ed482e2bf..900d91e0575 100644
--- a/lib/support/nginx/gitlab-pages-ssl
+++ b/lib/support/nginx/gitlab-pages-ssl
@@ -31,16 +31,16 @@ server {
 
   ## Strong SSL Security
   ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/
-  ssl on;
   ssl_certificate /etc/nginx/ssl/gitlab-pages.crt;
   ssl_certificate_key /etc/nginx/ssl/gitlab-pages.key;
 
-  # GitLab needs backwards compatible ciphers to retain compatibility with Java IDEs
-  ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4";
-  ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
-  ssl_prefer_server_ciphers on;
+  ssl_session_timeout 1d;
   ssl_session_cache shared:SSL:10m;
-  ssl_session_timeout 5m;
+  ssl_session_tickets off;
+
+  ssl_protocols TLSv1.2 TLSv1.3;
+  ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
+  ssl_prefer_server_ciphers off;
 
   ## See app/controllers/application_controller.rb for headers set
 
@@ -58,6 +58,9 @@ server {
   ##
   # ssl_dhparam /etc/ssl/certs/dhparam.pem;
 
+  ## [Optional] Enable HTTP Strict Transport Security
+  # add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
+
   ## Individual nginx logs for GitLab pages
   access_log  /var/log/nginx/gitlab_pages_access.log;
   error_log   /var/log/nginx/gitlab_pages_error.log;
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index 576c13d8d10..435b9055929 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -83,23 +83,23 @@ server {
 
 ## HTTPS host
 server {
-  listen 0.0.0.0:443 ssl;
-  listen [::]:443 ipv6only=on ssl default_server;
+  listen 0.0.0.0:443 ssl http2;
+  listen [::]:443 ipv6only=on ssl http2 default_server;
   server_name YOUR_SERVER_FQDN; ## Replace this with something like gitlab.example.com
   server_tokens off; ## Don't show the nginx version number, a security best practice
 
   ## Strong SSL Security
   ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/
-  ssl on;
   ssl_certificate /etc/nginx/ssl/gitlab.crt;
   ssl_certificate_key /etc/nginx/ssl/gitlab.key;
 
-  # GitLab needs backwards compatible ciphers to retain compatibility with Java IDEs
-  ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4";
-  ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
-  ssl_prefer_server_ciphers on;
+  ssl_session_timeout 1d;
   ssl_session_cache shared:SSL:10m;
-  ssl_session_timeout 5m;
+  ssl_session_tickets off;
+
+  ssl_protocols TLSv1.2 TLSv1.3;
+  ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
+  ssl_prefer_server_ciphers off;
 
   ## See app/controllers/application_controller.rb for headers set
 
@@ -120,7 +120,7 @@ server {
   # ssl_dhparam /etc/ssl/certs/dhparam.pem;
 
   ## [Optional] Enable HTTP Strict Transport Security
-  # add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
+  # add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
 
   ## Real IP Module Config
   ## http://nginx.org/en/docs/http/ngx_http_realip_module.html
diff --git a/lib/support/nginx/registry-ssl b/lib/support/nginx/registry-ssl
index df126919866..be16037629b 100644
--- a/lib/support/nginx/registry-ssl
+++ b/lib/support/nginx/registry-ssl
@@ -27,15 +27,24 @@ server {
 
   ## Strong SSL Security
   ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/
-  ssl on;
   ssl_certificate /etc/gitlab/ssl/registry.gitlab.example.com.crt
   ssl_certificate_key /etc/gitlab/ssl/registry.gitlab.example.com.key
 
-  ssl_ciphers 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4';
-  ssl_protocols  TLSv1 TLSv1.1 TLSv1.2;
-  ssl_prefer_server_ciphers on;
-  ssl_session_cache  builtin:1000  shared:SSL:10m;
-  ssl_session_timeout  5m;
+  ssl_session_timeout 1d;
+  ssl_session_cache shared:SSL:10m;
+  ssl_session_tickets off;
+
+  ssl_protocols TLSv1.2 TLSv1.3;
+  ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
+  ssl_prefer_server_ciphers off;
+
+  ## [Optional] Generate a stronger DHE parameter:
+  ##   sudo openssl dhparam -out /etc/ssl/certs/dhparam.pem 4096
+  ##
+  # ssl_dhparam /etc/ssl/certs/dhparam.pem;
+
+  ## [Optional] Enable HTTP Strict Transport Security
+  # add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
 
   access_log  /var/log/gitlab/nginx/gitlab_registry_access.log gitlab_access;
   error_log   /var/log/gitlab/nginx/gitlab_registry_error.log;
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index ed74dd472ff..cc10d73f76a 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -91,6 +91,8 @@ namespace :gitlab do
         backup.cleanup
       end
 
+      backup.remove_tmp
+
       puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \
            "and are not included in this backup. You will need to restore these files manually.".color(:red)
       puts "Restore task is done."
@@ -297,7 +299,7 @@ namespace :gitlab do
     end
 
     def repository_backup_strategy
-      if Feature.enabled?(:gitaly_backup)
+      if Feature.enabled?(:gitaly_backup, default_enabled: :yaml)
         max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence
         max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence
         Backup::GitalyBackup.new(progress, parallel: max_concurrency, parallel_storage: max_storage_concurrency)
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 2b508b341dd..51f15f5a56a 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -161,7 +161,7 @@ namespace :gitlab do
         exit
       end
 
-      indexes = Gitlab::Database::Reindexing.candidate_indexes
+      indexes = Gitlab::Database::PostgresIndex.reindexing_support
 
       if identifier = args[:index_name]
         raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/
@@ -173,6 +173,12 @@ namespace :gitlab do
 
       ActiveRecord::Base.logger = Logger.new($stdout) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false)
 
+      # Cleanup leftover temporary indexes from previous, possibly aborted runs (if any)
+      Gitlab::Database::Reindexing.cleanup_leftovers!
+
+      # Hack: Before we do actual reindexing work, create async indexes
+      Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops)
+
       Gitlab::Database::Reindexing.perform(indexes)
     rescue StandardError => e
       Gitlab::AppLogger.error(e)
@@ -217,7 +223,7 @@ namespace :gitlab do
       instrumentation = Gitlab::Database::Migrations::Instrumentation.new
 
       pending_migrations.each do |migration|
-        instrumentation.observe(migration.version) do
+        instrumentation.observe(version: migration.version, name: migration.name) do
           ActiveRecord::Migrator.new(:up, ctx.migrations, ctx.schema_migration, migration.version).run
         end
       end
diff --git a/lib/tasks/gitlab/docs/redirect.rake b/lib/tasks/gitlab/docs/redirect.rake
index 990ff723eeb..123a4775605 100644
--- a/lib/tasks/gitlab/docs/redirect.rake
+++ b/lib/tasks/gitlab/docs/redirect.rake
@@ -57,68 +57,5 @@ namespace :gitlab do
         post.puts "<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page -->"
       end
     end
-
-    desc 'GitLab | Docs | Clean up old redirects'
-    task :clean_redirects do
-      #
-      # Calculate new path from the redirect URL.
-      #
-      # If the redirect is not a full URL:
-      #   1. Create a new Pathname of the file
-      #   2. Use dirname to get all but the last component of the path
-      #   3. Join with the redirect_to entry
-      #   4. Substitute:
-      #      - '.md' => '.html'
-      #      - 'doc/' => '/ee/'
-      #
-      # If the redirect URL is a full URL pointing to the Docs site
-      # (cross-linking among the 4 products), remove the FQDN prefix:
-      #
-      #   From : https://docs.gitlab.com/ee/install/requirements.html
-      #   To   : /ee/install/requirements.html
-      #
-      def new_path(redirect, filename)
-        if !redirect.start_with?('http')
-          Pathname.new(filename).dirname.join(redirect).to_s.gsub(%r(\.md), '.html').gsub(%r(doc/), '/ee/')
-        elsif redirect.start_with?('https://docs.gitlab.com')
-          redirect.gsub('https://docs.gitlab.com', '')
-        else
-          redirect
-        end
-      end
-
-      today = Time.now.utc.to_date
-
-      #
-      # Find the files to be deleted.
-      # Exclude 'doc/development/documentation/index.md' because it
-      # contains an example of the YAML front matter.
-      #
-      files_to_be_deleted = `grep -Ir 'remove_date:' doc | grep -v doc/development/documentation/index.md | cut -d ":" -f 1`.split("\n")
-
-      #
-      # Iterate over the files to be deleted and print the needed
-      # YAML entries for the Docs site redirects.
-      #
-      files_to_be_deleted.each do |filename|
-        frontmatter = YAML.safe_load(File.read(filename))
-        remove_date = Date.parse(frontmatter['remove_date'])
-        old_path = filename.gsub(%r(\.md), '.html').gsub(%r(doc/), '/ee/')
-
-        #
-        # Check if the removal date is before today, and delete the file and
-        # print the content to be pasted in
-        # https://gitlab.com/gitlab-org/gitlab-docs/-/blob/master/content/_data/redirects.yaml.
-        # The remove_date of redirects.yaml should be nine months in the future.
-        # To not be confused with the remove_date of the Markdown page.
-        #
-        next unless remove_date < today
-
-        File.delete(filename) if File.exist?(filename)
-        puts "  - from: #{old_path}"
-        puts "    to: #{new_path(frontmatter['redirect_to'], filename)}"
-        puts "    remove_date: #{remove_date >> 9}"
-      end
-    end
   end
 end
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index df75b3cf716..6675439e430 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -2,6 +2,42 @@
 
 namespace :gitlab do
   namespace :gitaly do
+    desc 'Installs gitaly for running tests within gitlab-development-kit'
+    task :test_install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args|
+      inside_gdk = Rails.env.test? && File.exist?(Rails.root.join('../GDK_ROOT'))
+
+      if ENV['FORCE_GITALY_INSTALL'] || !inside_gdk
+        Rake::Task["gitlab:gitaly:install"].invoke(*args)
+
+        next
+      end
+
+      gdk_gitaly_dir = ENV.fetch('GDK_GITALY', Rails.root.join('../gitaly'))
+
+      # Our test setup expects a git repo, so clone rather than copy
+      version = Gitlab::GitalyClient.expected_server_version
+      checkout_or_clone_version(version: version, repo: gdk_gitaly_dir, target_dir: args.dir, clone_opts: %w[--depth 1])
+
+      # We assume the GDK gitaly already compiled binaries
+      build_dir = File.join(gdk_gitaly_dir, '_build')
+      FileUtils.cp_r(build_dir, args.dir)
+
+      # We assume the GDK gitaly already ran bundle install
+      bundle_dir = File.join(gdk_gitaly_dir, 'ruby', '.bundle')
+      FileUtils.cp_r(bundle_dir, File.join(args.dir, 'ruby'))
+
+      # For completeness we copy this for gitaly's make target
+      ruby_bundle_file = File.join(gdk_gitaly_dir, '.ruby-bundle')
+      FileUtils.cp_r(ruby_bundle_file, args.dir)
+
+      gitaly_binary = File.join(build_dir, 'bin', 'gitaly')
+      warn_gitaly_out_of_date!(gitaly_binary, version)
+    rescue Errno::ENOENT => e
+      puts "Could not copy files, did you run `gdk update`? Error: #{e.message}"
+
+      raise
+    end
+
     desc 'GitLab | Gitaly | Install or upgrade gitaly'
     task :install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args|
       warn_user_is_not_gitlab
@@ -41,5 +77,24 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
       _, status = Gitlab::Popen.popen(%w[which gmake])
       status == 0 ? 'gmake' : 'make'
     end
+
+    def warn_gitaly_out_of_date!(gitaly_binary, expected_version)
+      binary_version, exit_status = Gitlab::Popen.popen(%W[#{gitaly_binary} -version])
+
+      raise "Failed to run `#{gitaly_binary} -version`" unless exit_status == 0
+
+      binary_version = binary_version.strip
+
+      # See help for `git describe` for format
+      git_describe_sha = /g([a-f0-9]{5,40})\z/
+      match = binary_version.match(git_describe_sha)
+
+      # Just skip if the version does not have a sha component
+      return unless match
+
+      return if expected_version.start_with?(match[1])
+
+      puts "WARNING: #{binary_version.strip} does not exactly match repository version #{expected_version}"
+    end
   end
 end
diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake
index b405cbd3f68..52c5c680292 100644
--- a/lib/tasks/gitlab/graphql.rake
+++ b/lib/tasks/gitlab/graphql.rake
@@ -56,9 +56,9 @@ namespace :gitlab do
           color = case complexity
                   when 0..GitlabSchema::DEFAULT_MAX_COMPLEXITY
                     :green
-                  when GitlabSchema::DEFAULT_MAX_COMPLEXITY..GitlabSchema::AUTHENTICATED_COMPLEXITY
+                  when GitlabSchema::DEFAULT_MAX_COMPLEXITY..GitlabSchema::AUTHENTICATED_MAX_COMPLEXITY
                     :yellow
-                  when GitlabSchema::AUTHENTICATED_COMPLEXITY..GitlabSchema::ADMIN_COMPLEXITY
+                  when GitlabSchema::AUTHENTICATED_MAX_COMPLEXITY..GitlabSchema::ADMIN_MAX_COMPLEXITY
                     :orange
                   else
                     :red
diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake
index 2826002bdc2..68395d10d24 100644
--- a/lib/tasks/gitlab/info.rake
+++ b/lib/tasks/gitlab/info.rake
@@ -68,8 +68,8 @@ namespace :gitlab do
       puts "Version:\t#{Gitlab::VERSION}"
       puts "Revision:\t#{Gitlab.revision}"
       puts "Directory:\t#{Rails.root}"
-      puts "DB Adapter:\t#{Gitlab::Database.human_adapter_name}"
-      puts "DB Version:\t#{Gitlab::Database.version}"
+      puts "DB Adapter:\t#{Gitlab::Database.main.human_adapter_name}"
+      puts "DB Version:\t#{Gitlab::Database.main.version}"
       puts "URL:\t\t#{Gitlab.config.gitlab.url}"
       puts "HTTP Clone URL:\t#{http_clone_url}"
       puts "SSH Clone URL:\t#{ssh_clone_url}"
diff --git a/lib/tasks/gitlab/product_intelligence.rake b/lib/tasks/gitlab/product_intelligence.rake
new file mode 100644
index 00000000000..329cd9c8c2a
--- /dev/null
+++ b/lib/tasks/gitlab/product_intelligence.rake
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+namespace :gitlab do
+  namespace :product_intelligence do
+    # @example
+    #   bundle exec rake gitlab:product_intelligence:activate_metrics MILESTONE=14.0
+
+    desc 'GitLab | Product Intelligence | Update milestone metrics status to data_available'
+    task activate_metrics: :environment do
+      milestone = ENV['MILESTONE']
+      raise "Please supply the MILESTONE env var".color(:red) unless milestone.present?
+
+      Gitlab::Usage::MetricDefinition.definitions.values.each do |metric|
+        next if metric.attributes[:milestone] != milestone || metric.attributes[:status] != 'implemented'
+
+        metric.attributes[:status] = 'data_available'
+        path = metric.path
+        File.open(path, "w") { |file| file << metric.to_h.deep_stringify_keys.to_yaml }
+      end
+
+      puts "Task completed successfully"
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/smtp.rake b/lib/tasks/gitlab/smtp.rake
new file mode 100644
index 00000000000..23ad7577e3c
--- /dev/null
+++ b/lib/tasks/gitlab/smtp.rake
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+namespace :gitlab do
+  namespace :smtp do
+    namespace :secret do
+      desc 'GitLab | SMTP | Secret | Write SMTP secrets'
+      task write: [:environment] do
+        content = $stdin.tty? ? $stdin.gets : $stdin.read
+        Gitlab::EncryptedSmtpCommand.write(content)
+      end
+
+      desc 'GitLab | SMTP | Secret | Edit SMTP secrets'
+      task edit: [:environment] do
+        Gitlab::EncryptedSmtpCommand.edit
+      end
+
+      desc 'GitLab | SMTP | Secret | Show SMTP secrets'
+      task show: [:environment] do
+        Gitlab::EncryptedSmtpCommand.show
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake
index 6fa39a26488..fb9f9b9fe67 100644
--- a/lib/tasks/gitlab/storage.rake
+++ b/lib/tasks/gitlab/storage.rake
@@ -170,7 +170,7 @@ namespace :gitlab do
       inverval = (ENV['MAX_DATABASE_CONNECTION_CHECK_INTERVAL'] || 10).to_f
 
       attempts.to_i.times do
-        unless Gitlab::Database.exists?
+        unless Gitlab::Database.main.exists?
           puts "Waiting until database is ready before continuing...".color(:yellow)
           sleep inverval
         end
diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake
index 166f08ef16a..ddd3424acda 100644
--- a/lib/tasks/gitlab/usage_data.rake
+++ b/lib/tasks/gitlab/usage_data.rake
@@ -24,12 +24,6 @@ namespace :gitlab do
       puts Gitlab::Json.pretty_generate(result.attributes)
     end
 
-    desc 'GitLab | UsageData | Generate metrics dictionary'
-    task generate_metrics_dictionary: :environment do
-      items = Gitlab::Usage::MetricDefinition.definitions
-      Gitlab::Usage::Docs::Renderer.new(items).write
-    end
-
     desc 'GitLab | UsageDataMetrics | Generate usage ping from metrics definition YAML files in JSON'
     task generate_from_yaml: :environment do
       puts Gitlab::Json.pretty_generate(Gitlab::UsageDataMetrics.uncached_data)