From 9dc93a4519d9d5d7be48ff274127136236a3adb3 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 20 Apr 2021 23:50:22 +0000 Subject: Add latest changes from gitlab-org/gitlab@13-11-stable-ee --- lib/gitlab/alert_management/payload/base.rb | 2 +- .../analytics/cycle_analytics/records_fetcher.rb | 54 +++-- lib/gitlab/analytics/unique_visits.rb | 4 +- lib/gitlab/application_context.rb | 7 + lib/gitlab/auth/auth_finders.rb | 4 +- lib/gitlab/auth/ldap/adapter.rb | 2 +- lib/gitlab/auth/saml/origin_validator.rb | 2 +- .../backfill_design_internal_ids.rb | 4 +- ...ect_updated_at_after_repository_storage_move.rb | 2 +- .../copy_column_using_background_migration_job.rb | 8 +- ...uest_target_project_to_merge_request_metrics.rb | 2 +- .../fix_projects_without_project_feature.rb | 2 +- .../fix_projects_without_prometheus_service.rb | 4 +- .../fix_user_namespace_names.rb | 4 +- .../fix_user_project_route_names.rb | 2 +- .../migrate_pages_to_zip_storage.rb | 19 ++ ...ontainer_registry_enabled_to_project_feature.rb | 2 +- .../populate_has_vulnerabilities.rb | 30 +-- .../populate_merge_request_assignees_table.rb | 2 +- ...recalculate_vulnerabilities_occurrences_uuid.rb | 2 +- .../isolated_mentioned_project_parser.rb | 25 ++ .../isolated_mentioned_user_parser.rb | 25 ++ .../lib/gitlab/isolated_reference_extractor.rb | 2 +- .../lib/gitlab/isolated_visibility_level.rb | 60 +++++ .../user_mentions/models/commit_user_mention.rb | 1 + .../models/concerns/isolated_feature_gate.rb | 20 ++ .../models/concerns/isolated_mentionable.rb | 4 +- .../concerns/namespace/recursive_traversal.rb | 2 +- .../models/design_management/design.rb | 3 + .../user_mentions/models/design_user_mention.rb | 1 + .../user_mentions/models/epic.rb | 6 +- .../user_mentions/models/epic_user_mention.rb | 1 + .../user_mentions/models/group.rb | 2 + .../user_mentions/models/merge_request.rb | 7 +- .../models/merge_request_user_mention.rb | 1 + .../user_mentions/models/namespace.rb | 15 +- .../user_mentions/models/note.rb | 4 +- .../user_mentions/models/project.rb | 48 ++++ .../user_mentions/models/user.rb | 37 +++ .../wrongfully_confirmed_email_unconfirmer.rb | 4 +- lib/gitlab/batch_pop_queueing.rb | 3 +- lib/gitlab/bullet.rb | 16 ++ lib/gitlab/bullet/exclusions.rb | 37 +++ lib/gitlab/cache/ci/project_pipeline_status.rb | 4 +- lib/gitlab/changelog/config.rb | 24 +- lib/gitlab/chaos.rb | 6 +- lib/gitlab/ci/build/artifacts/metadata.rb | 4 +- lib/gitlab/ci/config.rb | 10 +- lib/gitlab/ci/config/entry/cache.rb | 6 +- lib/gitlab/ci/config/entry/processable.rb | 12 +- lib/gitlab/ci/config/entry/product/variables.rb | 3 +- lib/gitlab/ci/config/entry/variables.rb | 4 +- lib/gitlab/ci/config/external/mapper.rb | 12 + lib/gitlab/ci/config/normalizer/matrix_strategy.rb | 5 +- lib/gitlab/ci/features.rb | 12 +- lib/gitlab/ci/jwt.rb | 16 +- lib/gitlab/ci/pipeline/chain/command.rb | 9 +- lib/gitlab/ci/pipeline/chain/config/process.rb | 1 + .../ci/pipeline/chain/evaluate_workflow_rules.rb | 22 +- lib/gitlab/ci/pipeline/chain/helpers.rb | 20 +- lib/gitlab/ci/pipeline/chain/metrics.rb | 2 +- lib/gitlab/ci/pipeline/chain/pipeline/process.rb | 4 +- lib/gitlab/ci/pipeline/chain/seed.rb | 20 +- lib/gitlab/ci/pipeline/chain/validate/external.rb | 85 +++++-- lib/gitlab/ci/pipeline/metrics.rb | 72 +++--- lib/gitlab/ci/pipeline/seed/build.rb | 19 +- lib/gitlab/ci/pipeline/seed/context.rb | 18 ++ lib/gitlab/ci/pipeline/seed/pipeline.rb | 6 +- lib/gitlab/ci/pipeline/seed/stage.rb | 7 +- lib/gitlab/ci/queue/metrics.rb | 35 ++- lib/gitlab/ci/reports/codequality_reports.rb | 12 +- .../ci/reports/codequality_reports_comparer.rb | 5 + lib/gitlab/ci/reports/test_failure_history.rb | 18 +- lib/gitlab/ci/runner_instructions.rb | 31 +-- lib/gitlab/ci/status/build/failed.rb | 4 +- .../ci/templates/Android-Fastlane.gitlab-ci.yml | 7 +- lib/gitlab/ci/templates/Docker.gitlab-ci.yml | 42 ++-- lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml | 9 + .../ci/templates/Indeni.Cloudrail.gitlab-ci-.yml | 91 +++++++ .../Jobs/Browser-Performance-Testing.gitlab-ci.yml | 37 ++- lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml | 4 +- .../ci/templates/Jobs/Code-Quality.gitlab-ci.yml | 1 + .../templates/Security/API-Fuzzing.gitlab-ci.yml | 12 +- .../Security/API-Fuzzing.latest.gitlab-ci.yml | 270 +++++++++++++++++++++ .../Security/Container-Scanning.gitlab-ci.yml | 41 +++- .../templates/Security/DAST.latest.gitlab-ci.yml | 99 ++++++++ .../ci/templates/Security/SAST.gitlab-ci.yml | 9 +- .../Verify/Browser-Performance.gitlab-ci.yml | 25 +- lib/gitlab/ci/trace.rb | 33 ++- lib/gitlab/ci/variables/helpers.rb | 16 +- lib/gitlab/ci/yaml_processor/result.rb | 17 +- lib/gitlab/composer/version_index.rb | 22 +- lib/gitlab/conan_token.rb | 2 +- lib/gitlab/contributor.rb | 4 +- lib/gitlab/crypto_helper.rb | 28 +-- lib/gitlab/data_builder/build.rb | 6 +- lib/gitlab/data_builder/pipeline.rb | 1 - lib/gitlab/database/as_with_materialized.rb | 36 +++ .../database/background_migration/batch_metrics.rb | 33 +++ .../background_migration/batched_migration.rb | 20 +- .../batched_migration_runner.rb | 88 +++++++ .../batched_migration_wrapper.rb | 73 ++++++ .../database/background_migration/scheduler.rb | 60 ----- lib/gitlab/database/batch_count.rb | 39 ++- lib/gitlab/database/bulk_update.rb | 2 +- .../database/count/reltuples_count_strategy.rb | 6 +- .../database/loose_index_scan_distinct_count.rb | 102 ++++++++ lib/gitlab/database/migration_helpers.rb | 160 ++++++------ .../migrations/background_migration_helpers.rb | 21 +- .../foreign_key_helpers.rb | 74 ++++++ .../table_management_helpers.rb | 32 ++- lib/gitlab/database/pg_class.rb | 23 ++ .../postgres_hll/batch_distinct_counter.rb | 31 +-- lib/gitlab/database/similarity_score.rb | 2 +- lib/gitlab/database/unidirectional_copy_trigger.rb | 97 ++++++++ lib/gitlab/diff/highlight.rb | 106 ++++++-- lib/gitlab/diff/highlight_cache.rb | 7 +- lib/gitlab/diff/inline_diff.rb | 1 + lib/gitlab/diff/line.rb | 25 +- lib/gitlab/diff/suggestions_parser.rb | 7 +- lib/gitlab/downtime_check.rb | 73 ------ lib/gitlab/downtime_check/message.rb | 41 ---- lib/gitlab/error_tracking.rb | 10 +- .../processor/context_payload_processor.rb | 12 + .../processor/grpc_error_processor.rb | 166 +++++++++---- .../error_tracking/processor/sidekiq_processor.rb | 71 ++++-- lib/gitlab/exclusive_lease.rb | 4 +- lib/gitlab/experimentation.rb | 4 - lib/gitlab/external_authorization/access.rb | 3 +- lib/gitlab/external_authorization/cache.rb | 3 +- lib/gitlab/external_authorization/client.rb | 23 +- lib/gitlab/fogbugz_import/importer.rb | 6 +- lib/gitlab/git/blame.rb | 6 +- lib/gitlab/git/commit.rb | 2 +- lib/gitlab/git/diff_collection.rb | 49 +++- lib/gitlab/git/merge_base.rb | 3 +- lib/gitlab/git/patches/commit_patches.rb | 5 +- lib/gitlab/git/repository.rb | 8 +- lib/gitlab/git/tag.rb | 4 + lib/gitlab/git/wiki.rb | 13 - lib/gitlab/git/wiki_file.rb | 24 +- lib/gitlab/git_access.rb | 5 + lib/gitlab/gitaly_client.rb | 2 +- lib/gitlab/gitaly_client/attributes_bag.rb | 2 +- lib/gitlab/gitaly_client/blob_service.rb | 55 +++-- lib/gitlab/gitaly_client/call.rb | 14 +- lib/gitlab/gitaly_client/commit_service.rb | 2 + lib/gitlab/gitaly_client/operation_service.rb | 2 +- lib/gitlab/gitaly_client/repository_service.rb | 5 + lib/gitlab/gitaly_client/storage_settings.rb | 2 +- lib/gitlab/gitaly_client/wiki_file.rb | 11 - lib/gitlab/gitaly_client/wiki_service.rb | 26 -- lib/gitlab/golang.rb | 29 ++- lib/gitlab/gon_helper.rb | 2 +- lib/gitlab/grape_logging/loggers/context_logger.rb | 2 +- lib/gitlab/graphql/authorize.rb | 15 -- .../graphql/authorize/authorize_field_service.rb | 147 ----------- lib/gitlab/graphql/authorize/authorize_resource.rb | 44 ++-- .../authorize/connection_filter_extension.rb | 65 +++++ lib/gitlab/graphql/authorize/instrumentation.rb | 21 -- .../graphql/authorize/object_authorization.rb | 32 +++ lib/gitlab/graphql/deprecation.rb | 116 +++++++++ lib/gitlab/graphql/docs/helper.rb | 134 +++++++--- lib/gitlab/graphql/docs/renderer.rb | 7 +- lib/gitlab/graphql/docs/templates/default.md.haml | 18 +- lib/gitlab/graphql/loaders/batch_lfs_oid_loader.rb | 3 +- lib/gitlab/graphql/loaders/batch_model_loader.rb | 3 +- .../graphql/loaders/full_path_model_loader.rb | 3 +- lib/gitlab/graphql/negatable_arguments.rb | 53 ++++ .../pagination/keyset/conditions/base_condition.rb | 6 +- .../keyset/conditions/not_null_condition.rb | 4 +- .../pagination/keyset/conditions/null_condition.rb | 4 +- .../graphql/pagination/keyset/query_builder.rb | 5 +- lib/gitlab/graphql/queries.rb | 4 +- .../graphql/query_analyzers/logger_analyzer.rb | 7 +- lib/gitlab/health_checks/gitaly_check.rb | 2 +- lib/gitlab/highlight.rb | 14 +- lib/gitlab/hook_data/user_builder.rb | 53 ++++ lib/gitlab/http_connection_adapter.rb | 14 -- lib/gitlab/import_export/base/relation_factory.rb | 2 +- lib/gitlab/import_export/project/import_export.yml | 2 + lib/gitlab/import_export/uploads_manager.rb | 2 +- lib/gitlab/import_sources.rb | 2 +- lib/gitlab/instrumentation_helper.rb | 18 -- lib/gitlab/issuables_count_for_state.rb | 2 +- lib/gitlab/jira/dvcs.rb | 4 +- lib/gitlab/json.rb | 33 +++ lib/gitlab/kas.rb | 2 +- lib/gitlab/kubernetes/deployment.rb | 2 +- lib/gitlab/language_detection.rb | 5 +- lib/gitlab/manifest_import/manifest.rb | 8 + lib/gitlab/marker_range.rb | 6 + lib/gitlab/markup_helper.rb | 2 +- lib/gitlab/metrics/background_transaction.rb | 20 +- .../metrics/dashboard/stages/grafana_formatter.rb | 8 +- lib/gitlab/metrics/samplers/database_sampler.rb | 4 +- lib/gitlab/metrics/subscribers/active_record.rb | 21 +- lib/gitlab/metrics/subscribers/external_http.rb | 5 +- lib/gitlab/middleware/multipart.rb | 2 +- .../middleware/rack_multipart_tempfile_factory.rb | 25 ++ lib/gitlab/middleware/same_site_cookies.rb | 2 +- lib/gitlab/object_hierarchy.rb | 46 +++- lib/gitlab/pages.rb | 2 +- lib/gitlab/pages/migration_helper.rb | 53 ++++ lib/gitlab/pages/settings.rb | 22 +- lib/gitlab/pages/stores/local_store.rb | 15 ++ lib/gitlab/pages_transfer.rb | 4 +- lib/gitlab/pagination/keyset/order.rb | 8 +- lib/gitlab/pagination/offset_header_builder.rb | 21 +- lib/gitlab/performance_bar/stats.rb | 15 +- lib/gitlab/phabricator_import.rb | 2 +- lib/gitlab/phabricator_import/issues/importer.rb | 3 +- .../phabricator_import/issues/task_importer.rb | 3 +- lib/gitlab/phabricator_import/project_creator.rb | 13 +- lib/gitlab/phabricator_import/user_finder.rb | 3 +- lib/gitlab/project_template.rb | 6 +- lib/gitlab/prometheus/adapter.rb | 4 + .../prometheus/queries/matched_metric_query.rb | 2 +- lib/gitlab/prometheus_client.rb | 2 +- lib/gitlab/push_options.rb | 6 +- lib/gitlab/query_limiting.rb | 22 +- lib/gitlab/query_limiting/transaction.rb | 9 +- lib/gitlab/quick_actions/command_definition.rb | 11 +- .../issue_and_merge_request_actions.rb | 2 +- lib/gitlab/rack_attack/request.rb | 6 +- lib/gitlab/regex.rb | 6 +- lib/gitlab/relative_positioning/closed_range.rb | 3 +- lib/gitlab/relative_positioning/gap.rb | 3 +- lib/gitlab/repository_cache_adapter.rb | 9 +- lib/gitlab/repository_hash_cache.rb | 2 +- lib/gitlab/repository_set_cache.rb | 30 ++- lib/gitlab/search_context.rb | 7 +- lib/gitlab/set_cache.rb | 13 + lib/gitlab/setup_helper.rb | 12 +- lib/gitlab/sidekiq_cluster/cli.rb | 14 +- lib/gitlab/sidekiq_config.rb | 17 +- lib/gitlab/sidekiq_config/cli_methods.rb | 90 ++----- lib/gitlab/sidekiq_config/worker_matcher.rb | 86 +++++++ lib/gitlab/sidekiq_logging/structured_logger.rb | 4 +- lib/gitlab/sidekiq_middleware.rb | 2 + lib/gitlab/sidekiq_middleware/admin_mode/client.rb | 3 +- lib/gitlab/sidekiq_middleware/admin_mode/server.rb | 3 +- .../sidekiq_middleware/instrumentation_logger.rb | 23 +- lib/gitlab/sidekiq_middleware/metrics_helper.rb | 15 +- lib/gitlab/sidekiq_middleware/server_metrics.rb | 47 ++-- lib/gitlab/sidekiq_queue.rb | 2 +- lib/gitlab/slash_commands/base_command.rb | 4 +- lib/gitlab/slash_commands/presenters/issue_new.rb | 14 +- lib/gitlab/slash_commands/run.rb | 2 +- lib/gitlab/slug/environment.rb | 17 +- lib/gitlab/sql/cte.rb | 11 +- lib/gitlab/sql/recursive_cte.rb | 6 +- lib/gitlab/sql/set_operator.rb | 12 +- lib/gitlab/sql/union.rb | 4 +- .../static_site_editor/config/file_config.rb | 2 +- lib/gitlab/subscription_portal.rb | 7 +- lib/gitlab/template/base_template.rb | 4 +- lib/gitlab/tracking.rb | 25 +- lib/gitlab/tracking/destinations/snowplow.rb | 7 - lib/gitlab/tracking/standard_context.rb | 13 +- lib/gitlab/untrusted_regexp.rb | 4 + lib/gitlab/updated_notes_paginator.rb | 4 +- lib/gitlab/usage/docs/helper.rb | 4 + lib/gitlab/usage/docs/templates/default.md.haml | 3 + lib/gitlab/usage/metric_definition.rb | 32 ++- lib/gitlab/usage/metrics/aggregates/aggregate.rb | 2 +- .../usage/metrics/names_suggestions/generator.rb | 155 ++++++++++-- .../names_suggestions/relation_parsers/joins.rb | 74 ++++++ lib/gitlab/usage_data.rb | 92 ++----- .../aggregated_metrics/code_review.yml | 108 --------- .../aggregated_metrics/common.yml | 72 ------ lib/gitlab/usage_data_counters/base_counter.rb | 4 +- .../ci_template_unique_counter.rb | 4 +- .../usage_data_counters/hll_redis_counter.rb | 5 +- .../issue_activity_unique_counter.rb | 112 ++++----- .../known_events/ci_templates.yml | 15 -- .../usage_data_counters/known_events/common.yml | 42 +++- .../known_events/epic_events.yml | 142 +++++++++++ lib/gitlab/usage_data_counters/note_counter.rb | 6 +- .../quick_action_activity_unique_counter.rb | 2 +- lib/gitlab/usage_data_non_sql_metrics.rb | 35 +++ lib/gitlab/usage_data_queries.rb | 14 +- lib/gitlab/utils.rb | 2 + lib/gitlab/utils/usage_data.rb | 15 ++ lib/gitlab/uuid.rb | 4 +- lib/gitlab/web_ide/config/entry/terminal.rb | 18 +- lib/gitlab/word_diff/chunk_collection.rb | 21 ++ lib/gitlab/word_diff/parser.rb | 2 +- 288 files changed, 4408 insertions(+), 1835 deletions(-) create mode 100644 lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb create mode 100644 lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb create mode 100644 lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb create mode 100644 lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb create mode 100644 lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb create mode 100644 lib/gitlab/background_migration/user_mentions/models/project.rb create mode 100644 lib/gitlab/background_migration/user_mentions/models/user.rb create mode 100644 lib/gitlab/bullet.rb create mode 100644 lib/gitlab/bullet/exclusions.rb create mode 100644 lib/gitlab/ci/pipeline/seed/context.rb create mode 100644 lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml create mode 100644 lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml create mode 100644 lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml create mode 100644 lib/gitlab/database/as_with_materialized.rb create mode 100644 lib/gitlab/database/background_migration/batch_metrics.rb create mode 100644 lib/gitlab/database/background_migration/batched_migration_runner.rb delete mode 100644 lib/gitlab/database/background_migration/scheduler.rb create mode 100644 lib/gitlab/database/loose_index_scan_distinct_count.rb create mode 100644 lib/gitlab/database/pg_class.rb create mode 100644 lib/gitlab/database/unidirectional_copy_trigger.rb delete mode 100644 lib/gitlab/downtime_check.rb delete mode 100644 lib/gitlab/downtime_check/message.rb delete mode 100644 lib/gitlab/gitaly_client/wiki_file.rb delete mode 100644 lib/gitlab/graphql/authorize.rb delete mode 100644 lib/gitlab/graphql/authorize/authorize_field_service.rb create mode 100644 lib/gitlab/graphql/authorize/connection_filter_extension.rb delete mode 100644 lib/gitlab/graphql/authorize/instrumentation.rb create mode 100644 lib/gitlab/graphql/authorize/object_authorization.rb create mode 100644 lib/gitlab/graphql/deprecation.rb create mode 100644 lib/gitlab/graphql/negatable_arguments.rb create mode 100644 lib/gitlab/hook_data/user_builder.rb create mode 100644 lib/gitlab/middleware/rack_multipart_tempfile_factory.rb create mode 100644 lib/gitlab/pages/migration_helper.rb create mode 100644 lib/gitlab/pages/stores/local_store.rb create mode 100644 lib/gitlab/sidekiq_config/worker_matcher.rb create mode 100644 lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb delete mode 100644 lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml delete mode 100644 lib/gitlab/usage_data_counters/aggregated_metrics/common.yml create mode 100644 lib/gitlab/usage_data_counters/known_events/epic_events.yml create mode 100644 lib/gitlab/usage_data_non_sql_metrics.rb (limited to 'lib/gitlab') diff --git a/lib/gitlab/alert_management/payload/base.rb b/lib/gitlab/alert_management/payload/base.rb index c8b8d6c259d..786c5bf675b 100644 --- a/lib/gitlab/alert_management/payload/base.rb +++ b/lib/gitlab/alert_management/payload/base.rb @@ -132,7 +132,7 @@ module Gitlab EnvironmentsFinder .new(project, nil, { name: environment_name }) - .find + .execute .first end end diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb index 178ebe0d4d4..b4752ed9e5b 100644 --- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb @@ -31,14 +31,34 @@ module Gitlab @params = params @sort = params[:sort] || :end_event @direction = params[:direction] || :desc + @page = params[:page] || 1 + @per_page = MAX_RECORDS end + # rubocop: disable CodeReuse/ActiveRecord def serialized_records strong_memoize(:serialized_records) do # special case (legacy): 'Test' and 'Staging' stages should show Ci::Build records if default_test_stage? || default_staging_stage? + ci_build_join = mr_metrics_table + .join(build_table) + .on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) + .join_sources + + records = ordered_and_limited_query + .joins(ci_build_join) + .select(build_table[:id], *time_columns) + + yield records if block_given? + ci_build_records = preload_ci_build_associations(records) + AnalyticsBuildSerializer.new.represent(ci_build_records.map { |e| e['build'] }) else + records = ordered_and_limited_query.select(*columns, *time_columns) + + yield records if block_given? + records = preload_associations(records) + records.map do |record| project = record.project attributes = record.attributes.merge({ @@ -51,10 +71,11 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord private - attr_reader :stage, :query, :params, :sort, :direction + attr_reader :stage, :query, :params, :sort, :direction, :page, :per_page def columns MAPPINGS.fetch(subject_class).fetch(:columns_for_select).map do |column_name| @@ -74,41 +95,32 @@ module Gitlab MAPPINGS.fetch(subject_class).fetch(:serializer_class).new end - # Loading Ci::Build records instead of MergeRequest records # rubocop: disable CodeReuse/ActiveRecord - def ci_build_records - ci_build_join = mr_metrics_table - .join(build_table) - .on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) - .join_sources - - q = ordered_and_limited_query - .joins(ci_build_join) - .select(build_table[:id], *time_columns) - - results = execute_query(q).to_a + def preload_ci_build_associations(records) + results = records.map(&:attributes) Gitlab::CycleAnalytics::Updater.update!(results, from: 'id', to: 'build', klass: ::Ci::Build.includes({ project: [:namespace], user: [], pipeline: [] })) end + # rubocop: enable CodeReuse/ActiveRecord def ordered_and_limited_query - order_by(query, sort, direction, columns).limit(MAX_RECORDS) + strong_memoize(:ordered_and_limited_query) do + order_by(query, sort, direction, columns).page(page).per(per_page).without_count + end end - def records - results = ordered_and_limited_query - .select(*columns, *time_columns) - + # rubocop: disable CodeReuse/ActiveRecord + def preload_associations(records) # using preloader instead of includes to avoid AR generating a large column list ActiveRecord::Associations::Preloader.new.preload( - results, + records, MAPPINGS.fetch(subject_class).fetch(:includes_for_query) ) - results + records end - # rubocop: enable CodeReuse/ActiveRecord + # rubocop: enable CodeReuse/ActiveRecord def time_columns [ stage.start_event.timestamp_projection.as('start_event_timestamp'), diff --git a/lib/gitlab/analytics/unique_visits.rb b/lib/gitlab/analytics/unique_visits.rb index e367d33d743..723486231b1 100644 --- a/lib/gitlab/analytics/unique_visits.rb +++ b/lib/gitlab/analytics/unique_visits.rb @@ -3,8 +3,8 @@ module Gitlab module Analytics class UniqueVisits - def track_visit(visitor_id, target_id, time = Time.zone.now) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(target_id, values: visitor_id, time: time) + def track_visit(*args, **kwargs) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(*args, **kwargs) end # Returns number of unique visitors for given targets in given time frame diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index a75da3a682b..ceda82cb6f6 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -8,6 +8,9 @@ module Gitlab Attribute = Struct.new(:name, :type) + LOG_KEY = Labkit::Context::LOG_KEY + KNOWN_KEYS = Labkit::Context::KNOWN_KEYS + APPLICATION_ATTRIBUTES = [ Attribute.new(:project, Project), Attribute.new(:namespace, Namespace), @@ -24,6 +27,10 @@ module Gitlab application_context.use(&block) end + def self.with_raw_context(attributes = {}, &block) + Labkit::Context.with_context(attributes, &block) + end + def self.push(args) application_context = new(**args) Labkit::Context.push(application_context.to_lazy_hash) diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index 4c6254c9e69..6f6ac79c16b 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -24,9 +24,9 @@ module Gitlab PRIVATE_TOKEN_HEADER = 'HTTP_PRIVATE_TOKEN' PRIVATE_TOKEN_PARAM = :private_token - JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze + JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN' JOB_TOKEN_PARAM = :job_token - DEPLOY_TOKEN_HEADER = 'HTTP_DEPLOY_TOKEN'.freeze + DEPLOY_TOKEN_HEADER = 'HTTP_DEPLOY_TOKEN' RUNNER_TOKEN_PARAM = :token RUNNER_JOB_TOKEN_PARAM = :token diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index b7bb61f0677..7f85d3b1cd3 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -5,7 +5,7 @@ module Gitlab module Ldap class Adapter SEARCH_RETRY_FACTOR = [1, 1, 2, 3].freeze - MAX_SEARCH_RETRIES = Rails.env.test? ? 1 : SEARCH_RETRY_FACTOR.size.freeze + MAX_SEARCH_RETRIES = Rails.env.test? ? 1 : SEARCH_RETRY_FACTOR.size attr_reader :provider, :ldap diff --git a/lib/gitlab/auth/saml/origin_validator.rb b/lib/gitlab/auth/saml/origin_validator.rb index 4ecc688888f..ff0d25314f7 100644 --- a/lib/gitlab/auth/saml/origin_validator.rb +++ b/lib/gitlab/auth/saml/origin_validator.rb @@ -4,7 +4,7 @@ module Gitlab module Auth module Saml class OriginValidator - AUTH_REQUEST_SESSION_KEY = "last_authn_request_id".freeze + AUTH_REQUEST_SESSION_KEY = "last_authn_request_id" def initialize(session) @session = session || {} diff --git a/lib/gitlab/background_migration/backfill_design_internal_ids.rb b/lib/gitlab/background_migration/backfill_design_internal_ids.rb index 553571d5d00..6d1df95c66d 100644 --- a/lib/gitlab/background_migration/backfill_design_internal_ids.rb +++ b/lib/gitlab/background_migration/backfill_design_internal_ids.rb @@ -97,13 +97,13 @@ module Gitlab ActiveRecord::Base.connection.execute <<~SQL WITH - starting_iids(project_id, iid) as ( + starting_iids(project_id, iid) as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported}( SELECT project_id, MAX(COALESCE(iid, 0)) FROM #{table} WHERE project_id BETWEEN #{start_id} AND #{end_id} GROUP BY project_id ), - with_calculated_iid(id, iid) as ( + with_calculated_iid(id, iid) as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported}( SELECT design.id, init.iid + ROW_NUMBER() OVER (PARTITION BY design.project_id ORDER BY design.id ASC) FROM #{table} as design, starting_iids as init diff --git a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb index 7484027a0fa..030dfd2d99b 100644 --- a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb +++ b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb @@ -8,7 +8,7 @@ module Gitlab updated_repository_storages = Projects::RepositoryStorageMove.select("project_id, MAX(updated_at) as updated_at").where(project_id: project_ids).group(:project_id) Project.connection.execute <<-SQL - WITH repository_storage_cte as ( + WITH repository_storage_cte as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( #{updated_repository_storages.to_sql} ) UPDATE projects diff --git a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb index 60682bd2ec1..b89ea7dc250 100644 --- a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb +++ b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb @@ -34,12 +34,18 @@ module Gitlab parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| - sub_batch.update_all("#{quoted_copy_to}=#{quoted_copy_from}") + batch_metrics.time_operation(:update_all) do + sub_batch.update_all("#{quoted_copy_to}=#{quoted_copy_from}") + end sleep(PAUSE_SECONDS) end end + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + private def connection diff --git a/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb b/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb index 6014ccc12eb..691bdb457d7 100644 --- a/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb +++ b/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb @@ -8,7 +8,7 @@ module Gitlab def perform(start_id, stop_id) ActiveRecord::Base.connection.execute <<~SQL - WITH merge_requests_batch AS ( + WITH merge_requests_batch AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT id, target_project_id FROM merge_requests WHERE id BETWEEN #{Integer(start_id)} AND #{Integer(stop_id)} ) diff --git a/lib/gitlab/background_migration/fix_projects_without_project_feature.rb b/lib/gitlab/background_migration/fix_projects_without_project_feature.rb index 68665db522e..83c01afa432 100644 --- a/lib/gitlab/background_migration/fix_projects_without_project_feature.rb +++ b/lib/gitlab/background_migration/fix_projects_without_project_feature.rb @@ -22,7 +22,7 @@ module Gitlab def sql(from_id, to_id) <<~SQL - WITH created_records AS ( + WITH created_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( INSERT INTO project_features ( project_id, merge_requests_access_level, diff --git a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb index e750b8ca374..b8e4562b3bf 100644 --- a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb +++ b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb @@ -136,7 +136,7 @@ module Gitlab # there is no uniq constraint on project_id and type pair, which prevents us from using ON CONFLICT def create_sql(from_id, to_id) <<~SQL - WITH created_records AS ( + WITH created_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( INSERT INTO services (project_id, #{DEFAULTS.keys.map { |key| %("#{key}")}.join(',')}, created_at, updated_at) #{select_insert_values_sql(from_id, to_id)} RETURNING * @@ -149,7 +149,7 @@ module Gitlab # there is no uniq constraint on project_id and type pair, which prevents us from using ON CONFLICT def update_sql(from_id, to_id) <<~SQL - WITH updated_records AS ( + WITH updated_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( UPDATE services SET active = TRUE WHERE services.project_id BETWEEN #{Integer(from_id)} AND #{Integer(to_id)} AND services.properties = '{}' AND services.type = '#{Migratable::PrometheusService.type}' AND #{group_cluster_condition(from_id, to_id)} AND services.active = FALSE diff --git a/lib/gitlab/background_migration/fix_user_namespace_names.rb b/lib/gitlab/background_migration/fix_user_namespace_names.rb index d767cbfd8f5..cd5b4ab103d 100644 --- a/lib/gitlab/background_migration/fix_user_namespace_names.rb +++ b/lib/gitlab/background_migration/fix_user_namespace_names.rb @@ -14,7 +14,7 @@ module Gitlab def fix_namespace_names(from_id, to_id) ActiveRecord::Base.connection.execute <<~UPDATE_NAMESPACES - WITH namespaces_to_update AS ( + WITH namespaces_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT namespaces.id, users.name AS correct_name @@ -39,7 +39,7 @@ module Gitlab def fix_namespace_route_names(from_id, to_id) ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE - WITH routes_to_update AS ( + WITH routes_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT routes.id, users.name AS correct_name diff --git a/lib/gitlab/background_migration/fix_user_project_route_names.rb b/lib/gitlab/background_migration/fix_user_project_route_names.rb index 6b99685fd68..e534f2449aa 100644 --- a/lib/gitlab/background_migration/fix_user_project_route_names.rb +++ b/lib/gitlab/background_migration/fix_user_project_route_names.rb @@ -8,7 +8,7 @@ module Gitlab class FixUserProjectRouteNames def perform(from_id, to_id) ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE - WITH routes_to_update AS ( + WITH routes_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT routes.id, users.name || ' / ' || projects.name AS correct_name diff --git a/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb b/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb new file mode 100644 index 00000000000..b7a912da060 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # migrates pages from legacy storage to zip format + # we intentionally use application code here because + # it has a lot of dependencies including models, carrierwave uploaders and service objects + # and copying all or part of this code in the background migration doesn't add much value + # see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/54578 for discussion + class MigratePagesToZipStorage + def perform(start_id, stop_id) + ::Pages::MigrateFromLegacyStorageService.new(Gitlab::AppLogger, + ignore_invalid_entries: false, + mark_projects_as_not_deployed: false) + .execute_for_batch(start_id..stop_id) + end + end + end +end diff --git a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb index 4eaef26c9c6..9ecf53317d0 100644 --- a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb +++ b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb @@ -6,7 +6,7 @@ module Gitlab # project_features.container_registry_access_level for the projects within # the given range of ids. class MoveContainerRegistryEnabledToProjectFeature - MAX_BATCH_SIZE = 1_000 + MAX_BATCH_SIZE = 300 module Migratable # Migration model namespace isolated from application code. diff --git a/lib/gitlab/background_migration/populate_has_vulnerabilities.rb b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb index 78140b768fc..28ff2070209 100644 --- a/lib/gitlab/background_migration/populate_has_vulnerabilities.rb +++ b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb @@ -8,21 +8,23 @@ module Gitlab class ProjectSetting < ActiveRecord::Base # rubocop:disable Style/Documentation self.table_name = 'project_settings' - UPSERT_SQL = <<~SQL - WITH upsert_data (project_id, has_vulnerabilities, created_at, updated_at) AS ( - SELECT projects.id, true, current_timestamp, current_timestamp FROM projects WHERE projects.id IN (%{project_ids}) - ) - INSERT INTO project_settings - (project_id, has_vulnerabilities, created_at, updated_at) - (SELECT * FROM upsert_data) - ON CONFLICT (project_id) - DO UPDATE SET - has_vulnerabilities = true, - updated_at = EXCLUDED.updated_at - SQL - def self.upsert_for(project_ids) - connection.execute(UPSERT_SQL % { project_ids: project_ids.join(', ') }) + connection.execute(upsert_sql % { project_ids: project_ids.join(', ') }) + end + + def self.upsert_sql + <<~SQL + WITH upsert_data (project_id, has_vulnerabilities, created_at, updated_at) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + SELECT projects.id, true, current_timestamp, current_timestamp FROM projects WHERE projects.id IN (%{project_ids}) + ) + INSERT INTO project_settings + (project_id, has_vulnerabilities, created_at, updated_at) + (SELECT * FROM upsert_data) + ON CONFLICT (project_id) + DO UPDATE SET + has_vulnerabilities = true, + updated_at = EXCLUDED.updated_at + SQL end end diff --git a/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb b/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb index eb4bc0aaf28..28cc4a5e3fa 100644 --- a/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb +++ b/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb @@ -11,7 +11,7 @@ module Gitlab MergeRequest .where(merge_request_assignees_not_exists_clause) .where(id: from_id..to_id) - .where('assignee_id IS NOT NULL') + .where.not(assignee_id: nil) .select(:id, :assignee_id) .to_sql diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb index 7b18e617c81..888a12f2330 100644 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -32,7 +32,7 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid }.freeze NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze - PACK_PATTERN = "NnnnnN".freeze + PACK_PATTERN = "NnnnnN" def self.call(value) Digest::UUID.uuid_v5(namespace_id, value) diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb new file mode 100644 index 00000000000..5930d65bc2c --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Banzai + module ReferenceParser + # isolated Banzai::ReferenceParser::MentionedGroupParser + class IsolatedMentionedProjectParser < ::Banzai::ReferenceParser::MentionedProjectParser + extend ::Gitlab::Utils::Override + + self.reference_type = :user + + override :references_relation + def references_relation + ::Gitlab::BackgroundMigration::UserMentions::Models::Project + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb new file mode 100644 index 00000000000..f5f98517433 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Banzai + module ReferenceParser + # isolated Banzai::ReferenceParser::MentionedGroupParser + class IsolatedMentionedUserParser < ::Banzai::ReferenceParser::MentionedUserParser + extend ::Gitlab::Utils::Override + + self.reference_type = :user + + override :references_relation + def references_relation + ::Gitlab::BackgroundMigration::UserMentions::Models::User + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb index 1d3a3af81a1..8610129533d 100644 --- a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb +++ b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb @@ -7,7 +7,7 @@ module Gitlab module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class IsolatedReferenceExtractor < ::Gitlab::ReferenceExtractor - REFERABLES = %i(isolated_mentioned_group).freeze + REFERABLES = %i(isolated_mentioned_group isolated_mentioned_user isolated_mentioned_project).freeze REFERABLES.each do |type| define_method("#{type}s") do diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb new file mode 100644 index 00000000000..0334ea1dd08 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Gitlab + # Gitlab::IsolatedVisibilityLevel module + # + # Define allowed public modes that can be used for + # GitLab projects to determine project public mode + # + module IsolatedVisibilityLevel + extend ::ActiveSupport::Concern + + included do + scope :public_to_user, -> (user = nil) do + where(visibility_level: IsolatedVisibilityLevel.levels_for_user(user)) + end + end + + PRIVATE = 0 unless const_defined?(:PRIVATE) + INTERNAL = 10 unless const_defined?(:INTERNAL) + PUBLIC = 20 unless const_defined?(:PUBLIC) + + class << self + def levels_for_user(user = nil) + return [PUBLIC] unless user + + if user.can_read_all_resources? + [PRIVATE, INTERNAL, PUBLIC] + elsif user.external? + [PUBLIC] + else + [INTERNAL, PUBLIC] + end + end + end + + def private? + visibility_level_value == PRIVATE + end + + def internal? + visibility_level_value == INTERNAL + end + + def public? + visibility_level_value == PUBLIC + end + + def visibility_level_value + self[visibility_level_field] + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb index bdb4d6c7d48..f4cc96c8bc0 100644 --- a/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb @@ -7,6 +7,7 @@ module Gitlab module Models class CommitUserMention < ActiveRecord::Base self.table_name = 'commit_user_mentions' + self.inheritance_column = :_type_disabled def self.resource_foreign_key :commit_id diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb new file mode 100644 index 00000000000..ba6b783f9f1 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + module Concerns + # isolated FeatureGate module + module IsolatedFeatureGate + def flipper_id + return if new_record? + + "#{self.class.name}:#{id}" + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb index be9c0ad2b3a..f684f789ea9 100644 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb @@ -70,8 +70,8 @@ module Gitlab def build_mention_values(resource_foreign_key) refs = all_references(author) - mentioned_users_ids = array_to_sql(refs.mentioned_users.pluck(:id)) - mentioned_projects_ids = array_to_sql(refs.mentioned_projects.pluck(:id)) + mentioned_users_ids = array_to_sql(refs.isolated_mentioned_users.pluck(:id)) + mentioned_projects_ids = array_to_sql(refs.isolated_mentioned_projects.pluck(:id)) mentioned_groups_ids = array_to_sql(refs.isolated_mentioned_groups.pluck(:id)) return if mentioned_users_ids.blank? && mentioned_projects_ids.blank? && mentioned_groups_ids.blank? diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb index 5cadfa45b5b..75759ed0111 100644 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb @@ -6,7 +6,7 @@ module Gitlab module Models module Concerns module Namespace - # extracted methods for recursive traversing of namespace hierarchy + # isolate recursive traversal code for namespace hierarchy module RecursiveTraversal extend ActiveSupport::Concern diff --git a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb index bdb90b5d2b9..d010d68600d 100644 --- a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb +++ b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb @@ -10,6 +10,9 @@ module Gitlab include EachBatch include Concerns::MentionableMigrationMethods + self.table_name = 'design_management_designs' + self.inheritance_column = :_type_disabled + def self.user_mention_model Gitlab::BackgroundMigration::UserMentions::Models::DesignUserMention end diff --git a/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb index 68205ecd3c2..eb00f6cfa3f 100644 --- a/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb @@ -7,6 +7,7 @@ module Gitlab module Models class DesignUserMention < ActiveRecord::Base self.table_name = 'design_user_mentions' + self.inheritance_column = :_type_disabled def self.resource_foreign_key :design_id diff --git a/lib/gitlab/background_migration/user_mentions/models/epic.rb b/lib/gitlab/background_migration/user_mentions/models/epic.rb index 61d9244a4c9..cfd9a4faa9b 100644 --- a/lib/gitlab/background_migration/user_mentions/models/epic.rb +++ b/lib/gitlab/background_migration/user_mentions/models/epic.rb @@ -17,10 +17,10 @@ module Gitlab cache_markdown_field :description, issuable_state_filter_enabled: true self.table_name = 'epics' + self.inheritance_column = :_type_disabled - belongs_to :author, class_name: "User" - belongs_to :project - belongs_to :group + belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User" + belongs_to :group, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Group" def self.user_mention_model Gitlab::BackgroundMigration::UserMentions::Models::EpicUserMention diff --git a/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb index 4e3ce9bf3a7..579e4d99612 100644 --- a/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb @@ -7,6 +7,7 @@ module Gitlab module Models class EpicUserMention < ActiveRecord::Base self.table_name = 'epic_user_mentions' + self.inheritance_column = :_type_disabled def self.resource_foreign_key :epic_id diff --git a/lib/gitlab/background_migration/user_mentions/models/group.rb b/lib/gitlab/background_migration/user_mentions/models/group.rb index bc04172b9a2..a8b4b59b06c 100644 --- a/lib/gitlab/background_migration/user_mentions/models/group.rb +++ b/lib/gitlab/background_migration/user_mentions/models/group.rb @@ -7,6 +7,8 @@ module Gitlab # isolated Group model class Group < ::Gitlab::BackgroundMigration::UserMentions::Models::Namespace self.store_full_sti_class = false + self.inheritance_column = :_type_disabled + has_one :saml_provider def self.declarative_policy_class diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb index 6b52afea17c..13addcc3c55 100644 --- a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb +++ b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb @@ -17,10 +17,11 @@ module Gitlab cache_markdown_field :description, issuable_state_filter_enabled: true self.table_name = 'merge_requests' + self.inheritance_column = :_type_disabled - belongs_to :author, class_name: "User" - belongs_to :target_project, class_name: "Project" - belongs_to :source_project, class_name: "Project" + belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User" + belongs_to :target_project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" + belongs_to :source_project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" alias_attribute :project, :target_project diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb index e9b85e9cb8c..4a85892d7b8 100644 --- a/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb @@ -7,6 +7,7 @@ module Gitlab module Models class MergeRequestUserMention < ActiveRecord::Base self.table_name = 'merge_request_user_mentions' + self.inheritance_column = :_type_disabled def self.resource_foreign_key :merge_request_id diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb index 8fa0db5fd4b..a2b50c41f4a 100644 --- a/lib/gitlab/background_migration/user_mentions/models/namespace.rb +++ b/lib/gitlab/background_migration/user_mentions/models/namespace.rb @@ -5,9 +5,11 @@ module Gitlab module UserMentions module Models # isolated Namespace model - class Namespace < ApplicationRecord - include FeatureGate - include ::Gitlab::VisibilityLevel + class Namespace < ActiveRecord::Base + self.inheritance_column = :_type_disabled + + include Concerns::IsolatedFeatureGate + include Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedVisibilityLevel include ::Gitlab::Utils::StrongMemoize include Gitlab::BackgroundMigration::UserMentions::Models::Concerns::Namespace::RecursiveTraversal @@ -21,8 +23,13 @@ module Gitlab parent_id.present? || parent.present? end + # Deprecated, use #licensed_feature_available? instead. Remove once Namespace#feature_available? isn't used anymore. + def feature_available?(feature) + licensed_feature_available?(feature) + end + # Overridden in EE::Namespace - def feature_available?(_feature) + def licensed_feature_available?(_feature) false end end diff --git a/lib/gitlab/background_migration/user_mentions/models/note.rb b/lib/gitlab/background_migration/user_mentions/models/note.rb index a3224c8c456..7da933c7b11 100644 --- a/lib/gitlab/background_migration/user_mentions/models/note.rb +++ b/lib/gitlab/background_migration/user_mentions/models/note.rb @@ -16,9 +16,9 @@ module Gitlab attr_mentionable :note, pipeline: :note cache_markdown_field :note, pipeline: :note, issuable_state_filter_enabled: true - belongs_to :author, class_name: "User" + belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User" belongs_to :noteable, polymorphic: true - belongs_to :project + belongs_to :project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" def for_personal_snippet? noteable && noteable.class.name == 'PersonalSnippet' diff --git a/lib/gitlab/background_migration/user_mentions/models/project.rb b/lib/gitlab/background_migration/user_mentions/models/project.rb new file mode 100644 index 00000000000..4e02bf97d12 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/project.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + # isolated Namespace model + class Project < ActiveRecord::Base + include Concerns::IsolatedFeatureGate + include Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedVisibilityLevel + + self.table_name = 'projects' + self.inheritance_column = :_type_disabled + + belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'namespace_id', class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Group" + belongs_to :namespace, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Namespace" + alias_method :parent, :namespace + + # Returns a collection of projects that is either public or visible to the + # logged in user. + def self.public_or_visible_to_user(user = nil, min_access_level = nil) + min_access_level = nil if user&.can_read_all_resources? + + return public_to_user unless user + + if user.is_a?(::Gitlab::BackgroundMigration::UserMentions::Models::User) + where('EXISTS (?) OR projects.visibility_level IN (?)', + user.authorizations_for_projects(min_access_level: min_access_level), + levels_for_user(user)) + end + end + + def grafana_integration + nil + end + + def default_issues_tracker? + true # we do not care of the issue tracker type(internal or external) when parsing mentions + end + + def visibility_level_field + :visibility_level + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/user.rb b/lib/gitlab/background_migration/user_mentions/models/user.rb new file mode 100644 index 00000000000..a30220b6934 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/user.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + # isolated Namespace model + class User < ActiveRecord::Base + include Concerns::IsolatedFeatureGate + + self.table_name = 'users' + self.inheritance_column = :_type_disabled + + has_many :project_authorizations, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent + + def authorizations_for_projects(min_access_level: nil, related_project_column: 'projects.id') + authorizations = project_authorizations + .select(1) + .where("project_authorizations.project_id = #{related_project_column}") + + return authorizations unless min_access_level.present? + + authorizations.where('project_authorizations.access_level >= ?', min_access_level) + end + + def can_read_all_resources? + can?(:read_all_resources) + end + + def can?(action, subject = :global) + Ability.allowed?(self, action, subject) + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb b/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb index baacc912df3..665ad7abcbb 100644 --- a/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb +++ b/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb @@ -27,7 +27,7 @@ module Gitlab joins(:user) .merge(UserModel.active) .where(id: (start_id..stop_id)) - .where('emails.confirmed_at IS NOT NULL') + .where.not('emails.confirmed_at' => nil) .where('emails.confirmed_at = users.confirmed_at') .where('emails.email <> users.email') .where('NOT EXISTS (SELECT 1 FROM user_synced_attributes_metadata WHERE user_id=users.id AND email_synced IS true)') @@ -57,7 +57,7 @@ module Gitlab def update_email_records(start_id, stop_id) EmailModel.connection.execute <<-SQL - WITH md5_strings as ( + WITH md5_strings as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( #{email_query_for_update(start_id, stop_id).to_sql} ) UPDATE #{EmailModel.connection.quote_table_name(EmailModel.table_name)} diff --git a/lib/gitlab/batch_pop_queueing.rb b/lib/gitlab/batch_pop_queueing.rb index e18f1320ea4..62fc8cd048e 100644 --- a/lib/gitlab/batch_pop_queueing.rb +++ b/lib/gitlab/batch_pop_queueing.rb @@ -46,7 +46,8 @@ module Gitlab def initialize(namespace, queue_id) raise ArgumentError if namespace.empty? || queue_id.empty? - @namespace, @queue_id = namespace, queue_id + @namespace = namespace + @queue_id = queue_id end ## diff --git a/lib/gitlab/bullet.rb b/lib/gitlab/bullet.rb new file mode 100644 index 00000000000..f5f8a316855 --- /dev/null +++ b/lib/gitlab/bullet.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module Bullet + extend self + + def enabled? + Gitlab::Utils.to_boolean(ENV['ENABLE_BULLET'], default: false) + end + alias_method :extra_logging_enabled?, :enabled? + + def configure_bullet? + defined?(::Bullet) && (enabled? || Rails.env.development?) + end + end +end diff --git a/lib/gitlab/bullet/exclusions.rb b/lib/gitlab/bullet/exclusions.rb new file mode 100644 index 00000000000..f897ff492d9 --- /dev/null +++ b/lib/gitlab/bullet/exclusions.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Bullet + class Exclusions + def initialize(config_file = Gitlab.root.join('config/bullet.yml')) + @config_file = config_file + end + + def execute + exclusions.map { |v| v['exclude'] } + end + + def validate_paths! + exclusions.each do |properties| + next unless properties['path_with_method'] + + file = properties['exclude'].first + + raise "Bullet: File used by #{config_file} doesn't exist, validate the #{file} exclusion!" unless File.exist?(file) + end + end + + private + + attr_reader :config_file + + def exclusions + @exclusions ||= if File.exist?(config_file) + YAML.load_file(config_file)['exclusions']&.values || [] + else + [] + end + end + end + end +end diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index d981f263c5e..9e958eb52fb 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -69,7 +69,9 @@ module Gitlab def load_from_project return unless commit - self.sha, self.status, self.ref = commit.sha, commit.status, project.default_branch + self.sha = commit.sha + self.status = commit.status + self.ref = project.default_branch end # We only cache the status for the HEAD commit of a project diff --git a/lib/gitlab/changelog/config.rb b/lib/gitlab/changelog/config.rb index 105050936ce..be8009750da 100644 --- a/lib/gitlab/changelog/config.rb +++ b/lib/gitlab/changelog/config.rb @@ -17,7 +17,24 @@ module Gitlab # The default template to use for generating release sections. DEFAULT_TEMPLATE = File.read(File.join(__dir__, 'template.tpl')) - attr_accessor :date_format, :categories, :template + # The regex to use for extracting the version from a Git tag. + # + # This regex is based on the official semantic versioning regex (as found + # on https://semver.org/), with the addition of allowing a "v" at the + # start of a tag name. + # + # We default to a strict regex as we simply don't know what kind of data + # users put in their tags. As such, using simpler patterns (e.g. just + # `\d+` for the major version) could lead to unexpected results. + # + # We use a String here as `Gitlab::UntrustedRegexp` is a mutable object. + DEFAULT_TAG_REGEX = '^v?(?P0|[1-9]\d*)' \ + '\.(?P0|[1-9]\d*)' \ + '\.(?P0|[1-9]\d*)' \ + '(?:-(?P
(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))' \
+        '?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$'
+
+      attr_accessor :date_format, :categories, :template, :tag_regex
 
       def self.from_git(project)
         if (yaml = project.repository.changelog_config)
@@ -46,6 +63,10 @@ module Gitlab
           end
         end
 
+        if (regex = hash['tag_regex'])
+          config.tag_regex = regex
+        end
+
         config
       end
 
@@ -54,6 +75,7 @@ module Gitlab
         @date_format = DEFAULT_DATE_FORMAT
         @template = Parser.new.parse_and_transform(DEFAULT_TEMPLATE)
         @categories = {}
+        @tag_regex = DEFAULT_TAG_REGEX
       end
 
       def contributor?(user)
diff --git a/lib/gitlab/chaos.rb b/lib/gitlab/chaos.rb
index 029a9210dc9..495f12882e5 100644
--- a/lib/gitlab/chaos.rb
+++ b/lib/gitlab/chaos.rb
@@ -43,9 +43,9 @@ module Gitlab
       Kernel.sleep(duration_s)
     end
 
-    # Kill will send a SIGKILL signal to the current process
-    def self.kill
-      Process.kill("KILL", Process.pid)
+    # Kill will send the given signal to the current process.
+    def self.kill(signal)
+      Process.kill(signal, Process.pid)
     end
 
     def self.run_gc
diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb
index c5afb16ab1a..88d624503df 100644
--- a/lib/gitlab/ci/build/artifacts/metadata.rb
+++ b/lib/gitlab/ci/build/artifacts/metadata.rb
@@ -17,7 +17,9 @@ module Gitlab
           attr_reader :stream, :path, :full_version
 
           def initialize(stream, path, **opts)
-            @stream, @path, @opts = stream, path, opts
+            @stream = stream
+            @path = path
+            @opts = opts
             @full_version = read_version
           end
 
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index d3f030c3b36..23b0c93a3ee 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -17,12 +17,14 @@ module Gitlab
         Config::Yaml::Tags::TagError
       ].freeze
 
-      attr_reader :root
+      attr_reader :root, :context, :ref
 
-      def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil)
+      def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, ref: nil)
         @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline)
         @context.set_deadline(TIMEOUT_SECONDS)
 
+        @ref = ref
+
         @config = expand_config(config)
 
         @root = Entry::Root.new(@config)
@@ -94,9 +96,7 @@ module Gitlab
         initial_config = Config::External::Processor.new(initial_config, @context).perform
         initial_config = Config::Extendable.new(initial_config).to_hash
         initial_config = Config::Yaml::Tags::Resolver.new(initial_config).to_hash
-        initial_config = Config::EdgeStagesInjector.new(initial_config).to_hash
-
-        initial_config
+        Config::EdgeStagesInjector.new(initial_config).to_hash
       end
 
       def find_sha(project)
diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb
index cf599ce5294..f9688c500d2 100644
--- a/lib/gitlab/ci/config/entry/cache.rb
+++ b/lib/gitlab/ci/config/entry/cache.rb
@@ -8,8 +8,8 @@ module Gitlab
         # Entry that represents a cache configuration
         #
         class Cache < ::Gitlab::Config::Entry::Simplifiable
-          strategy :Caches, if: -> (config) { Feature.enabled?(:multiple_cache_per_job) }
-          strategy :Cache, if: -> (config) { Feature.disabled?(:multiple_cache_per_job) }
+          strategy :Caches, if: -> (config) { Feature.enabled?(:multiple_cache_per_job, default_enabled: :yaml) }
+          strategy :Cache, if: -> (config) { Feature.disabled?(:multiple_cache_per_job, default_enabled: :yaml) }
 
           class Caches < ::Gitlab::Config::Entry::ComposableArray
             include ::Gitlab::Config::Entry::Validatable
@@ -17,8 +17,6 @@ module Gitlab
             MULTIPLE_CACHE_LIMIT = 4
 
             validations do
-              validates :config, presence: true
-
               validate do
                 unless config.is_a?(Hash) || config.is_a?(Array)
                   errors.add(:config, 'can only be a Hash or an Array')
diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb
index 9584d19bdec..947b6787aa0 100644
--- a/lib/gitlab/ci/config/entry/processable.rb
+++ b/lib/gitlab/ci/config/entry/processable.rb
@@ -124,7 +124,9 @@ module Gitlab
               stage: stage_value,
               extends: extends,
               rules: rules_value,
-              variables: root_and_job_variables_value,
+              variables: root_and_job_variables_value, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581
+              job_variables: job_variables,
+              root_variables_inheritance: root_variables_inheritance,
               only: only_value,
               except: except_value,
               resource_group: resource_group }.compact
@@ -139,6 +141,14 @@ module Gitlab
             root_variables.merge(variables_value.to_h)
           end
 
+          def job_variables
+            variables_value.to_h
+          end
+
+          def root_variables_inheritance
+            inherit_entry&.variables_entry&.value
+          end
+
           def manual_action?
             self.when == 'manual'
           end
diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb
index aa34cfb3acc..e869e0bbb31 100644
--- a/lib/gitlab/ci/config/entry/product/variables.rb
+++ b/lib/gitlab/ci/config/entry/product/variables.rb
@@ -25,8 +25,7 @@ module Gitlab
 
             def value
               @config
-                .map { |key, value| [key.to_s, Array(value).map(&:to_s)] }
-                .to_h
+                .to_h { |key, value| [key.to_s, Array(value).map(&:to_s)] }
             end
           end
         end
diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb
index dc164d752be..efb469ee32a 100644
--- a/lib/gitlab/ci/config/entry/variables.rb
+++ b/lib/gitlab/ci/config/entry/variables.rb
@@ -18,7 +18,7 @@ module Gitlab
           end
 
           def value
-            Hash[@config.map { |key, value| [key.to_s, expand_value(value)[:value]] }]
+            @config.to_h { |key, value| [key.to_s, expand_value(value)[:value]] }
           end
 
           def self.default(**)
@@ -26,7 +26,7 @@ module Gitlab
           end
 
           def value_with_data
-            Hash[@config.map { |key, value| [key.to_s, expand_value(value)] }]
+            @config.to_h { |key, value| [key.to_s, expand_value(value)] }
           end
 
           def use_value_data?
diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb
index b85b7a9edeb..3216d4eaac4 100644
--- a/lib/gitlab/ci/config/external/mapper.rb
+++ b/lib/gitlab/ci/config/external/mapper.rb
@@ -34,6 +34,7 @@ module Gitlab
               .compact
               .map(&method(:normalize_location))
               .flat_map(&method(:expand_project_files))
+              .flat_map(&method(:expand_wildcard_paths))
               .map(&method(:expand_variables))
               .each(&method(:verify_duplicates!))
               .map(&method(:select_first_matching))
@@ -63,6 +64,17 @@ module Gitlab
             end
           end
 
+          def expand_wildcard_paths(location)
+            return location unless ::Feature.enabled?(:ci_wildcard_file_paths, context.project, default_enabled: :yaml)
+
+            # We only support local files for wildcard paths
+            return location unless location[:local] && location[:local].include?('*')
+
+            context.project.repository.search_files_by_wildcard_path(location[:local], context.sha).map do |path|
+              { local: path }
+            end
+          end
+
           def normalize_location_string(location)
             if ::Gitlab::UrlSanitizer.valid?(location)
               { remote: location }
diff --git a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb
index 5a23836d8a0..5cabbc86d3e 100644
--- a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb
+++ b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb
@@ -43,9 +43,10 @@ module Gitlab
             {
               name: name,
               instance: instance,
-              variables: variables,
+              variables: variables, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581
+              job_variables: variables,
               parallel: { total: total }
-            }
+            }.compact
           end
 
           def name
diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb
index c811ef211d6..12e182b38fc 100644
--- a/lib/gitlab/ci/features.rb
+++ b/lib/gitlab/ci/features.rb
@@ -10,10 +10,6 @@ module Gitlab
         ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: true)
       end
 
-      def self.instance_variables_ui_enabled?
-        ::Feature.enabled?(:ci_instance_variables_ui, default_enabled: true)
-      end
-
       def self.pipeline_latest?
         ::Feature.enabled?(:ci_pipeline_latest, default_enabled: true)
       end
@@ -60,16 +56,12 @@ module Gitlab
         ::Feature.enabled?(:codequality_mr_diff, project, default_enabled: false)
       end
 
-      def self.display_codequality_backend_comparison?(project)
-        ::Feature.enabled?(:codequality_backend_comparison, project, default_enabled: :yaml)
-      end
-
       def self.multiple_cache_per_job?
         ::Feature.enabled?(:multiple_cache_per_job, default_enabled: :yaml)
       end
 
-      def self.ci_commit_pipeline_mini_graph_vue_enabled?(project)
-        ::Feature.enabled?(:ci_commit_pipeline_mini_graph_vue, project, default_enabled: :yaml)
+      def self.gldropdown_tags_enabled?
+        ::Feature.enabled?(:gldropdown_tags, default_enabled: :yaml)
       end
     end
   end
diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb
index af06e124736..a6ae249fa58 100644
--- a/lib/gitlab/ci/jwt.rb
+++ b/lib/gitlab/ci/jwt.rb
@@ -72,16 +72,16 @@ module Gitlab
 
       def key
         @key ||= begin
-                   key_data = if Feature.enabled?(:ci_jwt_signing_key, build.project, default_enabled: true)
-                                Gitlab::CurrentSettings.ci_jwt_signing_key
-                              else
-                                Rails.application.secrets.openid_connect_signing_key
-                              end
+          key_data = if Feature.enabled?(:ci_jwt_signing_key, build.project, default_enabled: true)
+                       Gitlab::CurrentSettings.ci_jwt_signing_key
+                     else
+                       Rails.application.secrets.openid_connect_signing_key
+                     end
 
-                   raise NoSigningKeyError unless key_data
+          raise NoSigningKeyError unless key_data
 
-                   OpenSSL::PKey::RSA.new(key_data)
-                 end
+          OpenSSL::PKey::RSA.new(key_data)
+        end
       end
 
       def public_key
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index 815fe6bac6d..c3c1728602c 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -12,7 +12,7 @@ module Gitlab
           :seeds_block, :variables_attributes, :push_options,
           :chat_data, :allow_mirror_update, :bridge, :content, :dry_run,
           # These attributes are set by Chains during processing:
-          :config_content, :yaml_processor_result, :pipeline_seed
+          :config_content, :yaml_processor_result, :workflow_rules_result, :pipeline_seed
         ) do
           include Gitlab::Utils::StrongMemoize
 
@@ -84,7 +84,7 @@ module Gitlab
           end
 
           def metrics
-            @metrics ||= ::Gitlab::Ci::Pipeline::Metrics.new
+            @metrics ||= ::Gitlab::Ci::Pipeline::Metrics
           end
 
           def observe_creation_duration(duration)
@@ -97,6 +97,11 @@ module Gitlab
               .observe({ source: pipeline.source.to_s }, pipeline.total_size)
           end
 
+          def increment_pipeline_failure_reason_counter(reason)
+            metrics.pipeline_failure_reason_counter
+              .increment(reason: (reason || :unknown_failure).to_s)
+          end
+
           def dangling_build?
             %i[ondemand_dast_scan webide].include?(source)
           end
diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb
index c3fbd0c9e24..8f1c49563f2 100644
--- a/lib/gitlab/ci/pipeline/chain/config/process.rb
+++ b/lib/gitlab/ci/pipeline/chain/config/process.rb
@@ -14,6 +14,7 @@ module Gitlab
               result = ::Gitlab::Ci::YamlProcessor.new(
                 @command.config_content, {
                   project: project,
+                  ref: @pipeline.ref,
                   sha: @pipeline.sha,
                   user: current_user,
                   parent_pipeline: parent_pipeline
diff --git a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb
index 3c910963a2a..cceaa52de16 100644
--- a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb
+++ b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb
@@ -9,6 +9,8 @@ module Gitlab
           include Chain::Helpers
 
           def perform!
+            @command.workflow_rules_result = workflow_rules_result
+
             error('Pipeline filtered out by workflow rules.') unless workflow_passed?
           end
 
@@ -19,27 +21,33 @@ module Gitlab
           private
 
           def workflow_passed?
-            strong_memoize(:workflow_passed) do
-              workflow_rules.evaluate(@pipeline, global_context).pass?
+            workflow_rules_result.pass?
+          end
+
+          def workflow_rules_result
+            strong_memoize(:workflow_rules_result) do
+              workflow_rules.evaluate(@pipeline, global_context)
             end
           end
 
           def workflow_rules
             Gitlab::Ci::Build::Rules.new(
-              workflow_config[:rules], default_when: 'always')
+              workflow_rules_config, default_when: 'always')
           end
 
           def global_context
             Gitlab::Ci::Build::Context::Global.new(
-              @pipeline, yaml_variables: workflow_config[:yaml_variables])
+              @pipeline, yaml_variables: @command.yaml_processor_result.root_variables)
           end
 
           def has_workflow_rules?
-            workflow_config[:rules].present?
+            workflow_rules_config.present?
           end
 
-          def workflow_config
-            @command.yaml_processor_result.workflow_attributes || {}
+          def workflow_rules_config
+            strong_memoize(:workflow_rules_config) do
+              @command.yaml_processor_result.workflow_rules
+            end
           end
         end
       end
diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb
index d7271df1694..9988b6f18ed 100644
--- a/lib/gitlab/ci/pipeline/chain/helpers.rb
+++ b/lib/gitlab/ci/pipeline/chain/helpers.rb
@@ -12,7 +12,8 @@ module Gitlab
             end
 
             pipeline.add_error_message(message)
-            pipeline.drop!(drop_reason) if drop_reason && persist_pipeline?
+
+            drop_pipeline!(drop_reason)
 
             # TODO: consider not to rely on AR errors directly as they can be
             # polluted with other unrelated errors (e.g. state machine)
@@ -24,8 +25,21 @@ module Gitlab
             pipeline.add_warning_message(message)
           end
 
-          def persist_pipeline?
-            command.save_incompleted && !pipeline.readonly?
+          private
+
+          def drop_pipeline!(drop_reason)
+            return if pipeline.readonly?
+
+            if drop_reason && command.save_incompleted
+              # Project iid must be called outside a transaction, so we ensure it is set here
+              # otherwise it may be set within the state transition transaction of the drop! call
+              # which it will lock the InternalId row for the whole transaction
+              pipeline.ensure_project_iid!
+
+              pipeline.drop!(drop_reason)
+            else
+              command.increment_pipeline_failure_reason_counter(drop_reason)
+            end
           end
         end
       end
diff --git a/lib/gitlab/ci/pipeline/chain/metrics.rb b/lib/gitlab/ci/pipeline/chain/metrics.rb
index 0d7449813b4..b17ae77d445 100644
--- a/lib/gitlab/ci/pipeline/chain/metrics.rb
+++ b/lib/gitlab/ci/pipeline/chain/metrics.rb
@@ -14,7 +14,7 @@ module Gitlab
           end
 
           def counter
-            ::Gitlab::Ci::Pipeline::Metrics.new.pipelines_created_counter
+            ::Gitlab::Ci::Pipeline::Metrics.pipelines_created_counter
           end
         end
       end
diff --git a/lib/gitlab/ci/pipeline/chain/pipeline/process.rb b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb
index 1eb7474e915..c1b6dfb7e36 100644
--- a/lib/gitlab/ci/pipeline/chain/pipeline/process.rb
+++ b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb
@@ -8,9 +8,7 @@ module Gitlab
           # After pipeline has been successfully created we can start processing it.
           class Process < Chain::Base
             def perform!
-              ::Ci::ProcessPipelineService
-                .new(@pipeline)
-                .execute
+              ::Ci::InitialPipelineProcessWorker.perform_async(pipeline.id)
             end
 
             def break?
diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb
index 7b537125b9b..66fc6741252 100644
--- a/lib/gitlab/ci/pipeline/chain/seed.rb
+++ b/lib/gitlab/ci/pipeline/chain/seed.rb
@@ -11,6 +11,10 @@ module Gitlab
           def perform!
             raise ArgumentError, 'missing YAML processor result' unless @command.yaml_processor_result
 
+            if ::Feature.enabled?(:ci_workflow_rules_variables, pipeline.project, default_enabled: :yaml)
+              raise ArgumentError, 'missing workflow rules result' unless @command.workflow_rules_result
+            end
+
             # Allocate next IID. This operation must be outside of transactions of pipeline creations.
             pipeline.ensure_project_iid!
             pipeline.ensure_ci_ref!
@@ -38,7 +42,21 @@ module Gitlab
           def pipeline_seed
             strong_memoize(:pipeline_seed) do
               stages_attributes = @command.yaml_processor_result.stages_attributes
-              Gitlab::Ci::Pipeline::Seed::Pipeline.new(pipeline, stages_attributes)
+              Gitlab::Ci::Pipeline::Seed::Pipeline.new(context, stages_attributes)
+            end
+          end
+
+          def context
+            Gitlab::Ci::Pipeline::Seed::Context.new(pipeline, root_variables: root_variables)
+          end
+
+          def root_variables
+            if ::Feature.enabled?(:ci_workflow_rules_variables, pipeline.project, default_enabled: :yaml)
+              ::Gitlab::Ci::Variables::Helpers.merge_variables(
+                @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables
+              )
+            else
+              @command.yaml_processor_result.root_variables
             end
           end
         end
diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb
index d056501a6d3..6149d2f04d7 100644
--- a/lib/gitlab/ci/pipeline/chain/validate/external.rb
+++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb
@@ -10,77 +10,116 @@ module Gitlab
 
             InvalidResponseCode = Class.new(StandardError)
 
-            VALIDATION_REQUEST_TIMEOUT = 5
+            DEFAULT_VALIDATION_REQUEST_TIMEOUT = 5
+            ACCEPTED_STATUS = 200
+            DOT_COM_REJECTED_STATUS = 406
+            GENERAL_REJECTED_STATUS = (400..499).freeze
 
             def perform!
+              return unless enabled?
+
               pipeline_authorized = validate_external
 
               log_message = pipeline_authorized ? 'authorized' : 'not authorized'
-              Gitlab::AppLogger.info(message: "Pipeline #{log_message}", project_id: @pipeline.project.id, user_id: @pipeline.user.id)
+              Gitlab::AppLogger.info(message: "Pipeline #{log_message}", project_id: project.id, user_id: current_user.id)
 
               error('External validation failed', drop_reason: :external_validation_failure) unless pipeline_authorized
             end
 
             def break?
-              @pipeline.errors.any?
+              pipeline.errors.any?
             end
 
             private
 
+            def enabled?
+              return true unless Gitlab.com?
+
+              ::Feature.enabled?(:ci_external_validation_service, project, default_enabled: :yaml)
+            end
+
             def validate_external
               return true unless validation_service_url
 
               # 200 - accepted
-              # 4xx - not accepted
+              # 406 - not accepted on GitLab.com
+              # 4XX - not accepted for other installations
               # everything else - accepted and logged
               response_code = validate_service_request.code
               case response_code
-              when 200
+              when ACCEPTED_STATUS
                 true
-              when 400..499
+              when rejected_status
                 false
               else
                 raise InvalidResponseCode, "Unsupported response code received from Validation Service: #{response_code}"
               end
             rescue => ex
-              Gitlab::ErrorTracking.track_exception(ex)
+              Gitlab::ErrorTracking.track_exception(ex, project_id: project.id)
 
               true
             end
 
+            def rejected_status
+              if Gitlab.com?
+                DOT_COM_REJECTED_STATUS
+              else
+                GENERAL_REJECTED_STATUS
+              end
+            end
+
             def validate_service_request
+              headers = {
+                'X-Gitlab-Correlation-id' => Labkit::Correlation::CorrelationId.current_id,
+                'X-Gitlab-Token' => validation_service_token
+              }.compact
+
               Gitlab::HTTP.post(
-                validation_service_url, timeout: VALIDATION_REQUEST_TIMEOUT,
-                body: validation_service_payload(@pipeline, @command.yaml_processor_result.stages_attributes)
+                validation_service_url, timeout: validation_service_timeout,
+                headers: headers,
+                body: validation_service_payload.to_json
               )
             end
 
+            def validation_service_timeout
+              timeout = Gitlab::CurrentSettings.external_pipeline_validation_service_timeout || ENV['EXTERNAL_VALIDATION_SERVICE_TIMEOUT'].to_i
+              return timeout if timeout > 0
+
+              DEFAULT_VALIDATION_REQUEST_TIMEOUT
+            end
+
             def validation_service_url
-              ENV['EXTERNAL_VALIDATION_SERVICE_URL']
+              Gitlab::CurrentSettings.external_pipeline_validation_service_url || ENV['EXTERNAL_VALIDATION_SERVICE_URL']
+            end
+
+            def validation_service_token
+              Gitlab::CurrentSettings.external_pipeline_validation_service_token || ENV['EXTERNAL_VALIDATION_SERVICE_TOKEN']
             end
 
-            def validation_service_payload(pipeline, stages_attributes)
+            def validation_service_payload
               {
                 project: {
-                  id: pipeline.project.id,
-                  path: pipeline.project.full_path
+                  id: project.id,
+                  path: project.full_path,
+                  created_at: project.created_at&.iso8601
                 },
                 user: {
-                  id: pipeline.user.id,
-                  username: pipeline.user.username,
-                  email: pipeline.user.email
+                  id: current_user.id,
+                  username: current_user.username,
+                  email: current_user.email,
+                  created_at: current_user.created_at&.iso8601
                 },
                 pipeline: {
                   sha: pipeline.sha,
                   ref: pipeline.ref,
                   type: pipeline.source
                 },
-                builds: builds_validation_payload(stages_attributes)
-              }.to_json
+                builds: builds_validation_payload
+              }
             end
 
-            def builds_validation_payload(stages_attributes)
-              stages_attributes.map { |stage| stage[:builds] }.flatten
+            def builds_validation_payload
+              stages_attributes.flat_map { |stage| stage[:builds] }
                 .map(&method(:build_validation_payload))
             end
 
@@ -97,9 +136,15 @@ module Gitlab
                 ].flatten.compact
               }
             end
+
+            def stages_attributes
+              command.yaml_processor_result.stages_attributes
+            end
           end
         end
       end
     end
   end
 end
+
+Gitlab::Ci::Pipeline::Chain::Validate::External.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Validate::External')
diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb
index c77f4dcca5a..6cb6fd3920d 100644
--- a/lib/gitlab/ci/pipeline/metrics.rb
+++ b/lib/gitlab/ci/pipeline/metrics.rb
@@ -4,55 +4,57 @@ module Gitlab
   module Ci
     module Pipeline
       class Metrics
-        include Gitlab::Utils::StrongMemoize
+        def self.pipeline_creation_duration_histogram
+          name = :gitlab_ci_pipeline_creation_duration_seconds
+          comment = 'Pipeline creation duration'
+          labels = {}
+          buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0]
 
-        def pipeline_creation_duration_histogram
-          strong_memoize(:pipeline_creation_duration_histogram) do
-            name = :gitlab_ci_pipeline_creation_duration_seconds
-            comment = 'Pipeline creation duration'
-            labels = {}
-            buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0]
+          ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
+        end
+
+        def self.pipeline_size_histogram
+          name = :gitlab_ci_pipeline_size_builds
+          comment = 'Pipeline size'
+          labels = { source: nil }
+          buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000]
+
+          ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
+        end
+
+        def self.pipeline_processing_events_counter
+          name = :gitlab_ci_pipeline_processing_events_total
+          comment = 'Total amount of pipeline processing events'
 
-            ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
-          end
+          Gitlab::Metrics.counter(name, comment)
         end
 
-        def pipeline_size_histogram
-          strong_memoize(:pipeline_size_histogram) do
-            name = :gitlab_ci_pipeline_size_builds
-            comment = 'Pipeline size'
-            labels = { source: nil }
-            buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000]
+        def self.pipelines_created_counter
+          name = :pipelines_created_total
+          comment = 'Counter of pipelines created'
 
-            ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
-          end
+          Gitlab::Metrics.counter(name, comment)
         end
 
-        def pipeline_processing_events_counter
-          strong_memoize(:pipeline_processing_events_counter) do
-            name = :gitlab_ci_pipeline_processing_events_total
-            comment = 'Total amount of pipeline processing events'
+        def self.legacy_update_jobs_counter
+          name = :ci_legacy_update_jobs_as_retried_total
+          comment = 'Counter of occurrences when jobs were not being set as retried before update_retried'
 
-            Gitlab::Metrics.counter(name, comment)
-          end
+          Gitlab::Metrics.counter(name, comment)
         end
 
-        def pipelines_created_counter
-          strong_memoize(:pipelines_created_count) do
-            name = :pipelines_created_total
-            comment = 'Counter of pipelines created'
+        def self.pipeline_failure_reason_counter
+          name = :gitlab_ci_pipeline_failure_reasons
+          comment = 'Counter of pipeline failure reasons'
 
-            Gitlab::Metrics.counter(name, comment)
-          end
+          Gitlab::Metrics.counter(name, comment)
         end
 
-        def legacy_update_jobs_counter
-          strong_memoize(:legacy_update_jobs_counter) do
-            name = :ci_legacy_update_jobs_as_retried_total
-            comment = 'Counter of occurrences when jobs were not being set as retried before update_retried'
+        def self.job_failure_reason_counter
+          name = :gitlab_ci_job_failure_reasons
+          comment = 'Counter of job failure reasons'
 
-            Gitlab::Metrics.counter(name, comment)
-          end
+          Gitlab::Metrics.counter(name, comment)
         end
       end
     end
diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index 11b01822e4b..39dee7750d6 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -11,12 +11,15 @@ module Gitlab
 
           delegate :dig, to: :@seed_attributes
 
-          def initialize(pipeline, attributes, previous_stages)
-            @pipeline = pipeline
+          def initialize(context, attributes, previous_stages)
+            @context = context
+            @pipeline = context.pipeline
             @seed_attributes = attributes
             @previous_stages = previous_stages
             @needs_attributes = dig(:needs_attributes)
             @resource_group_key = attributes.delete(:resource_group_key)
+            @job_variables = @seed_attributes.delete(:job_variables)
+            @root_variables_inheritance = @seed_attributes.delete(:root_variables_inheritance) { true }
 
             @using_rules  = attributes.key?(:rules)
             @using_only   = attributes.key?(:only)
@@ -29,7 +32,9 @@ module Gitlab
             @rules = Gitlab::Ci::Build::Rules
               .new(attributes.delete(:rules), default_when: 'on_success')
             @cache = Gitlab::Ci::Build::Cache
-              .new(attributes.delete(:cache), pipeline)
+              .new(attributes.delete(:cache), @pipeline)
+
+            recalculate_yaml_variables!
           end
 
           def name
@@ -206,6 +211,14 @@ module Gitlab
 
             { options: { allow_failure_criteria: nil } }
           end
+
+          def recalculate_yaml_variables!
+            return unless ::Feature.enabled?(:ci_workflow_rules_variables, @pipeline.project, default_enabled: :yaml)
+
+            @seed_attributes[:yaml_variables] = Gitlab::Ci::Variables::Helpers.inherit_yaml_variables(
+              from: @context.root_variables, to: @job_variables, inheritance: @root_variables_inheritance
+            )
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/pipeline/seed/context.rb b/lib/gitlab/ci/pipeline/seed/context.rb
new file mode 100644
index 00000000000..6194a78f682
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/seed/context.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Seed
+        class Context
+          attr_reader :pipeline, :root_variables
+
+          def initialize(pipeline, root_variables: [])
+            @pipeline = pipeline
+            @root_variables = root_variables
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/seed/pipeline.rb b/lib/gitlab/ci/pipeline/seed/pipeline.rb
index da9d853cf68..e1a15fb8d5b 100644
--- a/lib/gitlab/ci/pipeline/seed/pipeline.rb
+++ b/lib/gitlab/ci/pipeline/seed/pipeline.rb
@@ -7,8 +7,8 @@ module Gitlab
         class Pipeline
           include Gitlab::Utils::StrongMemoize
 
-          def initialize(pipeline, stages_attributes)
-            @pipeline = pipeline
+          def initialize(context, stages_attributes)
+            @context = context
             @stages_attributes = stages_attributes
           end
 
@@ -37,7 +37,7 @@ module Gitlab
           def stage_seeds
             strong_memoize(:stage_seeds) do
               seeds = @stages_attributes.inject([]) do |previous_stages, attributes|
-                seed = Gitlab::Ci::Pipeline::Seed::Stage.new(@pipeline, attributes, previous_stages)
+                seed = Gitlab::Ci::Pipeline::Seed::Stage.new(@context, attributes, previous_stages)
                 previous_stages + [seed]
               end
 
diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb
index b600df2f656..c988ea10e41 100644
--- a/lib/gitlab/ci/pipeline/seed/stage.rb
+++ b/lib/gitlab/ci/pipeline/seed/stage.rb
@@ -10,13 +10,14 @@ module Gitlab
           delegate :size, to: :seeds
           delegate :dig, to: :seeds
 
-          def initialize(pipeline, attributes, previous_stages)
-            @pipeline = pipeline
+          def initialize(context, attributes, previous_stages)
+            @context = context
+            @pipeline = context.pipeline
             @attributes = attributes
             @previous_stages = previous_stages
 
             @builds = attributes.fetch(:builds).map do |attributes|
-              Seed::Build.new(@pipeline, attributes, previous_stages)
+              Seed::Build.new(context, attributes, previous_stages)
             end
           end
 
diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb
index 5398c19e536..7ecb9a1db16 100644
--- a/lib/gitlab/ci/queue/metrics.rb
+++ b/lib/gitlab/ci/queue/metrics.rb
@@ -9,12 +9,12 @@ module Gitlab
         QUEUE_DURATION_SECONDS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze
         QUEUE_ACTIVE_RUNNERS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze
         QUEUE_DEPTH_TOTAL_BUCKETS = [1, 2, 3, 5, 8, 16, 32, 50, 100, 250, 500, 1000, 2000, 5000].freeze
-        QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000].freeze
-        QUEUE_ITERATION_DURATION_SECONDS_BUCKETS = [0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze
+        QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000, 7500, 10000, 15000, 20000].freeze
+        QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS = [0.01, 0.05, 0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze
 
         METRICS_SHARD_TAG_PREFIX = 'metrics_shard::'
         DEFAULT_METRICS_SHARD = 'default'
-        JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET = 5.freeze
+        JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET = 5
 
         OPERATION_COUNTERS = [
           :build_can_pick,
@@ -94,13 +94,13 @@ module Gitlab
           self.class.queue_depth_total.observe({ queue: queue }, size.to_f)
         end
 
-        def observe_queue_size(size_proc)
+        def observe_queue_size(size_proc, runner_type)
           return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false)
 
-          self.class.queue_size_total.observe({}, size_proc.call.to_f)
+          self.class.queue_size_total.observe({ runner_type: runner_type }, size_proc.call.to_f)
         end
 
-        def observe_queue_time
+        def observe_queue_time(metric, runner_type)
           start_time = ::Gitlab::Metrics::System.monotonic_time
 
           result = yield
@@ -108,7 +108,15 @@ module Gitlab
           return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false)
 
           seconds = ::Gitlab::Metrics::System.monotonic_time - start_time
-          self.class.queue_iteration_duration_seconds.observe({}, seconds.to_f)
+
+          case metric
+          when :process
+            self.class.queue_iteration_duration_seconds.observe({ runner_type: runner_type }, seconds.to_f)
+          when :retrieve
+            self.class.queue_retrieval_duration_seconds.observe({ runner_type: runner_type }, seconds.to_f)
+          else
+            raise ArgumentError unless Rails.env.production?
+          end
 
           result
         end
@@ -187,7 +195,18 @@ module Gitlab
           strong_memoize(:queue_iteration_duration_seconds) do
             name = :gitlab_ci_queue_iteration_duration_seconds
             comment = 'Time it takes to find a build in CI/CD queue'
-            buckets = QUEUE_ITERATION_DURATION_SECONDS_BUCKETS
+            buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS
+            labels = {}
+
+            Gitlab::Metrics.histogram(name, comment, labels, buckets)
+          end
+        end
+
+        def self.queue_retrieval_duration_seconds
+          strong_memoize(:queue_retrieval_duration_seconds) do
+            name = :gitlab_ci_queue_retrieval_duration_seconds
+            comment = 'Time it takes to execute a SQL query to retrieve builds queue'
+            buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS
             labels = {}
 
             Gitlab::Metrics.histogram(name, comment, labels, buckets)
diff --git a/lib/gitlab/ci/reports/codequality_reports.rb b/lib/gitlab/ci/reports/codequality_reports.rb
index 060a1e2399b..27c41c384b8 100644
--- a/lib/gitlab/ci/reports/codequality_reports.rb
+++ b/lib/gitlab/ci/reports/codequality_reports.rb
@@ -6,6 +6,7 @@ module Gitlab
       class CodequalityReports
         attr_reader :degradations, :error_message
 
+        SEVERITY_PRIORITIES = %w(blocker critical major minor info).map.with_index.to_h.freeze # { "blocker" => 0, "critical" => 1 ... }
         CODECLIMATE_SCHEMA_PATH = Rails.root.join('app', 'validators', 'json_schemas', 'codeclimate.json').to_s
 
         def initialize
@@ -29,12 +30,17 @@ module Gitlab
           @degradations.values
         end
 
+        def sort_degradations!
+          @degradations = @degradations.sort_by do |_fingerprint, degradation|
+            SEVERITY_PRIORITIES[degradation.dig(:severity)]
+          end.to_h
+        end
+
         private
 
         def valid_degradation?(degradation)
-          JSON::Validator.validate!(CODECLIMATE_SCHEMA_PATH, degradation)
-        rescue JSON::Schema::ValidationError => e
-          set_error_message("Invalid degradation format: #{e.message}")
+          JSONSchemer.schema(Pathname.new(CODECLIMATE_SCHEMA_PATH)).valid?(degradation)
+        rescue StandardError => _
           false
         end
       end
diff --git a/lib/gitlab/ci/reports/codequality_reports_comparer.rb b/lib/gitlab/ci/reports/codequality_reports_comparer.rb
index 10748b8ca02..e34d9675c10 100644
--- a/lib/gitlab/ci/reports/codequality_reports_comparer.rb
+++ b/lib/gitlab/ci/reports/codequality_reports_comparer.rb
@@ -7,6 +7,11 @@ module Gitlab
         def initialize(base_report, head_report)
           @base_report = base_report
           @head_report = head_report
+
+          unless not_found?
+            @base_report.sort_degradations!
+            @head_report.sort_degradations!
+          end
         end
 
         def success?
diff --git a/lib/gitlab/ci/reports/test_failure_history.rb b/lib/gitlab/ci/reports/test_failure_history.rb
index c024e794ad5..37d0da38065 100644
--- a/lib/gitlab/ci/reports/test_failure_history.rb
+++ b/lib/gitlab/ci/reports/test_failure_history.rb
@@ -6,32 +6,32 @@ module Gitlab
       class TestFailureHistory
         include Gitlab::Utils::StrongMemoize
 
-        def initialize(failed_test_cases, project)
-          @failed_test_cases = build_map(failed_test_cases)
+        def initialize(failed_junit_tests, project)
+          @failed_junit_tests = build_map(failed_junit_tests)
           @project = project
         end
 
         def load!
           recent_failures_count.each do |key_hash, count|
-            failed_test_cases[key_hash].set_recent_failures(count, project.default_branch_or_master)
+            failed_junit_tests[key_hash].set_recent_failures(count, project.default_branch_or_master)
           end
         end
 
         private
 
-        attr_reader :report, :project, :failed_test_cases
+        attr_reader :report, :project, :failed_junit_tests
 
         def recent_failures_count
-          ::Ci::TestCaseFailure.recent_failures_count(
+          ::Ci::UnitTestFailure.recent_failures_count(
             project: project,
-            test_case_keys: failed_test_cases.keys
+            unit_test_keys: failed_junit_tests.keys
           )
         end
 
-        def build_map(test_cases)
+        def build_map(junit_tests)
           {}.tap do |hash|
-            test_cases.each do |test_case|
-              hash[test_case.key] = test_case
+            junit_tests.each do |test|
+              hash[test.key] = test
             end
           end
         end
diff --git a/lib/gitlab/ci/runner_instructions.rb b/lib/gitlab/ci/runner_instructions.rb
index dd0bfa768a8..365864d3317 100644
--- a/lib/gitlab/ci/runner_instructions.rb
+++ b/lib/gitlab/ci/runner_instructions.rb
@@ -51,10 +51,7 @@ module Gitlab
 
       attr_reader :errors
 
-      def initialize(current_user:, group: nil, project: nil, os:, arch:)
-        @current_user = current_user
-        @group = group
-        @project = project
+      def initialize(os:, arch:)
         @os = os
         @arch = arch
         @errors = []
@@ -77,7 +74,7 @@ module Gitlab
           server_url = Gitlab::Routing.url_helpers.root_url(only_path: false)
           runner_executable = environment[:runner_executable]
 
-          "#{runner_executable} register --url #{server_url} --registration-token #{registration_token}"
+          "#{runner_executable} register --url #{server_url} --registration-token $REGISTRATION_TOKEN"
         end
       end
 
@@ -108,30 +105,6 @@ module Gitlab
       def get_file(path)
         File.read(Rails.root.join(path).to_s)
       end
-
-      def registration_token
-        project_token || group_token || instance_token
-      end
-
-      def project_token
-        return unless @project
-        raise Gitlab::Access::AccessDeniedError unless can?(@current_user, :admin_pipeline, @project)
-
-        @project.runners_token
-      end
-
-      def group_token
-        return unless @group
-        raise Gitlab::Access::AccessDeniedError unless can?(@current_user, :admin_group, @group)
-
-        @group.runners_token
-      end
-
-      def instance_token
-        raise Gitlab::Access::AccessDeniedError unless @current_user&.admin?
-
-        Gitlab::CurrentSettings.runners_registration_token
-      end
     end
   end
 end
diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb
index f6562737838..787dee3b267 100644
--- a/lib/gitlab/ci/status/build/failed.rb
+++ b/lib/gitlab/ci/status/build/failed.rb
@@ -26,7 +26,9 @@ module Gitlab
             bridge_pipeline_is_child_pipeline: 'creation of child pipeline not allowed from another child pipeline',
             downstream_pipeline_creation_failed: 'downstream pipeline can not be created',
             secrets_provider_not_found: 'secrets provider can not be found',
-            reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines'
+            reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines',
+            project_deleted: 'pipeline project was deleted',
+            user_blocked: 'pipeline user was blocked'
           }.freeze
 
           private_constant :REASONS
diff --git a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml
index 5ebbbf15682..2ff36bcc657 100644
--- a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml
@@ -113,9 +113,10 @@ promoteBeta:
 promoteProduction:
   extends: .promote_job
   stage: production
-  # We only allow production promotion on `master` because
-  # it has its own production scoped secret variables
+  # We only allow production promotion on the default branch because
+  # it has its own production scoped secret variables.
   only:
-    - master
+    variables:
+      - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
   script:
     - bundle exec fastlane promote_beta_to_production
diff --git a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml
index 15cdbf63cb1..d0c63ab6edf 100644
--- a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml
@@ -1,27 +1,31 @@
-docker-build-master:
-  # Official docker image.
-  image: docker:latest
-  stage: build
-  services:
-    - docker:dind
-  before_script:
-    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
-  script:
-    - docker build --pull -t "$CI_REGISTRY_IMAGE" .
-    - docker push "$CI_REGISTRY_IMAGE"
-  only:
-    - master
-
+# Build a Docker image with CI/CD and push to the GitLab registry.
+# Docker-in-Docker documentation: https://docs.gitlab.com/ee/ci/docker/using_docker_build.html
+#
+# This template uses one generic job with conditional builds
+# for the default branch and all other (MR) branches.
 docker-build:
-  # Official docker image.
+  # Use the official docker image.
   image: docker:latest
   stage: build
   services:
     - docker:dind
   before_script:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+  # Default branch leaves tag empty (= latest tag)
+  # All other branches are tagged with the escaped branch name (commit ref slug)
   script:
-    - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" .
-    - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
-  except:
-    - master
+    - |
+      if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then
+        tag=""
+        echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest'"
+      else
+        tag=":$CI_COMMIT_REF_SLUG"
+        echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag"
+      fi
+    - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" .
+    - docker push "$CI_REGISTRY_IMAGE${tag}"
+  # Run this job in a branch where a Dockerfile exists
+  rules:
+    - if: $CI_COMMIT_BRANCH
+      exists:
+        - Dockerfile
diff --git a/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml b/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml
new file mode 100644
index 00000000000..90812083917
--- /dev/null
+++ b/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml
@@ -0,0 +1,9 @@
+# This file is a template demonstrating the `script` keyword.
+# Learn more about this keyword here: https://docs.gitlab.com/ee/ci/yaml/README.html#script
+
+# After committing this template, visit CI/CD > Jobs to see the script output.
+
+job:
+  script:
+    # provide a shell script as argument for this keyword.
+    - echo "Hello World"
diff --git a/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml
new file mode 100644
index 00000000000..c7fb1321055
--- /dev/null
+++ b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml
@@ -0,0 +1,91 @@
+# This template is provided and maintained by Indeni, an official Technology Partner with GitLab.
+# See https://about.gitlab.com/partners/technology-partners/#security for more information.
+
+# For more information about Indeni Cloudrail: https://indeni.com/cloudrail/
+#
+# This file shows an example of using Indeni Cloudrail with GitLab CI/CD.
+# It is not designed to be included in an existing CI/CD configuration with the "include:" keyword.
+# Documentation about this integration: https://indeni.com/doc-indeni-cloudrail/integrate-with-ci-cd/gitlab-instructions
+#
+# For an example of this used in a GitLab repository, see: https://gitlab.com/indeni/cloudrail-demo/-/blob/master/.gitlab-ci.yml
+
+# The sast-report output complies with GitLab's format. This report displays Cloudrail's
+# results in the Security tab in the pipeline view, if you have that feature enabled
+# (GitLab Ultimate only). Otherwise, Cloudrail generates a JUnit report, which displays
+# in the "Test summary" in merge requests.
+
+# Note that Cloudrail's input is the Terraform plan. That is why we've included in this
+# template an example of doing that. You are welcome to replace it with your own way
+# of generating a Terraform plan.
+
+# Before you can use this template, get a Cloudrail API key from the Cloudrail web
+# user interface. Save it as a CI/CD variable named CLOUDRAIL_API_KEY in your project
+# settings.
+
+variables:
+  TEST_ROOT: ${CI_PROJECT_DIR}/my_folder_with_terraform_content
+
+default:
+  before_script:
+    - cd ${CI_PROJECT_DIR}/my_folder_with_terraform_content
+
+stages:
+  - init_and_plan
+  - cloudrail
+
+init_and_plan:
+  stage: init_and_plan
+  image: registry.gitlab.com/gitlab-org/terraform-images/releases/0.13
+  rules:
+    - if: $SAST_DISABLED
+      when: never
+    - if: $CI_COMMIT_BRANCH
+      exists:
+        - '**/*.tf'
+  script:
+    - terraform init
+    - terraform plan -out=plan.out
+  artifacts:
+    name: "$CI_COMMIT_BRANCH-terraform_plan"
+    paths:
+      - ./**/plan.out
+      - ./**/.terraform
+
+cloudrail_scan:
+  stage: cloudrail
+  image: indeni/cloudrail-cli:1.2.44
+  rules:
+    - if: $SAST_DISABLED
+      when: never
+    - if: $CI_COMMIT_BRANCH
+      exists:
+        - '**/*.tf'
+  script:
+    - |
+      if [[ "${GITLAB_FEATURES}" == *"security_dashboard"* ]]; then
+        echo "You are licensed for GitLab Security Dashboards. Your scan results will display in the Security Dashboard."
+        cloudrail run --tf-plan plan.out \
+                      --directory . \
+                      --api-key ${CLOUDRAIL_API_KEY} \
+                      --origin ci \
+                      --build-link "$CI_PROJECT_URL/-/jobs/$CI_JOB_ID" \
+                      --execution-source-identifier "$CI_COMMIT_BRANCH - $CI_JOB_ID" \
+                      --output-format json-gitlab-sast \
+                      --output-file ${CI_PROJECT_DIR}/cloudrail-sast-report.json \
+                      --auto-approve
+      else
+        echo "Your scan results will display in the GitLab Test results visualization panel."
+        cloudrail run --tf-plan plan.out \
+                      --directory . \
+                      --api-key ${CLOUDRAIL_API_KEY} \
+                      --origin ci \
+                      --build-link "$CI_PROJECT_URL/-/jobs/$CI_JOB_ID" \
+                      --execution-source-identifier "$CI_COMMIT_BRANCH - $CI_JOB_ID" \
+                      --output-format junit \
+                      --output-file ${CI_PROJECT_DIR}/cloudrail-junit-report.xml \
+                      --auto-approve
+      fi
+  artifacts:
+    reports:
+      sast: cloudrail-sast-report.json
+      junit: cloudrail-junit-report.xml
diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml
index 5edb26a0b56..01907ef9e2e 100644
--- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml
@@ -20,15 +20,48 @@ performance:
       fi
     - export CI_ENVIRONMENT_URL=$(cat environment_url.txt)
     - mkdir gitlab-exporter
+    # Busybox wget does not support proxied HTTPS, get the real thing.
+    # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611.
+    - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget
     - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js
     - mkdir sitespeed-results
+    - |
+      function propagate_env_vars() {
+        CURRENT_ENV=$(printenv)
+
+        for VAR_NAME; do
+          echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME "
+        done
+      }
     - |
       if [ -f .gitlab-urls.txt ]
       then
         sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt
-        docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results .gitlab-urls.txt $SITESPEED_OPTIONS
+        docker run \
+          $(propagate_env_vars \
+            auto_proxy \
+            https_proxy \
+            http_proxy \
+            no_proxy \
+            AUTO_PROXY \
+            HTTPS_PROXY \
+            HTTP_PROXY \
+            NO_PROXY \
+          ) \
+          --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results .gitlab-urls.txt $SITESPEED_OPTIONS
       else
-        docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" $SITESPEED_OPTIONS
+        docker run \
+          $(propagate_env_vars \
+            auto_proxy \
+            https_proxy \
+            http_proxy \
+            no_proxy \
+            AUTO_PROXY \
+            HTTPS_PROXY \
+            HTTP_PROXY \
+            NO_PROXY \
+          ) \
+          --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" $SITESPEED_OPTIONS
       fi
     - mv sitespeed-results/data/performance.json browser-performance.json
   artifacts:
diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
index 1c25d9d583b..196d42f3e3a 100644
--- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
@@ -1,10 +1,10 @@
 build:
   stage: build
-  image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.4.0"
+  image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.6.0"
   variables:
     DOCKER_TLS_CERTDIR: ""
   services:
-    - docker:19.03.12-dind
+    - docker:20.10.6-dind
   script:
     - |
       if [[ -z "$CI_COMMIT_TAG" ]]; then
diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
index fd6c51ea350..b29342216fc 100644
--- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
@@ -36,6 +36,7 @@ code_quality:
           REPORT_STDOUT \
           REPORT_FORMAT \
           ENGINE_MEMORY_LIMIT_BYTES \
+          CODECLIMATE_PREFIX \
         ) \
         --volume "$PWD":/code \
         --volume /var/run/docker.sock:/var/run/docker.sock \
diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
index 654a03ced5f..bf42cd52605 100644
--- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
@@ -12,7 +12,7 @@ stages:
 
 variables:
     FUZZAPI_PROFILE: Quick
-    FUZZAPI_VERSION: latest
+    FUZZAPI_VERSION: "1.6"
     FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml
     FUZZAPI_TIMEOUT: 30
     FUZZAPI_REPORT: gl-api-fuzzing-report.json
@@ -45,7 +45,7 @@ apifuzzer_fuzz:
         entrypoint: ["/bin/bash", "-l", "-c"]
     variables:
         FUZZAPI_PROJECT: $CI_PROJECT_PATH
-        FUZZAPI_API: http://localhost:80
+        FUZZAPI_API: http://localhost:5000
         FUZZAPI_NEW_REPORT: 1
         FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log
         TZ: America/Los_Angeles
@@ -107,7 +107,7 @@ apifuzzer_fuzz_dnd:
         DOCKER_DRIVER: overlay2
         DOCKER_TLS_CERTDIR: ""
         FUZZAPI_PROJECT: $CI_PROJECT_PATH
-        FUZZAPI_API: http://apifuzzer:80
+        FUZZAPI_API: http://apifuzzer:5000
     allow_failure: true
     rules:
         - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null
@@ -142,6 +142,7 @@ apifuzzer_fuzz_dnd:
             -e TZ=America/Los_Angeles \
             -e GITLAB_FEATURES \
             -p 80:80 \
+            -p 5000:5000 \
             -p 8000:8000 \
             -p 514:514 \
             --restart=no \
@@ -168,7 +169,7 @@ apifuzzer_fuzz_dnd:
                 docker run \
                     --name worker \
                     --network $FUZZAPI_D_NETWORK \
-                    -e FUZZAPI_API=http://apifuzzer:80 \
+                    -e FUZZAPI_API=http://apifuzzer:5000 \
                     -e FUZZAPI_PROJECT \
                     -e FUZZAPI_PROFILE \
                     -e FUZZAPI_CONFIG \
@@ -211,7 +212,7 @@ apifuzzer_fuzz_dnd:
                     --name worker \
                     --network $FUZZAPI_D_NETWORK \
                     -e TZ=America/Los_Angeles \
-                    -e FUZZAPI_API=http://apifuzzer:80 \
+                    -e FUZZAPI_API=http://apifuzzer:5000 \
                     -e FUZZAPI_PROJECT \
                     -e FUZZAPI_PROFILE \
                     -e FUZZAPI_CONFIG \
@@ -237,6 +238,7 @@ apifuzzer_fuzz_dnd:
                     -v $CI_PROJECT_DIR:/app \
                     -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \
                     -p 81:80 \
+                    -p 5001:5000 \
                     -p 8001:8000 \
                     -p 515:514 \
                     --restart=no \
diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml
new file mode 100644
index 00000000000..215029dc952
--- /dev/null
+++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml
@@ -0,0 +1,270 @@
+# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/
+
+# Configure the scanning tool through the environment variables.
+# List of the variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-variables
+# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
+
+variables:
+    FUZZAPI_PROFILE: Quick
+    FUZZAPI_VERSION: latest
+    FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml
+    FUZZAPI_TIMEOUT: 30
+    FUZZAPI_REPORT: gl-api-fuzzing-report.json
+    FUZZAPI_REPORT_ASSET_PATH: assets
+    #
+    FUZZAPI_D_NETWORK: testing-net
+    #
+    # Wait up to 5 minutes for API Fuzzer and target url to become
+    # available (non 500 response to HTTP(s))
+    FUZZAPI_SERVICE_START_TIMEOUT: "300"
+    #
+    FUZZAPI_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine
+    #
+
+apifuzzer_fuzz_unlicensed:
+    stage: fuzz
+    allow_failure: true
+    rules:
+        - if: '$GITLAB_FEATURES !~ /\bapi_fuzzing\b/ && $API_FUZZING_DISABLED == null'
+        - when: never
+    script:
+        - |
+            echo "Error: Your GitLab project is not licensed for API Fuzzing."
+        - exit 1
+
+apifuzzer_fuzz:
+    stage: fuzz
+    image:
+        name: $FUZZAPI_IMAGE
+        entrypoint: ["/bin/bash", "-l", "-c"]
+    variables:
+        FUZZAPI_PROJECT: $CI_PROJECT_PATH
+        FUZZAPI_API: http://localhost:80
+        FUZZAPI_NEW_REPORT: 1
+        FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log
+        TZ: America/Los_Angeles
+    allow_failure: true
+    rules:
+        - if: $FUZZAPI_D_TARGET_IMAGE
+          when: never
+        - if: $FUZZAPI_D_WORKER_IMAGE
+          when: never
+        - if: $API_FUZZING_DISABLED
+          when: never
+        - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH &&
+                $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+          when: never
+        - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/
+    script:
+        #
+        # Validate options
+        - |
+            if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \
+                echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \
+                echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \
+                exit 1; \
+            fi
+        #
+        # Run user provided pre-script
+        - sh -c "$FUZZAPI_PRE_SCRIPT"
+        #
+        # Make sure asset path exists
+        - mkdir -p $FUZZAPI_REPORT_ASSET_PATH
+        #
+        # Start API Security background process
+        - dotnet /peach/Peach.Web.dll &> $FUZZAPI_LOG_SCANNER &
+        - APISEC_PID=$!
+        #
+        # Start scanning
+        - worker-entry
+        #
+        # Run user provided post-script
+        - sh -c "$FUZZAPI_POST_SCRIPT"
+        #
+        # Shutdown API Security
+        - kill $APISEC_PID
+        - wait $APISEC_PID
+        #
+    artifacts:
+        when: always
+        paths:
+            - $FUZZAPI_REPORT_ASSET_PATH
+            - $FUZZAPI_REPORT
+            - $FUZZAPI_LOG_SCANNER
+        reports:
+            api_fuzzing: $FUZZAPI_REPORT
+
+apifuzzer_fuzz_dnd:
+    stage: fuzz
+    image: docker:19.03.12
+    variables:
+        DOCKER_DRIVER: overlay2
+        DOCKER_TLS_CERTDIR: ""
+        FUZZAPI_PROJECT: $CI_PROJECT_PATH
+        FUZZAPI_API: http://apifuzzer:80
+    allow_failure: true
+    rules:
+        - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null
+          when: never
+        - if: $API_FUZZING_DISABLED
+          when: never
+        - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH &&
+                $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+          when: never
+        - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/
+    services:
+        - docker:19.03.12-dind
+    script:
+        #
+        #
+        - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+        #
+        - docker network create --driver bridge $FUZZAPI_D_NETWORK
+        #
+        # Run user provided pre-script
+        - sh -c "$FUZZAPI_PRE_SCRIPT"
+        #
+        # Make sure asset path exists
+        - mkdir -p $FUZZAPI_REPORT_ASSET_PATH
+        #
+        # Start peach testing engine container
+        - |
+            docker run -d \
+            --name apifuzzer \
+            --network $FUZZAPI_D_NETWORK \
+            -e Proxy:Port=8000 \
+            -e TZ=America/Los_Angeles \
+            -e GITLAB_FEATURES \
+            -p 80:80 \
+            -p 8000:8000 \
+            -p 514:514 \
+            --restart=no \
+            $FUZZAPI_IMAGE \
+            dotnet /peach/Peach.Web.dll
+        #
+        # Start target container
+        - |
+            if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then \
+                docker run -d \
+                    --name target \
+                    --network $FUZZAPI_D_NETWORK \
+                    $FUZZAPI_D_TARGET_ENV \
+                    $FUZZAPI_D_TARGET_PORTS \
+                    $FUZZAPI_D_TARGET_VOLUME \
+                    --restart=no \
+                    $FUZZAPI_D_TARGET_IMAGE \
+                ; fi
+        #
+        # Start worker container if provided
+        - |
+            if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \
+                echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE"; \
+                docker run \
+                    --name worker \
+                    --network $FUZZAPI_D_NETWORK \
+                    -e FUZZAPI_API=http://apifuzzer:80 \
+                    -e FUZZAPI_PROJECT \
+                    -e FUZZAPI_PROFILE \
+                    -e FUZZAPI_CONFIG \
+                    -e FUZZAPI_REPORT \
+                    -e FUZZAPI_REPORT_ASSET_PATH \
+                    -e FUZZAPI_NEW_REPORT=1 \
+                    -e FUZZAPI_HAR \
+                    -e FUZZAPI_OPENAPI \
+                    -e FUZZAPI_POSTMAN_COLLECTION \
+                    -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \
+                    -e FUZZAPI_TARGET_URL \
+                    -e FUZZAPI_OVERRIDES_FILE \
+                    -e FUZZAPI_OVERRIDES_ENV \
+                    -e FUZZAPI_OVERRIDES_CMD \
+                    -e FUZZAPI_OVERRIDES_INTERVAL \
+                    -e FUZZAPI_TIMEOUT \
+                    -e FUZZAPI_VERBOSE \
+                    -e FUZZAPI_SERVICE_START_TIMEOUT \
+                    -e FUZZAPI_HTTP_USERNAME \
+                    -e FUZZAPI_HTTP_PASSWORD \
+                    -e CI_PROJECT_URL \
+                    -e CI_JOB_ID \
+                    -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \
+                    $FUZZAPI_D_WORKER_ENV \
+                    $FUZZAPI_D_WORKER_PORTS \
+                    $FUZZAPI_D_WORKER_VOLUME \
+                    --restart=no \
+                    $FUZZAPI_D_WORKER_IMAGE \
+                ; fi
+        #
+        # Start API Fuzzing provided worker if no other worker present
+        - |
+            if [ "$FUZZAPI_D_WORKER_IMAGE" == "" ]; then \
+                if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \
+                    echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \
+                    echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \
+                    exit 1; \
+                fi; \
+                docker run \
+                    --name worker \
+                    --network $FUZZAPI_D_NETWORK \
+                    -e TZ=America/Los_Angeles \
+                    -e FUZZAPI_API=http://apifuzzer:80 \
+                    -e FUZZAPI_PROJECT \
+                    -e FUZZAPI_PROFILE \
+                    -e FUZZAPI_CONFIG \
+                    -e FUZZAPI_REPORT \
+                    -e FUZZAPI_REPORT_ASSET_PATH \
+                    -e FUZZAPI_NEW_REPORT=1 \
+                    -e FUZZAPI_HAR \
+                    -e FUZZAPI_OPENAPI \
+                    -e FUZZAPI_POSTMAN_COLLECTION \
+                    -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \
+                    -e FUZZAPI_TARGET_URL \
+                    -e FUZZAPI_OVERRIDES_FILE \
+                    -e FUZZAPI_OVERRIDES_ENV \
+                    -e FUZZAPI_OVERRIDES_CMD \
+                    -e FUZZAPI_OVERRIDES_INTERVAL \
+                    -e FUZZAPI_TIMEOUT \
+                    -e FUZZAPI_VERBOSE \
+                    -e FUZZAPI_SERVICE_START_TIMEOUT \
+                    -e FUZZAPI_HTTP_USERNAME \
+                    -e FUZZAPI_HTTP_PASSWORD \
+                    -e CI_PROJECT_URL \
+                    -e CI_JOB_ID \
+                    -v $CI_PROJECT_DIR:/app \
+                    -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \
+                    -p 81:80 \
+                    -p 8001:8000 \
+                    -p 515:514 \
+                    --restart=no \
+                    $FUZZAPI_IMAGE \
+                    worker-entry \
+                ; fi
+        #
+        # Propagate exit code from api fuzzing scanner (if any)
+        - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing scanner exited with an error. Logs are available as job artifacts."; exit 1; fi
+        #
+        # Run user provided post-script
+        - sh -c "$FUZZAPI_POST_SCRIPT"
+        #
+    after_script:
+        #
+        # Shutdown all containers
+        - echo "Stopping all containers"
+        - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi
+        - docker stop worker
+        - docker stop apifuzzer
+        #
+        # Save docker logs
+        - docker logs apifuzzer &> gl-api_fuzzing-logs.log
+        - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi
+        - docker logs worker &> gl-api_fuzzing-worker-logs.log
+        #
+    artifacts:
+        when: always
+        paths:
+            - ./gl-api_fuzzing*.log
+            - ./gl-api_fuzzing*.zip
+            - $FUZZAPI_REPORT_ASSET_PATH
+            - $FUZZAPI_REPORT
+        reports:
+            api_fuzzing: $FUZZAPI_REPORT
+
+# end
diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
index 64001c2828a..c628e30b2c7 100644
--- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
@@ -6,14 +6,10 @@ variables:
   SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
   CS_MAJOR_VERSION: 3
 
-container_scanning:
+.cs_common:
   stage: test
   image: "$CS_ANALYZER_IMAGE"
   variables:
-    # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image
-    # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes
-    CLAIR_DB_IMAGE_TAG: "latest"
-    CLAIR_DB_IMAGE: "$SECURE_ANALYZERS_PREFIX/clair-vulnerabilities-db:$CLAIR_DB_IMAGE_TAG"
     # Override the GIT_STRATEGY variable in your `.gitlab-ci.yml` file and set it to `fetch` if you want to provide a `clair-whitelist.yml`
     # file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template
     # for details
@@ -21,19 +17,44 @@ container_scanning:
     # CS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to
     # override the analyzer image with a custom value. This may be subject to change or
     # breakage across GitLab releases.
-    CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION
+    CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/$CS_PROJECT:$CS_MAJOR_VERSION
   allow_failure: true
+  artifacts:
+    reports:
+      container_scanning: gl-container-scanning-report.json
+  dependencies: []
+
+container_scanning:
+  extends: .cs_common
+  variables:
+    # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image
+    # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes
+    CLAIR_DB_IMAGE_TAG: "latest"
+    CLAIR_DB_IMAGE: "$SECURE_ANALYZERS_PREFIX/clair-vulnerabilities-db:$CLAIR_DB_IMAGE_TAG"
+    CS_PROJECT: 'klar'
   services:
     - name: $CLAIR_DB_IMAGE
       alias: clair-vulnerabilities-db
   script:
     - /analyzer run
+  rules:
+    - if: $CONTAINER_SCANNING_DISABLED
+      when: never
+    - if: $CI_COMMIT_BRANCH &&
+          $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ &&
+          $CS_MAJOR_VERSION =~ /^[0-3]$/
+
+container_scanning_new:
+  extends: .cs_common
+  variables:
+    CS_PROJECT: 'container-scanning'
+  script:
+    - gtcs scan
   artifacts:
-    reports:
-      container_scanning: gl-container-scanning-report.json
-  dependencies: []
+    paths: [gl-container-scanning-report.json]
   rules:
     - if: $CONTAINER_SCANNING_DISABLED
       when: never
     - if: $CI_COMMIT_BRANCH &&
-          $GITLAB_FEATURES =~ /\bcontainer_scanning\b/
+          $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ &&
+          $CS_MAJOR_VERSION !~ /^[0-3]$/
diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml
index fc1acd09714..533f8bb25f8 100644
--- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml
@@ -1,3 +1,16 @@
+# To use this template, add the following to your .gitlab-ci.yml file:
+#
+# include:
+#   template: DAST.latest.gitlab-ci.yml
+#
+# You also need to add a `dast` stage to your `stages:` configuration. A sample configuration for DAST:
+#
+# stages:
+#   - build
+#   - test
+#   - deploy
+#   - dast
+
 # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/
 
 # Configure the scanning tool through the environment variables.
@@ -9,6 +22,19 @@ variables:
   # Setting this variable will affect all Security templates
   # (SAST, Dependency Scanning, ...)
   SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
+  #
+  DAST_API_PROFILE: Full
+  DAST_API_VERSION: latest
+  DAST_API_CONFIG: .gitlab-dast-api.yml
+  DAST_API_TIMEOUT: 30
+  DAST_API_REPORT: gl-dast-api-report.json
+  DAST_API_REPORT_ASSET_PATH: assets
+  #
+  # Wait up to 5 minutes for API Security and target url to become
+  # available (non 500 response to HTTP(s))
+  DAST_API_SERVICE_START_TIMEOUT: "300"
+  #
+  DAST_API_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${DAST_API_VERSION}-engine
 
 dast:
   stage: dast
@@ -25,6 +51,11 @@ dast:
     reports:
       dast: gl-dast-report.json
   rules:
+    - if: $DAST_API_BETA && ( $DAST_API_SPECIFICATION ||
+          $DAST_API_OPENAPI ||
+          $DAST_API_POSTMAN_COLLECTION ||
+          $DAST_API_HAR )
+      when: never
     - if: $DAST_DISABLED
       when: never
     - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH &&
@@ -40,4 +71,72 @@ dast:
     - if: $CI_COMMIT_BRANCH &&
           $DAST_WEBSITE
     - if: $CI_COMMIT_BRANCH &&
+          $DAST_API_BETA == null &&
           $DAST_API_SPECIFICATION
+
+dast_api:
+  stage: dast
+  image:
+    name: $DAST_API_IMAGE
+    entrypoint: ["/bin/bash", "-l", "-c"]
+  variables:
+    API_SECURITY_MODE: DAST
+    DAST_API_NEW_REPORT: 1
+    DAST_API_PROJECT: $CI_PROJECT_PATH
+    DAST_API_API: http://127.0.0.1:5000
+    DAST_API_LOG_SCANNER: gl-dast-api-scanner.log
+    TZ: America/Los_Angeles
+  allow_failure: true
+  rules:
+    - if: $DAST_API_BETA == null
+      when: never
+    - if: $DAST_DISABLED
+      when: never
+    - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH &&
+          $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+      when: never
+    - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME &&
+          $REVIEW_DISABLED &&
+          $DAST_API_SPECIFICATION == null &&
+          $DAST_API_OPENAPI == null &&
+          $DAST_API_POSTMAN_COLLECTION == null &&
+          $DAST_API_HAR == null
+      when: never
+    - if: $DAST_API_SPECIFICATION == null &&
+          $DAST_API_OPENAPI == null &&
+          $DAST_API_POSTMAN_COLLECTION == null &&
+          $DAST_API_HAR == null
+      when: never
+    - if: $CI_COMMIT_BRANCH &&
+          $GITLAB_FEATURES =~ /\bdast\b/
+  script:
+    #
+    # Run user provided pre-script
+    - sh -c "$DAST_API_PRE_SCRIPT"
+    #
+    # Make sure asset path exists
+    - mkdir -p $DAST_API_REPORT_ASSET_PATH
+    #
+    # Start API Security background process
+    - dotnet /peach/Peach.Web.dll &> $DAST_API_LOG_SCANNER &
+    - APISEC_PID=$!
+    #
+    # Start scanning
+    - worker-entry
+    #
+    # Run user provided post-script
+    - sh -c "$DAST_API_POST_SCRIPT"
+    #
+    # Shutdown API Security
+    - kill $APISEC_PID
+    - wait $APISEC_PID
+    #
+  artifacts:
+    when: always
+    paths:
+      - $DAST_API_REPORT_ASSET_PATH
+      - $DAST_API_REPORT
+      - $DAST_API_LOG_SCANNER
+      - gl-*.log
+    reports:
+      dast: $DAST_API_REPORT
diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
index 9693a4fbca2..3ebccfbba4a 100644
--- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
@@ -160,7 +160,7 @@ mobsf-android-sast:
   services:
     # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile
     # Unfortunately, we need to keep track of mobsf version in 2 different places for now.
-    - name: opensecurity/mobile-security-framework-mobsf:v3.3.3
+    - name: opensecurity/mobile-security-framework-mobsf:v3.4.0
       alias: mobsf
   image:
     name: "$SAST_ANALYZER_IMAGE"
@@ -186,7 +186,7 @@ mobsf-ios-sast:
   services:
     # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile
     # Unfortunately, we need to keep track of mobsf version in 2 different places for now.
-    - name: opensecurity/mobile-security-framework-mobsf:v3.3.3
+    - name: opensecurity/mobile-security-framework-mobsf:v3.4.0
       alias: mobsf
   image:
     name: "$SAST_ANALYZER_IMAGE"
@@ -303,6 +303,10 @@ semgrep-sast:
           $SAST_EXPERIMENTAL_FEATURES == 'true'
       exists:
         - '**/*.py'
+        - '**/*.js'
+        - '**/*.jsx'
+        - '**/*.ts'
+        - '**/*.tsx'
 
 sobelow-sast:
   extends: .sast-analyzer
@@ -348,3 +352,4 @@ spotbugs-sast:
         - '**/*.groovy'
         - '**/*.java'
         - '**/*.scala'
+        - '**/*.kt'
diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml
index e591e3cc1e2..404d4a4c6db 100644
--- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml
@@ -18,9 +18,32 @@ performance:
     - docker:stable-dind
   script:
     - mkdir gitlab-exporter
+    # Busybox wget does not support proxied HTTPS, get the real thing.
+    # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611.
+    - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget
     - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js
     - mkdir sitespeed-results
-    - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS
+    - |
+      function propagate_env_vars() {
+        CURRENT_ENV=$(printenv)
+
+        for VAR_NAME; do
+          echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME "
+        done
+      }
+    - |
+      docker run \
+        $(propagate_env_vars \
+          auto_proxy \
+          https_proxy \
+          http_proxy \
+          no_proxy \
+          AUTO_PROXY \
+          HTTPS_PROXY \
+          HTTP_PROXY \
+          NO_PROXY \
+        ) \
+        --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS
     - mv sitespeed-results/data/performance.json browser-performance.json
   artifacts:
     paths:
diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb
index 3258d965c93..c25c4339c35 100644
--- a/lib/gitlab/ci/trace.rb
+++ b/lib/gitlab/ci/trace.rb
@@ -11,7 +11,7 @@ module Gitlab
       LOCK_SLEEP = 0.001.seconds
       WATCH_FLAG_TTL = 10.seconds
 
-      UPDATE_FREQUENCY_DEFAULT = 30.seconds
+      UPDATE_FREQUENCY_DEFAULT = 60.seconds
       UPDATE_FREQUENCY_WHEN_BEING_WATCHED = 3.seconds
 
       ArchiveError = Class.new(StandardError)
@@ -93,6 +93,10 @@ module Gitlab
         end
       end
 
+      def erase_trace_chunks!
+        job.trace_chunks.fast_destroy_all # Destroy chunks of a live trace
+      end
+
       def erase!
         ##
         # Erase the archived trace
@@ -100,7 +104,7 @@ module Gitlab
 
         ##
         # Erase the live trace
-        job.trace_chunks.fast_destroy_all # Destroy chunks of a live trace
+        erase_trace_chunks!
         FileUtils.rm_f(current_path) if current_path # Remove a trace file of a live trace
         job.erase_old_trace! if job.has_old_trace? # Remove a trace in database of a live trace
       ensure
@@ -114,7 +118,11 @@ module Gitlab
       end
 
       def update_interval
-        being_watched? ? UPDATE_FREQUENCY_WHEN_BEING_WATCHED : UPDATE_FREQUENCY_DEFAULT
+        if being_watched?
+          UPDATE_FREQUENCY_WHEN_BEING_WATCHED
+        else
+          UPDATE_FREQUENCY_DEFAULT
+        end
       end
 
       def being_watched!
@@ -176,9 +184,14 @@ module Gitlab
       end
 
       def unsafe_archive!
-        raise AlreadyArchivedError, 'Could not archive again' if trace_artifact
         raise ArchiveError, 'Job is not finished yet' unless job.complete?
 
+        if trace_artifact
+          unsafe_trace_cleanup! if Feature.enabled?(:erase_traces_from_already_archived_jobs_when_archiving_again, job.project, default_enabled: :yaml)
+
+          raise AlreadyArchivedError, 'Could not archive again'
+        end
+
         if job.trace_chunks.any?
           Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream|
             archive_stream!(stream)
@@ -197,6 +210,18 @@ module Gitlab
         end
       end
 
+      def unsafe_trace_cleanup!
+        return unless trace_artifact
+
+        if trace_artifact.archived_trace_exists?
+          # An archive already exists, so make sure to remove the trace chunks
+          erase_trace_chunks!
+        else
+          # An archive already exists, but its associated file does not, so remove it
+          trace_artifact.destroy!
+        end
+      end
+
       def in_write_lock(&blk)
         lock_key = "trace:write:lock:#{job.id}"
         in_lock(lock_key, ttl: LOCK_TTL, retries: LOCK_RETRIES, sleep_sec: LOCK_SLEEP, &blk)
diff --git a/lib/gitlab/ci/variables/helpers.rb b/lib/gitlab/ci/variables/helpers.rb
index e2a54f90ecb..3a62f01e2e3 100644
--- a/lib/gitlab/ci/variables/helpers.rb
+++ b/lib/gitlab/ci/variables/helpers.rb
@@ -23,7 +23,21 @@ module Gitlab
           def transform_from_yaml_variables(vars)
             return vars.stringify_keys if vars.is_a?(Hash)
 
-            vars.to_a.map { |var| [var[:key].to_s, var[:value]] }.to_h
+            vars.to_a.to_h { |var| [var[:key].to_s, var[:value]] }
+          end
+
+          def inherit_yaml_variables(from:, to:, inheritance:)
+            merge_variables(apply_inheritance(from, inheritance), to)
+          end
+
+          private
+
+          def apply_inheritance(variables, inheritance)
+            case inheritance
+            when true then variables
+            when false then {}
+            when Array then variables.select { |var| inheritance.include?(var[:key]) }
+            end
           end
         end
       end
diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb
index 3459b69bebc..f96a6629849 100644
--- a/lib/gitlab/ci/yaml_processor/result.rb
+++ b/lib/gitlab/ci/yaml_processor/result.rb
@@ -38,11 +38,12 @@ module Gitlab
             .map { |job| build_attributes(job[:name]) }
         end
 
-        def workflow_attributes
-          {
-            rules: hash_config.dig(:workflow, :rules),
-            yaml_variables: transform_to_yaml_variables(variables)
-          }
+        def workflow_rules
+          @workflow_rules ||= hash_config.dig(:workflow, :rules)
+        end
+
+        def root_variables
+          @root_variables ||= transform_to_yaml_variables(variables)
         end
 
         def jobs
@@ -68,7 +69,9 @@ module Gitlab
             when: job[:when] || 'on_success',
             environment: job[:environment_name],
             coverage_regex: job[:coverage],
-            yaml_variables: transform_to_yaml_variables(job[:variables]),
+            yaml_variables: transform_to_yaml_variables(job[:variables]), # https://gitlab.com/gitlab-org/gitlab/-/issues/300581
+            job_variables: transform_to_yaml_variables(job[:job_variables]),
+            root_variables_inheritance: job[:root_variables_inheritance],
             needs_attributes: job.dig(:needs, :job),
             interruptible: job[:interruptible],
             only: job[:only],
@@ -101,7 +104,7 @@ module Gitlab
         end
 
         def merged_yaml
-          @ci_config&.to_hash&.to_yaml
+          @ci_config&.to_hash&.deep_stringify_keys&.to_yaml
         end
 
         def variables_with_data
diff --git a/lib/gitlab/composer/version_index.rb b/lib/gitlab/composer/version_index.rb
index ac0071cdc53..fdff8fb32d3 100644
--- a/lib/gitlab/composer/version_index.rb
+++ b/lib/gitlab/composer/version_index.rb
@@ -28,20 +28,34 @@ module Gitlab
       def package_metadata(package)
         json = package.composer_metadatum.composer_json
 
-        json.merge('dist' => package_dist(package), 'uid' => package.id, 'version' => package.version)
+        json.merge(
+          'dist' => package_dist(package),
+          'source' => package_source(package),
+          'uid' => package.id,
+          'version' => package.version
+        )
       end
 
       def package_dist(package)
-        sha = package.composer_metadatum.target_sha
         archive_api_path = api_v4_projects_packages_composer_archives_package_name_path({ id: package.project_id, package_name: package.name, format: '.zip' }, true)
 
         {
           'type' => 'zip',
-          'url' => expose_url(archive_api_path) + "?sha=#{sha}",
-          'reference' => sha,
+          'url' => expose_url(archive_api_path) + "?sha=#{package.composer_target_sha}",
+          'reference' => package.composer_target_sha,
           'shasum' => ''
         }
       end
+
+      def package_source(package)
+        git_url = package.project.http_url_to_repo
+
+        {
+          'type' => 'git',
+          'url' => git_url,
+          'reference' => package.composer_target_sha
+        }
+      end
     end
   end
 end
diff --git a/lib/gitlab/conan_token.rb b/lib/gitlab/conan_token.rb
index d03997b4158..c3d90aa78fb 100644
--- a/lib/gitlab/conan_token.rb
+++ b/lib/gitlab/conan_token.rb
@@ -7,7 +7,7 @@
 
 module Gitlab
   class ConanToken
-    HMAC_KEY = 'gitlab-conan-packages'.freeze
+    HMAC_KEY = 'gitlab-conan-packages'
 
     attr_reader :access_token_id, :user_id
 
diff --git a/lib/gitlab/contributor.rb b/lib/gitlab/contributor.rb
index d74d5a86aa0..c1c270bc9e6 100644
--- a/lib/gitlab/contributor.rb
+++ b/lib/gitlab/contributor.rb
@@ -5,7 +5,9 @@ module Gitlab
     attr_accessor :email, :name, :commits, :additions, :deletions
 
     def initialize
-      @commits, @additions, @deletions = 0, 0, 0
+      @commits = 0
+      @additions = 0
+      @deletions = 0
     end
   end
 end
diff --git a/lib/gitlab/crypto_helper.rb b/lib/gitlab/crypto_helper.rb
index 4428354642d..c113cebd72f 100644
--- a/lib/gitlab/crypto_helper.rb
+++ b/lib/gitlab/crypto_helper.rb
@@ -16,34 +16,16 @@ module Gitlab
       ::Digest::SHA256.base64digest("#{value}#{salt}")
     end
 
-    def aes256_gcm_encrypt(value, nonce: nil)
-      aes256_gcm_encrypt_using_static_nonce(value)
+    def aes256_gcm_encrypt(value, nonce: AES256_GCM_IV_STATIC)
+      encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value, iv: nonce))
+      Base64.strict_encode64(encrypted_token)
     end
 
-    def aes256_gcm_decrypt(value)
+    def aes256_gcm_decrypt(value, nonce: AES256_GCM_IV_STATIC)
       return unless value
 
-      nonce = Feature.enabled?(:dynamic_nonce_creation) ? dynamic_nonce(value) : AES256_GCM_IV_STATIC
       encrypted_token = Base64.decode64(value)
-      decrypted_token = Encryptor.decrypt(AES256_GCM_OPTIONS.merge(value: encrypted_token, iv: nonce))
-      decrypted_token
-    end
-
-    def dynamic_nonce(value)
-      TokenWithIv.find_nonce_by_hashed_token(value) || AES256_GCM_IV_STATIC
-    end
-
-    def aes256_gcm_encrypt_using_static_nonce(value)
-      create_encrypted_token(value, AES256_GCM_IV_STATIC)
-    end
-
-    def read_only?
-      Gitlab::Database.read_only?
-    end
-
-    def create_encrypted_token(value, iv)
-      encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value, iv: iv))
-      Base64.strict_encode64(encrypted_token)
+      Encryptor.decrypt(AES256_GCM_OPTIONS.merge(value: encrypted_token, iv: nonce))
     end
   end
 end
diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb
index c4af5e6608e..0e4fc8efa95 100644
--- a/lib/gitlab/data_builder/build.rb
+++ b/lib/gitlab/data_builder/build.rb
@@ -12,7 +12,7 @@ module Gitlab
 
         author_url = build_author_url(build.commit, commit)
 
-        data = {
+        {
           object_kind: 'build',
 
           ref: build.ref,
@@ -26,6 +26,7 @@ module Gitlab
           build_name: build.name,
           build_stage: build.stage,
           build_status: build.status,
+          build_created_at: build.created_at,
           build_started_at: build.started_at,
           build_finished_at: build.finished_at,
           build_duration: build.duration,
@@ -66,8 +67,6 @@ module Gitlab
 
           environment: build_environment(build)
         }
-
-        data
       end
 
       private
@@ -84,7 +83,6 @@ module Gitlab
           id: runner.id,
           description: runner.description,
           active: runner.active?,
-          is_shared: runner.instance_type?,
           tags: runner.tags&.map(&:name)
         }
       end
diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb
index 7fd1b9cd228..a56029c0d1d 100644
--- a/lib/gitlab/data_builder/pipeline.rb
+++ b/lib/gitlab/data_builder/pipeline.rb
@@ -77,7 +77,6 @@ module Gitlab
           id: runner.id,
           description: runner.description,
           active: runner.active?,
-          is_shared: runner.instance_type?,
           tags: runner.tags&.map(&:name)
         }
       end
diff --git a/lib/gitlab/database/as_with_materialized.rb b/lib/gitlab/database/as_with_materialized.rb
new file mode 100644
index 00000000000..7c45f416638
--- /dev/null
+++ b/lib/gitlab/database/as_with_materialized.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    # This class is a special Arel node which allows optionally define the `MATERIALIZED` keyword for CTE and Recursive CTE queries.
+    class AsWithMaterialized < Arel::Nodes::Binary
+      extend Gitlab::Utils::StrongMemoize
+
+      MATERIALIZED = Arel.sql(' MATERIALIZED')
+      EMPTY_STRING = Arel.sql('')
+      attr_reader :expr
+
+      def initialize(left, right, materialized: true)
+        @expr = if materialized && self.class.materialized_supported?
+                  MATERIALIZED
+                else
+                  EMPTY_STRING
+                end
+
+        super(left, right)
+      end
+
+      # Note: to be deleted after the minimum PG version is set to 12.0
+      def self.materialized_supported?
+        strong_memoize(:materialized_supported) do
+          Gitlab::Database.version.match?(/^1[2-9]\./) # version 12.x and above
+        end
+      end
+
+      # Note: to be deleted after the minimum PG version is set to 12.0
+      def self.materialized_if_supported
+        materialized_supported? ? 'MATERIALIZED' : ''
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/background_migration/batch_metrics.rb b/lib/gitlab/database/background_migration/batch_metrics.rb
new file mode 100644
index 00000000000..3e6d7ac3c9f
--- /dev/null
+++ b/lib/gitlab/database/background_migration/batch_metrics.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module BackgroundMigration
+      class BatchMetrics
+        attr_reader :timings
+
+        def initialize
+          @timings = {}
+        end
+
+        def time_operation(label)
+          start_time = monotonic_time
+
+          yield
+
+          timings_for_label(label) << monotonic_time - start_time
+        end
+
+        private
+
+        def timings_for_label(label)
+          timings[label] ||= []
+        end
+
+        def monotonic_time
+          Gitlab::Metrics::System.monotonic_time
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb
index 0c9add9b355..4aa33ed7946 100644
--- a/lib/gitlab/database/background_migration/batched_migration.rb
+++ b/lib/gitlab/database/background_migration/batched_migration.rb
@@ -5,7 +5,7 @@ module Gitlab
     module BackgroundMigration
       class BatchedMigration < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord
         JOB_CLASS_MODULE = 'Gitlab::BackgroundMigration'
-        BATCH_CLASS_MODULE = "#{JOB_CLASS_MODULE}::BatchingStrategies".freeze
+        BATCH_CLASS_MODULE = "#{JOB_CLASS_MODULE}::BatchingStrategies"
 
         self.table_name = :batched_background_migrations
 
@@ -23,8 +23,15 @@ module Gitlab
           finished: 3
         }
 
-        def interval_elapsed?
-          last_job.nil? || last_job.created_at <= Time.current - interval
+        def self.active_migration
+          active.queue_order.first
+        end
+
+        def interval_elapsed?(variance: 0)
+          return true unless last_job
+
+          interval_with_variance = interval - variance
+          last_job.created_at <= Time.current - interval_with_variance
         end
 
         def create_batched_job!(min, max)
@@ -50,6 +57,13 @@ module Gitlab
         def batch_class_name=(class_name)
           write_attribute(:batch_class_name, class_name.demodulize)
         end
+
+        def prometheus_labels
+          @prometheus_labels ||= {
+            migration_id: id,
+            migration_identifier: "%s/%s.%s" % [job_class_name, table_name, column_name]
+          }
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb
new file mode 100644
index 00000000000..cf8b61f5feb
--- /dev/null
+++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb
@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module BackgroundMigration
+      class BatchedMigrationRunner
+        def initialize(migration_wrapper = BatchedMigrationWrapper.new)
+          @migration_wrapper = migration_wrapper
+        end
+
+        # Runs the next batched_job for a batched_background_migration.
+        #
+        # The batch bounds of the next job are calculated at runtime, based on the migration
+        # configuration and the bounds of the most recently created batched_job. Updating the
+        # migration configuration will cause future jobs to use the updated batch sizes.
+        #
+        # The job instance will automatically receive a set of arguments based on the migration
+        # configuration. For more details, see the BatchedMigrationWrapper class.
+        #
+        # Note that this method is primarily intended to called by a scheduled worker.
+        def run_migration_job(active_migration)
+          if next_batched_job = create_next_batched_job!(active_migration)
+            migration_wrapper.perform(next_batched_job)
+          else
+            finish_active_migration(active_migration)
+          end
+        end
+
+        # Runs all remaining batched_jobs for a batched_background_migration.
+        #
+        # This method is intended to be used in a test/dev environment to execute the background
+        # migration inline. It should NOT be used in a real environment for any non-trivial migrations.
+        def run_entire_migration(migration)
+          unless Rails.env.development? || Rails.env.test?
+            raise 'this method is not intended for use in real environments'
+          end
+
+          while migration.active?
+            run_migration_job(migration)
+
+            migration.reload_last_job
+          end
+        end
+
+        private
+
+        attr_reader :migration_wrapper
+
+        def create_next_batched_job!(active_migration)
+          next_batch_range = find_next_batch_range(active_migration)
+
+          return if next_batch_range.nil?
+
+          active_migration.create_batched_job!(next_batch_range.min, next_batch_range.max)
+        end
+
+        def find_next_batch_range(active_migration)
+          batching_strategy = active_migration.batch_class.new
+          batch_min_value = active_migration.next_min_value
+
+          next_batch_bounds = batching_strategy.next_batch(
+            active_migration.table_name,
+            active_migration.column_name,
+            batch_min_value: batch_min_value,
+            batch_size: active_migration.batch_size)
+
+          return if next_batch_bounds.nil?
+
+          clamped_batch_range(active_migration, next_batch_bounds)
+        end
+
+        def clamped_batch_range(active_migration, next_bounds)
+          min_value, max_value = next_bounds
+
+          return if min_value > active_migration.max_value
+
+          max_value = max_value.clamp(min_value, active_migration.max_value)
+
+          (min_value..max_value)
+        end
+
+        def finish_active_migration(active_migration)
+          active_migration.finished!
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
index 299bd992197..c276f8ce75b 100644
--- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
+++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
@@ -4,6 +4,15 @@ module Gitlab
   module Database
     module BackgroundMigration
       class BatchedMigrationWrapper
+        extend Gitlab::Utils::StrongMemoize
+
+        # Wraps the execution of a batched_background_migration.
+        #
+        # Updates the job's tracking records with the status of the migration
+        # when starting and finishing execution, and optionally saves batch_metrics
+        # the migration provides, if any are given.
+        #
+        # The job's batch_metrics are serialized to JSON for storage.
         def perform(batch_tracking_record)
           start_tracking_execution(batch_tracking_record)
 
@@ -16,6 +25,7 @@ module Gitlab
           raise e
         ensure
           finish_tracking_execution(batch_tracking_record)
+          track_prometheus_metrics(batch_tracking_record)
         end
 
         private
@@ -34,12 +44,75 @@ module Gitlab
             tracking_record.migration_column_name,
             tracking_record.sub_batch_size,
             *tracking_record.migration_job_arguments)
+
+          if job_instance.respond_to?(:batch_metrics)
+            tracking_record.metrics = job_instance.batch_metrics
+          end
         end
 
         def finish_tracking_execution(tracking_record)
           tracking_record.finished_at = Time.current
           tracking_record.save!
         end
+
+        def track_prometheus_metrics(tracking_record)
+          migration = tracking_record.batched_migration
+          base_labels = migration.prometheus_labels
+
+          metric_for(:gauge_batch_size).set(base_labels, tracking_record.batch_size)
+          metric_for(:gauge_sub_batch_size).set(base_labels, tracking_record.sub_batch_size)
+          metric_for(:counter_updated_tuples).increment(base_labels, tracking_record.batch_size)
+
+          # Time efficiency: Ratio of duration to interval (ideal: less than, but close to 1)
+          efficiency = (tracking_record.finished_at - tracking_record.started_at).to_i / migration.interval.to_f
+          metric_for(:histogram_time_efficiency).observe(base_labels, efficiency)
+
+          if metrics = tracking_record.metrics
+            metrics['timings']&.each do |key, timings|
+              summary = metric_for(:histogram_timings)
+              labels = base_labels.merge(operation: key)
+
+              timings.each do |timing|
+                summary.observe(labels, timing)
+              end
+            end
+          end
+        end
+
+        def metric_for(name)
+          self.class.metrics[name]
+        end
+
+        def self.metrics
+          strong_memoize(:metrics) do
+            {
+              gauge_batch_size: Gitlab::Metrics.gauge(
+                :batched_migration_job_batch_size,
+                'Batch size for a batched migration job'
+              ),
+              gauge_sub_batch_size: Gitlab::Metrics.gauge(
+                :batched_migration_job_sub_batch_size,
+                'Sub-batch size for a batched migration job'
+              ),
+              counter_updated_tuples: Gitlab::Metrics.counter(
+                :batched_migration_job_updated_tuples_total,
+                'Number of tuples updated by batched migration job'
+              ),
+              histogram_timings: Gitlab::Metrics.histogram(
+                :batched_migration_job_duration_seconds,
+                'Timings for a batched migration job',
+                {},
+                [0.1, 0.25, 0.5, 1, 5].freeze
+              ),
+              histogram_time_efficiency: Gitlab::Metrics.histogram(
+                :batched_migration_job_time_efficiency,
+                'Ratio of job duration to interval',
+                {},
+                [0.5, 0.9, 1, 1.5, 2].freeze
+              )
+            }
+          end
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/background_migration/scheduler.rb b/lib/gitlab/database/background_migration/scheduler.rb
deleted file mode 100644
index 5f8a5ec06a5..00000000000
--- a/lib/gitlab/database/background_migration/scheduler.rb
+++ /dev/null
@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Database
-    module BackgroundMigration
-      class Scheduler
-        def perform(migration_wrapper: BatchedMigrationWrapper.new)
-          active_migration = BatchedMigration.active.queue_order.first
-
-          return unless active_migration&.interval_elapsed?
-
-          if next_batched_job = create_next_batched_job!(active_migration)
-            migration_wrapper.perform(next_batched_job)
-          else
-            finish_active_migration(active_migration)
-          end
-        end
-
-        private
-
-        def create_next_batched_job!(active_migration)
-          next_batch_range = find_next_batch_range(active_migration)
-
-          return if next_batch_range.nil?
-
-          active_migration.create_batched_job!(next_batch_range.min, next_batch_range.max)
-        end
-
-        def find_next_batch_range(active_migration)
-          batching_strategy = active_migration.batch_class.new
-          batch_min_value = active_migration.next_min_value
-
-          next_batch_bounds = batching_strategy.next_batch(
-            active_migration.table_name,
-            active_migration.column_name,
-            batch_min_value: batch_min_value,
-            batch_size: active_migration.batch_size)
-
-          return if next_batch_bounds.nil?
-
-          clamped_batch_range(active_migration, next_batch_bounds)
-        end
-
-        def clamped_batch_range(active_migration, next_bounds)
-          min_value, max_value = next_bounds
-
-          return if min_value > active_migration.max_value
-
-          max_value = max_value.clamp(min_value, active_migration.max_value)
-
-          (min_value..max_value)
-        end
-
-        def finish_active_migration(active_migration)
-          active_migration.finished!
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb
index 5a506da0d05..9002d39e1ee 100644
--- a/lib/gitlab/database/batch_count.rb
+++ b/lib/gitlab/database/batch_count.rb
@@ -88,11 +88,16 @@ module Gitlab
         batch_start = start
 
         while batch_start < finish
-          batch_end = [batch_start + batch_size, finish].min
-          batch_relation = build_relation_batch(batch_start, batch_end, mode)
-
           begin
-            results = merge_results(results, batch_relation.send(@operation, *@operation_args)) # rubocop:disable GitlabSecurity/PublicSend
+            batch_end = [batch_start + batch_size, finish].min
+            batch_relation = build_relation_batch(batch_start, batch_end, mode)
+
+            op_args = @operation_args
+            if @operation == :count && @operation_args.blank? && use_loose_index_scan_for_distinct_values?(mode)
+              op_args = [Gitlab::Database::LooseIndexScanDistinctCount::COLUMN_ALIAS]
+            end
+
+            results = merge_results(results, batch_relation.send(@operation, *op_args)) # rubocop:disable GitlabSecurity/PublicSend
             batch_start = batch_end
           rescue ActiveRecord::QueryCanceled => error
             # retry with a safe batch size & warmer cache
@@ -102,6 +107,18 @@ module Gitlab
               log_canceled_batch_fetch(batch_start, mode, batch_relation.to_sql, error)
               return FALLBACK
             end
+          rescue Gitlab::Database::LooseIndexScanDistinctCount::ColumnConfigurationError => error
+            Gitlab::AppJsonLogger
+              .error(
+                event: 'batch_count',
+                relation: @relation.table_name,
+                operation: @operation,
+                operation_args: @operation_args,
+                mode: mode,
+                message: "LooseIndexScanDistinctCount column error: #{error.message}"
+              )
+
+            return FALLBACK
           end
 
           sleep(SLEEP_TIME_IN_SECONDS)
@@ -123,7 +140,11 @@ module Gitlab
       private
 
       def build_relation_batch(start, finish, mode)
-        @relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend
+        if use_loose_index_scan_for_distinct_values?(mode)
+          Gitlab::Database::LooseIndexScanDistinctCount.new(@relation, @column).build_query(from: start, to: finish)
+        else
+          @relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend
+        end
       end
 
       def batch_size_for_mode_and_operation(mode, operation)
@@ -165,6 +186,14 @@ module Gitlab
             message: "Query has been canceled with message: #{error.message}"
           )
       end
+
+      def use_loose_index_scan_for_distinct_values?(mode)
+        Feature.enabled?(:loose_index_scan_for_distinct_values) && not_group_by_query? && mode == :distinct
+      end
+
+      def not_group_by_query?
+        !@relation.is_a?(ActiveRecord::Relation) || @relation.group_values.blank?
+      end
     end
   end
 end
diff --git a/lib/gitlab/database/bulk_update.rb b/lib/gitlab/database/bulk_update.rb
index 1403d561890..b1f9da30585 100644
--- a/lib/gitlab/database/bulk_update.rb
+++ b/lib/gitlab/database/bulk_update.rb
@@ -130,7 +130,7 @@ module Gitlab
 
         def sql
           <<~SQL
-            WITH cte(#{list_of(cte_columns)}) AS (VALUES #{list_of(values)})
+            WITH cte(#{list_of(cte_columns)}) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (VALUES #{list_of(values)})
             UPDATE #{table_name} SET #{list_of(updates)} FROM cte WHERE cte_id = id
           SQL
         end
diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb
index 89190320cf9..a7bfafe2815 100644
--- a/lib/gitlab/database/count/reltuples_count_strategy.rb
+++ b/lib/gitlab/database/count/reltuples_count_strategy.rb
@@ -3,10 +3,6 @@
 module Gitlab
   module Database
     module Count
-      class PgClass < ActiveRecord::Base
-        self.table_name = 'pg_class'
-      end
-
       # This strategy counts based on PostgreSQL's statistics in pg_stat_user_tables.
       #
       # Specifically, it relies on the column reltuples in said table. An additional
@@ -74,7 +70,7 @@ module Gitlab
         def get_statistics(table_names, check_statistics: true)
           time = 6.hours.ago
 
-          query = PgClass.joins("LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relid = pg_class.oid")
+          query = ::Gitlab::Database::PgClass.joins("LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relid = pg_class.oid")
             .where(relname: table_names)
             .where('schemaname = current_schema()')
             .select('pg_class.relname AS table_name, reltuples::bigint AS estimate')
diff --git a/lib/gitlab/database/loose_index_scan_distinct_count.rb b/lib/gitlab/database/loose_index_scan_distinct_count.rb
new file mode 100644
index 00000000000..884f4d47ff8
--- /dev/null
+++ b/lib/gitlab/database/loose_index_scan_distinct_count.rb
@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    # This class builds efficient batched distinct query by using loose index scan.
+    # Consider the following example:
+    # > Issue.distinct(:project_id).where(project_id: (1...100)).count
+    #
+    # Note: there is an index on project_id
+    #
+    # This query will read each element in the index matching the project_id filter.
+    # If for a project_id has 100_000 issues, all 100_000 elements will be read.
+    #
+    # A loose index scan will read only one entry from the index for each project_id to reduce the number of disk reads.
+    #
+    # Usage:
+    #
+    # Gitlab::Database::LooseIndexScanDisctinctCount.new(Issue, :project_id).count(from: 1, to: 100)
+    #
+    # The query will return the number of distinct projects_ids between 1 and 100
+    #
+    # Getting the Arel query:
+    #
+    # Gitlab::Database::LooseIndexScanDisctinctCount.new(Issue, :project_id).build_query(from: 1, to: 100)
+    class LooseIndexScanDistinctCount
+      COLUMN_ALIAS = 'distinct_count_column'
+
+      ColumnConfigurationError = Class.new(StandardError)
+
+      def initialize(scope, column)
+        if scope.is_a?(ActiveRecord::Relation)
+          @scope = scope
+          @model = scope.model
+        else
+          @scope = scope.where({})
+          @model = scope
+        end
+
+        @column = transform_column(column)
+      end
+
+      def count(from:, to:)
+        build_query(from: from, to: to).count(COLUMN_ALIAS)
+      end
+
+      def build_query(from:, to:) # rubocop:disable Metrics/AbcSize
+        cte = Gitlab::SQL::RecursiveCTE.new(:counter_cte, union_args: { remove_order: false })
+        table = model.arel_table
+
+        cte << @scope
+          .dup
+          .select(column.as(COLUMN_ALIAS))
+          .where(column.gteq(from))
+          .where(column.lt(to))
+          .order(column)
+          .limit(1)
+
+        inner_query = @scope
+          .dup
+          .where(column.gt(cte.table[COLUMN_ALIAS]))
+          .where(column.lt(to))
+          .select(column.as(COLUMN_ALIAS))
+          .order(column)
+          .limit(1)
+
+        cte << cte.table
+          .project(Arel::Nodes::Grouping.new(Arel.sql(inner_query.to_sql)).as(COLUMN_ALIAS))
+          .where(cte.table[COLUMN_ALIAS].lt(to))
+
+        model
+          .with
+          .recursive(cte.to_arel)
+          .from(cte.alias_to(table))
+          .unscope(where: :source_type)
+          .unscope(where: model.inheritance_column) # Remove STI query, not needed here
+      end
+
+      private
+
+      attr_reader :column, :model
+
+      # Transforms the column so it can be used in Arel expressions
+      #
+      # 'table.column' => 'table.column'
+      # 'column' => 'table_name.column'
+      # :column => 'table_name.column'
+      # Arel::Attributes::Attribute => name of the column
+      def transform_column(column)
+        if column.is_a?(String) || column.is_a?(Symbol)
+          column_as_string = column.to_s
+          column_as_string = "#{model.table_name}.#{column_as_string}" unless column_as_string.include?('.')
+
+          Arel.sql(column_as_string)
+        elsif column.is_a?(Arel::Attributes::Attribute)
+          column
+        else
+          raise ColumnConfigurationError.new("Cannot transform the column: #{column.inspect}, please provide the column name as string")
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 31e733050e1..d06a73da8ac 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -4,6 +4,7 @@ module Gitlab
   module Database
     module MigrationHelpers
       include Migrations::BackgroundMigrationHelpers
+      include DynamicModelHelpers
 
       # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
       MAX_IDENTIFIER_NAME_LENGTH = 63
@@ -576,17 +577,7 @@ module Gitlab
       # old_column - The name of the old column.
       # new_column - The name of the new column.
       def install_rename_triggers(table, old_column, new_column)
-        trigger_name = rename_trigger_name(table, old_column, new_column)
-        quoted_table = quote_table_name(table)
-        quoted_old = quote_column_name(old_column)
-        quoted_new = quote_column_name(new_column)
-
-        install_rename_triggers_for_postgresql(
-          trigger_name,
-          quoted_table,
-          quoted_old,
-          quoted_new
-        )
+        install_rename_triggers_for_postgresql(table, old_column, new_column)
       end
 
       # Changes the type of a column concurrently.
@@ -927,19 +918,67 @@ module Gitlab
       #   This is crucial for Primary Key conversions, because setting a column
       #    as the PK converts even check constraints to NOT NULL constraints
       #    and forces an inline re-verification of the whole table.
-      # - It backfills the new column with the values of the existing primary key
-      #    by scheduling background jobs.
-      # - It tracks the scheduled background jobs through the use of
-      #    Gitlab::Database::BackgroundMigrationJob
+      # - It sets up a trigger to keep the two columns in sync.
+      #
+      #   Note: this helper is intended to be used in a regular (pre-deployment) migration.
+      #
+      #   This helper is part 1 of a multi-step migration process:
+      #   1. initialize_conversion_of_integer_to_bigint to create the new column and database triggers
+      #   2. backfill_conversion_of_integer_to_bigint to copy historic data using background migrations
+      #   3. remaining steps TBD, see #288005
+      #
+      # table - The name of the database table containing the column
+      # column - The name of the column that we want to convert to bigint.
+      # primary_key - The name of the primary key column (most often :id)
+      def initialize_conversion_of_integer_to_bigint(table, column, primary_key: :id)
+        unless table_exists?(table)
+          raise "Table #{table} does not exist"
+        end
+
+        unless column_exists?(table, primary_key)
+          raise "Column #{primary_key} does not exist on #{table}"
+        end
+
+        unless column_exists?(table, column)
+          raise "Column #{column} does not exist on #{table}"
+        end
+
+        check_trigger_permissions!(table)
+
+        old_column = column_for(table, column)
+        tmp_column = "#{column}_convert_to_bigint"
+
+        with_lock_retries do
+          if (column.to_s == primary_key.to_s) || !old_column.null
+            # If the column to be converted is either a PK or is defined as NOT NULL,
+            # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow
+            # That way, we skip the expensive validation step required to add
+            #  a NOT NULL constraint at the end of the process
+            add_column(table, tmp_column, :bigint, default: old_column.default || 0, null: false)
+          else
+            add_column(table, tmp_column, :bigint, default: old_column.default)
+          end
+
+          install_rename_triggers(table, column, tmp_column)
+        end
+      end
+
+      # Backfills the new column used in the conversion of an integer column to bigint using background migrations.
+      #
+      # - This helper should be called from a post-deployment migration.
+      # - In order for this helper to work properly,  the new column must be first initialized with
+      #   the `initialize_conversion_of_integer_to_bigint` helper.
+      # - It tracks the scheduled background jobs through Gitlab::Database::BackgroundMigration::BatchedMigration,
       #   which allows a more thorough check that all jobs succeeded in the
       #   cleanup migration and is way faster for very large tables.
-      # - It sets up a trigger to keep the two columns in sync
-      # - It does not schedule a cleanup job: we have to do that with followup
-      #    post deployment migrations in the next release.
       #
-      #   This needs to be done manually by using the
-      #    `cleanup_initialize_conversion_of_integer_to_bigint`
-      #   (not yet implemented - check #288005)
+      #   Note: this helper is intended to be used in a post-deployment migration, to ensure any new code is
+      #   deployed (including background job changes) before we begin processing the background migration.
+      #
+      #   This helper is part 2 of a multi-step migration process:
+      #   1. initialize_conversion_of_integer_to_bigint to create the new column and database triggers
+      #   2. backfill_conversion_of_integer_to_bigint to copy historic data using background migrations
+      #   3. remaining steps TBD, see #288005
       #
       # table - The name of the database table containing the column
       # column - The name of the column that we want to convert to bigint.
@@ -960,7 +999,7 @@ module Gitlab
       #  and set the batch_size to 50_000 which will require
       #  ~50s = (50000 / 200) * (0.1 + 0.1) to complete and leaves breathing space
       #  between the scheduled jobs
-      def initialize_conversion_of_integer_to_bigint(
+      def backfill_conversion_of_integer_to_bigint(
         table,
         column,
         primary_key: :id,
@@ -969,10 +1008,6 @@ module Gitlab
         interval: 2.minutes
       )
 
-        if transaction_open?
-          raise 'initialize_conversion_of_integer_to_bigint can not be run inside a transaction'
-        end
-
         unless table_exists?(table)
           raise "Table #{table} does not exist"
         end
@@ -985,87 +1020,42 @@ module Gitlab
           raise "Column #{column} does not exist on #{table}"
         end
 
-        check_trigger_permissions!(table)
-
-        old_column = column_for(table, column)
         tmp_column = "#{column}_convert_to_bigint"
 
-        with_lock_retries do
-          if (column.to_s == primary_key.to_s) || !old_column.null
-            # If the column to be converted is either a PK or is defined as NOT NULL,
-            # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow
-            # That way, we skip the expensive validation step required to add
-            #  a NOT NULL constraint at the end of the process
-            add_column(table, tmp_column, :bigint, default: old_column.default || 0, null: false)
-          else
-            add_column(table, tmp_column, :bigint, default: old_column.default)
-          end
-
-          install_rename_triggers(table, column, tmp_column)
-        end
-
-        source_model = Class.new(ActiveRecord::Base) do
-          include EachBatch
-
-          self.table_name = table
-          self.inheritance_column = :_type_disabled
+        unless column_exists?(table, tmp_column)
+          raise 'The temporary column does not exist, initialize it with `initialize_conversion_of_integer_to_bigint`'
         end
 
-        queue_background_migration_jobs_by_range_at_intervals(
-          source_model,
+        batched_migration = queue_batched_background_migration(
           'CopyColumnUsingBackgroundMigrationJob',
-          interval,
+          table,
+          primary_key,
+          column,
+          tmp_column,
+          job_interval: interval,
           batch_size: batch_size,
-          other_job_arguments: [table, primary_key, sub_batch_size, column, tmp_column],
-          track_jobs: true,
-          primary_column_name: primary_key
-        )
+          sub_batch_size: sub_batch_size)
 
         if perform_background_migration_inline?
           # To ensure the schema is up to date immediately we perform the
           # migration inline in dev / test environments.
-          Gitlab::BackgroundMigration.steal('CopyColumnUsingBackgroundMigrationJob')
+          Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new.run_entire_migration(batched_migration)
         end
       end
 
       # Performs a concurrent column rename when using PostgreSQL.
-      def install_rename_triggers_for_postgresql(trigger, table, old, new)
-        execute <<-EOF.strip_heredoc
-        CREATE OR REPLACE FUNCTION #{trigger}()
-        RETURNS trigger AS
-        $BODY$
-        BEGIN
-          NEW.#{new} := NEW.#{old};
-          RETURN NEW;
-        END;
-        $BODY$
-        LANGUAGE 'plpgsql'
-        VOLATILE
-        EOF
-
-        execute <<-EOF.strip_heredoc
-        DROP TRIGGER IF EXISTS #{trigger}
-        ON #{table}
-        EOF
-
-        execute <<-EOF.strip_heredoc
-        CREATE TRIGGER #{trigger}
-        BEFORE INSERT OR UPDATE
-        ON #{table}
-        FOR EACH ROW
-        EXECUTE FUNCTION #{trigger}()
-        EOF
+      def install_rename_triggers_for_postgresql(table, old, new, trigger_name: nil)
+        Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).create(old, new, trigger_name: trigger_name)
       end
 
       # Removes the triggers used for renaming a PostgreSQL column concurrently.
       def remove_rename_triggers_for_postgresql(table, trigger)
-        execute("DROP TRIGGER IF EXISTS #{trigger} ON #{table}")
-        execute("DROP FUNCTION IF EXISTS #{trigger}()")
+        Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).drop(trigger)
       end
 
       # Returns the (base) name to use for triggers when renaming columns.
       def rename_trigger_name(table, old, new)
-        'trigger_' + Digest::SHA256.hexdigest("#{table}_#{old}_#{new}").first(12)
+        Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).name(old, new)
       end
 
       # Returns an Array containing the indexes for the given column
diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb
index e8cbea72887..8d5ea652bfc 100644
--- a/lib/gitlab/database/migrations/background_migration_helpers.rb
+++ b/lib/gitlab/database/migrations/background_migration_helpers.rb
@@ -190,7 +190,7 @@ module Gitlab
           migration_status = batch_max_value.nil? ? :finished : :active
           batch_max_value ||= batch_min_value
 
-          Gitlab::Database::BackgroundMigration::BatchedMigration.create!(
+          migration = Gitlab::Database::BackgroundMigration::BatchedMigration.create!(
             job_class_name: job_class_name,
             table_name: batch_table_name,
             column_name: batch_column_name,
@@ -202,6 +202,17 @@ module Gitlab
             sub_batch_size: sub_batch_size,
             job_arguments: job_arguments,
             status: migration_status)
+
+          # This guard is necessary since #total_tuple_count was only introduced schema-wise,
+          # after this migration helper had been used for the first time.
+          return migration unless migration.respond_to?(:total_tuple_count)
+
+          # We keep track of the estimated number of tuples to reason later
+          # about the overall progress of a migration.
+          migration.total_tuple_count = Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate
+          migration.save!
+
+          migration
         end
 
         def perform_background_migration_inline?
@@ -236,6 +247,14 @@ module Gitlab
           Gitlab::ApplicationContext.with_context(caller_id: self.class.to_s, &block)
         end
 
+        def delete_queued_jobs(class_name)
+          Gitlab::BackgroundMigration.steal(class_name) do |job|
+            job.delete
+
+            false
+          end
+        end
+
         private
 
         def track_in_database(class_name, arguments)
diff --git a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb
index 2def3a4d3a9..4402c42b136 100644
--- a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb
+++ b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb
@@ -6,6 +6,80 @@ module Gitlab
       module ForeignKeyHelpers
         include ::Gitlab::Database::SchemaHelpers
 
+        # Adds a foreign key with only minimal locking on the tables involved.
+        #
+        # In concept it works similarly to add_concurrent_foreign_key, but we have
+        # to add a special helper for partitioned tables for the following reasons:
+        # - add_concurrent_foreign_key sets the constraint to `NOT VALID`
+        #   before validating it
+        # - Setting an FK to NOT VALID is not supported currently in Postgres (up to PG13)
+        # - Also, PostgreSQL will currently ignore NOT VALID constraints on partitions
+        #   when adding a valid FK to the partitioned table, so they have to
+        #   also be validated before we can add the final FK.
+        # Solution:
+        # - Add the foreign key first to each partition by using
+        #   add_concurrent_foreign_key and validating it
+        # - Once all partitions have a foreign key, add it also to the partitioned
+        #   table (there will be no need for a validation at that level)
+        # For those reasons, this method does not include an option to delay the
+        # validation, we have to force validate: true.
+        #
+        # source - The source (partitioned) table containing the foreign key.
+        # target - The target table the key points to.
+        # column - The name of the column to create the foreign key on.
+        # on_delete - The action to perform when associated data is removed,
+        #             defaults to "CASCADE".
+        # name - The name of the foreign key.
+        #
+        def add_concurrent_partitioned_foreign_key(source, target, column:, on_delete: :cascade, name: nil)
+          partition_options = {
+            column: column,
+            on_delete: on_delete,
+
+            # We'll use the same FK name for all partitions and match it to
+            # the name used for the partitioned table to follow the convention
+            # used by PostgreSQL when adding FKs to new partitions
+            name: name.presence || concurrent_partitioned_foreign_key_name(source, column),
+
+            # Force the FK validation to true for partitions (and the partitioned table)
+            validate: true
+          }
+
+          if foreign_key_exists?(source, target, **partition_options)
+            warning_message = "Foreign key not created because it exists already " \
+              "(this may be due to an aborted migration or similar): " \
+              "source: #{source}, target: #{target}, column: #{partition_options[:column]}, "\
+              "name: #{partition_options[:name]}, on_delete: #{partition_options[:on_delete]}"
+
+            Gitlab::AppLogger.warn warning_message
+
+            return
+          end
+
+          partitioned_table = find_partitioned_table(source)
+
+          partitioned_table.postgres_partitions.order(:name).each do |partition|
+            add_concurrent_foreign_key(partition.identifier, target, **partition_options)
+          end
+
+          with_lock_retries do
+            add_foreign_key(source, target, **partition_options)
+          end
+        end
+
+        # Returns the name for a concurrent partitioned foreign key.
+        #
+        # Similar to concurrent_foreign_key_name (Gitlab::Database::MigrationHelpers)
+        # we just keep a separate method in case we want a different behavior
+        # for partitioned tables
+        #
+        def concurrent_partitioned_foreign_key_name(table, column, prefix: 'fk_rails_')
+          identifier = "#{table}_#{column}_fk"
+          hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10)
+
+          "#{prefix}#{hashed_identifier}"
+        end
+
         # Creates a "foreign key" that references a partitioned table. Because foreign keys referencing partitioned
         # tables are not supported in PG11, this does not create a true database foreign key, but instead implements the
         # same functionality at the database level by using triggers.
diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
index 1c289391e21..9ccbdc9930e 100644
--- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
+++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
@@ -223,6 +223,28 @@ module Gitlab
           replace_table(table_name, archived_table_name, partitioned_table_name, primary_key_name)
         end
 
+        def drop_nonpartitioned_archive_table(table_name)
+          assert_table_is_allowed(table_name)
+
+          archived_table_name = make_archived_table_name(table_name)
+
+          with_lock_retries do
+            drop_sync_trigger(table_name)
+          end
+
+          drop_table(archived_table_name)
+        end
+
+        def create_trigger_to_sync_tables(source_table_name, partitioned_table_name, unique_key)
+          function_name = make_sync_function_name(source_table_name)
+          trigger_name = make_sync_trigger_name(source_table_name)
+
+          create_sync_function(function_name, partitioned_table_name, unique_key)
+          create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table_name} table")
+
+          create_sync_trigger(source_table_name, trigger_name, function_name)
+        end
+
         private
 
         def assert_table_is_allowed(table_name)
@@ -316,16 +338,6 @@ module Gitlab
           create_range_partition(partition_name, table_name, lower_bound, upper_bound)
         end
 
-        def create_trigger_to_sync_tables(source_table_name, partitioned_table_name, unique_key)
-          function_name = make_sync_function_name(source_table_name)
-          trigger_name = make_sync_trigger_name(source_table_name)
-
-          create_sync_function(function_name, partitioned_table_name, unique_key)
-          create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table_name} table")
-
-          create_sync_trigger(source_table_name, trigger_name, function_name)
-        end
-
         def drop_sync_trigger(source_table_name)
           trigger_name = make_sync_trigger_name(source_table_name)
           drop_trigger(source_table_name, trigger_name)
diff --git a/lib/gitlab/database/pg_class.rb b/lib/gitlab/database/pg_class.rb
new file mode 100644
index 00000000000..0ce9eebc14c
--- /dev/null
+++ b/lib/gitlab/database/pg_class.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    class PgClass < ActiveRecord::Base
+      self.table_name = 'pg_class'
+
+      def self.for_table(relname)
+        joins("LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relid = pg_class.oid")
+          .where('schemaname = current_schema()')
+          .find_by(relname: relname)
+      end
+
+      def cardinality_estimate
+        tuples = reltuples.to_i
+
+        return if tuples < 1
+
+        tuples
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb
index 62dfaeeaae3..e8b49c7f62c 100644
--- a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb
+++ b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb
@@ -41,19 +41,6 @@ module Gitlab
         BUCKET_ID_MASK = (Buckets::TOTAL_BUCKETS - ZERO_OFFSET).to_s(2)
         BIT_31_MASK = "B'0#{'1' * 31}'"
         BIT_32_NORMALIZED_BUCKET_ID_MASK = "B'#{'0' * (32 - BUCKET_ID_MASK.size)}#{BUCKET_ID_MASK}'"
-        # @example source_query
-        #   SELECT CAST(('X' || md5(CAST(%{column} as text))) as bit(32)) attr_hash_32_bits
-        #   FROM %{relation}
-        #   WHERE %{pkey} >= %{batch_start}
-        #   AND %{pkey} < %{batch_end}
-        #   AND %{column} IS NOT NULL
-        BUCKETED_DATA_SQL = <<~SQL
-          WITH hashed_attributes AS (%{source_query})
-          SELECT (attr_hash_32_bits & #{BIT_32_NORMALIZED_BUCKET_ID_MASK})::int AS bucket_num,
-            (31 - floor(log(2, min((attr_hash_32_bits & #{BIT_31_MASK})::int))))::int as bucket_hash
-          FROM hashed_attributes
-          GROUP BY 1
-        SQL
 
         WRONG_CONFIGURATION_ERROR = Class.new(ActiveRecord::StatementInvalid)
 
@@ -103,7 +90,7 @@ module Gitlab
         def hll_buckets_for_batch(start, finish)
           @relation
             .connection
-            .execute(BUCKETED_DATA_SQL % { source_query: source_query(start, finish) })
+            .execute(bucketed_data_sql % { source_query: source_query(start, finish) })
             .map(&:values)
             .to_h
         end
@@ -139,6 +126,22 @@ module Gitlab
         def actual_finish(finish)
           finish || @relation.unscope(:group, :having).maximum(@relation.primary_key) || 0
         end
+
+        # @example source_query
+        #   SELECT CAST(('X' || md5(CAST(%{column} as text))) as bit(32)) attr_hash_32_bits
+        #   FROM %{relation}
+        #   WHERE %{pkey} >= %{batch_start}
+        #   AND %{pkey} < %{batch_end}
+        #   AND %{column} IS NOT NULL
+        def bucketed_data_sql
+          <<~SQL
+            WITH hashed_attributes AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (%{source_query})
+            SELECT (attr_hash_32_bits & #{BIT_32_NORMALIZED_BUCKET_ID_MASK})::int AS bucket_num,
+              (31 - floor(log(2, min((attr_hash_32_bits & #{BIT_31_MASK})::int))))::int as bucket_hash
+            FROM hashed_attributes
+            GROUP BY 1
+          SQL
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/similarity_score.rb b/lib/gitlab/database/similarity_score.rb
index 40845c0d5e0..20bf6fa4d30 100644
--- a/lib/gitlab/database/similarity_score.rb
+++ b/lib/gitlab/database/similarity_score.rb
@@ -10,7 +10,7 @@ module Gitlab
 
       # Adds a "magic" comment in the generated SQL expression in order to be able to tell if we're sorting by similarity.
       # Example: /* gitlab/database/similarity_score */ SIMILARITY(COALESCE...
-      SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION = "/* #{DISPLAY_NAME} */ SIMILARITY".freeze
+      SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION = "/* #{DISPLAY_NAME} */ SIMILARITY"
 
       # This method returns an Arel expression that can be used in an ActiveRecord query to order the resultset by similarity.
       #
diff --git a/lib/gitlab/database/unidirectional_copy_trigger.rb b/lib/gitlab/database/unidirectional_copy_trigger.rb
new file mode 100644
index 00000000000..029c894a5ff
--- /dev/null
+++ b/lib/gitlab/database/unidirectional_copy_trigger.rb
@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    class UnidirectionalCopyTrigger
+      def self.on_table(table_name, connection: ActiveRecord::Base.connection)
+        new(table_name, connection)
+      end
+
+      def name(from_column_names, to_column_names)
+        from_column_names, to_column_names = check_column_names!(from_column_names, to_column_names)
+
+        unchecked_name(from_column_names, to_column_names)
+      end
+
+      def create(from_column_names, to_column_names, trigger_name: nil)
+        from_column_names, to_column_names = check_column_names!(from_column_names, to_column_names)
+        trigger_name ||= unchecked_name(from_column_names, to_column_names)
+
+        assignment_clauses = assignment_clauses_for_columns(from_column_names, to_column_names)
+
+        connection.execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{trigger_name}()
+          RETURNS trigger AS
+          $BODY$
+          BEGIN
+            #{assignment_clauses};
+            RETURN NEW;
+          END;
+          $BODY$
+          LANGUAGE 'plpgsql'
+          VOLATILE
+        SQL
+
+        connection.execute(<<~SQL)
+          DROP TRIGGER IF EXISTS #{trigger_name}
+          ON #{quoted_table_name}
+        SQL
+
+        connection.execute(<<~SQL)
+          CREATE TRIGGER #{trigger_name}
+          BEFORE INSERT OR UPDATE
+          ON #{quoted_table_name}
+          FOR EACH ROW
+          EXECUTE FUNCTION #{trigger_name}()
+        SQL
+      end
+
+      def drop(trigger_name)
+        connection.execute("DROP TRIGGER IF EXISTS #{trigger_name} ON #{quoted_table_name}")
+        connection.execute("DROP FUNCTION IF EXISTS #{trigger_name}()")
+      end
+
+      private
+
+      attr_reader :table_name, :connection
+
+      def initialize(table_name, connection)
+        @table_name = table_name
+        @connection = connection
+      end
+
+      def quoted_table_name
+        @quoted_table_name ||= connection.quote_table_name(table_name)
+      end
+
+      def check_column_names!(from_column_names, to_column_names)
+        from_column_names = Array.wrap(from_column_names)
+        to_column_names = Array.wrap(to_column_names)
+
+        unless from_column_names.size == to_column_names.size
+          raise ArgumentError, 'number of source and destination columns must match'
+        end
+
+        [from_column_names, to_column_names]
+      end
+
+      def unchecked_name(from_column_names, to_column_names)
+        joined_column_names = from_column_names.zip(to_column_names).flatten.join('_')
+        'trigger_' + Digest::SHA256.hexdigest("#{table_name}_#{joined_column_names}").first(12)
+      end
+
+      def assignment_clauses_for_columns(from_column_names, to_column_names)
+        combined_column_names = to_column_names.zip(from_column_names)
+
+        assignment_clauses = combined_column_names.map do |(new_name, old_name)|
+          new_name = connection.quote_column_name(new_name)
+          old_name = connection.quote_column_name(old_name)
+
+          "NEW.#{new_name} := NEW.#{old_name}"
+        end
+
+        assignment_clauses.join(";\n  ")
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb
index baa46e7e306..8385bbbb3de 100644
--- a/lib/gitlab/diff/highlight.rb
+++ b/lib/gitlab/diff/highlight.rb
@@ -3,7 +3,7 @@
 module Gitlab
   module Diff
     class Highlight
-      attr_reader :diff_file, :diff_lines, :raw_lines, :repository, :project
+      attr_reader :diff_file, :diff_lines, :repository, :project
 
       delegate :old_path, :new_path, :old_sha, :new_sha, to: :diff_file, prefix: :diff
 
@@ -22,29 +22,15 @@ module Gitlab
       end
 
       def highlight
-        @diff_lines.map.with_index do |diff_line, i|
+        populate_marker_ranges if Feature.enabled?(:use_marker_ranges, project, default_enabled: :yaml)
+
+        @diff_lines.map.with_index do |diff_line, index|
           diff_line = diff_line.dup
           # ignore highlighting for "match" lines
           next diff_line if diff_line.meta?
 
-          rich_line = highlight_line(diff_line) || ERB::Util.html_escape(diff_line.text)
-
-          if line_inline_diffs = inline_diffs[i]
-            begin
-              # MarkerRange objects are converted to Ranges to keep the previous behavior
-              # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/324068
-              if Feature.disabled?(:introduce_marker_ranges, project, default_enabled: :yaml)
-                line_inline_diffs = line_inline_diffs.map { |marker_range| marker_range.to_range }
-              end
-
-              rich_line = InlineDiffMarker.new(diff_line.text, rich_line).mark(line_inline_diffs)
-            # This should only happen when the encoding of the diff doesn't
-            # match the blob, which is a bug. But we shouldn't fail to render
-            # completely in that case, even though we want to report the error.
-            rescue RangeError => e
-              Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, issue_url: 'https://gitlab.com/gitlab-org/gitlab-foss/issues/45441')
-            end
-          end
+          rich_line = apply_syntax_highlight(diff_line)
+          rich_line = apply_marker_ranges_highlight(diff_line, rich_line, index)
 
           diff_line.rich_text = rich_line
 
@@ -54,9 +40,87 @@ module Gitlab
 
       private
 
+      def populate_marker_ranges
+        pair_selector = Gitlab::Diff::PairSelector.new(@raw_lines)
+
+        pair_selector.each do |old_index, new_index|
+          old_line = diff_lines[old_index]
+          new_line = diff_lines[new_index]
+
+          old_diffs, new_diffs = Gitlab::Diff::InlineDiff.new(old_line.text, new_line.text, offset: 1).inline_diffs
+
+          old_line.set_marker_ranges(old_diffs)
+          new_line.set_marker_ranges(new_diffs)
+        end
+      end
+
+      def apply_syntax_highlight(diff_line)
+        highlight_line(diff_line) || ERB::Util.html_escape(diff_line.text)
+      end
+
+      def apply_marker_ranges_highlight(diff_line, rich_line, index)
+        marker_ranges = if Feature.enabled?(:use_marker_ranges, project, default_enabled: :yaml)
+                          diff_line.marker_ranges
+                        else
+                          inline_diffs[index]
+                        end
+
+        return rich_line if marker_ranges.blank?
+
+        begin
+          # MarkerRange objects are converted to Ranges to keep the previous behavior
+          # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/324068
+          if Feature.disabled?(:introduce_marker_ranges, project, default_enabled: :yaml)
+            marker_ranges = marker_ranges.map { |marker_range| marker_range.to_range }
+          end
+
+          InlineDiffMarker.new(diff_line.text, rich_line).mark(marker_ranges)
+        # This should only happen when the encoding of the diff doesn't
+        # match the blob, which is a bug. But we shouldn't fail to render
+        # completely in that case, even though we want to report the error.
+        rescue RangeError => e
+          Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, issue_url: 'https://gitlab.com/gitlab-org/gitlab-foss/issues/45441')
+        end
+      end
+
       def highlight_line(diff_line)
         return unless diff_file && diff_file.diff_refs
 
+        if Feature.enabled?(:diff_line_syntax_highlighting, project, default_enabled: :yaml)
+          diff_line_highlighting(diff_line)
+        else
+          blob_highlighting(diff_line)
+        end
+      end
+
+      def diff_line_highlighting(diff_line)
+        rich_line = syntax_highlighter(diff_line).highlight(
+          diff_line.text(prefix: false),
+          context: { line_number: diff_line.line }
+        )&.html_safe
+
+        # Only update text if line is found. This will prevent
+        # issues with submodules given the line only exists in diff content.
+        if rich_line
+          line_prefix = diff_line.text =~ /\A(.)/ ? Regexp.last_match(1) : ' '
+          rich_line.prepend(line_prefix).concat("\n")
+        end
+      end
+
+      def syntax_highlighter(diff_line)
+        path = diff_line.removed? ? diff_file.old_path : diff_file.new_path
+
+        @syntax_highlighter ||= {}
+        @syntax_highlighter[path] ||= Gitlab::Highlight.new(
+          path,
+          @raw_lines,
+          language: repository&.gitattribute(path, 'gitlab-language')
+        )
+      end
+
+      # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/324159
+      # ------------------------------------------------------------------------
+      def blob_highlighting(diff_line)
         rich_line =
           if diff_line.unchanged? || diff_line.added?
             new_lines[diff_line.new_pos - 1]&.html_safe
@@ -72,6 +136,8 @@ module Gitlab
         end
       end
 
+      # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/324638
+      # ------------------------------------------------------------------------
       def inline_diffs
         @inline_diffs ||= InlineDiff.for_lines(@raw_lines)
       end
diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb
index c5e9bfdc321..209462fd6e9 100644
--- a/lib/gitlab/diff/highlight_cache.rb
+++ b/lib/gitlab/diff/highlight_cache.rb
@@ -71,9 +71,12 @@ module Gitlab
         strong_memoize(:redis_key) do
           [
             'highlighted-diff-files',
-            diffable.cache_key, VERSION,
+            diffable.cache_key,
+            VERSION,
             diff_options,
-            Feature.enabled?(:introduce_marker_ranges, diffable.project, default_enabled: :yaml)
+            Feature.enabled?(:introduce_marker_ranges, diffable.project, default_enabled: :yaml),
+            Feature.enabled?(:use_marker_ranges, diffable.project, default_enabled: :yaml),
+            Feature.enabled?(:diff_line_syntax_highlighting, diffable.project, default_enabled: :yaml)
           ].join(":")
         end
       end
diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb
index dd73e4d6c15..f70618195d0 100644
--- a/lib/gitlab/diff/inline_diff.rb
+++ b/lib/gitlab/diff/inline_diff.rb
@@ -18,6 +18,7 @@ module Gitlab
         CharDiff.new(old_line, new_line).changed_ranges(offset: offset)
       end
 
+      # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/324638
       class << self
         def for_lines(lines)
           pair_selector = Gitlab::Diff::PairSelector.new(lines)
diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb
index 98ed2400d82..6cf414e29cc 100644
--- a/lib/gitlab/diff/line.rb
+++ b/lib/gitlab/diff/line.rb
@@ -8,19 +8,24 @@ module Gitlab
       #
       SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze
 
-      attr_reader :line_code
-      attr_writer :rich_text
-      attr_accessor :text, :index, :type, :old_pos, :new_pos
+      attr_reader :line_code, :marker_ranges
+      attr_writer :text, :rich_text
+      attr_accessor :index, :type, :old_pos, :new_pos
 
       def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil)
-        @text, @type, @index = text, type, index
-        @old_pos, @new_pos = old_pos, new_pos
+        @text = text
+        @type = type
+        @index = index
+        @old_pos = old_pos
+        @new_pos = new_pos
         @parent_file = parent_file
         @rich_text = rich_text
 
         # When line code is not provided from cache store we build it
         # using the parent_file(Diff::File or Conflict::File).
         @line_code = line_code || calculate_line_code
+
+        @marker_ranges = []
       end
 
       def self.init_from_hash(hash)
@@ -48,6 +53,16 @@ module Gitlab
         hash
       end
 
+      def set_marker_ranges(marker_ranges)
+        @marker_ranges = marker_ranges
+      end
+
+      def text(prefix: true)
+        return @text if prefix
+
+        @text&.slice(1..).to_s
+      end
+
       def old_line
         old_pos unless added? || meta?
       end
diff --git a/lib/gitlab/diff/suggestions_parser.rb b/lib/gitlab/diff/suggestions_parser.rb
index 6e17ffaf6ff..f3e6fc455ac 100644
--- a/lib/gitlab/diff/suggestions_parser.rb
+++ b/lib/gitlab/diff/suggestions_parser.rb
@@ -17,7 +17,7 @@ module Gitlab
                                      no_original_data: true,
                                      suggestions_filter_enabled: supports_suggestion)
           doc = Nokogiri::HTML(html)
-          suggestion_nodes = doc.search('pre.suggestion')
+          suggestion_nodes = doc.search('pre.language-suggestion')
 
           return [] if suggestion_nodes.empty?
 
@@ -29,9 +29,8 @@ module Gitlab
             lines_above, lines_below = nil
 
             if lang_param && suggestion_params = fetch_suggestion_params(lang_param)
-              lines_above, lines_below =
-                suggestion_params[:above],
-                suggestion_params[:below]
+              lines_above = suggestion_params[:above]
+              lines_below = suggestion_params[:below]
             end
 
             Gitlab::Diff::Suggestion.new(node.text,
diff --git a/lib/gitlab/downtime_check.rb b/lib/gitlab/downtime_check.rb
deleted file mode 100644
index 457a3c12206..00000000000
--- a/lib/gitlab/downtime_check.rb
+++ /dev/null
@@ -1,73 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  # Checks if a set of migrations requires downtime or not.
-  class DowntimeCheck
-    # The constant containing the boolean that indicates if downtime is needed
-    # or not.
-    DOWNTIME_CONST = :DOWNTIME
-
-    # The constant that specifies the reason for the migration requiring
-    # downtime.
-    DOWNTIME_REASON_CONST = :DOWNTIME_REASON
-
-    # Checks the given migration paths and returns an Array of
-    # `Gitlab::DowntimeCheck::Message` instances.
-    #
-    # migrations - The migration file paths to check.
-    def check(migrations)
-      migrations.map do |path|
-        require(path)
-
-        migration_class = class_for_migration_file(path)
-
-        unless migration_class.const_defined?(DOWNTIME_CONST)
-          raise "The migration in #{path} does not specify if it requires " \
-            "downtime or not"
-        end
-
-        if online?(migration_class)
-          Message.new(path)
-        else
-          reason = downtime_reason(migration_class)
-
-          unless reason
-            raise "The migration in #{path} requires downtime but no reason " \
-              "was given"
-          end
-
-          Message.new(path, true, reason)
-        end
-      end
-    end
-
-    # Checks the given migrations and prints the results to STDOUT/STDERR.
-    #
-    # migrations - The migration file paths to check.
-    def check_and_print(migrations)
-      check(migrations).each do |message|
-        puts message.to_s # rubocop: disable Rails/Output
-      end
-    end
-
-    # Returns the class for the given migration file path.
-    def class_for_migration_file(path)
-      File.basename(path, File.extname(path)).split('_', 2).last.camelize
-        .constantize
-    end
-
-    # Returns true if the given migration can be performed without downtime.
-    def online?(migration)
-      migration.const_get(DOWNTIME_CONST, false) == false
-    end
-
-    # Returns the downtime reason, or nil if none was defined.
-    def downtime_reason(migration)
-      if migration.const_defined?(DOWNTIME_REASON_CONST)
-        migration.const_get(DOWNTIME_REASON_CONST, false)
-      else
-        nil
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/downtime_check/message.rb b/lib/gitlab/downtime_check/message.rb
deleted file mode 100644
index 5debb754943..00000000000
--- a/lib/gitlab/downtime_check/message.rb
+++ /dev/null
@@ -1,41 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  class DowntimeCheck
-    class Message
-      attr_reader :path, :offline
-
-      OFFLINE = "\e[31moffline\e[0m"
-      ONLINE = "\e[32monline\e[0m"
-
-      # path - The file path of the migration.
-      # offline - When set to `true` the migration will require downtime.
-      # reason - The reason as to why the migration requires downtime.
-      def initialize(path, offline = false, reason = nil)
-        @path = path
-        @offline = offline
-        @reason = reason
-      end
-
-      def to_s
-        label = offline ? OFFLINE : ONLINE
-
-        message = ["[#{label}]: #{path}"]
-
-        if reason?
-          message << ":\n\n#{reason}\n\n"
-        end
-
-        message.join
-      end
-
-      def reason?
-        @reason.present?
-      end
-
-      def reason
-        @reason.strip.lines.map(&:strip).join("\n")
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb
index dfed8db8df0..47d361fb95c 100644
--- a/lib/gitlab/error_tracking.rb
+++ b/lib/gitlab/error_tracking.rb
@@ -16,6 +16,12 @@ module Gitlab
       Rack::Timeout::RequestTimeoutException
     ].freeze
 
+    PROCESSORS = [
+      ::Gitlab::ErrorTracking::Processor::SidekiqProcessor,
+      ::Gitlab::ErrorTracking::Processor::GrpcErrorProcessor,
+      ::Gitlab::ErrorTracking::Processor::ContextPayloadProcessor
+    ].freeze
+
     class << self
       def configure
         Raven.configure do |config|
@@ -97,7 +103,9 @@ module Gitlab
         inject_context_for_exception(event, hint[:exception])
         custom_fingerprinting(event, hint[:exception])
 
-        event
+        PROCESSORS.reduce(event) do |processed_event, processor|
+          processor.call(processed_event)
+        end
       end
 
       def process_exception(exception, sentry: false, logging: true, extra:)
diff --git a/lib/gitlab/error_tracking/processor/context_payload_processor.rb b/lib/gitlab/error_tracking/processor/context_payload_processor.rb
index 5185205e94e..758f6aa11d7 100644
--- a/lib/gitlab/error_tracking/processor/context_payload_processor.rb
+++ b/lib/gitlab/error_tracking/processor/context_payload_processor.rb
@@ -9,9 +9,21 @@ module Gitlab
         # integrations are re-implemented and use Gitlab::ErrorTracking, this
         # processor should be removed.
         def process(payload)
+          return payload if ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml)
+
           context_payload = Gitlab::ErrorTracking::ContextPayloadGenerator.generate(nil, {})
           payload.deep_merge!(context_payload)
         end
+
+        def self.call(event)
+          return event unless ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml)
+
+          Gitlab::ErrorTracking::ContextPayloadGenerator.generate(nil, {}).each do |key, value|
+            event.public_send(key).deep_merge!(value) # rubocop:disable GitlabSecurity/PublicSend
+          end
+
+          event
+        end
       end
     end
   end
diff --git a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb
index 871e9c4b7c8..419098dbd09 100644
--- a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb
+++ b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb
@@ -6,60 +6,126 @@ module Gitlab
       class GrpcErrorProcessor < ::Raven::Processor
         DEBUG_ERROR_STRING_REGEX = RE2('(.*) debug_error_string:(.*)')
 
-        def process(value)
-          process_first_exception_value(value)
-          process_custom_fingerprint(value)
+        def process(payload)
+          return payload if ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml)
 
-          value
-        end
-
-        # Sentry can report multiple exceptions in an event. Sanitize
-        # only the first one since that's what is used for grouping.
-        def process_first_exception_value(value)
-          exceptions = value.dig(:exception, :values)
-
-          return unless exceptions.is_a?(Array)
-
-          entry = exceptions.first
-
-          return unless entry.is_a?(Hash)
-
-          exception_type = entry[:type]
-          raw_message = entry[:value]
-
-          return unless exception_type&.start_with?('GRPC::')
-          return unless raw_message.present?
-
-          message, debug_str = split_debug_error_string(raw_message)
-
-          entry[:value] = message if message
-          extra = value[:extra] || {}
-          extra[:grpc_debug_error_string] = debug_str if debug_str
-        end
-
-        def process_custom_fingerprint(value)
-          fingerprint = value[:fingerprint]
-
-          return value unless custom_grpc_fingerprint?(fingerprint)
+          self.class.process_first_exception_value(payload)
+          self.class.process_custom_fingerprint(payload)
 
-          message, _ = split_debug_error_string(fingerprint[1])
-          fingerprint[1] = message if message
+          payload
         end
 
-        private
-
-        def custom_grpc_fingerprint?(fingerprint)
-          fingerprint.is_a?(Array) && fingerprint.length == 2 && fingerprint[0].start_with?('GRPC::')
-        end
-
-        def split_debug_error_string(message)
-          return unless message
-
-          match = DEBUG_ERROR_STRING_REGEX.match(message)
-
-          return unless match
-
-          [match[1], match[2]]
+        class << self
+          def call(event)
+            return event unless ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml)
+
+            process_first_exception_value(event)
+            process_custom_fingerprint(event)
+
+            event
+          end
+
+          # Sentry can report multiple exceptions in an event. Sanitize
+          # only the first one since that's what is used for grouping.
+          def process_first_exception_value(event_or_payload)
+            exceptions = exceptions(event_or_payload)
+
+            return unless exceptions.is_a?(Array)
+
+            exception = exceptions.first
+
+            return unless valid_exception?(exception)
+
+            exception_type, raw_message = type_and_value(exception)
+
+            return unless exception_type&.start_with?('GRPC::')
+            return unless raw_message.present?
+
+            message, debug_str = split_debug_error_string(raw_message)
+
+            set_new_values!(event_or_payload, exception, message, debug_str)
+          end
+
+          def process_custom_fingerprint(event)
+            fingerprint = fingerprint(event)
+
+            return event unless custom_grpc_fingerprint?(fingerprint)
+
+            message, _ = split_debug_error_string(fingerprint[1])
+            fingerprint[1] = message if message
+          end
+
+          private
+
+          def custom_grpc_fingerprint?(fingerprint)
+            fingerprint.is_a?(Array) && fingerprint.length == 2 && fingerprint[0].start_with?('GRPC::')
+          end
+
+          def split_debug_error_string(message)
+            return unless message
+
+            match = DEBUG_ERROR_STRING_REGEX.match(message)
+
+            return unless match
+
+            [match[1], match[2]]
+          end
+
+          # The below methods can be removed once we remove the
+          # sentry_processors_before_send feature flag, and we can
+          # assume we always have an Event object
+          def exceptions(event_or_payload)
+            case event_or_payload
+            when Raven::Event
+              # Better in new version, will be event_or_payload.exception.values
+              event_or_payload.instance_variable_get(:@interfaces)[:exception]&.values
+            when Hash
+              event_or_payload.dig(:exception, :values)
+            end
+          end
+
+          def valid_exception?(exception)
+            case exception
+            when Raven::SingleExceptionInterface
+              exception&.value
+            when Hash
+              true
+            else
+              false
+            end
+          end
+
+          def type_and_value(exception)
+            case exception
+            when Raven::SingleExceptionInterface
+              [exception.type, exception.value]
+            when Hash
+              exception.values_at(:type, :value)
+            end
+          end
+
+          def set_new_values!(event_or_payload, exception, message, debug_str)
+            case event_or_payload
+            when Raven::Event
+              # Worse in new version, no setter! Have to poke at the
+              # instance variable
+              exception.value = message if message
+              event_or_payload.extra[:grpc_debug_error_string] = debug_str if debug_str
+            when Hash
+              exception[:value] = message if message
+              extra = event_or_payload[:extra] || {}
+              extra[:grpc_debug_error_string] = debug_str if debug_str
+            end
+          end
+
+          def fingerprint(event_or_payload)
+            case event_or_payload
+            when Raven::Event
+              event_or_payload.fingerprint
+            when Hash
+              event_or_payload[:fingerprint]
+            end
+          end
         end
       end
     end
diff --git a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb
index 272cb689ad5..93310745ece 100644
--- a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb
+++ b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb
@@ -8,39 +8,66 @@ module Gitlab
       class SidekiqProcessor < ::Raven::Processor
         FILTERED_STRING = '[FILTERED]'
 
-        def self.filter_arguments(args, klass)
-          args.lazy.with_index.map do |arg, i|
-            case arg
-            when Numeric
-              arg
-            else
-              if permitted_arguments_for_worker(klass).include?(i)
+        class << self
+          def filter_arguments(args, klass)
+            args.lazy.with_index.map do |arg, i|
+              case arg
+              when Numeric
                 arg
               else
-                FILTERED_STRING
+                if permitted_arguments_for_worker(klass).include?(i)
+                  arg
+                else
+                  FILTERED_STRING
+                end
               end
             end
           end
-        end
 
-        def self.permitted_arguments_for_worker(klass)
-          @permitted_arguments_for_worker ||= {}
-          @permitted_arguments_for_worker[klass] ||=
-            begin
-              klass.constantize&.loggable_arguments&.to_set
-            rescue
-              Set.new
+          def permitted_arguments_for_worker(klass)
+            @permitted_arguments_for_worker ||= {}
+            @permitted_arguments_for_worker[klass] ||=
+              begin
+                klass.constantize&.loggable_arguments&.to_set
+              rescue
+                Set.new
+              end
+          end
+
+          def loggable_arguments(args, klass)
+            Gitlab::Utils::LogLimitedArray
+              .log_limited_array(filter_arguments(args, klass))
+              .map(&:to_s)
+              .to_a
+          end
+
+          def call(event)
+            return event unless ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml)
+
+            sidekiq = event&.extra&.dig(:sidekiq)
+
+            return event unless sidekiq
+
+            sidekiq = sidekiq.deep_dup
+            sidekiq.delete(:jobstr)
+
+            # 'args' in this hash => from Gitlab::ErrorTracking.track_*
+            # 'args' in :job => from default error handler
+            job_holder = sidekiq.key?('args') ? sidekiq : sidekiq[:job]
+
+            if job_holder['args']
+              job_holder['args'] = filter_arguments(job_holder['args'], job_holder['class']).to_a
             end
-        end
 
-        def self.loggable_arguments(args, klass)
-          Gitlab::Utils::LogLimitedArray
-            .log_limited_array(filter_arguments(args, klass))
-            .map(&:to_s)
-            .to_a
+            event.extra[:sidekiq] = sidekiq
+
+            event
+          end
         end
 
         def process(value, key = nil)
+          return value if ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml)
+
           sidekiq = value.dig(:extra, :sidekiq)
 
           return value unless sidekiq
diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb
index b602393b59e..ef0236f8275 100644
--- a/lib/gitlab/exclusive_lease.rb
+++ b/lib/gitlab/exclusive_lease.rb
@@ -15,14 +15,14 @@ module Gitlab
     PREFIX = 'gitlab:exclusive_lease'
     NoKey = Class.new(ArgumentError)
 
-    LUA_CANCEL_SCRIPT = <<~EOS.freeze
+    LUA_CANCEL_SCRIPT = <<~EOS
       local key, uuid = KEYS[1], ARGV[1]
       if redis.call("get", key) == uuid then
         redis.call("del", key)
       end
     EOS
 
-    LUA_RENEW_SCRIPT = <<~EOS.freeze
+    LUA_RENEW_SCRIPT = <<~EOS
       local key, uuid, ttl = KEYS[1], ARGV[1], ARGV[2]
       if redis.call("get", key) == uuid then
         redis.call("expire", key, ttl)
diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb
index 1bb29ba3eac..145bb6d7b8f 100644
--- a/lib/gitlab/experimentation.rb
+++ b/lib/gitlab/experimentation.rb
@@ -34,10 +34,6 @@
 module Gitlab
   module Experimentation
     EXPERIMENTS = {
-      upgrade_link_in_user_menu_a: {
-        tracking_category: 'Growth::Expansion::Experiment::UpgradeLinkInUserMenuA',
-        use_backwards_compatible_subject_index: true
-      },
       invite_members_version_b: {
         tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionB',
         use_backwards_compatible_subject_index: true
diff --git a/lib/gitlab/external_authorization/access.rb b/lib/gitlab/external_authorization/access.rb
index e111c41fcc2..21fa728fd3a 100644
--- a/lib/gitlab/external_authorization/access.rb
+++ b/lib/gitlab/external_authorization/access.rb
@@ -10,7 +10,8 @@ module Gitlab
                   :load_type
 
       def initialize(user, label)
-        @user, @label = user, label
+        @user = user
+        @label = label
       end
 
       def loaded?
diff --git a/lib/gitlab/external_authorization/cache.rb b/lib/gitlab/external_authorization/cache.rb
index acdc028b4dc..509daeb0248 100644
--- a/lib/gitlab/external_authorization/cache.rb
+++ b/lib/gitlab/external_authorization/cache.rb
@@ -6,7 +6,8 @@ module Gitlab
       VALIDITY_TIME = 6.hours
 
       def initialize(user, label)
-        @user, @label = user, label
+        @user = user
+        @label = label
       end
 
       def load
diff --git a/lib/gitlab/external_authorization/client.rb b/lib/gitlab/external_authorization/client.rb
index fc859304eab..582051010d3 100644
--- a/lib/gitlab/external_authorization/client.rb
+++ b/lib/gitlab/external_authorization/client.rb
@@ -13,7 +13,8 @@ module Gitlab
       }.freeze
 
       def initialize(user, label)
-        @user, @label = user, label
+        @user = user
+        @label = label
       end
 
       def request_access
@@ -51,18 +52,18 @@ module Gitlab
 
       def body
         @body ||= begin
-                    body = {
-                      user_identifier: @user.email,
-                      project_classification_label: @label,
-                      identities: @user.identities.map { |identity| { provider: identity.provider, extern_uid: identity.extern_uid } }
-                    }
+          body = {
+            user_identifier: @user.email,
+            project_classification_label: @label,
+            identities: @user.identities.map { |identity| { provider: identity.provider, extern_uid: identity.extern_uid } }
+          }
 
-                    if @user.ldap_identity
-                      body[:user_ldap_dn] = @user.ldap_identity.extern_uid
-                    end
+          if @user.ldap_identity
+            body[:user_ldap_dn] = @user.ldap_identity.extern_uid
+          end
 
-                    body
-                  end
+          body
+        end
       end
     end
   end
diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb
index bd5d2e53180..612865ed1be 100644
--- a/lib/gitlab/fogbugz_import/importer.rb
+++ b/lib/gitlab/fogbugz_import/importer.rb
@@ -199,8 +199,7 @@ module Gitlab
 
       def linkify_issues(str)
         str = str.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2')
-        str = str.gsub(/([Cc]ase) ([0-9]+)/, '\1 #\2')
-        str
+        str.gsub(/([Cc]ase) ([0-9]+)/, '\1 #\2')
       end
 
       def escape_for_markdown(str)
@@ -208,8 +207,7 @@ module Gitlab
         str = str.gsub(/^-/, "\\-")
         str = str.gsub("`", "\\~")
         str = str.delete("\r")
-        str = str.gsub("\n", "  \n")
-        str
+        str.gsub("\n", "  \n")
       end
 
       def format_content(raw_content)
diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb
index 9e24306c05e..a5b1b7d914b 100644
--- a/lib/gitlab/git/blame.rb
+++ b/lib/gitlab/git/blame.rb
@@ -30,8 +30,10 @@ module Gitlab
       end
 
       def process_raw_blame(output)
-        lines, final = [], []
-        info, commits = {}, {}
+        lines = []
+        final = []
+        info = {}
+        commits = {}
 
         # process the output
         output.split("\n").each do |line|
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index ff99803d8de..51baed32935 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -390,7 +390,7 @@ module Gitlab
         @committer_name = commit.committer.name.dup
         @committer_email = commit.committer.email.dup
         @parent_ids = Array(commit.parent_ids)
-        @trailers = Hash[commit.trailers.map { |t| [t.key, t.value] }]
+        @trailers = commit.trailers.to_h { |t| [t.key, t.value] }
       end
 
       # Gitaly provides a UNIX timestamp in author.date.seconds, and a timezone
diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb
index 19462e6cb02..fb947c80b7e 100644
--- a/lib/gitlab/git/diff_collection.rb
+++ b/lib/gitlab/git/diff_collection.rb
@@ -82,6 +82,30 @@ module Gitlab
         !!@overflow
       end
 
+      def overflow_max_lines?
+        !!@overflow_max_lines
+      end
+
+      def overflow_max_bytes?
+        !!@overflow_max_bytes
+      end
+
+      def overflow_max_files?
+        !!@overflow_max_files
+      end
+
+      def collapsed_safe_lines?
+        !!@collapsed_safe_lines
+      end
+
+      def collapsed_safe_files?
+        !!@collapsed_safe_files
+      end
+
+      def collapsed_safe_bytes?
+        !!@collapsed_safe_bytes
+      end
+
       def size
         @size ||= count # forces a loop using each method
       end
@@ -103,10 +127,9 @@ module Gitlab
       end
 
       def decorate!
-        collection = each_with_index do |element, i|
+        each_with_index do |element, i|
           @array[i] = yield(element)
         end
-        collection
       end
 
       alias_method :to_ary, :to_a
@@ -121,7 +144,15 @@ module Gitlab
       end
 
       def over_safe_limits?(files)
-        files >= safe_max_files || @line_count > safe_max_lines || @byte_count >= safe_max_bytes
+        if files >= safe_max_files
+          @collapsed_safe_files = true
+        elsif @line_count > safe_max_lines
+          @collapsed_safe_lines = true
+        elsif @byte_count >= safe_max_bytes
+          @collapsed_safe_bytes = true
+        end
+
+        @collapsed_safe_files || @collapsed_safe_lines || @collapsed_safe_bytes
       end
 
       def expand_diff?
@@ -154,6 +185,7 @@ module Gitlab
 
           if @enforce_limits && i >= max_files
             @overflow = true
+            @overflow_max_files = true
             break
           end
 
@@ -166,10 +198,19 @@ module Gitlab
           @line_count += diff.line_count
           @byte_count += diff.diff.bytesize
 
-          if @enforce_limits && (@line_count >= max_lines || @byte_count >= max_bytes)
+          if @enforce_limits && @line_count >= max_lines
+            # This last Diff instance pushes us over the lines limit. We stop and
+            # discard it.
+            @overflow = true
+            @overflow_max_lines = true
+            break
+          end
+
+          if @enforce_limits && @byte_count >= max_bytes
             # This last Diff instance pushes us over the lines limit. We stop and
             # discard it.
             @overflow = true
+            @overflow_max_bytes = true
             break
           end
 
diff --git a/lib/gitlab/git/merge_base.rb b/lib/gitlab/git/merge_base.rb
index b27f7038c26..905d72cadbf 100644
--- a/lib/gitlab/git/merge_base.rb
+++ b/lib/gitlab/git/merge_base.rb
@@ -6,7 +6,8 @@ module Gitlab
       include Gitlab::Utils::StrongMemoize
 
       def initialize(repository, refs)
-        @repository, @refs = repository, refs
+        @repository = repository
+        @refs = refs
       end
 
       # Returns the SHA of the first common ancestor
diff --git a/lib/gitlab/git/patches/commit_patches.rb b/lib/gitlab/git/patches/commit_patches.rb
index c62994432d3..1182db10c34 100644
--- a/lib/gitlab/git/patches/commit_patches.rb
+++ b/lib/gitlab/git/patches/commit_patches.rb
@@ -7,7 +7,10 @@ module Gitlab
         include Gitlab::Git::WrapsGitalyErrors
 
         def initialize(user, repository, branch, patch_collection)
-          @user, @repository, @branch, @patches = user, repository, branch, patch_collection
+          @user = user
+          @repository = repository
+          @branch = branch
+          @patches = patch_collection
         end
 
         def commit
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index e316d52ac05..3361cee733b 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -599,9 +599,9 @@ module Gitlab
         tags.find { |tag| tag.name == name }
       end
 
-      def merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref, allow_conflicts)
+      def merge_to_ref(user, **kwargs)
         wrapped_gitaly_errors do
-          gitaly_operation_client.user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref, allow_conflicts)
+          gitaly_operation_client.user_merge_to_ref(user, **kwargs)
         end
       end
 
@@ -1017,6 +1017,10 @@ module Gitlab
         gitaly_repository_client.search_files_by_name(ref, safe_query)
       end
 
+      def search_files_by_regexp(filter, ref = 'HEAD')
+        gitaly_repository_client.search_files_by_regexp(ref, filter)
+      end
+
       def find_commits_by_message(query, ref, path, limit, offset)
         wrapped_gitaly_errors do
           gitaly_commit_client
diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb
index da86d6baf4a..568e894a02f 100644
--- a/lib/gitlab/git/tag.rb
+++ b/lib/gitlab/git/tag.rb
@@ -87,6 +87,10 @@ module Gitlab
         end
       end
 
+      def cache_key
+        "tag:" + Digest::SHA1.hexdigest([name, message, target, target_commit&.sha].join)
+      end
+
       private
 
       def message_from_gitaly_tag
diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb
index 55ff3c6caf1..75d6b949874 100644
--- a/lib/gitlab/git/wiki.rb
+++ b/lib/gitlab/git/wiki.rb
@@ -102,12 +102,6 @@ module Gitlab
         end
       end
 
-      def file(name, version)
-        wrapped_gitaly_errors do
-          gitaly_find_file(name, version)
-        end
-      end
-
       # options:
       #  :page     - The Integer page number.
       #  :per_page - The number of items per page.
@@ -161,13 +155,6 @@ module Gitlab
         nil
       end
 
-      def gitaly_find_file(name, version)
-        wiki_file = gitaly_wiki_client.find_file(name, version)
-        return unless wiki_file
-
-        Gitlab::Git::WikiFile.new(wiki_file)
-      end
-
       def gitaly_list_pages(limit: 0, sort: nil, direction_desc: false, load_content: false)
         params = { limit: limit, sort: sort, direction_desc: direction_desc }
 
diff --git a/lib/gitlab/git/wiki_file.rb b/lib/gitlab/git/wiki_file.rb
index 7f09173f05c..c56a17c52f3 100644
--- a/lib/gitlab/git/wiki_file.rb
+++ b/lib/gitlab/git/wiki_file.rb
@@ -5,25 +5,11 @@ module Gitlab
     class WikiFile
       attr_reader :mime_type, :raw_data, :name, :path
 
-      # This class wraps Gitlab::GitalyClient::WikiFile
-      def initialize(gitaly_file)
-        @mime_type = gitaly_file.mime_type
-        @raw_data = gitaly_file.raw_data
-        @name = gitaly_file.name
-        @path = gitaly_file.path
-      end
-
-      def self.from_blob(blob)
-        hash = {
-          name: File.basename(blob.name),
-          mime_type: blob.mime_type,
-          path: blob.path,
-          raw_data: blob.data
-        }
-
-        gitaly_file = Gitlab::GitalyClient::WikiFile.new(hash)
-
-        Gitlab::Git::WikiFile.new(gitaly_file)
+      def initialize(blob)
+        @mime_type = blob.mime_type
+        @raw_data = blob.data
+        @name = File.basename(blob.name)
+        @path = blob.path
       end
     end
   end
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index c5ca46827cb..31e4755192e 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -91,6 +91,7 @@ module Gitlab
       when *PUSH_COMMANDS
         check_push_access!
       end
+      check_additional_conditions!
 
       success_result
     end
@@ -530,6 +531,10 @@ module Gitlab
     def size_checker
       container.repository_size_checker
     end
+
+    # overriden in EE
+    def check_additional_conditions!
+    end
   end
 end
 
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index e3788814dd5..f4a89edecd1 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -215,7 +215,7 @@ module Gitlab
         'client_name' => CLIENT_NAME
       }
 
-      context_data = Labkit::Context.current&.to_h
+      context_data = Gitlab::ApplicationContext.current
 
       feature_stack = Thread.current[:gitaly_feature_stack]
       feature = feature_stack && feature_stack[0]
diff --git a/lib/gitlab/gitaly_client/attributes_bag.rb b/lib/gitlab/gitaly_client/attributes_bag.rb
index f935281ac2e..74e6279708e 100644
--- a/lib/gitlab/gitaly_client/attributes_bag.rb
+++ b/lib/gitlab/gitaly_client/attributes_bag.rb
@@ -3,7 +3,7 @@
 module Gitlab
   module GitalyClient
     # This module expects an `ATTRS` const to be defined on the subclass
-    # See GitalyClient::WikiFile for an example
+    # See GitalyClient::WikiPage for an example
     module AttributesBag
       extend ActiveSupport::Concern
 
diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb
index c66b3335d89..19a473e4785 100644
--- a/lib/gitlab/gitaly_client/blob_service.rb
+++ b/lib/gitlab/gitaly_client/blob_service.rb
@@ -78,17 +78,7 @@ module Gitlab
       end
 
       def get_new_lfs_pointers(revision, limit, not_in, dynamic_timeout = nil)
-        request = Gitaly::GetNewLFSPointersRequest.new(
-          repository: @gitaly_repo,
-          revision: encode_binary(revision),
-          limit: limit || 0
-        )
-
-        if not_in.nil? || not_in == :all
-          request.not_in_all = true
-        else
-          request.not_in_refs += not_in
-        end
+        request, rpc = create_new_lfs_pointers_request(revision, limit, not_in)
 
         timeout =
           if dynamic_timeout
@@ -100,7 +90,7 @@ module Gitlab
         response = GitalyClient.call(
           @gitaly_repo.storage_name,
           :blob_service,
-          :get_new_lfs_pointers,
+          rpc,
           request,
           timeout: timeout
         )
@@ -108,16 +98,51 @@ module Gitlab
       end
 
       def get_all_lfs_pointers
-        request = Gitaly::GetAllLFSPointersRequest.new(
-          repository: @gitaly_repo
+        request = Gitaly::ListLFSPointersRequest.new(
+          repository: @gitaly_repo,
+          revisions: [encode_binary("--all")]
         )
 
-        response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_all_lfs_pointers, request, timeout: GitalyClient.medium_timeout)
+        response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :list_lfs_pointers, request, timeout: GitalyClient.medium_timeout)
         map_lfs_pointers(response)
       end
 
       private
 
+      def create_new_lfs_pointers_request(revision, limit, not_in)
+        # If the check happens for a change which is using a quarantine
+        # environment for incoming objects, then we can avoid doing the
+        # necessary graph walk to detect only new LFS pointers and instead scan
+        # through all quarantined objects.
+        git_env = ::Gitlab::Git::HookEnv.all(@gitaly_repo.gl_repository)
+        if Feature.enabled?(:lfs_integrity_inspect_quarantined_objects, @project, default_enabled: :yaml) && git_env['GIT_OBJECT_DIRECTORY_RELATIVE'].present?
+          repository = @gitaly_repo.dup
+          repository.git_alternate_object_directories = Google::Protobuf::RepeatedField.new(:string)
+
+          request = Gitaly::ListAllLFSPointersRequest.new(
+            repository: repository,
+            limit: limit || 0
+          )
+
+          [request, :list_all_lfs_pointers]
+        else
+          revisions = [revision]
+          revisions += if not_in.nil? || not_in == :all
+                         ["--not", "--all"]
+                       else
+                         not_in.prepend "--not"
+                       end
+
+          request = Gitaly::ListLFSPointersRequest.new(
+            repository: @gitaly_repo,
+            limit: limit || 0,
+            revisions: revisions.map { |rev| encode_binary(rev) }
+          )
+
+          [request, :list_lfs_pointers]
+        end
+      end
+
       def consume_blob_response(response)
         data = []
         blob = nil
diff --git a/lib/gitlab/gitaly_client/call.rb b/lib/gitlab/gitaly_client/call.rb
index 9d4d86997ad..4bb184bee2f 100644
--- a/lib/gitlab/gitaly_client/call.rb
+++ b/lib/gitlab/gitaly_client/call.rb
@@ -50,11 +50,11 @@ module Gitlab
       end
 
       def recording_request
-        start = Gitlab::Metrics::System.monotonic_time
+        @start = Gitlab::Metrics::System.monotonic_time
 
         yield
       ensure
-        @duration += Gitlab::Metrics::System.monotonic_time - start
+        @duration += Gitlab::Metrics::System.monotonic_time - @start
       end
 
       def store_timings
@@ -64,8 +64,14 @@ module Gitlab
 
         request_hash = @request.is_a?(Google::Protobuf::MessageExts) ? @request.to_h : {}
 
-        GitalyClient.add_call_details(feature: "#{@service}##{@rpc}", duration: @duration, request: request_hash, rpc: @rpc,
-                                      backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller))
+        GitalyClient.add_call_details(
+          start: @start,
+          feature: "#{@service}##{@rpc}",
+          duration: @duration,
+          request: request_hash,
+          rpc: @rpc,
+          backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller)
+        )
       end
     end
   end
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index ef5221a8042..3d24b4d53a4 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -107,6 +107,8 @@ module Gitlab
         entry.data = data.join
 
         entry unless entry.oid.blank?
+      rescue GRPC::NotFound
+        nil
       end
 
       def tree_entries(repository, revision, path, recursive)
diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb
index 6f302b2c4e7..5ce1b1f0c87 100644
--- a/lib/gitlab/gitaly_client/operation_service.rb
+++ b/lib/gitlab/gitaly_client/operation_service.rb
@@ -103,7 +103,7 @@ module Gitlab
         end
       end
 
-      def user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref, allow_conflicts)
+      def user_merge_to_ref(user, source_sha:, branch:, target_ref:, message:, first_parent_ref:, allow_conflicts: false)
         request = Gitaly::UserMergeToRefRequest.new(
           repository: @gitaly_repo,
           source_sha: source_sha,
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index bd450249355..a93f4071efc 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -339,6 +339,11 @@ module Gitlab
         search_results_from_response(response, options)
       end
 
+      def search_files_by_regexp(ref, filter)
+        request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: '.', filter: filter)
+        GitalyClient.call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files)
+      end
+
       def disconnect_alternates
         request = Gitaly::DisconnectGitAlternatesRequest.new(
           repository: @gitaly_repo
diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb
index 7edd42f9ef7..dd9e3d5d28b 100644
--- a/lib/gitlab/gitaly_client/storage_settings.rb
+++ b/lib/gitlab/gitaly_client/storage_settings.rb
@@ -11,7 +11,7 @@ module Gitlab
       DirectPathAccessError = Class.new(StandardError)
       InvalidConfigurationError = Class.new(StandardError)
 
-      INVALID_STORAGE_MESSAGE = <<~MSG.freeze
+      INVALID_STORAGE_MESSAGE = <<~MSG
         Storage is invalid because it has no `path` key.
 
         For source installations, update your config/gitlab.yml Refer to gitlab.yml.example for an updated example.
diff --git a/lib/gitlab/gitaly_client/wiki_file.rb b/lib/gitlab/gitaly_client/wiki_file.rb
deleted file mode 100644
index ef2b23732d1..00000000000
--- a/lib/gitlab/gitaly_client/wiki_file.rb
+++ /dev/null
@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module GitalyClient
-    class WikiFile
-      ATTRS = %i(name mime_type path raw_data).freeze
-
-      include AttributesBag
-    end
-  end
-end
diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb
index 9034edb6263..fecc2b7023d 100644
--- a/lib/gitlab/gitaly_client/wiki_service.rb
+++ b/lib/gitlab/gitaly_client/wiki_service.rb
@@ -153,32 +153,6 @@ module Gitlab
         versions
       end
 
-      def find_file(name, revision)
-        request = Gitaly::WikiFindFileRequest.new(
-          repository: @gitaly_repo,
-          name: encode_binary(name),
-          revision: encode_binary(revision)
-        )
-
-        response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_file, request, timeout: GitalyClient.fast_timeout)
-        wiki_file = nil
-
-        response.each do |message|
-          next unless message.name.present? || wiki_file
-
-          if wiki_file
-            wiki_file.raw_data = "#{wiki_file.raw_data}#{message.raw_data}"
-          else
-            wiki_file = GitalyClient::WikiFile.new(message.to_h)
-            # All gRPC strings in a response are frozen, so we get
-            # an unfrozen version here so appending in the else clause below doesn't blow up.
-            wiki_file.raw_data = wiki_file.raw_data.dup
-          end
-        end
-
-        wiki_file
-      end
-
       private
 
       # If a block is given and the yielded value is truthy, iteration will be
diff --git a/lib/gitlab/golang.rb b/lib/gitlab/golang.rb
index f2dc668c482..31b7a198b92 100644
--- a/lib/gitlab/golang.rb
+++ b/lib/gitlab/golang.rb
@@ -2,10 +2,12 @@
 
 module Gitlab
   module Golang
+    PseudoVersion = Struct.new(:semver, :timestamp, :commit_id)
+
     extend self
 
     def local_module_prefix
-      @gitlab_prefix ||= "#{Settings.build_gitlab_go_url}/".freeze
+      @gitlab_prefix ||= "#{Settings.build_gitlab_go_url}/"
     end
 
     def semver_tag?(tag)
@@ -37,11 +39,11 @@ module Gitlab
       end
 
       # This pattern is intentionally more forgiving than the patterns
-      # above. Correctness is verified by #pseudo_version_commit.
+      # above. Correctness is verified by #validate_pseudo_version.
       /\A\d{14}-\h+\z/.freeze.match? pre
     end
 
-    def pseudo_version_commit(project, semver)
+    def parse_pseudo_version(semver)
       # Per Go's implementation of pseudo-versions, a tag should be
       # considered a pseudo-version if it matches one of the patterns
       # listed in #pseudo_version?, regardless of the content of the
@@ -55,9 +57,14 @@ module Gitlab
       # - [Pseudo-version request processing](https://github.com/golang/go/blob/master/src/cmd/go/internal/modfetch/coderepo.go)
 
       # Go ignores anything before '.' or after the second '-', so we will do the same
-      timestamp, sha = semver.prerelease.split('-').last 2
+      timestamp, commit_id = semver.prerelease.split('-').last 2
       timestamp = timestamp.split('.').last
-      commit = project.repository.commit_by(oid: sha)
+
+      PseudoVersion.new(semver, timestamp, commit_id)
+    end
+
+    def validate_pseudo_version(project, version, commit = nil)
+      commit ||= project.repository.commit_by(oid: version.commit_id)
 
       # Error messages are based on the responses of proxy.golang.org
 
@@ -65,10 +72,10 @@ module Gitlab
       raise ArgumentError.new 'invalid pseudo-version: unknown commit' unless commit
 
       # Require the SHA fragment to be 12 characters long
-      raise ArgumentError.new 'invalid pseudo-version: revision is shorter than canonical' unless sha.length == 12
+      raise ArgumentError.new 'invalid pseudo-version: revision is shorter than canonical' unless version.commit_id.length == 12
 
       # Require the timestamp to match that of the commit
-      raise ArgumentError.new 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == timestamp
+      raise ArgumentError.new 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == version.timestamp
 
       commit
     end
@@ -77,6 +84,14 @@ module Gitlab
       Packages::SemVer.parse(str, prefixed: true)
     end
 
+    def go_path(project, path = nil)
+      if path.blank?
+        "#{local_module_prefix}/#{project.full_path}"
+      else
+        "#{local_module_prefix}/#{project.full_path}/#{path}"
+      end
+    end
+
     def pkg_go_dev_url(name, version = nil)
       if version
         "https://pkg.go.dev/#{name}@#{version}"
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index c7e215c143f..08c17058fcb 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -45,7 +45,7 @@ module Gitlab
       # Initialize gon.features with any flags that should be
       # made globally available to the frontend
       push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false)
-      push_frontend_feature_flag(:usage_data_api, default_enabled: true)
+      push_frontend_feature_flag(:usage_data_api, type: :ops, default_enabled: :yaml)
       push_frontend_feature_flag(:security_auto_fix, default_enabled: false)
     end
 
diff --git a/lib/gitlab/grape_logging/loggers/context_logger.rb b/lib/gitlab/grape_logging/loggers/context_logger.rb
index 0a8f0872fbe..468a296886e 100644
--- a/lib/gitlab/grape_logging/loggers/context_logger.rb
+++ b/lib/gitlab/grape_logging/loggers/context_logger.rb
@@ -6,7 +6,7 @@ module Gitlab
     module Loggers
       class ContextLogger < ::GrapeLogging::Loggers::Base
         def parameters(_, _)
-          Labkit::Context.current.to_h
+          Gitlab::ApplicationContext.current
         end
       end
     end
diff --git a/lib/gitlab/graphql/authorize.rb b/lib/gitlab/graphql/authorize.rb
deleted file mode 100644
index e83b567308b..00000000000
--- a/lib/gitlab/graphql/authorize.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Graphql
-    # Allow fields to declare permissions their objects must have. The field
-    # will be set to nil unless all required permissions are present.
-    module Authorize
-      extend ActiveSupport::Concern
-
-      def self.use(schema_definition)
-        schema_definition.instrument(:field, Gitlab::Graphql::Authorize::Instrumentation.new, after_built_ins: true)
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/graphql/authorize/authorize_field_service.rb b/lib/gitlab/graphql/authorize/authorize_field_service.rb
deleted file mode 100644
index e8db619f88a..00000000000
--- a/lib/gitlab/graphql/authorize/authorize_field_service.rb
+++ /dev/null
@@ -1,147 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Graphql
-    module Authorize
-      class AuthorizeFieldService
-        def initialize(field)
-          @field = field
-          @old_resolve_proc = @field.resolve_proc
-        end
-
-        def authorizations?
-          authorizations.present?
-        end
-
-        def authorized_resolve
-          proc do |parent_typed_object, args, ctx|
-            resolved_type = @old_resolve_proc.call(parent_typed_object, args, ctx)
-            authorizing_object = authorize_against(parent_typed_object, resolved_type)
-
-            filter_allowed(ctx[:current_user], resolved_type, authorizing_object)
-          end
-        end
-
-        private
-
-        def authorizations
-          @authorizations ||= (type_authorizations + field_authorizations).uniq
-        end
-
-        # Returns any authorize metadata from the return type of @field
-        def type_authorizations
-          type = @field.type
-
-          # When the return type of @field is a collection, find the singular type
-          if @field.connection?
-            type = node_type_for_relay_connection(type)
-          elsif type.list?
-            type = node_type_for_basic_connection(type)
-          end
-
-          type = type.unwrap if type.kind.non_null?
-
-          Array.wrap(type.metadata[:authorize])
-        end
-
-        # Returns any authorize metadata from @field
-        def field_authorizations
-          return [] if @field.metadata[:authorize] == true
-
-          Array.wrap(@field.metadata[:authorize])
-        end
-
-        def authorize_against(parent_typed_object, resolved_type)
-          if scalar_type?
-            # The field is a built-in/scalar type, or a list of scalars
-            # authorize using the parent's object
-            parent_typed_object.object
-          elsif @field.connection? || @field.type.list? || resolved_type.is_a?(Array)
-            # The field is a connection or a list of non-built-in types, we'll
-            # authorize each element when rendering
-            nil
-          elsif resolved_type.respond_to?(:object)
-            # The field is a type representing a single object, we'll authorize
-            # against the object directly
-            resolved_type.object
-          else
-            # Resolved type is a single object that might not be loaded yet by
-            # the batchloader, we'll authorize that
-            resolved_type
-          end
-        end
-
-        def filter_allowed(current_user, resolved_type, authorizing_object)
-          if resolved_type.nil?
-            # We're not rendering anything, for example when a record was not found
-            # no need to do anything
-          elsif authorizing_object
-            # Authorizing fields representing scalars, or a simple field with an object
-            ::Gitlab::Graphql::Lazy.with_value(authorizing_object) do |object|
-              resolved_type if allowed_access?(current_user, object)
-            end
-          elsif @field.connection?
-            ::Gitlab::Graphql::Lazy.with_value(resolved_type) do |type|
-              # A connection with pagination, modify the visible nodes on the
-              # connection type in place
-              nodes = to_nodes(type)
-              nodes.keep_if { |node| allowed_access?(current_user, node) } if nodes
-              type
-            end
-          elsif @field.type.list? || resolved_type.is_a?(Array)
-            # A simple list of rendered types  each object being an object to authorize
-            ::Gitlab::Graphql::Lazy.with_value(resolved_type) do |items|
-              items.select do |single_object_type|
-                object_type = realized(single_object_type)
-                object = object_type.try(:object) || object_type
-                allowed_access?(current_user, object)
-              end
-            end
-          else
-            raise "Can't authorize #{@field}"
-          end
-        end
-
-        # Ensure that we are dealing with realized objects, not delayed promises
-        def realized(thing)
-          ::Gitlab::Graphql::Lazy.force(thing)
-        end
-
-        # Try to get the connection
-        # can be at type.object or at type
-        def to_nodes(type)
-          if type.respond_to?(:nodes)
-            type.nodes
-          elsif type.respond_to?(:object)
-            to_nodes(type.object)
-          else
-            nil
-          end
-        end
-
-        def allowed_access?(current_user, object)
-          object = realized(object)
-
-          authorizations.all? do |ability|
-            Ability.allowed?(current_user, ability, object)
-          end
-        end
-
-        # Returns the singular type for relay connections.
-        # This will be the type class of edges.node
-        def node_type_for_relay_connection(type)
-          type.unwrap.get_field('edges').type.unwrap.get_field('node').type
-        end
-
-        # Returns the singular type for basic connections, for example `[Types::ProjectType]`
-        def node_type_for_basic_connection(type)
-          type.unwrap
-        end
-
-        def scalar_type?
-          node_type_for_basic_connection(@field.type).kind.scalar?
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb
index 6ee446011d4..4d575b964e5 100644
--- a/lib/gitlab/graphql/authorize/authorize_resource.rb
+++ b/lib/gitlab/graphql/authorize/authorize_resource.rb
@@ -5,15 +5,17 @@ module Gitlab
     module Authorize
       module AuthorizeResource
         extend ActiveSupport::Concern
+        ConfigurationError = Class.new(StandardError)
 
-        RESOURCE_ACCESS_ERROR = "The resource that you are attempting to access does not exist or you don't have permission to perform this action"
+        RESOURCE_ACCESS_ERROR = "The resource that you are attempting to access does " \
+          "not exist or you don't have permission to perform this action"
 
         class_methods do
           def required_permissions
             # If the `#authorize` call is used on multiple classes, we add the
             # permissions specified on a subclass, to the ones that were specified
-            # on it's superclass.
-            @required_permissions ||= if self.respond_to?(:superclass) && superclass.respond_to?(:required_permissions)
+            # on its superclass.
+            @required_permissions ||= if respond_to?(:superclass) && superclass.respond_to?(:required_permissions)
                                         superclass.required_permissions.dup
                                       else
                                         []
@@ -23,6 +25,18 @@ module Gitlab
           def authorize(*permissions)
             required_permissions.concat(permissions)
           end
+
+          def authorizes_object?
+            defined?(@authorizes_object) ? @authorizes_object : false
+          end
+
+          def authorizes_object!
+            @authorizes_object = true
+          end
+
+          def raise_resource_not_available_error!(msg = RESOURCE_ACCESS_ERROR)
+            raise ::Gitlab::Graphql::Errors::ResourceNotAvailable, msg
+          end
         end
 
         def find_object(*args)
@@ -37,33 +51,21 @@ module Gitlab
           object
         end
 
+        # authorizes the object using the current class authorization.
         def authorize!(object)
-          unless authorized_resource?(object)
-            raise_resource_not_available_error!
-          end
+          raise_resource_not_available_error! unless authorized_resource?(object)
         end
 
-        # this was named `#authorized?`, however it conflicts with the native
-        # graphql gem version
-        # TODO consider adopting the gem's built in authorization system
-        # https://gitlab.com/gitlab-org/gitlab/issues/13984
         def authorized_resource?(object)
           # Sanity check. We don't want to accidentally allow a developer to authorize
           # without first adding permissions to authorize against
-          if self.class.required_permissions.empty?
-            raise Gitlab::Graphql::Errors::ArgumentError, "#{self.class.name} has no authorizations"
-          end
+          raise ConfigurationError, "#{self.class.name} has no authorizations" if self.class.authorization.none?
 
-          self.class.required_permissions.all? do |ability|
-            # The actions could be performed across multiple objects. In which
-            # case the current user is common, and we could benefit from the
-            # caching in `DeclarativePolicy`.
-            Ability.allowed?(current_user, ability, object, scope: :user)
-          end
+          self.class.authorization.ok?(object, current_user)
         end
 
-        def raise_resource_not_available_error!(msg = RESOURCE_ACCESS_ERROR)
-          raise Gitlab::Graphql::Errors::ResourceNotAvailable, msg
+        def raise_resource_not_available_error!(*args)
+          self.class.raise_resource_not_available_error!(*args)
         end
       end
     end
diff --git a/lib/gitlab/graphql/authorize/connection_filter_extension.rb b/lib/gitlab/graphql/authorize/connection_filter_extension.rb
new file mode 100644
index 00000000000..c75510df3e3
--- /dev/null
+++ b/lib/gitlab/graphql/authorize/connection_filter_extension.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Authorize
+      class ConnectionFilterExtension < GraphQL::Schema::FieldExtension
+        class Redactor
+          include ::Gitlab::Graphql::Laziness
+
+          def initialize(type, context)
+            @type = type
+            @context = context
+          end
+
+          def redact(nodes)
+            remove_unauthorized(nodes)
+
+            nodes
+          end
+
+          def active?
+            # some scalar types (such as integers) do not respond to :authorized?
+            return false unless @type.respond_to?(:authorized?)
+
+            auth = @type.try(:authorization)
+
+            auth.nil? || auth.any?
+          end
+
+          private
+
+          def remove_unauthorized(nodes)
+            nodes
+              .map! { |lazy| force(lazy) }
+              .keep_if { |forced| @type.authorized?(forced, @context) }
+          end
+        end
+
+        def after_resolve(value:, context:, **rest)
+          return value if value.is_a?(GraphQL::Execution::Execute::Skip)
+
+          if @field.connection?
+            redact_connection(value, context)
+          elsif @field.type.list?
+            redact_list(value.to_a, context) unless value.nil?
+          end
+
+          value
+        end
+
+        def redact_connection(conn, context)
+          redactor = Redactor.new(@field.type.unwrap.node_type, context)
+          return unless redactor.active?
+
+          conn.redactor = redactor if conn.respond_to?(:redactor=)
+        end
+
+        def redact_list(list, context)
+          redactor = Redactor.new(@field.type.unwrap, context)
+          redactor.redact(list) if redactor.active?
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/authorize/instrumentation.rb b/lib/gitlab/graphql/authorize/instrumentation.rb
deleted file mode 100644
index 15ecc3b04f0..00000000000
--- a/lib/gitlab/graphql/authorize/instrumentation.rb
+++ /dev/null
@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Graphql
-    module Authorize
-      class Instrumentation
-        # Replace the resolver for the field with one that will only return the
-        # resolved object if the permissions check is successful.
-        def instrument(_type, field)
-          service = AuthorizeFieldService.new(field)
-
-          if service.authorizations?
-            field.redefine { resolve(service.authorized_resolve) }
-          else
-            field
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/graphql/authorize/object_authorization.rb b/lib/gitlab/graphql/authorize/object_authorization.rb
new file mode 100644
index 00000000000..0bc87108871
--- /dev/null
+++ b/lib/gitlab/graphql/authorize/object_authorization.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Authorize
+      class ObjectAuthorization
+        attr_reader :abilities
+
+        def initialize(abilities)
+          @abilities = Array.wrap(abilities).flatten
+        end
+
+        def none?
+          abilities.empty?
+        end
+
+        def any?
+          abilities.present?
+        end
+
+        def ok?(object, current_user)
+          return true if none?
+
+          subject = object.try(:declarative_policy_subject) || object
+          abilities.all? do |ability|
+            Ability.allowed?(current_user, ability, subject)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/deprecation.rb b/lib/gitlab/graphql/deprecation.rb
new file mode 100644
index 00000000000..e0176e2d6e0
--- /dev/null
+++ b/lib/gitlab/graphql/deprecation.rb
@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    class Deprecation
+      REASONS = {
+        renamed: 'This was renamed.',
+        discouraged: 'Use of this is not recommended.'
+      }.freeze
+
+      include ActiveModel::Validations
+
+      validates :milestone, presence: true, format: { with: /\A\d+\.\d+\z/, message: 'must be milestone-ish' }
+      validates :reason, presence: true
+      validates :reason,
+                format: { with: /.*[^.]\z/, message: 'must not end with a period' },
+                if: :reason_is_string?
+      validate :milestone_is_string
+      validate :reason_known_or_string
+
+      def self.parse(options)
+        new(**options) if options
+      end
+
+      def initialize(reason: nil, milestone: nil, replacement: nil)
+        @reason = reason.presence
+        @milestone = milestone.presence
+        @replacement = replacement.presence
+      end
+
+      def ==(other)
+        return false unless other.is_a?(self.class)
+
+        [reason_text, milestone, replacement] == [:reason_text, :milestone, :replacement].map do |attr|
+          other.send(attr) # rubocop: disable GitlabSecurity/PublicSend
+        end
+      end
+      alias_method :eql, :==
+
+      def markdown(context: :inline)
+        parts = [
+          "#{deprecated_in(format: :markdown)}.",
+          reason_text,
+          replacement.then { |r| "Use: `#{r}`." if r }
+        ].compact
+
+        case context
+        when :block
+          ['WARNING:', *parts].join("\n")
+        when :inline
+          parts.join(' ')
+        end
+      end
+
+      def edit_description(original_description)
+        @original_description = original_description
+        return unless original_description
+
+        original_description + description_suffix
+      end
+
+      def original_description
+        return unless @original_description
+        return @original_description if @original_description.ends_with?('.')
+
+        "#{@original_description}."
+      end
+
+      def deprecation_reason
+        [
+          reason_text,
+          replacement && "Please use `#{replacement}`.",
+          "#{deprecated_in}."
+        ].compact.join(' ')
+      end
+
+      private
+
+      attr_reader :reason, :milestone, :replacement
+
+      def milestone_is_string
+        return if milestone.is_a?(String)
+
+        errors.add(:milestone, 'must be a string')
+      end
+
+      def reason_known_or_string
+        return if REASONS.key?(reason)
+        return if reason_is_string?
+
+        errors.add(:reason, 'must be a known reason or a string')
+      end
+
+      def reason_is_string?
+        reason.is_a?(String)
+      end
+
+      def reason_text
+        @reason_text ||= REASONS[reason] || "#{reason.to_s.strip}."
+      end
+
+      def description_suffix
+        " #{deprecated_in}: #{reason_text}"
+      end
+
+      def deprecated_in(format: :plain)
+        case format
+        when :plain
+          "Deprecated in #{milestone}"
+        when :markdown
+          "**Deprecated** in #{milestone}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb
index e9ff85d9ca9..f4173e26224 100644
--- a/lib/gitlab/graphql/docs/helper.rb
+++ b/lib/gitlab/graphql/docs/helper.rb
@@ -27,7 +27,10 @@ module Gitlab
           MD
         end
 
-        def render_name_and_description(object, level = 3)
+        # Template methods:
+        # Methods that return chunks of Markdown for insertion into the document
+
+        def render_name_and_description(object, owner: nil, level: 3)
           content = []
 
           content << "#{'#' * level} `#{object[:name]}`"
@@ -35,10 +38,22 @@ module Gitlab
           if object[:description].present?
             desc = object[:description].strip
             desc += '.' unless desc.ends_with?('.')
+          end
+
+          if object[:is_deprecated]
+            owner = Array.wrap(owner)
+            deprecation = schema_deprecation(owner, object[:name])
+            content << (deprecation&.original_description || desc)
+            content << render_deprecation(object, owner, :block)
+          else
             content << desc
           end
 
-          content.join("\n\n")
+          content.compact.join("\n\n")
+        end
+
+        def render_return_type(query)
+          "Returns #{render_field_type(query[:type])}.\n"
         end
 
         def sorted_by_name(objects)
@@ -47,39 +62,25 @@ module Gitlab
           objects.sort_by { |o| o[:name] }
         end
 
-        def render_field(field)
-          row(render_name(field), render_field_type(field[:type]), render_description(field))
+        def render_field(field, owner)
+          render_row(
+            render_name(field, owner),
+            render_field_type(field[:type]),
+            render_description(field, owner, :inline)
+          )
         end
 
-        def render_enum_value(value)
-          row(render_name(value), render_description(value))
+        def render_enum_value(enum, value)
+          render_row(render_name(value, enum[:name]), render_description(value, enum[:name], :inline))
         end
 
-        def row(*values)
-          "| #{values.join(' | ')} |"
+        def render_union_member(member)
+          "- [`#{member}`](##{member.downcase})"
         end
 
-        def render_name(object)
-          rendered_name = "`#{object[:name]}`"
-          rendered_name += ' **{warning-solid}**' if object[:is_deprecated]
-          rendered_name
-        end
+        # QUERIES:
 
-        # Returns the object description. If the object has been deprecated,
-        # the deprecation reason will be returned in place of the description.
-        def render_description(object)
-          return object[:description] unless object[:is_deprecated]
-
-          "**Deprecated:** #{object[:deprecation_reason]}"
-        end
-
-        def render_field_type(type)
-          "[`#{type[:info]}`](##{type[:name].downcase})"
-        end
-
-        def render_return_type(query)
-          "Returns #{render_field_type(query[:type])}.\n"
-        end
+        # Methods that return parts of the schema, or related information:
 
         # We are ignoring connections and built in types for now,
         # they should be added when queries are generated.
@@ -103,6 +104,83 @@ module Gitlab
             !enum_type[:name].in?(%w[__DirectiveLocation __TypeKind])
           end
         end
+
+        private # DO NOT CALL THESE METHODS IN TEMPLATES
+
+        # Template methods
+
+        def render_row(*values)
+          "| #{values.map { |val| val.to_s.squish }.join(' | ')} |"
+        end
+
+        def render_name(object, owner = nil)
+          rendered_name = "`#{object[:name]}`"
+          rendered_name += ' **{warning-solid}**' if object[:is_deprecated]
+          rendered_name
+        end
+
+        # Returns the object description. If the object has been deprecated,
+        # the deprecation reason will be returned in place of the description.
+        def render_description(object, owner = nil, context = :block)
+          owner = Array.wrap(owner)
+          return render_deprecation(object, owner, context) if object[:is_deprecated]
+          return if object[:description].blank?
+
+          desc = object[:description].strip
+          desc += '.' unless desc.ends_with?('.')
+          desc
+        end
+
+        def render_deprecation(object, owner, context)
+          deprecation = schema_deprecation(owner, object[:name])
+          return deprecation.markdown(context: context) if deprecation
+
+          reason = object[:deprecation_reason] || 'Use of this is deprecated.'
+          "**Deprecated:** #{reason}"
+        end
+
+        def render_field_type(type)
+          "[`#{type[:info]}`](##{type[:name].downcase})"
+        end
+
+        # Queries
+
+        # returns the deprecation information for a field or argument
+        # See: Gitlab::Graphql::Deprecation
+        def schema_deprecation(type_name, field_name)
+          schema_member(type_name, field_name)&.deprecation
+        end
+
+        # Return a part of the schema.
+        #
+        # This queries the Schema by owner and name to find:
+        #
+        # - fields (e.g. `schema_member('Query', 'currentUser')`)
+        # - arguments (e.g. `schema_member(['Query', 'project], 'fullPath')`)
+        def schema_member(type_name, field_name)
+          type_name = Array.wrap(type_name)
+          if type_name.size == 2
+            arg_name = field_name
+            type_name, field_name = type_name
+          else
+            type_name = type_name.first
+            arg_name = nil
+          end
+
+          return if type_name.nil? || field_name.nil?
+
+          type = schema.types[type_name]
+          return unless type && type.kind.fields?
+
+          field = type.fields[field_name]
+          return field if arg_name.nil?
+
+          args = field.arguments
+          is_mutation = field.mutation && field.mutation <= ::Mutations::BaseMutation
+          args = args['input'].type.unwrap.arguments if is_mutation
+
+          args[arg_name]
+        end
       end
     end
   end
diff --git a/lib/gitlab/graphql/docs/renderer.rb b/lib/gitlab/graphql/docs/renderer.rb
index 6abd56c89c6..497567f9389 100644
--- a/lib/gitlab/graphql/docs/renderer.rb
+++ b/lib/gitlab/graphql/docs/renderer.rb
@@ -10,17 +10,20 @@ module Gitlab
       # It uses graphql-docs helpers and schema parser, more information in https://github.com/gjtorikian/graphql-docs.
       #
       # Arguments:
-      #   schema - the GraphQL schema definition. For GitLab should be: GitlabSchema.graphql_definition
+      #   schema - the GraphQL schema definition. For GitLab should be: GitlabSchema
       #   output_dir: The folder where the markdown files will be saved
       #   template: The path of the haml template to be parsed
       class Renderer
         include Gitlab::Graphql::Docs::Helper
 
+        attr_reader :schema
+
         def initialize(schema, output_dir:, template:)
           @output_dir = output_dir
           @template = template
           @layout = Haml::Engine.new(File.read(template))
-          @parsed_schema = GraphQLDocs::Parser.new(schema, {}).parse
+          @parsed_schema = GraphQLDocs::Parser.new(schema.graphql_definition, {}).parse
+          @schema = schema
         end
 
         def contents
diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml
index 847f1777b08..fe73297d0d9 100644
--- a/lib/gitlab/graphql/docs/templates/default.md.haml
+++ b/lib/gitlab/graphql/docs/templates/default.md.haml
@@ -27,7 +27,7 @@
 \
 
 - sorted_by_name(queries).each do |query|
-  = render_name_and_description(query)
+  = render_name_and_description(query, owner: 'Query')
   \
   = render_return_type(query)
   - unless query[:arguments].empty?
@@ -35,7 +35,7 @@
     ~ "| Name | Type | Description |"
     ~ "| ---- | ---- | ----------- |"
     - sorted_by_name(query[:arguments]).each do |argument|
-      = render_field(argument)
+      = render_field(argument, query[:type][:name])
     \
 
 :plain
@@ -58,7 +58,7 @@
     ~ "| Field | Type | Description |"
     ~ "| ----- | ---- | ----------- |"
     - sorted_by_name(type[:fields]).each do |field|
-      = render_field(field)
+      = render_field(field, type[:name])
     \
 
 :plain
@@ -79,7 +79,7 @@
     ~ "| Value | Description |"
     ~ "| ----- | ----------- |"
     - sorted_by_name(enum[:values]).each do |value|
-      = render_enum_value(value)
+      = render_enum_value(enum, value)
     \
 
 :plain
@@ -121,12 +121,12 @@
 \
 
 - graphql_union_types.each do |type|
-  = render_name_and_description(type, 4)
+  = render_name_and_description(type, level: 4)
   \
   One of:
   \
-  - type[:possible_types].each do |type_name|
-    ~ "- [`#{type_name}`](##{type_name.downcase})"
+  - type[:possible_types].each do |member|
+    = render_union_member(member)
   \
 
 :plain
@@ -134,7 +134,7 @@
 \
 
 - graphql_interface_types.each do |type|
-  = render_name_and_description(type, 4)
+  = render_name_and_description(type, level: 4)
   \
   Implementations:
   \
@@ -144,5 +144,5 @@
   ~ "| Field | Type | Description |"
   ~ "| ----- | ---- | ----------- |"
   - sorted_by_name(type[:fields] + type[:connections]).each do |field|
-    = render_field(field)
+    = render_field(field, type[:name])
   \
diff --git a/lib/gitlab/graphql/loaders/batch_lfs_oid_loader.rb b/lib/gitlab/graphql/loaders/batch_lfs_oid_loader.rb
index 67511c124e4..1945388cdd4 100644
--- a/lib/gitlab/graphql/loaders/batch_lfs_oid_loader.rb
+++ b/lib/gitlab/graphql/loaders/batch_lfs_oid_loader.rb
@@ -5,7 +5,8 @@ module Gitlab
     module Loaders
       class BatchLfsOidLoader
         def initialize(repository, blob_id)
-          @repository, @blob_id = repository, blob_id
+          @repository = repository
+          @blob_id = blob_id
         end
 
         def find
diff --git a/lib/gitlab/graphql/loaders/batch_model_loader.rb b/lib/gitlab/graphql/loaders/batch_model_loader.rb
index 9b85ba164d4..805864cdd4c 100644
--- a/lib/gitlab/graphql/loaders/batch_model_loader.rb
+++ b/lib/gitlab/graphql/loaders/batch_model_loader.rb
@@ -7,7 +7,8 @@ module Gitlab
         attr_reader :model_class, :model_id
 
         def initialize(model_class, model_id)
-          @model_class, @model_id = model_class, model_id
+          @model_class = model_class
+          @model_id = model_id
         end
 
         # rubocop: disable CodeReuse/ActiveRecord
diff --git a/lib/gitlab/graphql/loaders/full_path_model_loader.rb b/lib/gitlab/graphql/loaders/full_path_model_loader.rb
index 0aa237c78de..26c1ce64a83 100644
--- a/lib/gitlab/graphql/loaders/full_path_model_loader.rb
+++ b/lib/gitlab/graphql/loaders/full_path_model_loader.rb
@@ -9,7 +9,8 @@ module Gitlab
         attr_reader :model_class, :full_path
 
         def initialize(model_class, full_path)
-          @model_class, @full_path = model_class, full_path
+          @model_class = model_class
+          @full_path = full_path
         end
 
         def find
diff --git a/lib/gitlab/graphql/negatable_arguments.rb b/lib/gitlab/graphql/negatable_arguments.rb
new file mode 100644
index 00000000000..b4ab31ed51a
--- /dev/null
+++ b/lib/gitlab/graphql/negatable_arguments.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module NegatableArguments
+      class TypeDefiner
+        def initialize(resolver_class, type_definition)
+          @resolver_class = resolver_class
+          @type_definition = type_definition
+        end
+
+        def define!
+          negated_params_type.instance_eval(&@type_definition)
+        end
+
+        def negated_params_type
+          @negated_params_type ||= existing_type || build_type
+        end
+
+        private
+
+        def existing_type
+          ::Types.const_get(type_class_name, false) if ::Types.const_defined?(type_class_name)
+        end
+
+        def build_type
+          klass = Class.new(::Types::BaseInputObject)
+          ::Types.const_set(type_class_name, klass)
+          klass
+        end
+
+        def type_class_name
+          @type_class_name ||= begin
+            base_name = @resolver_class.name.sub('Resolvers::', '')
+            base_name + 'NegatedParamsType'
+          end
+        end
+      end
+
+      def negated(param_key: :not, &block)
+        definer = ::Gitlab::Graphql::NegatableArguments::TypeDefiner.new(self, block)
+        definer.define!
+
+        argument param_key, definer.negated_params_type,
+                 required: false,
+                 description: <<~MD
+                     List of negated arguments.
+                     Warning: this argument is experimental and a subject to change in future.
+                 MD
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb b/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb
index bd785880b57..6645dac36fa 100644
--- a/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb
+++ b/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb
@@ -13,7 +13,11 @@ module Gitlab
             # @param [Symbol] before_or_after indicates whether we want
             #        items :before the cursor or :after the cursor
             def initialize(arel_table, order_list, values, operators, before_or_after)
-              @arel_table, @order_list, @values, @operators, @before_or_after = arel_table, order_list, values, operators, before_or_after
+              @arel_table = arel_table
+              @order_list = order_list
+              @values = values
+              @operators = operators
+              @before_or_after = before_or_after
 
               @before_or_after = :after unless [:after, :before].include?(@before_or_after)
             end
diff --git a/lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb b/lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb
index 3164598b7b9..ec70f5c5a24 100644
--- a/lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb
+++ b/lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb
@@ -30,15 +30,13 @@ module Gitlab
 
             # ex: " OR (relative_position = 23 AND id > 500)"
             def second_attribute_condition
-              condition = <<~SQL
+              <<~SQL
                 OR (
                   #{table_condition(order_list.first, values.first, '=').to_sql}
                   AND
                   #{table_condition(order_list[1], values[1], operators[1]).to_sql}
                 )
               SQL
-
-              condition
             end
 
             # ex: " OR (relative_position IS NULL)"
diff --git a/lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb b/lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb
index fa25181d663..1aae1020e79 100644
--- a/lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb
+++ b/lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb
@@ -14,15 +14,13 @@ module Gitlab
 
             # ex: "(relative_position IS NULL AND id > 500)"
             def first_attribute_condition
-              condition = <<~SQL
+              <<~SQL
                 (
                   #{table_condition(order_list.first, nil, 'is_null').to_sql}
                   AND
                   #{table_condition(order_list[1], values[1], operators[1]).to_sql}
                 )
               SQL
-
-              condition
             end
 
             # ex: " OR (relative_position IS NOT NULL)"
diff --git a/lib/gitlab/graphql/pagination/keyset/query_builder.rb b/lib/gitlab/graphql/pagination/keyset/query_builder.rb
index 29169449843..ee9c902c735 100644
--- a/lib/gitlab/graphql/pagination/keyset/query_builder.rb
+++ b/lib/gitlab/graphql/pagination/keyset/query_builder.rb
@@ -6,7 +6,10 @@ module Gitlab
       module Keyset
         class QueryBuilder
           def initialize(arel_table, order_list, decoded_cursor, before_or_after)
-            @arel_table, @order_list, @decoded_cursor, @before_or_after = arel_table, order_list, decoded_cursor, before_or_after
+            @arel_table = arel_table
+            @order_list = order_list
+            @decoded_cursor = decoded_cursor
+            @before_or_after = before_or_after
 
             if order_list.empty?
               raise ArgumentError.new('No ordering scopes have been supplied')
diff --git a/lib/gitlab/graphql/queries.rb b/lib/gitlab/graphql/queries.rb
index fcf293fb13e..74f55abccbc 100644
--- a/lib/gitlab/graphql/queries.rb
+++ b/lib/gitlab/graphql/queries.rb
@@ -224,11 +224,9 @@ module Gitlab
           frag_path = frag_path.gsub(DOTS_RE) do |dots|
             rel_dir(dots.split('/').count)
           end
-          frag_path = frag_path.gsub(IMPLICIT_ROOT) do
+          frag_path.gsub(IMPLICIT_ROOT) do
             (Rails.root / 'app').to_s + '/'
           end
-
-          frag_path
         end
 
         def rel_dir(n_steps_up)
diff --git a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb
index 8acd27869a9..c6f22e0bd4f 100644
--- a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb
+++ b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb
@@ -12,6 +12,7 @@ module Gitlab
         def initial_value(query)
           variables = process_variables(query.provided_variables)
           default_initial_values(query).merge({
+            operation_name: query.operation_name,
             query_string: query.query_string,
             variables: variables
           })
@@ -20,8 +21,8 @@ module Gitlab
           default_initial_values(query)
         end
 
-        def call(memo, visit_type, irep_node)
-          RequestStore.store[:graphql_logs] = memo
+        def call(memo, *)
+          memo
         end
 
         def final_value(memo)
@@ -37,6 +38,8 @@ module Gitlab
           memo[:used_fields] = field_usages.first
           memo[:used_deprecated_fields] = field_usages.second
 
+          RequestStore.store[:graphql_logs] ||= []
+          RequestStore.store[:graphql_logs] << memo
           GraphqlLogger.info(memo.except!(:time_started, :query))
         rescue => e
           Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
diff --git a/lib/gitlab/health_checks/gitaly_check.rb b/lib/gitlab/health_checks/gitaly_check.rb
index e780bf8a986..f5f142c251f 100644
--- a/lib/gitlab/health_checks/gitaly_check.rb
+++ b/lib/gitlab/health_checks/gitaly_check.rb
@@ -5,7 +5,7 @@ module Gitlab
     class GitalyCheck
       extend BaseAbstractCheck
 
-      METRIC_PREFIX = 'gitaly_health_check'.freeze
+      METRIC_PREFIX = 'gitaly_health_check'
 
       class << self
         def readiness
diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb
index 40dee0142b9..765d3dfca56 100644
--- a/lib/gitlab/highlight.rb
+++ b/lib/gitlab/highlight.rb
@@ -20,7 +20,9 @@ module Gitlab
       @blob_content = blob_content
     end
 
-    def highlight(text, continue: true, plain: false)
+    def highlight(text, continue: false, plain: false, context: {})
+      @context = context
+
       plain ||= text.length > MAXIMUM_TEXT_HIGHLIGHT_SIZE
 
       highlighted_text = highlight_text(text, continue: continue, plain: plain)
@@ -31,13 +33,15 @@ module Gitlab
     def lexer
       @lexer ||= custom_language || begin
         Rouge::Lexer.guess(filename: @blob_name, source: @blob_content).new
-                                    rescue Rouge::Guesser::Ambiguous => e
-                                      e.alternatives.min_by(&:tag)
+      rescue Rouge::Guesser::Ambiguous => e
+        e.alternatives.min_by(&:tag)
       end
     end
 
     private
 
+    attr_reader :context
+
     def custom_language
       return unless @language
 
@@ -53,13 +57,13 @@ module Gitlab
     end
 
     def highlight_plain(text)
-      @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe
+      @formatter.format(Rouge::Lexers::PlainText.lex(text), context).html_safe
     end
 
     def highlight_rich(text, continue: true)
       tag = lexer.tag
       tokens = lexer.lex(text, continue: continue)
-      Timeout.timeout(timeout_time) { @formatter.format(tokens, tag: tag).html_safe }
+      Timeout.timeout(timeout_time) { @formatter.format(tokens, context.merge(tag: tag)).html_safe }
     rescue Timeout::Error => e
       Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
       highlight_plain(text)
diff --git a/lib/gitlab/hook_data/user_builder.rb b/lib/gitlab/hook_data/user_builder.rb
new file mode 100644
index 00000000000..537245e948f
--- /dev/null
+++ b/lib/gitlab/hook_data/user_builder.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module HookData
+    class UserBuilder < BaseBuilder
+      alias_method :user, :object
+
+      # Sample data
+      # {
+      # :created_at=>"2021-04-02T10:00:26Z",
+      # :updated_at=>"2021-04-02T10:00:26Z",
+      # :event_name=>"user_create",
+      # :name=>"John Doe",
+      # :email=>"john@example.com",
+      # :user_id=>1,
+      # :username=>"johndoe"
+      # }
+
+      def build(event)
+        [
+          timestamps_data,
+          event_data(event),
+          user_data,
+          event_specific_user_data(event)
+        ].reduce(:merge)
+      end
+
+      private
+
+      def user_data
+        {
+          name: user.name,
+          email: user.email,
+          user_id: user.id,
+          username: user.username
+        }
+      end
+
+      def event_specific_user_data(event)
+        case event
+        when :rename
+          { old_username: user.username_before_last_save }
+        when :failed_login
+          { state: user.state }
+        else
+          {}
+        end
+      end
+    end
+  end
+end
+
+Gitlab::HookData::UserBuilder.prepend_if_ee('EE::Gitlab::HookData::UserBuilder')
diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb
index 37f618ae879..f7a3da53fdb 100644
--- a/lib/gitlab/http_connection_adapter.rb
+++ b/lib/gitlab/http_connection_adapter.rb
@@ -17,14 +17,6 @@ module Gitlab
     def connection
       @uri, hostname = validate_url!(uri)
 
-      if options.key?(:http_proxyaddr)
-        proxy_uri_with_port = uri_with_port(options[:http_proxyaddr], options[:http_proxyport])
-        proxy_uri_validated = validate_url!(proxy_uri_with_port).first
-
-        @options[:http_proxyaddr] = proxy_uri_validated.omit(:port).to_s
-        @options[:http_proxyport] = proxy_uri_validated.port
-      end
-
       super.tap do |http|
         http.hostname_override = hostname if hostname
       end
@@ -53,11 +45,5 @@ module Gitlab
     def allow_settings_local_requests?
       Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services?
     end
-
-    def uri_with_port(address, port)
-      uri = Addressable::URI.parse(address)
-      uri.port = port if port.present?
-      uri
-    end
   end
 end
diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb
index d60bc79df4c..05a4a8f4c93 100644
--- a/lib/gitlab/import_export/base/relation_factory.rb
+++ b/lib/gitlab/import_export/base/relation_factory.rb
@@ -6,7 +6,7 @@ module Gitlab
       class RelationFactory
         include Gitlab::Utils::StrongMemoize
 
-        IMPORTED_OBJECT_MAX_RETRIES = 5.freeze
+        IMPORTED_OBJECT_MAX_RETRIES = 5
 
         OVERRIDES = {}.freeze
         EXISTING_OBJECT_RELATIONS = %i[].freeze
diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml
index 778b42f4358..42d32593cbd 100644
--- a/lib/gitlab/import_export/project/import_export.yml
+++ b/lib/gitlab/import_export/project/import_export.yml
@@ -265,6 +265,7 @@ excluded_attributes:
     - :issue_id
   push_event_payload:
     - :event_id
+    - :event_id_convert_to_bigint
   project_badges:
     - :group_id
   resource_label_events:
@@ -287,6 +288,7 @@ excluded_attributes:
     - :label_id
   events:
     - :target_id
+    - :id_convert_to_bigint
   timelogs:
     - :issue_id
     - :merge_request_id
diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb
index 428bcbe8dc5..2f15cdd7506 100644
--- a/lib/gitlab/import_export/uploads_manager.rb
+++ b/lib/gitlab/import_export/uploads_manager.rb
@@ -76,7 +76,7 @@ module Gitlab
       def project_uploads_except_avatar(avatar_path)
         return @project.uploads unless avatar_path
 
-        @project.uploads.where("path != ?", avatar_path)
+        @project.uploads.where.not(path: avatar_path)
       end
 
       def download_and_copy(upload)
diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb
index 88753e80391..95c002edf0a 100644
--- a/lib/gitlab/import_sources.rb
+++ b/lib/gitlab/import_sources.rb
@@ -28,7 +28,7 @@ module Gitlab
       prepend_if_ee('EE::Gitlab::ImportSources') # rubocop: disable Cop/InjectEnterpriseEditionModule
 
       def options
-        Hash[import_table.map { |importer| [importer.title, importer.name] }]
+        import_table.to_h { |importer| [importer.title, importer.name] }
       end
 
       def values
diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb
index 61de6b02453..a865a6392f0 100644
--- a/lib/gitlab/instrumentation_helper.rb
+++ b/lib/gitlab/instrumentation_helper.rb
@@ -6,24 +6,6 @@ module Gitlab
 
     DURATION_PRECISION = 6 # microseconds
 
-    def keys
-      @keys ||= [
-        :cpu_s,
-        :gitaly_calls,
-        :gitaly_duration_s,
-        :rugged_calls,
-        :rugged_duration_s,
-        :elasticsearch_calls,
-        :elasticsearch_duration_s,
-        :elasticsearch_timed_out_count,
-        *::Gitlab::Memory::Instrumentation::KEY_MAPPING.values,
-        *::Gitlab::Instrumentation::Redis.known_payload_keys,
-        *::Gitlab::Metrics::Subscribers::ActiveRecord::DB_COUNTERS,
-        *::Gitlab::Metrics::Subscribers::ExternalHttp::KNOWN_PAYLOAD_KEYS,
-        *::Gitlab::Metrics::Subscribers::RackAttack::PAYLOAD_KEYS
-      ]
-    end
-
     def init_instrumentation_data(request_ip: nil)
       # Set `request_start_time` only if this is request
       # This is done, as `request_start_time` imply `request_deadline`
diff --git a/lib/gitlab/issuables_count_for_state.rb b/lib/gitlab/issuables_count_for_state.rb
index 945ab7f40c2..6b33b60e850 100644
--- a/lib/gitlab/issuables_count_for_state.rb
+++ b/lib/gitlab/issuables_count_for_state.rb
@@ -78,7 +78,7 @@ module Gitlab
       # to perform the calculation more efficiently. Until then, use a shorter
       # timeout and return -1 as a sentinel value if it is triggered
       begin
-        ApplicationRecord.with_fast_statement_timeout do
+        ApplicationRecord.with_fast_read_statement_timeout do
           finder.count_by_state
         end
       rescue ActiveRecord::QueryCanceled => err
diff --git a/lib/gitlab/jira/dvcs.rb b/lib/gitlab/jira/dvcs.rb
index 4415f98fc7f..ddf2cd76709 100644
--- a/lib/gitlab/jira/dvcs.rb
+++ b/lib/gitlab/jira/dvcs.rb
@@ -3,8 +3,8 @@
 module Gitlab
   module Jira
     module Dvcs
-      ENCODED_SLASH = '@'.freeze
-      SLASH = '/'.freeze
+      ENCODED_SLASH = '@'
+      SLASH = '/'
       ENCODED_ROUTE_REGEX = /[a-zA-Z0-9_\-\.#{ENCODED_SLASH}]+/.freeze
 
       def self.encode_slash(path)
diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb
index 8565f664cd4..b51c0a33457 100644
--- a/lib/gitlab/json.rb
+++ b/lib/gitlab/json.rb
@@ -186,9 +186,14 @@ module Gitlab
       # The `env` param is ignored because it's not needed in either our formatter or Grape's,
       # but it is passed through for consistency.
       #
+      # If explicitly supplied with a `PrecompiledJson` instance it will skip conversion
+      # and return it directly. This is mostly used in caching.
+      #
       # @param object [Object]
       # @return [String]
       def self.call(object, env = nil)
+        return object.to_s if object.is_a?(PrecompiledJson)
+
         if Feature.enabled?(:grape_gitlab_json, default_enabled: true)
           Gitlab::Json.dump(object)
         else
@@ -197,6 +202,34 @@ module Gitlab
       end
     end
 
+    # Wrapper class used to skip JSON dumping on Grape endpoints.
+
+    class PrecompiledJson
+      UnsupportedFormatError = Class.new(StandardError)
+
+      # @overload PrecompiledJson.new("foo")
+      #   @param value [String]
+      #
+      # @overload PrecompiledJson.new(["foo", "bar"])
+      #   @param value [Array]
+      def initialize(value)
+        @value = value
+      end
+
+      # Convert the value to a String. This will invoke
+      # `#to_s` on the members of the value if it's an array.
+      #
+      # @return [String]
+      # @raise [NoMethodError] if the objects in an array doesn't support to_s
+      # @raise [PrecompiledJson::UnsupportedFormatError] if the value is neither a String or Array
+      def to_s
+        return @value if @value.is_a?(String)
+        return "[#{@value.join(',')}]" if @value.is_a?(Array)
+
+        raise UnsupportedFormatError
+      end
+    end
+
     class LimitedEncoder
       LimitExceeded = Class.new(StandardError)
 
diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb
index 329c0f221b5..7a674cb5c21 100644
--- a/lib/gitlab/kas.rb
+++ b/lib/gitlab/kas.rb
@@ -27,7 +27,7 @@ module Gitlab
       def included_in_gitlab_com_rollout?(project)
         return true unless ::Gitlab.com?
 
-        Feature.enabled?(:kubernetes_agent_on_gitlab_com, project)
+        Feature.enabled?(:kubernetes_agent_on_gitlab_com, project, default_enabled: :yaml)
       end
     end
   end
diff --git a/lib/gitlab/kubernetes/deployment.rb b/lib/gitlab/kubernetes/deployment.rb
index 55ed9a7517e..f2e3a0e6810 100644
--- a/lib/gitlab/kubernetes/deployment.rb
+++ b/lib/gitlab/kubernetes/deployment.rb
@@ -5,7 +5,7 @@ module Gitlab
     class Deployment
       include Gitlab::Utils::StrongMemoize
 
-      STABLE_TRACK_VALUE = 'stable'.freeze
+      STABLE_TRACK_VALUE = 'stable'
 
       def initialize(attributes = {}, pods: [])
         @attributes = attributes
diff --git a/lib/gitlab/language_detection.rb b/lib/gitlab/language_detection.rb
index 7600e60b904..1e5edb79f10 100644
--- a/lib/gitlab/language_detection.rb
+++ b/lib/gitlab/language_detection.rb
@@ -20,7 +20,7 @@ module Gitlab
     # Newly detected languages, returned in a structure accepted by
     # Gitlab::Database.bulk_insert
     def insertions(programming_languages)
-      lang_to_id = programming_languages.map { |p| [p.name, p.id] }.to_h
+      lang_to_id = programming_languages.to_h { |p| [p.name, p.id] }
 
       (languages - previous_language_names).map do |new_lang|
         {
@@ -63,8 +63,7 @@ module Gitlab
         @repository
         .languages
         .first(MAX_LANGUAGES)
-        .map { |l| [l[:label], l] }
-        .to_h
+        .to_h { |l| [l[:label], l] }
     end
   end
 end
diff --git a/lib/gitlab/manifest_import/manifest.rb b/lib/gitlab/manifest_import/manifest.rb
index 7208fe5bbc5..618ddf37b88 100644
--- a/lib/gitlab/manifest_import/manifest.rb
+++ b/lib/gitlab/manifest_import/manifest.rb
@@ -47,6 +47,10 @@ module Gitlab
           @errors << 'Make sure every  tag has name and path attributes.'
         end
 
+        unless validate_scheme
+          @errors << 'Make sure the url does not start with javascript'
+        end
+
         @errors.empty?
       end
 
@@ -64,6 +68,10 @@ module Gitlab
         end
       end
 
+      def validate_scheme
+        remote !~ /\Ajavascript/i
+      end
+
       def repository_url(name)
         Gitlab::Utils.append_path(remote, name)
       end
diff --git a/lib/gitlab/marker_range.rb b/lib/gitlab/marker_range.rb
index 50a59adebdf..73e4a545679 100644
--- a/lib/gitlab/marker_range.rb
+++ b/lib/gitlab/marker_range.rb
@@ -24,6 +24,12 @@ module Gitlab
       Range.new(self.begin, self.end, self.exclude_end?)
     end
 
+    def ==(other)
+      return false unless other.is_a?(self.class)
+
+      self.mode == other.mode && super
+    end
+
     attr_reader :mode
   end
 end
diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb
index d419fa66e57..45c6205b36b 100644
--- a/lib/gitlab/markup_helper.rb
+++ b/lib/gitlab/markup_helper.rb
@@ -4,7 +4,7 @@ module Gitlab
   module MarkupHelper
     extend self
 
-    MARKDOWN_EXTENSIONS = %w[mdown mkd mkdn md markdown].freeze
+    MARKDOWN_EXTENSIONS = %w[mdown mkd mkdn md markdown rmd].freeze
     ASCIIDOC_EXTENSIONS = %w[adoc ad asciidoc].freeze
     OTHER_EXTENSIONS = %w[textile rdoc org creole wiki mediawiki rst].freeze
     EXTENSIONS = MARKDOWN_EXTENSIONS + ASCIIDOC_EXTENSIONS + OTHER_EXTENSIONS
diff --git a/lib/gitlab/metrics/background_transaction.rb b/lib/gitlab/metrics/background_transaction.rb
index 3dda68bf93f..a1fabe75a97 100644
--- a/lib/gitlab/metrics/background_transaction.rb
+++ b/lib/gitlab/metrics/background_transaction.rb
@@ -34,8 +34,9 @@ module Gitlab
 
       def labels
         @labels ||= {
-          endpoint_id: current_context&.get_attribute(:caller_id),
-          feature_category: current_context&.get_attribute(:feature_category)
+          endpoint_id: endpoint_id,
+          feature_category: feature_category,
+          queue: queue
         }
       end
 
@@ -44,6 +45,21 @@ module Gitlab
       def current_context
         Labkit::Context.current
       end
+
+      def feature_category
+        current_context&.get_attribute(:feature_category)
+      end
+
+      def endpoint_id
+        current_context&.get_attribute(:caller_id)
+      end
+
+      def queue
+        worker_class = endpoint_id.to_s.safe_constantize
+        return if worker_class.blank? || !worker_class.respond_to?(:queue)
+
+        worker_class.queue.to_s
+      end
     end
   end
 end
diff --git a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb
index c90c1e3f0bc..55d14d6f94a 100644
--- a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb
+++ b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb
@@ -104,9 +104,7 @@ module Gitlab
           def format_query(metric)
             expression = remove_new_lines(metric[:expr])
             expression = replace_variables(expression)
-            expression = replace_global_variables(expression, metric)
-
-            expression
+            replace_global_variables(expression, metric)
           end
 
           # Accomodates instance-defined Grafana variables.
@@ -135,9 +133,7 @@ module Gitlab
           def replace_global_variables(expression, metric)
             expression = expression.gsub('$__interval', metric[:interval]) if metric[:interval]
             expression = expression.gsub('$__from', query_params[:from])
-            expression = expression.gsub('$__to', query_params[:to])
-
-            expression
+            expression.gsub('$__to', query_params[:to])
           end
 
           # Removes new lines from expression.
diff --git a/lib/gitlab/metrics/samplers/database_sampler.rb b/lib/gitlab/metrics/samplers/database_sampler.rb
index 60ae22df607..c0336a4d0fb 100644
--- a/lib/gitlab/metrics/samplers/database_sampler.rb
+++ b/lib/gitlab/metrics/samplers/database_sampler.rb
@@ -32,9 +32,9 @@ module Gitlab
         private
 
         def init_metrics
-          METRIC_DESCRIPTIONS.map do |name, description|
+          METRIC_DESCRIPTIONS.to_h do |name, description|
             [name, ::Gitlab::Metrics.gauge(:"#{METRIC_PREFIX}#{name}", description)]
-          end.to_h
+          end
         end
 
         def host_stats
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
index 5eefef02507..0d1cd641ffe 100644
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -11,13 +11,16 @@ module Gitlab
         DB_COUNTERS = %i{db_count db_write_count db_cached_count}.freeze
         SQL_COMMANDS_WITH_COMMENTS_REGEX = /\A(\/\*.*\*\/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$/i.freeze
 
-        DURATION_BUCKET = [0.05, 0.1, 0.25].freeze
+        SQL_DURATION_BUCKET = [0.05, 0.1, 0.25].freeze
+        TRANSACTION_DURATION_BUCKET = [0.1, 0.25, 1].freeze
 
         # This event is published from ActiveRecordBaseTransactionMetrics and
         # used to record a database transaction duration when calling
         # ActiveRecord::Base.transaction {} block.
         def transaction(event)
-          observe(:gitlab_database_transaction_seconds, event)
+          observe(:gitlab_database_transaction_seconds, event) do
+            buckets TRANSACTION_DURATION_BUCKET
+          end
         end
 
         def sql(event)
@@ -33,7 +36,9 @@ module Gitlab
           increment(:db_cached_count) if cached_query?(payload)
           increment(:db_write_count) unless select_sql_command?(payload)
 
-          observe(:gitlab_sql_duration_seconds, event)
+          observe(:gitlab_sql_duration_seconds, event) do
+            buckets SQL_DURATION_BUCKET
+          end
         end
 
         def self.db_counter_payload
@@ -46,6 +51,10 @@ module Gitlab
           payload
         end
 
+        def self.known_payload_keys
+          DB_COUNTERS
+        end
+
         private
 
         def ignored_query?(payload)
@@ -66,10 +75,8 @@ module Gitlab
           Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1
         end
 
-        def observe(histogram, event)
-          current_transaction&.observe(histogram, event.duration / 1000.0) do
-            buckets DURATION_BUCKET
-          end
+        def observe(histogram, event, &block)
+          current_transaction&.observe(histogram, event.duration / 1000.0, &block)
         end
 
         def current_transaction
diff --git a/lib/gitlab/metrics/subscribers/external_http.rb b/lib/gitlab/metrics/subscribers/external_http.rb
index 94c5d965200..0df64f2897e 100644
--- a/lib/gitlab/metrics/subscribers/external_http.rb
+++ b/lib/gitlab/metrics/subscribers/external_http.rb
@@ -37,7 +37,7 @@ module Gitlab
 
         def request(event)
           payload = event.payload
-          add_to_detail_store(payload)
+          add_to_detail_store(event.time, payload)
           add_to_request_store(payload)
           expose_metrics(payload)
         end
@@ -48,10 +48,11 @@ module Gitlab
           ::Gitlab::Metrics::Transaction.current
         end
 
-        def add_to_detail_store(payload)
+        def add_to_detail_store(start, payload)
           return unless Gitlab::PerformanceBar.enabled_for_request?
 
           self.class.detail_store << {
+            start: start,
             duration: payload[:duration],
             scheme: payload[:scheme],
             method: payload[:method],
diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb
index 79f1abe820f..329041e3ba2 100644
--- a/lib/gitlab/middleware/multipart.rb
+++ b/lib/gitlab/middleware/multipart.rb
@@ -31,7 +31,7 @@ module Gitlab
       RACK_ENV_KEY = 'HTTP_GITLAB_WORKHORSE_MULTIPART_FIELDS'
       JWT_PARAM_SUFFIX = '.gitlab-workhorse-upload'
       JWT_PARAM_FIXED_KEY = 'upload'
-      REWRITTEN_FIELD_NAME_MAX_LENGTH = 10000.freeze
+      REWRITTEN_FIELD_NAME_MAX_LENGTH = 10000
 
       class Handler
         def initialize(env, message)
diff --git a/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb b/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb
new file mode 100644
index 00000000000..d16c068c3c0
--- /dev/null
+++ b/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Middleware
+    class RackMultipartTempfileFactory
+      # Immediately unlink the created temporary file so we don't have to rely
+      # on Rack::TempfileReaper catching this after the fact.
+      FACTORY = lambda do |filename, content_type|
+        Rack::Multipart::Parser::TEMPFILE_FACTORY.call(filename, content_type).tap(&:unlink)
+      end
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        if ENV['GITLAB_TEMPFILE_IMMEDIATE_UNLINK'] == '1'
+          env[Rack::RACK_MULTIPART_TEMPFILE_FACTORY] = FACTORY
+        end
+
+        @app.call(env)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/middleware/same_site_cookies.rb b/lib/gitlab/middleware/same_site_cookies.rb
index 37ccc5abb10..405732e8015 100644
--- a/lib/gitlab/middleware/same_site_cookies.rb
+++ b/lib/gitlab/middleware/same_site_cookies.rb
@@ -17,7 +17,7 @@
 module Gitlab
   module Middleware
     class SameSiteCookies
-      COOKIE_SEPARATOR = "\n".freeze
+      COOKIE_SEPARATOR = "\n"
 
       def initialize(app)
         @app = app
diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb
index b1a1045a1f0..9a74266693b 100644
--- a/lib/gitlab/object_hierarchy.rb
+++ b/lib/gitlab/object_hierarchy.rb
@@ -68,13 +68,22 @@ module Gitlab
         expose_depth = hierarchy_order.present?
         hierarchy_order ||= :asc
 
-        recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all).distinct
-
         # if hierarchy_order is given, the calculated `depth` should be present in SELECT
         if expose_depth
+          recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all).distinct
           read_only(model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)).order(depth: hierarchy_order))
         else
-          read_only(remove_depth_and_maintain_order(recursive_query, hierarchy_order: hierarchy_order))
+          recursive_query = base_and_ancestors_cte(upto).apply_to(model.all)
+
+          if skip_ordering?
+            recursive_query = recursive_query.distinct
+          else
+            recursive_query = recursive_query.reselect(*recursive_query.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct
+            recursive_query = model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table))
+            recursive_query = remove_depth_and_maintain_order(recursive_query, hierarchy_order: hierarchy_order)
+          end
+
+          read_only(recursive_query)
         end
       else
         recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all)
@@ -93,12 +102,21 @@ module Gitlab
     def base_and_descendants(with_depth: false)
       if use_distinct?
         # Always calculate `depth`, remove it later if with_depth is false
-        base_cte = base_and_descendants_cte(with_depth: true).apply_to(model.all).distinct
-
         if with_depth
-          read_only(model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)).order(depth: :asc))
+          base_cte = base_and_descendants_cte(with_depth: true).apply_to(model.all).distinct
+          read_only(model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)).order(depth: :asc))
         else
-          read_only(remove_depth_and_maintain_order(base_cte, hierarchy_order: :asc))
+          base_cte = base_and_descendants_cte.apply_to(model.all)
+
+          if skip_ordering?
+            base_cte = base_cte.distinct
+          else
+            base_cte = base_cte.reselect(*base_cte.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct
+            base_cte = model.from(Arel::Nodes::As.new(base_cte.arel, objects_table))
+            base_cte = remove_depth_and_maintain_order(base_cte, hierarchy_order: :asc)
+          end
+
+          read_only(base_cte)
         end
       else
         read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(model.all))
@@ -161,7 +179,19 @@ module Gitlab
 
     # Use distinct on the Namespace queries to avoid bad planner behavior in PG11.
     def use_distinct?
-      (model <= Namespace) && options[:use_distinct]
+      return unless model <= Namespace
+      # Global use_distinct_for_all_object_hierarchy takes precedence over use_distinct_in_object_hierarchy
+      return true if Feature.enabled?(:use_distinct_for_all_object_hierarchy)
+      return options[:use_distinct] if options.key?(:use_distinct)
+
+      false
+    end
+
+    # Skips the extra ordering when using distinct on the namespace queries
+    def skip_ordering?
+      return options[:skip_ordering] if options.key?(:skip_ordering)
+
+      false
     end
 
     # Remove the extra `depth` field using an INNER JOIN to avoid breaking UNION queries
diff --git a/lib/gitlab/pages.rb b/lib/gitlab/pages.rb
index 33e709360ad..98e87e9e915 100644
--- a/lib/gitlab/pages.rb
+++ b/lib/gitlab/pages.rb
@@ -3,7 +3,7 @@
 module Gitlab
   module Pages
     VERSION = File.read(Rails.root.join("GITLAB_PAGES_VERSION")).strip.freeze
-    INTERNAL_API_REQUEST_HEADER = 'Gitlab-Pages-Api-Request'.freeze
+    INTERNAL_API_REQUEST_HEADER = 'Gitlab-Pages-Api-Request'
     MAX_SIZE = 1.terabyte
 
     include JwtAuthenticatable
diff --git a/lib/gitlab/pages/migration_helper.rb b/lib/gitlab/pages/migration_helper.rb
new file mode 100644
index 00000000000..8f8667fafd9
--- /dev/null
+++ b/lib/gitlab/pages/migration_helper.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Pages
+    class MigrationHelper
+      def initialize(logger = nil)
+        @logger = logger
+      end
+
+      def migrate_to_remote_storage
+        deployments = ::PagesDeployment.with_files_stored_locally
+        migrate(deployments, ObjectStorage::Store::REMOTE)
+      end
+
+      def migrate_to_local_storage
+        deployments = ::PagesDeployment.with_files_stored_remotely
+        migrate(deployments, ObjectStorage::Store::LOCAL)
+      end
+
+      private
+
+      def batch_size
+        ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i
+      end
+
+      def migrate(deployments, store)
+        deployments.find_each(batch_size: batch_size) do |deployment| # rubocop:disable CodeReuse/ActiveRecord
+          deployment.file.migrate!(store)
+
+          log_success(deployment, store)
+        rescue => e
+          log_error(e, deployment)
+        end
+      end
+
+      def log_success(deployment, store)
+        logger.info("Transferred deployment ID #{deployment.id} of type #{deployment.file_type} with size #{deployment.size} to #{storage_label(store)} storage")
+      end
+
+      def log_error(err, deployment)
+        logger.warn("Failed to transfer deployment of type #{deployment.file_type} and ID #{deployment.id} with error: #{err.message}")
+      end
+
+      def storage_label(store)
+        if store == ObjectStorage::Store::LOCAL
+          'local'
+        else
+          'object'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/pages/settings.rb b/lib/gitlab/pages/settings.rb
index 8650a80a85e..be71018e851 100644
--- a/lib/gitlab/pages/settings.rb
+++ b/lib/gitlab/pages/settings.rb
@@ -6,12 +6,28 @@ module Gitlab
       DiskAccessDenied = Class.new(StandardError)
 
       def path
-        if ::Gitlab::Runtime.web_server? && !::Gitlab::Runtime.test_suite?
-          raise DiskAccessDenied
-        end
+        report_denied_disk_access
 
         super
       end
+
+      def local_store
+        @local_store ||= ::Gitlab::Pages::Stores::LocalStore.new(super)
+      end
+
+      private
+
+      def disk_access_denied?
+        return true unless ::Settings.pages.local_store&.enabled
+
+        ::Gitlab::Runtime.web_server? && !::Gitlab::Runtime.test_suite?
+      end
+
+      def report_denied_disk_access
+        raise DiskAccessDenied if disk_access_denied?
+      rescue => e
+        ::Gitlab::ErrorTracking.track_exception(e)
+      end
     end
   end
 end
diff --git a/lib/gitlab/pages/stores/local_store.rb b/lib/gitlab/pages/stores/local_store.rb
new file mode 100644
index 00000000000..68a7ebaceff
--- /dev/null
+++ b/lib/gitlab/pages/stores/local_store.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Pages
+    module Stores
+      class LocalStore < ::SimpleDelegator
+        def enabled
+          return false unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true)
+
+          super
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/pages_transfer.rb b/lib/gitlab/pages_transfer.rb
index c1ccfae3e1f..ae5539c03b1 100644
--- a/lib/gitlab/pages_transfer.rb
+++ b/lib/gitlab/pages_transfer.rb
@@ -12,7 +12,7 @@ module Gitlab
     class Async
       METHODS.each do |meth|
         define_method meth do |*args|
-          next unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true)
+          next unless Settings.pages.local_store.enabled
 
           PagesTransferWorker.perform_async(meth, args)
         end
@@ -21,7 +21,7 @@ module Gitlab
 
     METHODS.each do |meth|
       define_method meth do |*args|
-        next unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true)
+        next unless Settings.pages.local_store.enabled
 
         super(*args)
       end
diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb
index e8e68a5c4a5..e596e1bac9d 100644
--- a/lib/gitlab/pagination/keyset/order.rb
+++ b/lib/gitlab/pagination/keyset/order.rb
@@ -55,14 +55,14 @@ module Gitlab
       #   scope :created_at_ordered, -> {
       #     keyset_order = Gitlab::Pagination::Keyset::Order.build([
       #       Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
-      #         attribute: :created_at,
+      #         attribute_name: :created_at,
       #         column_expression: Project.arel_table[:created_at],
       #         order_expression: Project.arel_table[:created_at].asc,
       #         distinct: false, # values in the column are not unique
       #         nullable: :nulls_last # we might see NULL values (bottom)
       #       ),
       #       Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
-      #         attribute: :id,
+      #         attribute_name: :id,
       #         order_expression: Project.arel_table[:id].asc
       #       )
       #     ])
@@ -93,7 +93,7 @@ module Gitlab
         end
 
         def cursor_attributes_for_node(node)
-          column_definitions.each_with_object({}) do |column_definition, hash|
+          column_definitions.each_with_object({}.with_indifferent_access) do |column_definition, hash|
             field_value = node[column_definition.attribute_name]
             hash[column_definition.attribute_name] = if field_value.is_a?(Time)
                                                        field_value.strftime('%Y-%m-%d %H:%M:%S.%N %Z')
@@ -162,7 +162,7 @@ module Gitlab
         # rubocop: disable CodeReuse/ActiveRecord
         def apply_cursor_conditions(scope, values = {})
           scope = apply_custom_projections(scope)
-          scope.where(build_where_values(values))
+          scope.where(build_where_values(values.with_indifferent_access))
         end
         # rubocop: enable CodeReuse/ActiveRecord
 
diff --git a/lib/gitlab/pagination/offset_header_builder.rb b/lib/gitlab/pagination/offset_header_builder.rb
index 32089e40932..555f0e5a607 100644
--- a/lib/gitlab/pagination/offset_header_builder.rb
+++ b/lib/gitlab/pagination/offset_header_builder.rb
@@ -5,9 +5,9 @@ module Gitlab
     class OffsetHeaderBuilder
       attr_reader :request_context, :per_page, :page, :next_page, :prev_page, :total, :total_pages
 
-      delegate :params, :header, :request, to: :request_context
+      delegate :request, to: :request_context
 
-      def initialize(request_context:, per_page:, page:, next_page:, prev_page: nil, total:, total_pages:)
+      def initialize(request_context:, per_page:, page:, next_page:, prev_page: nil, total: nil, total_pages: nil, params: nil)
         @request_context = request_context
         @per_page = per_page
         @page = page
@@ -15,6 +15,7 @@ module Gitlab
         @prev_page = prev_page
         @total = total
         @total_pages = total_pages
+        @params = params
       end
 
       def execute(exclude_total_headers: false, data_without_counts: false)
@@ -56,10 +57,24 @@ module Gitlab
       end
 
       def page_href(next_page_params = {})
-        query_params = params.merge(**next_page_params, per_page: params[:per_page]).to_query
+        query_params = params.merge(**next_page_params, per_page: per_page).to_query
 
         build_page_url(query_params: query_params)
       end
+
+      def params
+        @params || request_context.params
+      end
+
+      def header(name, value)
+        if request_context.respond_to?(:header)
+          # For Grape API
+          request_context.header(name, value)
+        else
+          # For rails controllers
+          request_context.response.headers[name] = value
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/performance_bar/stats.rb b/lib/gitlab/performance_bar/stats.rb
index 380340b80be..c2a4602fd16 100644
--- a/lib/gitlab/performance_bar/stats.rb
+++ b/lib/gitlab/performance_bar/stats.rb
@@ -5,6 +5,12 @@ module Gitlab
     # This class fetches Peek stats stored in redis and logs them in a
     # structured log (so these can be then analyzed in Kibana)
     class Stats
+      IGNORED_BACKTRACE_LOCATIONS = %w[
+        ee/lib/ee/peek
+        lib/peek
+        lib/gitlab/database
+      ].freeze
+
       def initialize(redis)
         @redis = redis
       end
@@ -53,7 +59,8 @@ module Gitlab
       end
 
       def parse_backtrace(backtrace)
-        return unless match = /(?.*):(?\d+):in `(?.*)'/.match(backtrace.first)
+        return unless backtrace_row = find_caller(backtrace)
+        return unless match = /(?.*):(?\d+):in `(?.*)'/.match(backtrace_row)
 
         {
           filename: match[:filename],
@@ -65,6 +72,12 @@ module Gitlab
         }
       end
 
+      def find_caller(backtrace)
+        backtrace.find do |line|
+          !line.start_with?(*IGNORED_BACKTRACE_LOCATIONS)
+        end
+      end
+
       def logger
         @logger ||= Gitlab::PerformanceBar::Logger.build
       end
diff --git a/lib/gitlab/phabricator_import.rb b/lib/gitlab/phabricator_import.rb
index 3885a9934d5..4c9d54a93ce 100644
--- a/lib/gitlab/phabricator_import.rb
+++ b/lib/gitlab/phabricator_import.rb
@@ -5,7 +5,7 @@ module Gitlab
     BaseError = Class.new(StandardError)
 
     def self.available?
-      Feature.enabled?(:phabricator_import) &&
+      Feature.enabled?(:phabricator_import, default_enabled: :yaml) &&
         Gitlab::CurrentSettings.import_sources.include?('phabricator')
     end
   end
diff --git a/lib/gitlab/phabricator_import/issues/importer.rb b/lib/gitlab/phabricator_import/issues/importer.rb
index a58438452ff..478c26af030 100644
--- a/lib/gitlab/phabricator_import/issues/importer.rb
+++ b/lib/gitlab/phabricator_import/issues/importer.rb
@@ -4,7 +4,8 @@ module Gitlab
     module Issues
       class Importer
         def initialize(project, after = nil)
-          @project, @after = project, after
+          @project = project
+          @after = after
         end
 
         def execute
diff --git a/lib/gitlab/phabricator_import/issues/task_importer.rb b/lib/gitlab/phabricator_import/issues/task_importer.rb
index c17f3e1729a..9c419ecb700 100644
--- a/lib/gitlab/phabricator_import/issues/task_importer.rb
+++ b/lib/gitlab/phabricator_import/issues/task_importer.rb
@@ -4,7 +4,8 @@ module Gitlab
     module Issues
       class TaskImporter
         def initialize(project, task)
-          @project, @task = project, task
+          @project = project
+          @task = task
         end
 
         def execute
diff --git a/lib/gitlab/phabricator_import/project_creator.rb b/lib/gitlab/phabricator_import/project_creator.rb
index b37a5b44980..c842798ca74 100644
--- a/lib/gitlab/phabricator_import/project_creator.rb
+++ b/lib/gitlab/phabricator_import/project_creator.rb
@@ -55,12 +55,13 @@ module Gitlab
       end
 
       def project_feature_attributes
-        @project_features_attributes ||= begin
-                                           # everything disabled except for issues
-                                           ProjectFeature::FEATURES.map do |feature|
-                                             [ProjectFeature.access_level_attribute(feature), ProjectFeature::DISABLED]
-                                           end.to_h.merge(ProjectFeature.access_level_attribute(:issues) => ProjectFeature::ENABLED)
-                                         end
+        @project_features_attributes ||=
+          begin
+            # everything disabled except for issues
+            ProjectFeature::FEATURES.to_h do |feature|
+              [ProjectFeature.access_level_attribute(feature), ProjectFeature::DISABLED]
+            end.merge(ProjectFeature.access_level_attribute(:issues) => ProjectFeature::ENABLED)
+          end
       end
 
       def import_data
diff --git a/lib/gitlab/phabricator_import/user_finder.rb b/lib/gitlab/phabricator_import/user_finder.rb
index 4b50431e0e0..c6058d12527 100644
--- a/lib/gitlab/phabricator_import/user_finder.rb
+++ b/lib/gitlab/phabricator_import/user_finder.rb
@@ -4,7 +4,8 @@ module Gitlab
   module PhabricatorImport
     class UserFinder
       def initialize(project, phids)
-        @project, @phids = project, phids
+        @project = project
+        @phids = phids
         @loaded_phids = Set.new
       end
 
diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb
index 56eeea6e746..32d3eeb8cd2 100644
--- a/lib/gitlab/project_template.rb
+++ b/lib/gitlab/project_template.rb
@@ -5,7 +5,11 @@ module Gitlab
     attr_reader :title, :name, :description, :preview, :logo
 
     def initialize(name, title, description, preview, logo = 'illustrations/gitlab_logo.svg')
-      @name, @title, @description, @preview, @logo = name, title, description, preview, logo
+      @name = name
+      @title = title
+      @description = description
+      @preview = preview
+      @logo = logo
     end
 
     def file
diff --git a/lib/gitlab/prometheus/adapter.rb b/lib/gitlab/prometheus/adapter.rb
index ed10ef2917f..76e65d29c7a 100644
--- a/lib/gitlab/prometheus/adapter.rb
+++ b/lib/gitlab/prometheus/adapter.rb
@@ -19,6 +19,10 @@ module Gitlab
       end
 
       def cluster_prometheus_adapter
+        if cluster&.integration_prometheus
+          return cluster.integration_prometheus
+        end
+
         application = cluster&.application_prometheus
 
         application if application&.available?
diff --git a/lib/gitlab/prometheus/queries/matched_metric_query.rb b/lib/gitlab/prometheus/queries/matched_metric_query.rb
index e4d44df3baf..73de5a11998 100644
--- a/lib/gitlab/prometheus/queries/matched_metric_query.rb
+++ b/lib/gitlab/prometheus/queries/matched_metric_query.rb
@@ -4,7 +4,7 @@ module Gitlab
   module Prometheus
     module Queries
       class MatchedMetricQuery < BaseQuery
-        MAX_QUERY_ITEMS = 40.freeze
+        MAX_QUERY_ITEMS = 40
 
         def query
           groups_data.map do |group, data|
diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb
index 965349ad711..0fcf63d03fc 100644
--- a/lib/gitlab/prometheus_client.rb
+++ b/lib/gitlab/prometheus_client.rb
@@ -140,7 +140,7 @@ module Gitlab
     end
 
     def mapped_options
-      options.keys.map { |k| [gitlab_http_key(k), options[k]] }.to_h
+      options.keys.to_h { |k| [gitlab_http_key(k), options[k]] }
     end
 
     def http_options
diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb
index 02446a7953b..ce9fced9465 100644
--- a/lib/gitlab/push_options.rb
+++ b/lib/gitlab/push_options.rb
@@ -5,6 +5,7 @@ module Gitlab
     VALID_OPTIONS = HashWithIndifferentAccess.new({
       merge_request: {
         keys: [
+          :assign,
           :create,
           :description,
           :label,
@@ -12,6 +13,7 @@ module Gitlab
           :remove_source_branch,
           :target,
           :title,
+          :unassign,
           :unlabel
         ]
       },
@@ -23,7 +25,9 @@ module Gitlab
     MULTI_VALUE_OPTIONS = [
       %w[ci variable],
       %w[merge_request label],
-      %w[merge_request unlabel]
+      %w[merge_request unlabel],
+      %w[merge_request assign],
+      %w[merge_request unassign]
     ].freeze
 
     NAMESPACE_ALIASES = HashWithIndifferentAccess.new({
diff --git a/lib/gitlab/query_limiting.rb b/lib/gitlab/query_limiting.rb
index 5e46e26e14e..03386dca141 100644
--- a/lib/gitlab/query_limiting.rb
+++ b/lib/gitlab/query_limiting.rb
@@ -6,28 +6,36 @@ module Gitlab
     #
     # This is only enabled in development and test to ensure we don't produce
     # any errors that users of other environments can't do anything about themselves.
-    def self.enable?
+    def self.enabled_for_env?
       Rails.env.development? || Rails.env.test?
     end
 
+    def self.enabled?
+      enabled_for_env? &&
+        !Gitlab::SafeRequestStore[:query_limiting_disabled]
+    end
+
     # Allows the current request to execute any number of SQL queries.
     #
     # This method should _only_ be used when there's a corresponding issue to
     # reduce the number of queries.
     #
     # The issue URL is only meant to push developers into creating an issue
-    # instead of blindly whitelisting offending blocks of code.
-    def self.whitelist(issue_url)
-      return unless enable?
-
+    # instead of blindly disabling for offending blocks of code.
+    def self.disable!(issue_url)
       unless issue_url.start_with?('https://')
         raise(
           ArgumentError,
-          'You must provide a valid issue URL in order to whitelist a block of code'
+          'You must provide a valid issue URL in order to allow a block of code'
         )
       end
 
-      Transaction&.current&.whitelisted = true
+      Gitlab::SafeRequestStore[:query_limiting_disabled] = true
+    end
+
+    # Enables query limiting for the request.
+    def self.enable!
+      Gitlab::SafeRequestStore[:query_limiting_disabled] = nil
     end
   end
 end
diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb
index 196072dddda..643b2540c37 100644
--- a/lib/gitlab/query_limiting/transaction.rb
+++ b/lib/gitlab/query_limiting/transaction.rb
@@ -5,7 +5,7 @@ module Gitlab
     class Transaction
       THREAD_KEY = :__gitlab_query_counts_transaction
 
-      attr_accessor :count, :whitelisted
+      attr_accessor :count
 
       # The name of the action (e.g. `UsersController#show`) that is being
       # executed.
@@ -45,7 +45,6 @@ module Gitlab
       def initialize
         @action = nil
         @count = 0
-        @whitelisted = false
         @sql_executed = []
       end
 
@@ -59,7 +58,7 @@ module Gitlab
       end
 
       def increment
-        @count += 1 unless whitelisted
+        @count += 1 if enabled?
       end
 
       def executed_sql(sql)
@@ -83,6 +82,10 @@ module Gitlab
 
         ["#{header}: #{msg}", log, ellipsis].compact.join("\n")
       end
+
+      def enabled?
+        ::Gitlab::QueryLimiting.enabled?
+      end
     end
   end
 end
diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb
index b17a0208f95..8ce13db4c03 100644
--- a/lib/gitlab/quick_actions/command_definition.rb
+++ b/lib/gitlab/quick_actions/command_definition.rb
@@ -56,15 +56,18 @@ module Gitlab
       end
 
       def execute(context, arg)
-        return unless executable?(context)
+        return if noop?
 
         count_commands_executed_in(context)
 
+        return unless available?(context)
+
         execute_block(action_block, context, arg)
       end
 
       def execute_message(context, arg)
-        return unless executable?(context)
+        return if noop?
+        return _('Could not apply %{name} command.') % { name: name } unless available?(context)
 
         if execution_message.respond_to?(:call)
           execute_block(execution_message, context, arg)
@@ -101,10 +104,6 @@ module Gitlab
 
       private
 
-      def executable?(context)
-        !noop? && available?(context)
-      end
-
       def count_commands_executed_in(context)
         return unless context.respond_to?(:commands_executed_count=)
 
diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb
index 4934c12a339..b7d58e05651 100644
--- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb
+++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb
@@ -182,7 +182,7 @@ module Gitlab
         parse_params do |raw_time_date|
           Gitlab::QuickActions::SpendTimeAndDateSeparator.new(raw_time_date).execute
         end
-        command :spend do |time_spent, time_spent_date|
+        command :spend, :spent do |time_spent, time_spent_date|
           if time_spent
             @updates[:spend_time] = {
               duration: time_spent,
diff --git a/lib/gitlab/rack_attack/request.rb b/lib/gitlab/rack_attack/request.rb
index 67e3a5de223..bd6d2e016b4 100644
--- a/lib/gitlab/rack_attack/request.rb
+++ b/lib/gitlab/rack_attack/request.rb
@@ -34,12 +34,16 @@ module Gitlab
         path =~ %r{^/-/(health|liveness|readiness|metrics)}
       end
 
+      def container_registry_event?
+        path =~ %r{^/api/v\d+/container_registry_event/}
+      end
+
       def product_analytics_collector_request?
         path.start_with?('/-/collector/i')
       end
 
       def should_be_skipped?
-        api_internal_request? || health_check_request?
+        api_internal_request? || health_check_request? || container_registry_event?
       end
 
       def web_request?
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 00739c05386..488ba04f87c 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -181,7 +181,7 @@ module Gitlab
       end
 
       def generic_package_version_regex
-        /\A\d+\.\d+\.\d+\z/
+        maven_version_regex
       end
 
       def generic_package_name_regex
@@ -385,11 +385,11 @@ module Gitlab
     end
 
     def merge_request_wip
-      /(?i)(\[WIP\]\s*|WIP:\s*|WIP$)/
+      /(?i)(\[WIP\]\s*|WIP:\s*|\AWIP\z)/
     end
 
     def merge_request_draft
-      /(?i)(\[draft\]|\(draft\)|draft:|draft\s\-\s|draft$)/
+      /\A(?i)(\[draft\]|\(draft\)|draft:|draft\s\-\s|draft\z)/
     end
 
     def issue
diff --git a/lib/gitlab/relative_positioning/closed_range.rb b/lib/gitlab/relative_positioning/closed_range.rb
index 8916d1face5..11fba05edee 100644
--- a/lib/gitlab/relative_positioning/closed_range.rb
+++ b/lib/gitlab/relative_positioning/closed_range.rb
@@ -4,7 +4,8 @@ module Gitlab
   module RelativePositioning
     class ClosedRange < RelativePositioning::Range
       def initialize(lhs, rhs)
-        @lhs, @rhs = lhs, rhs
+        @lhs = lhs
+        @rhs = rhs
         raise IllegalRange, 'Either lhs or rhs is missing' unless lhs && rhs
         raise IllegalRange, 'lhs and rhs cannot be the same object' if lhs == rhs
       end
diff --git a/lib/gitlab/relative_positioning/gap.rb b/lib/gitlab/relative_positioning/gap.rb
index ab894141a60..2e30e598eb0 100644
--- a/lib/gitlab/relative_positioning/gap.rb
+++ b/lib/gitlab/relative_positioning/gap.rb
@@ -6,7 +6,8 @@ module Gitlab
       attr_reader :start_pos, :end_pos
 
       def initialize(start_pos, end_pos)
-        @start_pos, @end_pos = start_pos, end_pos
+        @start_pos = start_pos
+        @end_pos = end_pos
       end
 
       def ==(other)
diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb
index eb7c9bccf96..d0230c035cc 100644
--- a/lib/gitlab/repository_cache_adapter.rb
+++ b/lib/gitlab/repository_cache_adapter.rb
@@ -60,14 +60,17 @@ module Gitlab
         define_method("#{name}_include?") do |value|
           ivar = "@#{name}_include"
           memoized = instance_variable_get(ivar) || {}
+          lookup = proc { __send__(name).include?(value) } # rubocop:disable GitlabSecurity/PublicSend
 
           next memoized[value] if memoized.key?(value)
 
           memoized[value] =
-            if strong_memoized?(name) || !redis_set_cache.exist?(name)
-              __send__(name).include?(value) # rubocop:disable GitlabSecurity/PublicSend
+            if strong_memoized?(name)
+              lookup.call
             else
-              redis_set_cache.include?(name, value)
+              result, exists = redis_set_cache.try_include?(name, value)
+
+              exists ? result : lookup.call
             end
 
           instance_variable_set(ivar, memoized)[value]
diff --git a/lib/gitlab/repository_hash_cache.rb b/lib/gitlab/repository_hash_cache.rb
index d479d3115a6..430f3e8d162 100644
--- a/lib/gitlab/repository_hash_cache.rb
+++ b/lib/gitlab/repository_hash_cache.rb
@@ -148,7 +148,7 @@ module Gitlab
     # @param hash [Hash]
     # @return [Hash] the stringified hash
     def standardize_hash(hash)
-      hash.map { |k, v| [k.to_s, v.to_s] }.to_h
+      hash.to_h { |k, v| [k.to_s, v.to_s] }
     end
 
     # Record metrics in Prometheus.
diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb
index 69c1688767c..f73ac628bce 100644
--- a/lib/gitlab/repository_set_cache.rb
+++ b/lib/gitlab/repository_set_cache.rb
@@ -36,10 +36,32 @@ module Gitlab
     end
 
     def fetch(key, &block)
-      if exist?(key)
-        read(key)
-      else
-        write(key, yield)
+      full_key = cache_key(key)
+
+      smembers, exists = with do |redis|
+        redis.multi do
+          redis.smembers(full_key)
+          redis.exists(full_key)
+        end
+      end
+
+      return smembers if exists
+
+      write(key, yield)
+    end
+
+    # Searches the cache set using SSCAN with the MATCH option. The MATCH
+    # parameter is the pattern argument.
+    # See https://redis.io/commands/scan#the-match-option for more information.
+    # Returns an Enumerator that enumerates all SSCAN hits.
+    def search(key, pattern, &block)
+      full_key = cache_key(key)
+
+      with do |redis|
+        exists = redis.exists(full_key)
+        write(key, yield) unless exists
+
+        redis.sscan_each(full_key, match: pattern)
       end
     end
   end
diff --git a/lib/gitlab/search_context.rb b/lib/gitlab/search_context.rb
index c3bb0ff26f2..0323220690a 100644
--- a/lib/gitlab/search_context.rb
+++ b/lib/gitlab/search_context.rb
@@ -129,7 +129,10 @@ module Gitlab
           'wiki_blobs'
         elsif view_context.current_controller?(:commits)
           'commits'
-        else nil
+        elsif view_context.current_controller?(:groups)
+          if %w(issues merge_requests).include?(view_context.controller.action_name)
+            view_context.controller.action_name
+          end
         end
       end
     end
@@ -160,3 +163,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::SearchContext::Builder.prepend_ee_mod
diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb
index 591265d014e..0f2b7b194c9 100644
--- a/lib/gitlab/set_cache.rb
+++ b/lib/gitlab/set_cache.rb
@@ -51,6 +51,19 @@ module Gitlab
       with { |redis| redis.sismember(cache_key(key), value) }
     end
 
+    # Like include?, but also tells us if the cache was populated when it ran
+    # by returning two booleans: [member_exists, set_exists]
+    def try_include?(key, value)
+      full_key = cache_key(key)
+
+      with do |redis|
+        redis.multi do
+          redis.sismember(full_key, value)
+          redis.exists(full_key)
+        end
+      end
+    end
+
     def ttl(key)
       with { |redis| redis.ttl(cache_key(key)) }
     end
diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb
index 7561e36cc33..3ac20724403 100644
--- a/lib/gitlab/setup_helper.rb
+++ b/lib/gitlab/setup_helper.rb
@@ -98,6 +98,10 @@ module Gitlab
 
           if Rails.env.test?
             socket_filename = options[:gitaly_socket] || "gitaly.socket"
+            prometheus_listen_addr = options[:prometheus_listen_addr]
+
+            git_bin_path = File.expand_path('../gitaly/_build/deps/git/install/bin/git')
+            git_bin_path = nil unless File.exist?(git_bin_path)
 
             config = {
               # Override the set gitaly_address since Praefect is in the loop
@@ -106,8 +110,12 @@ module Gitlab
               # Compared to production, tests run in constrained environments. This
               # number is meant to grow with the number of concurrent rails requests /
               # sidekiq jobs, and concurrency will be low anyway in test.
-              git: { catfile_cache_size: 5 }
-            }
+              git: {
+                catfile_cache_size: 5,
+                bin_path: git_bin_path
+              }.compact,
+              prometheus_listen_addr: prometheus_listen_addr
+            }.compact
 
             storage_path = Rails.root.join('tmp', 'tests', 'second_storage').to_s
             storages << { name: 'test_second_storage', path: storage_path }
diff --git a/lib/gitlab/sidekiq_cluster/cli.rb b/lib/gitlab/sidekiq_cluster/cli.rb
index e471517c50a..9490d543dd1 100644
--- a/lib/gitlab/sidekiq_cluster/cli.rb
+++ b/lib/gitlab/sidekiq_cluster/cli.rb
@@ -53,11 +53,11 @@ module Gitlab
             'You cannot specify --queue-selector and --experimental-queue-selector together'
         end
 
-        all_queues = SidekiqConfig::CliMethods.all_queues(@rails_path)
-        queue_names = SidekiqConfig::CliMethods.worker_queues(@rails_path)
+        worker_metadatas = SidekiqConfig::CliMethods.worker_metadatas(@rails_path)
+        worker_queues = SidekiqConfig::CliMethods.worker_queues(@rails_path)
 
-        queue_groups = argv.map do |queues|
-          next queue_names if queues == '*'
+        queue_groups = argv.map do |queues_or_query_string|
+          next worker_queues if queues_or_query_string == SidekiqConfig::WorkerMatcher::WILDCARD_MATCH
 
           # When using the queue query syntax, we treat each queue group
           # as a worker attribute query, and resolve the queues for the
@@ -65,14 +65,14 @@ module Gitlab
 
           # Simplify with https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/646
           if @queue_selector || @experimental_queue_selector
-            SidekiqConfig::CliMethods.query_workers(queues, all_queues)
+            SidekiqConfig::CliMethods.query_queues(queues_or_query_string, worker_metadatas)
           else
-            SidekiqConfig::CliMethods.expand_queues(queues.split(','), queue_names)
+            SidekiqConfig::CliMethods.expand_queues(queues_or_query_string.split(','), worker_queues)
           end
         end
 
         if @negate_queues
-          queue_groups.map! { |queues| queue_names - queues }
+          queue_groups.map! { |queues| worker_queues - queues }
         end
 
         if queue_groups.all?(&:empty?)
diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb
index 633291dcdf3..78d45b5f3f0 100644
--- a/lib/gitlab/sidekiq_config.rb
+++ b/lib/gitlab/sidekiq_config.rb
@@ -13,10 +13,17 @@ module Gitlab
       (EE_QUEUE_CONFIG_PATH if Gitlab.ee?)
     ].compact.freeze
 
-    DEFAULT_WORKERS = [
-      DummyWorker.new('default', weight: 1, tags: []),
-      DummyWorker.new('mailers', weight: 2, tags: [])
-    ].map { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false) }.freeze
+    # This maps workers not in our application code to queues. We need
+    # these queues in our YAML files to ensure we don't accidentally
+    # miss jobs from these queues.
+    #
+    # The default queue should be unused, which is why it maps to an
+    # invalid class name. We keep it in the YAML file for safety, just
+    # in case anything does get scheduled to run there.
+    DEFAULT_WORKERS = {
+      '_' => DummyWorker.new('default', weight: 1, tags: []),
+      'ActionMailer::MailDeliveryJob' => DummyWorker.new('mailers', feature_category: :issue_tracking, urgency: 'low', weight: 2, tags: [])
+    }.transform_values { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false) }.freeze
 
     class << self
       include Gitlab::SidekiqConfig::CliMethods
@@ -40,7 +47,7 @@ module Gitlab
       def workers
         @workers ||= begin
           result = []
-          result.concat(DEFAULT_WORKERS)
+          result.concat(DEFAULT_WORKERS.values)
           result.concat(find_workers(Rails.root.join('app', 'workers'), ee: false))
 
           if Gitlab.ee?
diff --git a/lib/gitlab/sidekiq_config/cli_methods.rb b/lib/gitlab/sidekiq_config/cli_methods.rb
index a256632bc12..8eef15f9ccb 100644
--- a/lib/gitlab/sidekiq_config/cli_methods.rb
+++ b/lib/gitlab/sidekiq_config/cli_methods.rb
@@ -12,35 +12,19 @@ module Gitlab
       # rubocop:disable Gitlab/ModuleWithInstanceVariables
       extend self
 
+      # The file names are misleading. Those files contain the metadata of the
+      # workers. They should be renamed to all_workers instead.
+      # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1018
       QUEUE_CONFIG_PATHS = begin
         result = %w[app/workers/all_queues.yml]
         result << 'ee/app/workers/all_queues.yml' if Gitlab.ee?
         result
       end.freeze
 
-      QUERY_OR_OPERATOR = '|'
-      QUERY_AND_OPERATOR = '&'
-      QUERY_CONCATENATE_OPERATOR = ','
-      QUERY_TERM_REGEX = %r{^(\w+)(!?=)([\w:#{QUERY_CONCATENATE_OPERATOR}]+)}.freeze
+      def worker_metadatas(rails_path = Rails.root.to_s)
+        @worker_metadatas ||= {}
 
-      QUERY_PREDICATES = {
-        feature_category: :to_sym,
-        has_external_dependencies: lambda { |value| value == 'true' },
-        name: :to_s,
-        resource_boundary: :to_sym,
-        tags: :to_sym,
-        urgency: :to_sym
-      }.freeze
-
-      QueryError = Class.new(StandardError)
-      InvalidTerm = Class.new(QueryError)
-      UnknownOperator = Class.new(QueryError)
-      UnknownPredicate = Class.new(QueryError)
-
-      def all_queues(rails_path = Rails.root.to_s)
-        @worker_queues ||= {}
-
-        @worker_queues[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path|
+        @worker_metadatas[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path|
           full_path = File.join(rails_path, path)
 
           File.exist?(full_path) ? YAML.load_file(full_path) : []
@@ -49,7 +33,7 @@ module Gitlab
       # rubocop:enable Gitlab/ModuleWithInstanceVariables
 
       def worker_queues(rails_path = Rails.root.to_s)
-        worker_names(all_queues(rails_path))
+        worker_names(worker_metadatas(rails_path))
       end
 
       def expand_queues(queues, all_queues = self.worker_queues)
@@ -62,13 +46,18 @@ module Gitlab
         end
       end
 
-      def query_workers(query_string, queues)
-        worker_names(queues.select(&query_string_to_lambda(query_string)))
+      def query_queues(query_string, worker_metadatas)
+        matcher = SidekiqConfig::WorkerMatcher.new(query_string)
+        selected_metadatas = worker_metadatas.select do |worker_metadata|
+          matcher.match?(worker_metadata)
+        end
+
+        worker_names(selected_metadatas)
       end
 
       def clear_memoization!
-        if instance_variable_defined?('@worker_queues')
-          remove_instance_variable('@worker_queues')
+        if instance_variable_defined?('@worker_metadatas')
+          remove_instance_variable('@worker_metadatas')
         end
       end
 
@@ -77,53 +66,6 @@ module Gitlab
       def worker_names(workers)
         workers.map { |queue| queue[:name] }
       end
-
-      def query_string_to_lambda(query_string)
-        or_clauses = query_string.split(QUERY_OR_OPERATOR).map do |and_clauses_string|
-          and_clauses_predicates = and_clauses_string.split(QUERY_AND_OPERATOR).map do |term|
-            predicate_for_term(term)
-          end
-
-          lambda { |worker| and_clauses_predicates.all? { |predicate| predicate.call(worker) } }
-        end
-
-        lambda { |worker| or_clauses.any? { |predicate| predicate.call(worker) } }
-      end
-
-      def predicate_for_term(term)
-        match = term.match(QUERY_TERM_REGEX)
-
-        raise InvalidTerm.new("Invalid term: #{term}") unless match
-
-        _, lhs, op, rhs = *match
-
-        predicate_for_op(op, predicate_factory(lhs, rhs.split(QUERY_CONCATENATE_OPERATOR)))
-      end
-
-      def predicate_for_op(op, predicate)
-        case op
-        when '='
-          predicate
-        when '!='
-          lambda { |worker| !predicate.call(worker) }
-        else
-          # This is unreachable because InvalidTerm will be raised instead, but
-          # keeping it allows to guard against that changing in future.
-          raise UnknownOperator.new("Unknown operator: #{op}")
-        end
-      end
-
-      def predicate_factory(lhs, values)
-        values_block = QUERY_PREDICATES[lhs.to_sym]
-
-        raise UnknownPredicate.new("Unknown predicate: #{lhs}") unless values_block
-
-        lambda do |queue|
-          comparator = Array(queue[lhs.to_sym]).to_set
-
-          values.map(&values_block).to_set.intersect?(comparator)
-        end
-      end
     end
   end
 end
diff --git a/lib/gitlab/sidekiq_config/worker_matcher.rb b/lib/gitlab/sidekiq_config/worker_matcher.rb
new file mode 100644
index 00000000000..fe5ac10c65a
--- /dev/null
+++ b/lib/gitlab/sidekiq_config/worker_matcher.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqConfig
+    class WorkerMatcher
+      WILDCARD_MATCH = '*'
+      QUERY_OR_OPERATOR = '|'
+      QUERY_AND_OPERATOR = '&'
+      QUERY_CONCATENATE_OPERATOR = ','
+      QUERY_TERM_REGEX = %r{^(\w+)(!?=)([\w:#{QUERY_CONCATENATE_OPERATOR}]+)}.freeze
+
+      QUERY_PREDICATES = {
+        feature_category: :to_sym,
+        has_external_dependencies: lambda { |value| value == 'true' },
+        name: :to_s,
+        resource_boundary: :to_sym,
+        tags: :to_sym,
+        urgency: :to_sym
+      }.freeze
+
+      QueryError = Class.new(StandardError)
+      InvalidTerm = Class.new(QueryError)
+      UnknownOperator = Class.new(QueryError)
+      UnknownPredicate = Class.new(QueryError)
+
+      def initialize(query_string)
+        @match_lambda = query_string_to_lambda(query_string)
+      end
+
+      def match?(worker_metadata)
+        @match_lambda.call(worker_metadata)
+      end
+
+      private
+
+      def query_string_to_lambda(query_string)
+        return lambda { |_worker| true } if query_string.strip == WILDCARD_MATCH
+
+        or_clauses = query_string.split(QUERY_OR_OPERATOR).map do |and_clauses_string|
+          and_clauses_predicates = and_clauses_string.split(QUERY_AND_OPERATOR).map do |term|
+            predicate_for_term(term)
+          end
+
+          lambda { |worker| and_clauses_predicates.all? { |predicate| predicate.call(worker) } }
+        end
+
+        lambda { |worker| or_clauses.any? { |predicate| predicate.call(worker) } }
+      end
+
+      def predicate_for_term(term)
+        match = term.match(QUERY_TERM_REGEX)
+
+        raise InvalidTerm.new("Invalid term: #{term}") unless match
+
+        _, lhs, op, rhs = *match
+
+        predicate_for_op(op, predicate_factory(lhs, rhs.split(QUERY_CONCATENATE_OPERATOR)))
+      end
+
+      def predicate_for_op(op, predicate)
+        case op
+        when '='
+          predicate
+        when '!='
+          lambda { |worker| !predicate.call(worker) }
+        else
+          # This is unreachable because InvalidTerm will be raised instead, but
+          # keeping it allows to guard against that changing in future.
+          raise UnknownOperator.new("Unknown operator: #{op}")
+        end
+      end
+
+      def predicate_factory(lhs, values)
+        values_block = QUERY_PREDICATES[lhs.to_sym]
+
+        raise UnknownPredicate.new("Unknown predicate: #{lhs}") unless values_block
+
+        lambda do |queue|
+          comparator = Array(queue[lhs.to_sym]).to_set
+
+          values.map(&values_block).to_set.intersect?(comparator)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb
index 654b17c5740..b1fb3771c78 100644
--- a/lib/gitlab/sidekiq_logging/structured_logger.rb
+++ b/lib/gitlab/sidekiq_logging/structured_logger.rb
@@ -39,9 +39,7 @@ module Gitlab
       private
 
       def add_instrumentation_keys!(job, output_payload)
-        instrumentation_values = job.slice(*::Gitlab::InstrumentationHelper.keys).stringify_keys
-
-        output_payload.merge!(instrumentation_values)
+        output_payload.merge!(job[:instrumentation].stringify_keys)
       end
 
       def add_logging_extras!(job, output_payload)
diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb
index a2696e17078..563a105484d 100644
--- a/lib/gitlab/sidekiq_middleware.rb
+++ b/lib/gitlab/sidekiq_middleware.rb
@@ -43,3 +43,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::SidekiqMiddleware.singleton_class.prepend_if_ee('EE::Gitlab::SidekiqMiddleware')
diff --git a/lib/gitlab/sidekiq_middleware/admin_mode/client.rb b/lib/gitlab/sidekiq_middleware/admin_mode/client.rb
index 36204e1bee0..1b33743a0e9 100644
--- a/lib/gitlab/sidekiq_middleware/admin_mode/client.rb
+++ b/lib/gitlab/sidekiq_middleware/admin_mode/client.rb
@@ -8,7 +8,8 @@ module Gitlab
       # If enabled then it injects a job field that persists through the job execution
       class Client
         def call(_worker_class, job, _queue, _redis_pool)
-          return yield unless ::Feature.enabled?(:user_mode_in_session)
+          # Not calling Gitlab::CurrentSettings.admin_mode on purpose on sidekiq middleware
+          # Only when admin mode application setting is enabled might the admin_mode_user_id be non-nil here
 
           # Admin mode enabled in the original request or in a nested sidekiq job
           admin_mode_user_id = find_admin_user_id
diff --git a/lib/gitlab/sidekiq_middleware/admin_mode/server.rb b/lib/gitlab/sidekiq_middleware/admin_mode/server.rb
index 6366867a0fa..c4e64705d6e 100644
--- a/lib/gitlab/sidekiq_middleware/admin_mode/server.rb
+++ b/lib/gitlab/sidekiq_middleware/admin_mode/server.rb
@@ -5,7 +5,8 @@ module Gitlab
     module AdminMode
       class Server
         def call(_worker, job, _queue)
-          return yield unless Feature.enabled?(:user_mode_in_session)
+          # Not calling Gitlab::CurrentSettings.admin_mode on purpose on sidekiq middleware
+          # Only when admin_mode setting is enabled can it be true here
 
           admin_mode_user_id = job['admin_mode_user_id']
 
diff --git a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb
index a66a4de4655..b542aa4fe4c 100644
--- a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb
+++ b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb
@@ -3,6 +3,24 @@
 module Gitlab
   module SidekiqMiddleware
     class InstrumentationLogger
+      def self.keys
+        @keys ||= [
+          :cpu_s,
+          :gitaly_calls,
+          :gitaly_duration_s,
+          :rugged_calls,
+          :rugged_duration_s,
+          :elasticsearch_calls,
+          :elasticsearch_duration_s,
+          :elasticsearch_timed_out_count,
+          *::Gitlab::Memory::Instrumentation::KEY_MAPPING.values,
+          *::Gitlab::Instrumentation::Redis.known_payload_keys,
+          *::Gitlab::Metrics::Subscribers::ActiveRecord.known_payload_keys,
+          *::Gitlab::Metrics::Subscribers::ExternalHttp::KNOWN_PAYLOAD_KEYS,
+          *::Gitlab::Metrics::Subscribers::RackAttack::PAYLOAD_KEYS
+        ]
+      end
+
       def call(worker, job, queue)
         ::Gitlab::InstrumentationHelper.init_instrumentation_data
 
@@ -17,7 +35,10 @@ module Gitlab
         # because Sidekiq keeps a pristine copy of the original hash
         # before sending it to the middleware:
         # https://github.com/mperham/sidekiq/blob/53bd529a0c3f901879925b8390353129c465b1f2/lib/sidekiq/processor.rb#L115-L118
-        ::Gitlab::InstrumentationHelper.add_instrumentation_data(job)
+        job[:instrumentation] = {}.tap do |instrumentation_values|
+          ::Gitlab::InstrumentationHelper.add_instrumentation_data(instrumentation_values)
+          instrumentation_values.slice!(*self.class.keys)
+        end
       end
     end
   end
diff --git a/lib/gitlab/sidekiq_middleware/metrics_helper.rb b/lib/gitlab/sidekiq_middleware/metrics_helper.rb
index 60e79ee1188..66930a34319 100644
--- a/lib/gitlab/sidekiq_middleware/metrics_helper.rb
+++ b/lib/gitlab/sidekiq_middleware/metrics_helper.rb
@@ -10,6 +10,7 @@ module Gitlab
 
       def create_labels(worker_class, queue, job)
         worker_name = (job['wrapped'].presence || worker_class).to_s
+        worker = find_worker(worker_name, worker_class)
 
         labels = { queue: queue.to_s,
                    worker: worker_name,
@@ -18,15 +19,15 @@ module Gitlab
                    feature_category: "",
                    boundary: "" }
 
-        return labels unless worker_class && worker_class.include?(WorkerAttributes)
+        return labels unless worker.respond_to?(:get_urgency)
 
-        labels[:urgency] = worker_class.get_urgency.to_s
-        labels[:external_dependencies] = bool_as_label(worker_class.worker_has_external_dependencies?)
+        labels[:urgency] = worker.get_urgency.to_s
+        labels[:external_dependencies] = bool_as_label(worker.worker_has_external_dependencies?)
 
-        feature_category = worker_class.get_feature_category
+        feature_category = worker.get_feature_category
         labels[:feature_category] = feature_category.to_s
 
-        resource_boundary = worker_class.get_worker_resource_boundary
+        resource_boundary = worker.get_worker_resource_boundary
         labels[:boundary] = resource_boundary == :unknown ? "" : resource_boundary.to_s
 
         labels
@@ -35,6 +36,10 @@ module Gitlab
       def bool_as_label(value)
         value ? TRUE_LABEL : FALSE_LABEL
       end
+
+      def find_worker(worker_name, worker_class)
+        Gitlab::SidekiqConfig::DEFAULT_WORKERS.fetch(worker_name, worker_class)
+      end
     end
   end
 end
diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb
index cf768811ffd..f5fee8050ac 100644
--- a/lib/gitlab/sidekiq_middleware/server_metrics.rb
+++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb
@@ -21,6 +21,16 @@ module Gitlab
         Thread.current.name ||= Gitlab::Metrics::Samplers::ThreadsSampler::SIDEKIQ_WORKER_THREAD_NAME
 
         labels = create_labels(worker.class, queue, job)
+        instrument(job, labels) do
+          yield
+        end
+      end
+
+      protected
+
+      attr_reader :metrics
+
+      def instrument(job, labels)
         queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job)
 
         @metrics[:sidekiq_jobs_queue_duration_seconds].observe(labels, queue_duration) if queue_duration
@@ -50,19 +60,18 @@ module Gitlab
 
           # job_status: done, fail match the job_status attribute in structured logging
           labels[:job_status] = job_succeeded ? "done" : "fail"
+          instrumentation = job[:instrumentation] || {}
           @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime)
           @metrics[:sidekiq_jobs_completion_seconds].observe(labels, monotonic_time)
           @metrics[:sidekiq_jobs_db_seconds].observe(labels, ActiveRecord::LogSubscriber.runtime / 1000)
-          @metrics[:sidekiq_jobs_gitaly_seconds].observe(labels, get_gitaly_time(job))
-          @metrics[:sidekiq_redis_requests_total].increment(labels, get_redis_calls(job))
-          @metrics[:sidekiq_redis_requests_duration_seconds].observe(labels, get_redis_time(job))
-          @metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(job))
-          @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(job))
+          @metrics[:sidekiq_jobs_gitaly_seconds].observe(labels, get_gitaly_time(instrumentation))
+          @metrics[:sidekiq_redis_requests_total].increment(labels, get_redis_calls(instrumentation))
+          @metrics[:sidekiq_redis_requests_duration_seconds].observe(labels, get_redis_time(instrumentation))
+          @metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(instrumentation))
+          @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(instrumentation))
         end
       end
 
-      private
-
       def init_metrics
         {
           sidekiq_jobs_cpu_seconds:                ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
@@ -81,29 +90,33 @@ module Gitlab
         }
       end
 
+      private
+
       def get_thread_cputime
         defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0
       end
 
-      def get_redis_time(job)
-        job.fetch(:redis_duration_s, 0)
+      def get_redis_time(payload)
+        payload.fetch(:redis_duration_s, 0)
       end
 
-      def get_redis_calls(job)
-        job.fetch(:redis_calls, 0)
+      def get_redis_calls(payload)
+        payload.fetch(:redis_calls, 0)
       end
 
-      def get_elasticsearch_time(job)
-        job.fetch(:elasticsearch_duration_s, 0)
+      def get_elasticsearch_time(payload)
+        payload.fetch(:elasticsearch_duration_s, 0)
       end
 
-      def get_elasticsearch_calls(job)
-        job.fetch(:elasticsearch_calls, 0)
+      def get_elasticsearch_calls(payload)
+        payload.fetch(:elasticsearch_calls, 0)
       end
 
-      def get_gitaly_time(job)
-        job.fetch(:gitaly_duration_s, 0)
+      def get_gitaly_time(payload)
+        payload.fetch(:gitaly_duration_s, 0)
       end
     end
   end
 end
+
+Gitlab::SidekiqMiddleware::ServerMetrics.prepend_if_ee('EE::Gitlab::SidekiqMiddleware::ServerMetrics')
diff --git a/lib/gitlab/sidekiq_queue.rb b/lib/gitlab/sidekiq_queue.rb
index 807c27a71ff..4b71dfc0c1b 100644
--- a/lib/gitlab/sidekiq_queue.rb
+++ b/lib/gitlab/sidekiq_queue.rb
@@ -21,7 +21,7 @@ module Gitlab
       job_search_metadata =
         search_metadata
           .stringify_keys
-          .slice(*Labkit::Context::KNOWN_KEYS)
+          .slice(*Gitlab::ApplicationContext::KNOWN_KEYS)
           .transform_keys { |key| "meta.#{key}" }
           .compact
 
diff --git a/lib/gitlab/slash_commands/base_command.rb b/lib/gitlab/slash_commands/base_command.rb
index fcc120112f2..e184afa0032 100644
--- a/lib/gitlab/slash_commands/base_command.rb
+++ b/lib/gitlab/slash_commands/base_command.rb
@@ -36,7 +36,9 @@ module Gitlab
       attr_accessor :project, :current_user, :params, :chat_name
 
       def initialize(project, chat_name, params = {})
-        @project, @current_user, @params = project, chat_name.user, params.dup
+        @project = project
+        @current_user = chat_name.user
+        @params = params.dup
         @chat_name = chat_name
       end
 
diff --git a/lib/gitlab/slash_commands/presenters/issue_new.rb b/lib/gitlab/slash_commands/presenters/issue_new.rb
index 552456f5836..8841fef702e 100644
--- a/lib/gitlab/slash_commands/presenters/issue_new.rb
+++ b/lib/gitlab/slash_commands/presenters/issue_new.rb
@@ -12,16 +12,18 @@ module Gitlab
 
         private
 
-        def fallback_message
-          "New issue #{issue.to_reference}: #{issue.title}"
+        def pretext
+          "I created an issue on #{author_profile_link}'s behalf: *#{issue_link}* in #{project_link}"
         end
 
-        def fields_with_markdown
-          %i(title pretext text fields)
+        def issue_link
+          "[#{issue.to_reference}](#{project_issue_url(issue.project, issue)})"
         end
 
-        def pretext
-          "I created an issue on #{author_profile_link}'s behalf: *#{issue.to_reference}* in #{project_link}"
+        def response_message(custom_pretext: pretext)
+          {
+            text: pretext
+          }
         end
       end
     end
diff --git a/lib/gitlab/slash_commands/run.rb b/lib/gitlab/slash_commands/run.rb
index 10a545e28ac..40fd7ee4f20 100644
--- a/lib/gitlab/slash_commands/run.rb
+++ b/lib/gitlab/slash_commands/run.rb
@@ -5,7 +5,7 @@ module Gitlab
     # Slash command for triggering chatops jobs.
     class Run < BaseCommand
       def self.match(text)
-        /\Arun\s+(?\S+)(\s+(?.+))?\z/.match(text)
+        /\Arun\s+(?\S+)(\s+(?.+))?\z/m.match(text)
       end
 
       def self.help_message
diff --git a/lib/gitlab/slug/environment.rb b/lib/gitlab/slug/environment.rb
index 1b87d3bb626..fd70def8e7c 100644
--- a/lib/gitlab/slug/environment.rb
+++ b/lib/gitlab/slug/environment.rb
@@ -26,16 +26,13 @@ module Gitlab
         # Repeated dashes are invalid (OpenShift limitation)
         slugified.squeeze!('-')
 
-        slugified =
-          if slugified.size > 24 || slugified != name
-            # Maximum length: 24 characters (OpenShift limitation)
-            shorten_and_add_suffix(slugified)
-          else
-            # Cannot end with a dash (Kubernetes label limitation)
-            slugified.chomp('-')
-          end
-
-        slugified
+        if slugified.size > 24 || slugified != name
+          # Maximum length: 24 characters (OpenShift limitation)
+          shorten_and_add_suffix(slugified)
+        else
+          # Cannot end with a dash (Kubernetes label limitation)
+          slugified.chomp('-')
+        end
       end
 
       private
diff --git a/lib/gitlab/sql/cte.rb b/lib/gitlab/sql/cte.rb
index 7817a2a1ce2..8f37602aeaa 100644
--- a/lib/gitlab/sql/cte.rb
+++ b/lib/gitlab/sql/cte.rb
@@ -15,20 +15,27 @@ module Gitlab
     #     Namespace
     #       with(cte.to_arel).
     #       from(cte.alias_to(ns))
+    #
+    # To skip materialization of the CTE query by passing materialized: false
+    # More context: https://www.postgresql.org/docs/12/queries-with.html
+    #
+    # cte = CTE.new(:my_cte_name, materialized: false)
+    #
     class CTE
       attr_reader :table, :query
 
       # name - The name of the CTE as a String or Symbol.
-      def initialize(name, query)
+      def initialize(name, query, materialized: true)
         @table = Arel::Table.new(name)
         @query = query
+        @materialized = materialized
       end
 
       # Returns the Arel relation for this CTE.
       def to_arel
         sql = Arel::Nodes::SqlLiteral.new("(#{query.to_sql})")
 
-        Arel::Nodes::As.new(table, sql)
+        Gitlab::Database::AsWithMaterialized.new(table, sql, materialized: @materialized)
       end
 
       # Returns an "AS" statement that aliases the CTE name as the given table
diff --git a/lib/gitlab/sql/recursive_cte.rb b/lib/gitlab/sql/recursive_cte.rb
index e45ac5d4765..607ce10d778 100644
--- a/lib/gitlab/sql/recursive_cte.rb
+++ b/lib/gitlab/sql/recursive_cte.rb
@@ -23,9 +23,11 @@ module Gitlab
       attr_reader :table
 
       # name - The name of the CTE as a String or Symbol.
-      def initialize(name)
+      # union_args - The arguments supplied to Gitlab::SQL::Union class when building inner recursive query
+      def initialize(name, union_args: {})
         @table = Arel::Table.new(name)
         @queries = []
+        @union_args = union_args
       end
 
       # Adds a query to the body of the CTE.
@@ -37,7 +39,7 @@ module Gitlab
 
       # Returns the Arel relation for this CTE.
       def to_arel
-        sql = Arel::Nodes::SqlLiteral.new(Union.new(@queries).to_sql)
+        sql = Arel::Nodes::SqlLiteral.new(Union.new(@queries, **@union_args).to_sql)
 
         Arel::Nodes::As.new(table, Arel::Nodes::Grouping.new(sql))
       end
diff --git a/lib/gitlab/sql/set_operator.rb b/lib/gitlab/sql/set_operator.rb
index d58a1415493..59a808eafa9 100644
--- a/lib/gitlab/sql/set_operator.rb
+++ b/lib/gitlab/sql/set_operator.rb
@@ -8,6 +8,9 @@ module Gitlab
     # ORDER BYs are dropped from the relations as the final sort order is not
     # guaranteed any way.
     #
+    # remove_order: false option can be used in special cases where the
+    # ORDER BY is necessary for the query.
+    #
     # Example usage:
     #
     #     union = Gitlab::SQL::Union.new([user.personal_projects, user.projects])
@@ -15,9 +18,10 @@ module Gitlab
     #
     #     Project.where("id IN (#{sql})")
     class SetOperator
-      def initialize(relations, remove_duplicates: true)
+      def initialize(relations, remove_duplicates: true, remove_order: true)
         @relations = relations
         @remove_duplicates = remove_duplicates
+        @remove_order = remove_order
       end
 
       def self.operator_keyword
@@ -30,7 +34,9 @@ module Gitlab
         # By using "unprepared_statements" we remove the usage of placeholders
         # (thus fixing this problem), at a slight performance cost.
         fragments = ActiveRecord::Base.connection.unprepared_statement do
-          relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?)
+          relations.map do |rel|
+            remove_order ? rel.reorder(nil).to_sql : rel.to_sql
+          end.reject(&:blank?)
         end
 
         if fragments.any?
@@ -47,7 +53,7 @@ module Gitlab
 
       private
 
-      attr_reader :relations, :remove_duplicates
+      attr_reader :relations, :remove_duplicates, :remove_order
     end
   end
 end
diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb
index 7fb3487a5e5..c4e95284c50 100644
--- a/lib/gitlab/sql/union.rb
+++ b/lib/gitlab/sql/union.rb
@@ -4,8 +4,8 @@ module Gitlab
   module SQL
     # Class for building SQL UNION statements.
     #
-    # ORDER BYs are dropped from the relations as the final sort order is not
-    # guaranteed any way.
+    # By default ORDER BYs are dropped from the relations as the final sort
+    # order is not guaranteed any way.
     #
     # Example usage:
     #
diff --git a/lib/gitlab/static_site_editor/config/file_config.rb b/lib/gitlab/static_site_editor/config/file_config.rb
index 315c603c1dd..4180f6ccf00 100644
--- a/lib/gitlab/static_site_editor/config/file_config.rb
+++ b/lib/gitlab/static_site_editor/config/file_config.rb
@@ -28,7 +28,7 @@ module Gitlab
         def to_hash_with_defaults
           # NOTE: The current approach of simply mapping all the descendents' keys and values ('config')
           #       into a flat hash may need to be enhanced as we add more complex, non-scalar entries.
-          @global.descendants.map { |descendant| [descendant.key, descendant.config] }.to_h
+          @global.descendants.to_h { |descendant| [descendant.key, descendant.config] }
         end
 
         private
diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb
index a918e7bec80..3072210d7c8 100644
--- a/lib/gitlab/subscription_portal.rb
+++ b/lib/gitlab/subscription_portal.rb
@@ -6,6 +6,11 @@ module Gitlab
       ::Gitlab.dev_or_test_env? ? 'https://customers.stg.gitlab.com' : 'https://customers.gitlab.com'
     end
 
-    SUBSCRIPTIONS_URL = ENV.fetch('CUSTOMER_PORTAL_URL', default_subscriptions_url).freeze
+    def self.subscriptions_url
+      ENV.fetch('CUSTOMER_PORTAL_URL', default_subscriptions_url)
+    end
   end
 end
+
+Gitlab::SubscriptionPortal.prepend_if_jh('JH::Gitlab::SubscriptionPortal')
+Gitlab::SubscriptionPortal::SUBSCRIPTIONS_URL = Gitlab::SubscriptionPortal.subscriptions_url.freeze
diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb
index dc006877129..31e11f73fe7 100644
--- a/lib/gitlab/template/base_template.rb
+++ b/lib/gitlab/template/base_template.rb
@@ -130,10 +130,10 @@ module Gitlab
           return [] if project && !project.repository.exists?
 
           if categories.any?
-            categories.keys.map do |category|
+            categories.keys.to_h do |category|
               files = self.by_category(category, project)
               [category, files.map { |t| { key: t.key, name: t.name, content: t.content } }]
-            end.to_h
+            end
           else
             files = self.all(project)
             files.map { |t| { key: t.key, name: t.name, content: t.content } }
diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb
index 9bb793a75cc..b16ae39bcee 100644
--- a/lib/gitlab/tracking.rb
+++ b/lib/gitlab/tracking.rb
@@ -4,35 +4,18 @@ module Gitlab
   module Tracking
     SNOWPLOW_NAMESPACE = 'gl'
 
-    module ControllerConcern
-      extend ActiveSupport::Concern
-
-      protected
-
-      def track_event(action = action_name, **args)
-        category = args.delete(:category) || self.class.name
-        Gitlab::Tracking.event(category, action.to_s, **args)
-      end
-
-      def track_self_describing_event(schema_url, data:, **args)
-        Gitlab::Tracking.self_describing_event(schema_url, data: data, **args)
-      end
-    end
-
     class << self
       def enabled?
         Gitlab::CurrentSettings.snowplow_enabled?
       end
 
-      def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil) # rubocop:disable Metrics/ParameterLists
-        contexts = [Tracking::StandardContext.new(project: project, user: user, namespace: namespace).to_context, *context]
+      def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists
+        contexts = [Tracking::StandardContext.new(project: project, user: user, namespace: namespace, **extra).to_context, *context]
 
         snowplow.event(category, action, label: label, property: property, value: value, context: contexts)
         product_analytics.event(category, action, label: label, property: property, value: value, context: contexts)
-      end
-
-      def self_describing_event(schema_url, data:, context: nil)
-        snowplow.self_describing_event(schema_url, data: data, context: context)
+      rescue => error
+        Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action)
       end
 
       def snowplow_options(group)
diff --git a/lib/gitlab/tracking/destinations/snowplow.rb b/lib/gitlab/tracking/destinations/snowplow.rb
index 4fa844de325..e548532e061 100644
--- a/lib/gitlab/tracking/destinations/snowplow.rb
+++ b/lib/gitlab/tracking/destinations/snowplow.rb
@@ -15,13 +15,6 @@ module Gitlab
           tracker.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i)
         end
 
-        def self_describing_event(schema_url, data:, context: nil)
-          return unless enabled?
-
-          event_json = SnowplowTracker::SelfDescribingJson.new(schema_url, data)
-          tracker.track_self_describing_event(event_json, context, (Time.now.to_f * 1000).to_i)
-        end
-
         private
 
         def enabled?
diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb
index 8ce16c11267..da030649f76 100644
--- a/lib/gitlab/tracking/standard_context.rb
+++ b/lib/gitlab/tracking/standard_context.rb
@@ -3,11 +3,11 @@
 module Gitlab
   module Tracking
     class StandardContext
-      GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-3'.freeze
-      GITLAB_RAILS_SOURCE = 'gitlab-rails'.freeze
+      GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-4'
+      GITLAB_RAILS_SOURCE = 'gitlab-rails'
 
-      def initialize(namespace: nil, project: nil, user: nil, **data)
-        @data = data
+      def initialize(namespace: nil, project: nil, user: nil, **extra)
+        @extra = extra
       end
 
       def to_context
@@ -35,8 +35,9 @@ module Gitlab
       def to_h
         {
           environment: environment,
-          source: source
-        }.merge(@data)
+          source: source,
+          extra: @extra
+        }
       end
     end
   end
diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb
index 6a3e2062070..706c0925302 100644
--- a/lib/gitlab/untrusted_regexp.rb
+++ b/lib/gitlab/untrusted_regexp.rb
@@ -35,6 +35,10 @@ module Gitlab
       matches
     end
 
+    def match(text)
+      scan_regexp.match(text)
+    end
+
     def match?(text)
       text.present? && scan(text).present?
     end
diff --git a/lib/gitlab/updated_notes_paginator.rb b/lib/gitlab/updated_notes_paginator.rb
index 3d3d0e5bf9e..d5c01bde6b3 100644
--- a/lib/gitlab/updated_notes_paginator.rb
+++ b/lib/gitlab/updated_notes_paginator.rb
@@ -37,8 +37,8 @@ module Gitlab
     end
 
     def fetch_page(relation)
-      relation = relation.by_updated_at
-      notes = relation.at_most(LIMIT + 1).to_a
+      relation = relation.order_updated_asc.with_order_id_asc
+      notes = relation.limit(LIMIT + 1).to_a
 
       return [notes, false] unless notes.size > LIMIT
 
diff --git a/lib/gitlab/usage/docs/helper.rb b/lib/gitlab/usage/docs/helper.rb
index 1dc660e574b..6b185a5a1e9 100644
--- a/lib/gitlab/usage/docs/helper.rb
+++ b/lib/gitlab/usage/docs/helper.rb
@@ -33,6 +33,10 @@ module Gitlab
           object[:description]
         end
 
+        def render_object_schema(object)
+          "[Object JSON schema](#{object.json_schema_path})"
+        end
+
         def render_yaml_link(yaml_path)
           "[YAML definition](#{yaml_path})"
         end
diff --git a/lib/gitlab/usage/docs/templates/default.md.haml b/lib/gitlab/usage/docs/templates/default.md.haml
index 19ad668019e..26f1aa4396d 100644
--- a/lib/gitlab/usage/docs/templates/default.md.haml
+++ b/lib/gitlab/usage/docs/templates/default.md.haml
@@ -27,6 +27,9 @@
   = render_name(name)
   \
   = render_description(object.attributes)
+  - if object.has_json_schema?
+    \
+    = render_object_schema(object)
   \
   = render_yaml_link(object.yaml_path)
   \
diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb
index 4cb83348478..9c4255a7c92 100644
--- a/lib/gitlab/usage/metric_definition.rb
+++ b/lib/gitlab/usage/metric_definition.rb
@@ -5,6 +5,7 @@ module Gitlab
     class MetricDefinition
       METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json')
       BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master'
+      SKIP_VALIDATION_STATUSES = %w[deprecated removed].to_set.freeze
 
       attr_reader :path
       attr_reader :attributes
@@ -22,6 +23,16 @@ module Gitlab
         attributes
       end
 
+      def json_schema_path
+        return '' unless has_json_schema?
+
+        "#{BASE_REPO_PATH}/#{attributes[:object_json_schema]}"
+      end
+
+      def has_json_schema?
+        attributes[:value_type] == 'object' && attributes[:object_json_schema].present?
+      end
+
       def yaml_path
         "#{BASE_REPO_PATH}#{path.delete_prefix(Rails.root.to_s)}"
       end
@@ -29,7 +40,15 @@ module Gitlab
       def validate!
         unless skip_validation?
           self.class.schemer.validate(attributes.stringify_keys).each do |error|
-            Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new("#{error["details"] || error['data_pointer']} for `#{path}`"))
+            error_message = <<~ERROR_MSG
+              Error type: #{error['type']}
+              Data: #{error['data']}
+              Path: #{error['data_pointer']}
+              Details: #{error['details']}
+              Metric file: #{path}
+            ERROR_MSG
+
+            Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new(error_message))
           end
         end
       end
@@ -38,10 +57,11 @@ module Gitlab
 
       class << self
         def paths
-          @paths ||= [Rails.root.join('config', 'metrics', '**', '*.yml')]
+          @paths ||= [Rails.root.join('config', 'metrics', '[^agg]*', '*.yml')]
         end
 
-        def definitions
+        def definitions(skip_validation: false)
+          @skip_validation = skip_validation
           @definitions ||= load_all!
         end
 
@@ -49,6 +69,10 @@ module Gitlab
           @schemer ||= ::JSONSchemer.schema(Pathname.new(METRIC_SCHEMA_PATH))
         end
 
+        def dump_metrics_yaml
+          @metrics_yaml ||= definitions.values.map(&:to_h).map(&:deep_stringify_keys).to_yaml
+        end
+
         private
 
         def load_all!
@@ -87,7 +111,7 @@ module Gitlab
       end
 
       def skip_validation?
-        !!attributes[:skip_validation]
+        !!attributes[:skip_validation] || @skip_validation || SKIP_VALIDATION_STATUSES.include?(attributes[:status])
       end
     end
   end
diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb
index 1aeca87d849..f77c8cab39c 100644
--- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb
+++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb
@@ -7,7 +7,7 @@ module Gitlab
         UNION_OF_AGGREGATED_METRICS = 'OR'
         INTERSECTION_OF_AGGREGATED_METRICS = 'AND'
         ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze
-        AGGREGATED_METRICS_PATH = Rails.root.join('lib/gitlab/usage_data_counters/aggregated_metrics/*.yml')
+        AGGREGATED_METRICS_PATH = Rails.root.join('config/metrics/aggregates/*.yml')
         AggregatedMetricError = Class.new(StandardError)
         UnknownAggregationOperator = Class.new(AggregatedMetricError)
         UnknownAggregationSource = Class.new(AggregatedMetricError)
diff --git a/lib/gitlab/usage/metrics/names_suggestions/generator.rb b/lib/gitlab/usage/metrics/names_suggestions/generator.rb
index 33f025770e0..49581169452 100644
--- a/lib/gitlab/usage/metrics/names_suggestions/generator.rb
+++ b/lib/gitlab/usage/metrics/names_suggestions/generator.rb
@@ -6,6 +6,8 @@ module Gitlab
       module NamesSuggestions
         class Generator < ::Gitlab::UsageData
           FREE_TEXT_METRIC_NAME = ""
+          REDIS_EVENT_METRIC_NAME = ""
+          CONSTRAINTS_PROMPT_TEMPLATE = ""
 
           class << self
             def generate(key_path)
@@ -23,7 +25,7 @@ module Gitlab
             end
 
             def redis_usage_counter
-              FREE_TEXT_METRIC_NAME
+              REDIS_EVENT_METRIC_NAME
             end
 
             def alt_usage_data(*)
@@ -31,7 +33,7 @@ module Gitlab
             end
 
             def redis_usage_data_totals(counter)
-              counter.fallback_totals.transform_values { |_| FREE_TEXT_METRIC_NAME}
+              counter.fallback_totals.transform_values { |_| REDIS_EVENT_METRIC_NAME }
             end
 
             def sum(relation, column, *rest)
@@ -47,49 +49,160 @@ module Gitlab
             end
 
             def name_suggestion(relation:, column: nil, prefix: nil, distinct: nil)
-              parts = [prefix]
+              # rubocop: disable CodeReuse/ActiveRecord
+              relation = relation.unscope(where: :created_at)
+              # rubocop: enable CodeReuse/ActiveRecord
 
-              if column
-                parts << parse_target(column)
+              parts = [prefix]
+              arel_column = arelize_column(relation, column)
+
+              # nil as column indicates that the counting would use fallback value of primary key.
+              # Because counting primary key from relation is the conceptual equal to counting all
+              # records from given relation, in order to keep name suggestion more condensed
+              # primary key column is skipped.
+              # eg: SELECT COUNT(id) FROM issues would translate as count_issues and not
+              # as count_id_from_issues since it does not add more information to the name suggestion
+              if arel_column != Arel::Table.new(relation.table_name)[relation.primary_key]
+                parts << arel_column.name
                 parts << 'from'
               end
 
-              source = parse_source(relation)
-              constraints = parse_constraints(relation: relation, column: column, distinct: distinct)
+              arel = arel_query(relation: relation, column: arel_column, distinct: distinct)
+              constraints = parse_constraints(relation: relation, arel: arel)
+
+              # In some cases due to performance reasons metrics are instrumented with joined relations
+              # where relation listed in FROM statement is not the one that includes counted attribute
+              # in such situations to make name suggestion more intuitive source should be inferred based
+              # on the relation that provide counted attribute
+              # EG: SELECT COUNT(deployments.environment_id) FROM clusters
+              #       JOIN deployments ON deployments.cluster_id = cluster.id
+              # should be translated into:
+              #   count_environment_id_from_deployments_with_clusters
+              # instead of
+              #   count_environment_id_from_clusters_with_deployments
+              actual_source = parse_source(relation, arel_column)
+
+              append_constraints_prompt(actual_source, [constraints], parts)
+
+              parts << actual_source
+              parts += process_joined_relations(actual_source, arel, relation, constraints)
+              parts.compact.join('_').delete('"')
+            end
 
-              if constraints.include?(source)
-                parts << ""
-              end
+            def append_constraints_prompt(target, constraints, parts)
+              applicable_constraints = constraints.select { |constraint| constraint.include?(target) }
+              return unless applicable_constraints.any?
 
-              parts << source
-              parts.compact.join('_')
+              parts << CONSTRAINTS_PROMPT_TEMPLATE % { constraints: applicable_constraints.join(' AND ') }
             end
 
-            def parse_constraints(relation:, column: nil, distinct: nil)
+            def parse_constraints(relation:, arel:)
               connection = relation.connection
               ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Constraints
                 .new(connection)
-                .accept(arel(relation: relation, column: column, distinct: distinct), collector(connection))
+                .accept(arel, collector(connection))
                 .value
             end
 
-            def parse_target(column)
-              if column.is_a?(Arel::Attribute)
-                "#{column.relation.name}.#{column.name}"
-              else
+            # TODO: joins with `USING` keyword
+            def process_joined_relations(actual_source, arel, relation, where_constraints)
+              joins = parse_joins(connection: relation.connection, arel: arel)
+              return [] unless joins.any?
+
+              sources = [relation.table_name, *joins.map { |join| join[:source] }]
+              joins = extract_joins_targets(joins, sources)
+
+              relations = if actual_source != relation.table_name
+                            build_relations_tree(joins + [{ source: relation.table_name }], actual_source)
+                          else
+                            # in case where counter attribute comes from joined relations, the relations
+                            # diagram has to be built bottom up, thus source and target are reverted
+                            build_relations_tree(joins + [{ source: relation.table_name }], actual_source, source_key: :target, target_key: :source)
+                          end
+
+              collect_join_parts(relations: relations[actual_source], joins: joins, wheres: where_constraints)
+            end
+
+            def parse_joins(connection:, arel:)
+              ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Joins
+                .new(connection)
+                .accept(arel)
+            end
+
+            def extract_joins_targets(joins, sources)
+              joins.map do |join|
+                source_regex = /(#{join[:source]})\.(\w+_)*id/i
+
+                tables_except_src = (sources - [join[:source]]).join('|')
+                target_regex = /(?#{tables_except_src})\.(\w+_)*id/i
+
+                join_cond_regex = /(#{source_regex}\s+=\s+#{target_regex})|(#{target_regex}\s+=\s+#{source_regex})/i
+                matched = join_cond_regex.match(join[:constraints])
+
+                if matched
+                  join[:target] = matched[:target]
+                  join[:constraints].gsub!(/#{join_cond_regex}(\s+(and|or))*/i, '')
+                end
+
+                join
+              end
+            end
+
+            def build_relations_tree(joins, parent, source_key: :source, target_key: :target)
+              return [] if joins.blank?
+
+              tree = {}
+              tree[parent] = []
+
+              joins.each do |join|
+                if join[source_key] == parent
+                  tree[parent] << build_relations_tree(joins - [join], join[target_key], source_key: source_key, target_key: target_key)
+                end
+              end
+              tree
+            end
+
+            def collect_join_parts(relations:, joins:, wheres:, parts: [], conjunctions: %w[with having including].cycle)
+              conjunction = conjunctions.next
+              relations.each do |subtree|
+                subtree.each do |parent, children|
+                  parts << "<#{conjunction}>"
+                  join_constraints = joins.find { |join| join[:source] == parent }&.dig(:constraints)
+                  append_constraints_prompt(parent, [wheres, join_constraints].compact, parts)
+                  parts << parent
+                  collect_join_parts(relations: children, joins: joins, wheres: wheres, parts: parts, conjunctions: conjunctions)
+                end
+              end
+              parts
+            end
+
+            def arelize_column(relation, column)
+              case column
+              when Arel::Attribute
                 column
+              when NilClass
+                Arel::Table.new(relation.table_name)[relation.primary_key]
+              when String
+                if column.include?('.')
+                  table, col = column.split('.')
+                  Arel::Table.new(table)[col]
+                else
+                  Arel::Table.new(relation.table_name)[column]
+                end
+              when Symbol
+                arelize_column(relation, column.to_s)
               end
             end
 
-            def parse_source(relation)
-              relation.table_name
+            def parse_source(relation, column)
+              column.relation.name || relation.table_name
             end
 
             def collector(connection)
               Arel::Collectors::SubstituteBinds.new(connection, Arel::Collectors::SQLString.new)
             end
 
-            def arel(relation:, column: nil, distinct: nil)
+            def arel_query(relation:, column: nil, distinct: nil)
               column ||= relation.primary_key
 
               if column.is_a?(Arel::Attribute)
diff --git a/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb
new file mode 100644
index 00000000000..d52e4903f3c
--- /dev/null
+++ b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Usage
+    module Metrics
+      module NamesSuggestions
+        module RelationParsers
+          class Joins < ::Arel::Visitors::PostgreSQL
+            def accept(object)
+              object.source.right.map do |join|
+                visit(join, collector)
+              end
+            end
+
+            private
+
+            # rubocop:disable Naming/MethodName
+            def visit_Arel_Nodes_StringJoin(object, collector)
+              result = visit(object.left, collector)
+              source, constraints = result.value.split('ON')
+              {
+                source: source.split('JOIN').last&.strip,
+                constraints: constraints&.strip
+              }.compact
+            end
+
+            def visit_Arel_Nodes_FullOuterJoin(object, _)
+              parse_join(object)
+            end
+
+            def visit_Arel_Nodes_OuterJoin(object, _)
+              parse_join(object)
+            end
+
+            def visit_Arel_Nodes_RightOuterJoin(object, _)
+              parse_join(object)
+            end
+
+            def visit_Arel_Nodes_InnerJoin(object, _)
+              {
+                source: visit(object.left, collector).value,
+                constraints: object.right ? visit(object.right.expr, collector).value : nil
+              }.compact
+            end
+            # rubocop:enable Naming/MethodName
+
+            def parse_join(object)
+              {
+                source: visit(object.left, collector).value,
+                constraints: visit(object.right.expr, collector).value
+              }
+            end
+
+            def quote(value)
+              "#{value}"
+            end
+
+            def quote_table_name(name)
+              "#{name}"
+            end
+
+            def quote_column_name(name)
+              "#{name}"
+            end
+
+            def collector
+              Arel::Collectors::SubstituteBinds.new(@connection, Arel::Collectors::SQLString.new)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 5dc3f71329d..b36ca38cd64 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -87,7 +87,7 @@ module Gitlab
       # rubocop: disable Metrics/AbcSize
       # rubocop: disable CodeReuse/ActiveRecord
       def system_usage_data
-        issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id)
+        issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue))
 
         {
           counts: {
@@ -138,7 +138,7 @@ module Gitlab
             in_review_folder: count(::Environment.in_review_folder),
             grafana_integrated_projects: count(GrafanaIntegration.enabled),
             groups: count(Group),
-            issues: count(Issue, start: issue_minimum_id, finish: issue_maximum_id),
+            issues: count(Issue, start: minimum_id(Issue), finish: maximum_id(Issue)),
             issues_created_from_gitlab_error_tracking_ui: count(SentryIssue),
             issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue),
             issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
@@ -146,9 +146,9 @@ module Gitlab
             issues_created_from_alerts: total_alert_issues,
             issues_created_gitlab_alerts: issues_created_manually_from_alerts,
             issues_created_manually_from_alerts: issues_created_manually_from_alerts,
-            incident_issues: count(::Issue.incident, start: issue_minimum_id, finish: issue_maximum_id),
-            alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id),
-            incident_labeled_issues: count(::Issue.with_label_attributes(::IncidentManagement::CreateIncidentLabelService::LABEL_PROPERTIES), start: issue_minimum_id, finish: issue_maximum_id),
+            incident_issues: count(::Issue.incident, start: minimum_id(Issue), finish: maximum_id(Issue)),
+            alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)),
+            incident_labeled_issues: count(::Issue.with_label_attributes(::IncidentManagement::CreateIncidentLabelService::LABEL_PROPERTIES), start: minimum_id(Issue), finish: maximum_id(Issue)),
             keys: count(Key),
             label_lists: count(List.label),
             lfs_objects: count(LfsObject),
@@ -389,8 +389,8 @@ module Gitlab
       # rubocop: disable CodeReuse/ActiveRecord
       def container_expiration_policies_usage
         results = {}
-        start = ::Project.minimum(:id)
-        finish = ::Project.maximum(:id)
+        start = minimum_id(Project)
+        finish = maximum_id(Project)
 
         results[:projects_with_expiration_policy_disabled] = distinct_count(::ContainerExpirationPolicy.where(enabled: false), :project_id, start: start, finish: finish)
         # rubocop: disable UsageData/LargeTable
@@ -591,7 +591,7 @@ module Gitlab
         {
           events: distinct_count(::Event.where(time_period), :author_id),
           groups: distinct_count(::GroupMember.where(time_period), :user_id),
-          users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id),
+          users_created: count(::User.where(time_period), start: minimum_id(User), finish: maximum_id(User)),
           omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' },
           user_auth_by_provider: distinct_count_user_auth_by_provider(time_period),
           unique_users_all_imports: unique_users_all_imports(time_period),
@@ -636,8 +636,8 @@ module Gitlab
           clusters: distinct_count(::Clusters::Cluster.where(time_period), :user_id),
           clusters_applications_prometheus: cluster_applications_user_distinct_count(::Clusters::Applications::Prometheus, time_period),
           operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period),
-                                                        start: user_minimum_id,
-                                                        finish: user_maximum_id),
+                                                        start: minimum_id(User),
+                                                        finish: maximum_id(User)),
           projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id),
           projects_with_error_tracking_enabled: distinct_count(::Project.with_enabled_error_tracking.where(time_period), :creator_id),
           projects_with_incidents: distinct_count(::Issue.incident.where(time_period), :project_id),
@@ -691,12 +691,12 @@ module Gitlab
       def usage_activity_by_stage_verify(time_period)
         {
           ci_builds: distinct_count(::Ci::Build.where(time_period), :user_id),
-          ci_external_pipelines: distinct_count(::Ci::Pipeline.external.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
-          ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
-          ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
-          ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
+          ci_external_pipelines: distinct_count(::Ci::Pipeline.external.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
+          ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
+          ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
+          ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
           ci_pipeline_schedules: distinct_count(::Ci::PipelineSchedule.where(time_period), :owner_id),
-          ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
+          ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
           ci_triggers: distinct_count(::Ci::Trigger.where(time_period), :owner_id),
           clusters_applications_runner: cluster_applications_user_distinct_count(::Clusters::Applications::Runner, time_period)
         }
@@ -711,6 +711,8 @@ module Gitlab
       end
 
       def redis_hll_counters
+        return {} unless Feature.enabled?(:redis_hll_tracking, type: :ops, default_enabled: :yaml)
+
         { redis_hll_counters: ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events_data }
       end
 
@@ -799,8 +801,8 @@ module Gitlab
       end
 
       def distinct_count_service_desk_enabled_projects(time_period)
-        project_creator_id_start = user_minimum_id
-        project_creator_id_finish = user_maximum_id
+        project_creator_id_start = minimum_id(User)
+        project_creator_id_finish = maximum_id(User)
 
         distinct_count(::Project.service_desk_enabled.where(time_period), :creator_id, start: project_creator_id_start, finish: project_creator_id_finish) # rubocop: disable CodeReuse/ActiveRecord
       end
@@ -832,57 +834,9 @@ module Gitlab
       def total_alert_issues
         # Remove prometheus table queries once they are deprecated
         # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/217407.
-        add count(Issue.with_alert_management_alerts, start: issue_minimum_id, finish: issue_maximum_id),
-          count(::Issue.with_self_managed_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id),
-          count(::Issue.with_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id)
-      end
-
-      def user_minimum_id
-        strong_memoize(:user_minimum_id) do
-          ::User.minimum(:id)
-        end
-      end
-
-      def user_maximum_id
-        strong_memoize(:user_maximum_id) do
-          ::User.maximum(:id)
-        end
-      end
-
-      def issue_minimum_id
-        strong_memoize(:issue_minimum_id) do
-          ::Issue.minimum(:id)
-        end
-      end
-
-      def issue_maximum_id
-        strong_memoize(:issue_maximum_id) do
-          ::Issue.maximum(:id)
-        end
-      end
-
-      def deployment_minimum_id
-        strong_memoize(:deployment_minimum_id) do
-          ::Deployment.minimum(:id)
-        end
-      end
-
-      def deployment_maximum_id
-        strong_memoize(:deployment_maximum_id) do
-          ::Deployment.maximum(:id)
-        end
-      end
-
-      def project_minimum_id
-        strong_memoize(:project_minimum_id) do
-          ::Project.minimum(:id)
-        end
-      end
-
-      def project_maximum_id
-        strong_memoize(:project_maximum_id) do
-          ::Project.maximum(:id)
-        end
+        add count(Issue.with_alert_management_alerts, start: minimum_id(Issue), finish: maximum_id(Issue)),
+          count(::Issue.with_self_managed_prometheus_alert_events, start: minimum_id(Issue), finish: maximum_id(Issue)),
+          count(::Issue.with_prometheus_alert_events, start: minimum_id(Issue), finish: maximum_id(Issue))
       end
 
       def self_monitoring_project
@@ -916,7 +870,7 @@ module Gitlab
       end
 
       def deployment_count(relation)
-        count relation, start: deployment_minimum_id, finish: deployment_maximum_id
+        count relation, start: minimum_id(Deployment), finish: maximum_id(Deployment)
       end
 
       def project_imports(time_period)
diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml
deleted file mode 100644
index 4c2355d526a..00000000000
--- a/lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml
+++ /dev/null
@@ -1,108 +0,0 @@
-# code_review_extension_category_monthly_active_users
-# This is only metrics related to the VS Code Extension for now.
-#
-# code_review_category_monthly_active_users
-# This is the user based metrics. These should only be user based metrics and only be related to the Code Review things inside of GitLab.
-#
-# code_review_group_monthly_active_users
-# This is an aggregation of both of the above aggregations. It's intended to represent all users who interact with our group across all of our categories.
----
-- name: code_review_group_monthly_active_users
-  operator: OR
-  feature_flag: usage_data_code_review_aggregation
-  source: redis
-  time_frame: [7d, 28d]
-  events: [
-    'i_code_review_user_single_file_diffs',
-    'i_code_review_user_create_mr',
-    'i_code_review_user_close_mr',
-    'i_code_review_user_reopen_mr',
-    'i_code_review_user_resolve_thread',
-    'i_code_review_user_unresolve_thread',
-    'i_code_review_edit_mr_title',
-    'i_code_review_edit_mr_desc',
-    'i_code_review_user_merge_mr',
-    'i_code_review_user_create_mr_comment',
-    'i_code_review_user_edit_mr_comment',
-    'i_code_review_user_remove_mr_comment',
-    'i_code_review_user_create_review_note',
-    'i_code_review_user_publish_review',
-    'i_code_review_user_create_multiline_mr_comment',
-    'i_code_review_user_edit_multiline_mr_comment',
-    'i_code_review_user_remove_multiline_mr_comment',
-    'i_code_review_user_add_suggestion',
-    'i_code_review_user_apply_suggestion',
-    'i_code_review_user_assigned',
-    'i_code_review_user_review_requested',
-    'i_code_review_user_approve_mr',
-    'i_code_review_user_unapprove_mr',
-    'i_code_review_user_marked_as_draft',
-    'i_code_review_user_unmarked_as_draft',
-    'i_code_review_user_approval_rule_added',
-    'i_code_review_user_approval_rule_deleted',
-    'i_code_review_user_approval_rule_edited',
-    'i_code_review_user_vs_code_api_request',
-    'i_code_review_user_toggled_task_item_status',
-    'i_code_review_user_create_mr_from_issue',
-    'i_code_review_user_mr_discussion_locked',
-    'i_code_review_user_mr_discussion_unlocked',
-    'i_code_review_user_time_estimate_changed',
-    'i_code_review_user_time_spent_changed',
-    'i_code_review_user_assignees_changed',
-    'i_code_review_user_reviewers_changed',
-    'i_code_review_user_milestone_changed',
-    'i_code_review_user_labels_changed'
-  ]
-- name: code_review_category_monthly_active_users
-  operator: OR
-  feature_flag: usage_data_code_review_aggregation
-  source: redis
-  time_frame: [7d, 28d]
-  events: [
-    'i_code_review_user_single_file_diffs',
-    'i_code_review_user_create_mr',
-    'i_code_review_user_close_mr',
-    'i_code_review_user_reopen_mr',
-    'i_code_review_user_resolve_thread',
-    'i_code_review_user_unresolve_thread',
-    'i_code_review_edit_mr_title',
-    'i_code_review_edit_mr_desc',
-    'i_code_review_user_merge_mr',
-    'i_code_review_user_create_mr_comment',
-    'i_code_review_user_edit_mr_comment',
-    'i_code_review_user_remove_mr_comment',
-    'i_code_review_user_create_review_note',
-    'i_code_review_user_publish_review',
-    'i_code_review_user_create_multiline_mr_comment',
-    'i_code_review_user_edit_multiline_mr_comment',
-    'i_code_review_user_remove_multiline_mr_comment',
-    'i_code_review_user_add_suggestion',
-    'i_code_review_user_apply_suggestion',
-    'i_code_review_user_assigned',
-    'i_code_review_user_review_requested',
-    'i_code_review_user_approve_mr',
-    'i_code_review_user_unapprove_mr',
-    'i_code_review_user_marked_as_draft',
-    'i_code_review_user_unmarked_as_draft',
-    'i_code_review_user_approval_rule_added',
-    'i_code_review_user_approval_rule_deleted',
-    'i_code_review_user_approval_rule_edited',
-    'i_code_review_user_toggled_task_item_status',
-    'i_code_review_user_create_mr_from_issue',
-    'i_code_review_user_mr_discussion_locked',
-    'i_code_review_user_mr_discussion_unlocked',
-    'i_code_review_user_time_estimate_changed',
-    'i_code_review_user_time_spent_changed',
-    'i_code_review_user_assignees_changed',
-    'i_code_review_user_reviewers_changed',
-    'i_code_review_user_milestone_changed',
-    'i_code_review_user_labels_changed'
-  ]
-- name: code_review_extension_category_monthly_active_users
-  operator: OR
-  feature_flag: usage_data_code_review_aggregation
-  source: redis
-  time_frame: [7d, 28d]
-  events: [
-    'i_code_review_user_vs_code_api_request'
-  ]
diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml
deleted file mode 100644
index 73a55b5d5fa..00000000000
--- a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml
+++ /dev/null
@@ -1,72 +0,0 @@
-# Aggregated metrics that include EE only event names within `events:` attribute have to be defined at ee/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml
-# instead of this file.
-#- name: unique name of aggregated metric
-#  operator: aggregation operator. Valid values are:
-#             - "OR": counts unique elements that were observed triggering any of following events
-#             - "AND": counts unique elements that were observed triggering all of following events
-#  events: list of events names to aggregate into metric. All events in this list must have the same 'redis_slot' and 'aggregation' attributes
-#           see from lib/gitlab/usage_data_counters/known_events/ for the list of valid events.
-#  source: defines which datasource will be used to locate events that should be included in aggregated metric. Valid values are:
-#          - database
-#          - redis
-#  time_frame: defines time frames for aggregated metrics:
-#          - 7d - last 7 days
-#          - 28d - last 28 days
-#          - all - all historical available data, this time frame is not available for redis source
-#  feature_flag: name of development feature flag that will be checked before metrics aggregation is performed.
-#                Corresponding feature flag should have `default_enabled` attribute set to `false`.
-#                This attribute is OPTIONAL and can be omitted, when `feature_flag` is missing no feature flag will be checked.
----
-- name: compliance_features_track_unique_visits_union
-  operator: OR
-  source: redis
-  time_frame: [7d, 28d]
-  events: ['g_compliance_audit_events', 'g_compliance_dashboard', 'i_compliance_audit_events', 'a_compliance_audit_events_api', 'i_compliance_credential_inventory']
-- name: product_analytics_test_metrics_union
-  operator: OR
-  source: redis
-  time_frame: [7d, 28d]
-  events: ['i_search_total', 'i_search_advanced', 'i_search_paid']
-- name: product_analytics_test_metrics_intersection
-  operator: AND
-  source: redis
-  time_frame: [7d, 28d]
-  events: ['i_search_total', 'i_search_advanced', 'i_search_paid']
-- name: incident_management_alerts_total_unique_counts
-  operator: OR
-  source: redis
-  time_frame: [7d, 28d]
-  events: [
-    'incident_management_alert_status_changed',
-    'incident_management_alert_assigned',
-    'incident_management_alert_todo',
-    'incident_management_alert_create_incident'
-  ]
-- name: incident_management_incidents_total_unique_counts
-  operator: OR
-  source: redis
-  time_frame: [7d, 28d]
-  events: [
-    'incident_management_incident_created',
-    'incident_management_incident_reopened',
-    'incident_management_incident_closed',
-    'incident_management_incident_assigned',
-    'incident_management_incident_todo',
-    'incident_management_incident_comment',
-    'incident_management_incident_zoom_meeting',
-    'incident_management_incident_published',
-    'incident_management_incident_relate',
-    'incident_management_incident_unrelate',
-    'incident_management_incident_change_confidential'
-  ]
-- name: i_testing_paid_monthly_active_user_total
-  operator: OR
-  source: redis
-  time_frame: [7d, 28d]
-  events: [
-    'i_testing_web_performance_widget_total',
-    'i_testing_full_code_quality_report_total',
-    'i_testing_group_code_coverage_visit_total',
-    'i_testing_load_performance_widget_total',
-    'i_testing_metrics_report_widget_total'
-  ]
diff --git a/lib/gitlab/usage_data_counters/base_counter.rb b/lib/gitlab/usage_data_counters/base_counter.rb
index d28fd17a989..4ab310a2519 100644
--- a/lib/gitlab/usage_data_counters/base_counter.rb
+++ b/lib/gitlab/usage_data_counters/base_counter.rb
@@ -22,11 +22,11 @@ module Gitlab::UsageDataCounters
       end
 
       def totals
-        known_events.map { |event| [counter_key(event), read(event)] }.to_h
+        known_events.to_h { |event| [counter_key(event), read(event)] }
       end
 
       def fallback_totals
-        known_events.map { |event| [counter_key(event), -1] }.to_h
+        known_events.to_h { |event| [counter_key(event), -1] }
       end
 
       def fetch_supported_event(event_name)
diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb
index 772a4623280..c9106d7c6b8 100644
--- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb
+++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb
@@ -2,7 +2,7 @@
 
 module Gitlab::UsageDataCounters
   class CiTemplateUniqueCounter
-    REDIS_SLOT = 'ci_templates'.freeze
+    REDIS_SLOT = 'ci_templates'
 
     # NOTE: Events originating from implicit Auto DevOps pipelines get prefixed with `implicit_`
     TEMPLATE_TO_EVENT = {
@@ -20,8 +20,6 @@ module Gitlab::UsageDataCounters
 
     class << self
       def track_unique_project_event(project_id:, template:, config_source:)
-        return if Feature.disabled?(:usage_data_track_ci_templates_unique_projects, default_enabled: :yaml)
-
         if event = unique_project_event(template, config_source)
           Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: project_id)
         end
diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
index 336bef081a6..a8691169fb8 100644
--- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb
+++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
@@ -151,13 +151,16 @@ module Gitlab
           aggregation = events.first[:aggregation]
 
           keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date, context: context)
+
+          return FALLBACK unless keys.any?
+
           redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) }
         end
 
         def feature_enabled?(event)
           return true if event[:feature_flag].blank?
 
-          Feature.enabled?(event[:feature_flag], default_enabled: :yaml)
+          Feature.enabled?(event[:feature_flag], default_enabled: :yaml) && Feature.enabled?(:redis_hll_tracking, type: :ops, default_enabled: :yaml)
         end
 
         # Allow to add totals for events that are in the same redis slot, category and have the same aggregation level
diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb
index c2662a74432..6f5f878501f 100644
--- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb
+++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb
@@ -34,120 +34,120 @@ module Gitlab
       ISSUE_COMMENT_REMOVED = 'g_project_management_issue_comment_removed'
 
       class << self
-        def track_issue_created_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_CREATED, author, time)
+        def track_issue_created_action(author:)
+          track_unique_action(ISSUE_CREATED, author)
         end
 
-        def track_issue_title_changed_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_TITLE_CHANGED, author, time)
+        def track_issue_title_changed_action(author:)
+          track_unique_action(ISSUE_TITLE_CHANGED, author)
         end
 
-        def track_issue_description_changed_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_DESCRIPTION_CHANGED, author, time)
+        def track_issue_description_changed_action(author:)
+          track_unique_action(ISSUE_DESCRIPTION_CHANGED, author)
         end
 
-        def track_issue_assignee_changed_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_ASSIGNEE_CHANGED, author, time)
+        def track_issue_assignee_changed_action(author:)
+          track_unique_action(ISSUE_ASSIGNEE_CHANGED, author)
         end
 
-        def track_issue_made_confidential_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_MADE_CONFIDENTIAL, author, time)
+        def track_issue_made_confidential_action(author:)
+          track_unique_action(ISSUE_MADE_CONFIDENTIAL, author)
         end
 
-        def track_issue_made_visible_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_MADE_VISIBLE, author, time)
+        def track_issue_made_visible_action(author:)
+          track_unique_action(ISSUE_MADE_VISIBLE, author)
         end
 
-        def track_issue_closed_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_CLOSED, author, time)
+        def track_issue_closed_action(author:)
+          track_unique_action(ISSUE_CLOSED, author)
         end
 
-        def track_issue_reopened_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_REOPENED, author, time)
+        def track_issue_reopened_action(author:)
+          track_unique_action(ISSUE_REOPENED, author)
         end
 
-        def track_issue_label_changed_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_LABEL_CHANGED, author, time)
+        def track_issue_label_changed_action(author:)
+          track_unique_action(ISSUE_LABEL_CHANGED, author)
         end
 
-        def track_issue_milestone_changed_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_MILESTONE_CHANGED, author, time)
+        def track_issue_milestone_changed_action(author:)
+          track_unique_action(ISSUE_MILESTONE_CHANGED, author)
         end
 
-        def track_issue_cross_referenced_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_CROSS_REFERENCED, author, time)
+        def track_issue_cross_referenced_action(author:)
+          track_unique_action(ISSUE_CROSS_REFERENCED, author)
         end
 
-        def track_issue_moved_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_MOVED, author, time)
+        def track_issue_moved_action(author:)
+          track_unique_action(ISSUE_MOVED, author)
         end
 
-        def track_issue_related_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_RELATED, author, time)
+        def track_issue_related_action(author:)
+          track_unique_action(ISSUE_RELATED, author)
         end
 
-        def track_issue_unrelated_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_UNRELATED, author, time)
+        def track_issue_unrelated_action(author:)
+          track_unique_action(ISSUE_UNRELATED, author)
         end
 
-        def track_issue_marked_as_duplicate_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_MARKED_AS_DUPLICATE, author, time)
+        def track_issue_marked_as_duplicate_action(author:)
+          track_unique_action(ISSUE_MARKED_AS_DUPLICATE, author)
         end
 
-        def track_issue_locked_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_LOCKED, author, time)
+        def track_issue_locked_action(author:)
+          track_unique_action(ISSUE_LOCKED, author)
         end
 
-        def track_issue_unlocked_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_UNLOCKED, author, time)
+        def track_issue_unlocked_action(author:)
+          track_unique_action(ISSUE_UNLOCKED, author)
         end
 
-        def track_issue_designs_added_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_DESIGNS_ADDED, author, time)
+        def track_issue_designs_added_action(author:)
+          track_unique_action(ISSUE_DESIGNS_ADDED, author)
         end
 
-        def track_issue_designs_modified_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_DESIGNS_MODIFIED, author, time)
+        def track_issue_designs_modified_action(author:)
+          track_unique_action(ISSUE_DESIGNS_MODIFIED, author)
         end
 
-        def track_issue_designs_removed_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_DESIGNS_REMOVED, author, time)
+        def track_issue_designs_removed_action(author:)
+          track_unique_action(ISSUE_DESIGNS_REMOVED, author)
         end
 
-        def track_issue_due_date_changed_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_DUE_DATE_CHANGED, author, time)
+        def track_issue_due_date_changed_action(author:)
+          track_unique_action(ISSUE_DUE_DATE_CHANGED, author)
         end
 
-        def track_issue_time_estimate_changed_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_TIME_ESTIMATE_CHANGED, author, time)
+        def track_issue_time_estimate_changed_action(author:)
+          track_unique_action(ISSUE_TIME_ESTIMATE_CHANGED, author)
         end
 
-        def track_issue_time_spent_changed_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_TIME_SPENT_CHANGED, author, time)
+        def track_issue_time_spent_changed_action(author:)
+          track_unique_action(ISSUE_TIME_SPENT_CHANGED, author)
         end
 
-        def track_issue_comment_added_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_COMMENT_ADDED, author, time)
+        def track_issue_comment_added_action(author:)
+          track_unique_action(ISSUE_COMMENT_ADDED, author)
         end
 
-        def track_issue_comment_edited_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_COMMENT_EDITED, author, time)
+        def track_issue_comment_edited_action(author:)
+          track_unique_action(ISSUE_COMMENT_EDITED, author)
         end
 
-        def track_issue_comment_removed_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_COMMENT_REMOVED, author, time)
+        def track_issue_comment_removed_action(author:)
+          track_unique_action(ISSUE_COMMENT_REMOVED, author)
         end
 
-        def track_issue_cloned_action(author:, time: Time.zone.now)
-          track_unique_action(ISSUE_CLONED, author, time)
+        def track_issue_cloned_action(author:)
+          track_unique_action(ISSUE_CLONED, author)
         end
 
         private
 
-        def track_unique_action(action, author, time)
+        def track_unique_action(action, author)
           return unless author
 
-          Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id, time: time)
+          Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id)
         end
       end
     end
diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml
index 9c19c9e8b8c..3c692f2b1af 100644
--- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml
+++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml
@@ -3,89 +3,74 @@
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
 
 - name: p_ci_templates_implicit_auto_devops_build
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
 
 - name: p_ci_templates_implicit_auto_devops_deploy
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
 
 - name: p_ci_templates_implicit_security_sast
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
 
 - name: p_ci_templates_implicit_security_secret_detection
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
 
 # Explicit include:template pipeline events
 - name: p_ci_templates_5_min_production_app
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
 
 - name: p_ci_templates_auto_devops
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
 
 - name: p_ci_templates_aws_cf_deploy_ec2
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
 
 - name: p_ci_templates_aws_deploy_ecs
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
 
 - name: p_ci_templates_auto_devops_build
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
 
 - name: p_ci_templates_auto_devops_deploy
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
 
 - name: p_ci_templates_auto_devops_deploy_latest
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
 
 - name: p_ci_templates_security_sast
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
 
 - name: p_ci_templates_security_secret_detection
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
 
 - name: p_ci_templates_terraform_base_latest
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-  feature_flag: usage_data_track_ci_templates_unique_projects
diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml
index 80a79682338..077864032e8 100644
--- a/lib/gitlab/usage_data_counters/known_events/common.yml
+++ b/lib/gitlab/usage_data_counters/known_events/common.yml
@@ -91,6 +91,11 @@
   redis_slot: analytics
   aggregation: weekly
   feature_flag: track_unique_visits
+- name: i_analytics_dev_ops_adoption
+  category: analytics
+  redis_slot: analytics
+  aggregation: weekly
+  feature_flag: track_unique_visits
 - name: g_analytics_merge_request
   category: analytics
   redis_slot: analytics
@@ -242,6 +247,12 @@
   category: incident_management_alerts
   aggregation: weekly
   feature_flag: usage_data_incident_management_alert_create_incident
+# Incident management on-call
+- name: i_incident_management_oncall_notification_sent
+  redis_slot: incident_management
+  category: incident_management_oncall
+  aggregation: weekly
+  feature_flag: usage_data_i_incident_management_oncall_notification_sent
 # Testing category
 - name: i_testing_test_case_parsed
   category: testing
@@ -283,6 +294,11 @@
   redis_slot: testing
   aggregation: weekly
   feature_flag: usage_data_i_testing_metrics_report_artifact_uploaders
+- name: i_testing_summary_widget_total
+  category: testing
+  redis_slot: testing
+  aggregation: weekly
+  feature_flag: usage_data_i_testing_summary_widget_total
 # Project Management group
 - name: g_project_management_issue_title_changed
   category: issues_edit
@@ -444,13 +460,19 @@
   redis_slot: pipeline_authoring
   aggregation: weekly
   feature_flag: usage_data_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile
-# Epic events
-#
-# We are using the same slot of issue events 'project_management' for
-# epic events to allow data aggregation.
-# More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405
-- name: g_project_management_epic_created
-  category: epics_usage
-  redis_slot: project_management
-  aggregation: daily
-  feature_flag: track_epics_activity
+# Merge request widgets
+- name: users_expanding_secure_security_report
+  redis_slot: secure
+  category: secure
+  aggregation: weekly
+  feature_flag: users_expanding_widgets_usage_data
+- name: users_expanding_testing_code_quality_report
+  redis_slot: testing
+  category: testing
+  aggregation: weekly
+  feature_flag: users_expanding_widgets_usage_data
+- name: users_expanding_testing_accessibility_report
+  redis_slot: testing
+  category: testing
+  aggregation: weekly
+  feature_flag: users_expanding_widgets_usage_data
diff --git a/lib/gitlab/usage_data_counters/known_events/epic_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_events.yml
new file mode 100644
index 00000000000..80460dbe4d2
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/known_events/epic_events.yml
@@ -0,0 +1,142 @@
+# Epic events
+#
+# We are using the same slot of issue events 'project_management' for
+# epic events to allow data aggregation.
+# More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405
+- name: g_project_management_epic_created
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+- name: g_project_management_users_updating_epic_titles
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+- name: g_project_management_users_updating_epic_descriptions
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+# epic notes
+
+- name: g_project_management_users_creating_epic_notes
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+- name: g_project_management_users_updating_epic_notes
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+- name: g_project_management_users_destroying_epic_notes
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+# start date events
+
+- name: g_project_management_users_setting_epic_start_date_as_fixed
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+- name: g_project_management_users_updating_fixed_epic_start_date
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+- name: g_project_management_users_setting_epic_start_date_as_inherited
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+# due date events
+
+- name: g_project_management_users_setting_epic_due_date_as_fixed
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+- name: g_project_management_users_updating_fixed_epic_due_date
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+- name: g_project_management_users_setting_epic_due_date_as_inherited
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+- name: g_project_management_epic_issue_added
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+- name: g_project_management_epic_issue_removed
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+- name: g_project_management_epic_issue_moved_from_project
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+- name: g_project_management_epic_closed
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+- name: g_project_management_epic_reopened
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+- name: 'g_project_management_issue_promoted_to_epic'
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+- name: g_project_management_users_setting_epic_confidential
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+- name: g_project_management_users_setting_epic_visible
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+- name: g_project_management_epic_users_changing_labels
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
+
+- name: g_project_management_epic_destroyed
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity
diff --git a/lib/gitlab/usage_data_counters/note_counter.rb b/lib/gitlab/usage_data_counters/note_counter.rb
index 7a76180cb08..aae2d144c5b 100644
--- a/lib/gitlab/usage_data_counters/note_counter.rb
+++ b/lib/gitlab/usage_data_counters/note_counter.rb
@@ -24,13 +24,13 @@ module Gitlab::UsageDataCounters
       end
 
       def totals
-        COUNTABLE_TYPES.map do |countable_type|
+        COUNTABLE_TYPES.to_h do |countable_type|
           [counter_key(countable_type), read(:create, countable_type)]
-        end.to_h
+        end
       end
 
       def fallback_totals
-        COUNTABLE_TYPES.map { |counter_key| [counter_key(counter_key), -1] }.to_h
+        COUNTABLE_TYPES.to_h { |counter_key| [counter_key(counter_key), -1] }
       end
 
       private
diff --git a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb
index 15c68fb3945..ed3df7dcf75 100644
--- a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb
+++ b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb
@@ -28,7 +28,7 @@ module Gitlab
             'unassign_reviewer'
           when 'request_review', 'reviewer'
             'assign_reviewer'
-          when 'spend'
+          when 'spend', 'spent'
             event_name_for_spend(args)
           when 'unassign'
             event_name_for_unassign(args)
diff --git a/lib/gitlab/usage_data_non_sql_metrics.rb b/lib/gitlab/usage_data_non_sql_metrics.rb
new file mode 100644
index 00000000000..1f72bf4ce26
--- /dev/null
+++ b/lib/gitlab/usage_data_non_sql_metrics.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Gitlab
+  class UsageDataNonSqlMetrics < UsageData
+    SQL_METRIC_DEFAULT = -3
+
+    class << self
+      def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
+        SQL_METRIC_DEFAULT
+      end
+
+      def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
+        SQL_METRIC_DEFAULT
+      end
+
+      def estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil)
+        SQL_METRIC_DEFAULT
+      end
+
+      def sum(relation, column, batch_size: nil, start: nil, finish: nil)
+        SQL_METRIC_DEFAULT
+      end
+
+      def histogram(relation, column, buckets:, bucket_size: buckets.size)
+        SQL_METRIC_DEFAULT
+      end
+
+      def maximum_id(model)
+      end
+
+      def minimum_id(model)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb
index c00e7a2aa13..c0dfae88fc7 100644
--- a/lib/gitlab/usage_data_queries.rb
+++ b/lib/gitlab/usage_data_queries.rb
@@ -5,11 +5,11 @@ module Gitlab
   # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41091
   class UsageDataQueries < UsageData
     class << self
-      def count(relation, column = nil, *rest)
+      def count(relation, column = nil, *args, **kwargs)
         raw_sql(relation, column)
       end
 
-      def distinct_count(relation, column = nil, *rest)
+      def distinct_count(relation, column = nil, *args, **kwargs)
         raw_sql(relation, column, :distinct)
       end
 
@@ -21,14 +21,14 @@ module Gitlab
         end
       end
 
-      def sum(relation, column, *rest)
+      def sum(relation, column, *args, **kwargs)
         relation.select(relation.all.table[column].sum).to_sql
       end
 
       # For estimated distinct count use exact query instead of hll
       # buckets query, because it can't be used to obtain estimations without
       # supplementary ruby code present in Gitlab::Database::PostgresHll::BatchDistinctCounter
-      def estimate_batch_distinct_count(relation, column = nil, *rest)
+      def estimate_batch_distinct_count(relation, column = nil, *args, **kwargs)
         raw_sql(relation, column, :distinct)
       end
 
@@ -36,6 +36,12 @@ module Gitlab
         'SELECT ' + args.map {|arg| "(#{arg})" }.join(' + ')
       end
 
+      def maximum_id(model)
+      end
+
+      def minimum_id(model)
+      end
+
       private
 
       def raw_sql(relation, column, distinct = nil)
diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb
index 29f02a5912a..c1a57566640 100644
--- a/lib/gitlab/utils.rb
+++ b/lib/gitlab/utils.rb
@@ -99,6 +99,8 @@ module Gitlab
     end
 
     def to_boolean(value, default: nil)
+      value = value.to_s if [0, 1].include?(value)
+
       return value if [true, false].include?(value)
       return true if value =~ /^(true|t|yes|y|1|on)$/i
       return false if value =~ /^(false|f|no|n|0|off)$/i
diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb
index 854fc5c917d..efa2f7a943f 100644
--- a/lib/gitlab/utils/usage_data.rb
+++ b/lib/gitlab/utils/usage_data.rb
@@ -36,6 +36,7 @@
 module Gitlab
   module Utils
     module UsageData
+      include Gitlab::Utils::StrongMemoize
       extend self
 
       FALLBACK = -1
@@ -209,6 +210,20 @@ module Gitlab
         Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name.to_s, values: values)
       end
 
+      def maximum_id(model)
+        key = :"#{model.name.downcase}_maximum_id"
+        strong_memoize(key) do
+          model.maximum(:id)
+        end
+      end
+
+      def minimum_id(model)
+        key = :"#{model.name.downcase}_minimum_id"
+        strong_memoize(key) do
+          model.minimum(:id)
+        end
+      end
+
       private
 
       def prometheus_client(verify:)
diff --git a/lib/gitlab/uuid.rb b/lib/gitlab/uuid.rb
index 80caf2c6788..016c25eb94b 100644
--- a/lib/gitlab/uuid.rb
+++ b/lib/gitlab/uuid.rb
@@ -9,9 +9,9 @@ module Gitlab
       production: "58dc0f06-936c-43b3-93bb-71693f1b6570"
     }.freeze
 
-    UUID_V5_PATTERN = /\h{8}-\h{4}-5\h{3}-\h{4}-\h{4}\h{8}/.freeze
+    UUID_V5_PATTERN = /\h{8}-\h{4}-5\h{3}-\h{4}-\h{12}/.freeze
     NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze
-    PACK_PATTERN = "NnnnnN".freeze
+    PACK_PATTERN = "NnnnnN"
 
     class << self
       def v5(name, namespace_id: default_namespace_id)
diff --git a/lib/gitlab/web_ide/config/entry/terminal.rb b/lib/gitlab/web_ide/config/entry/terminal.rb
index 403e308d45b..ec07023461f 100644
--- a/lib/gitlab/web_ide/config/entry/terminal.rb
+++ b/lib/gitlab/web_ide/config/entry/terminal.rb
@@ -10,6 +10,7 @@ module Gitlab
         class Terminal < ::Gitlab::Config::Entry::Node
           include ::Gitlab::Config::Entry::Configurable
           include ::Gitlab::Config::Entry::Attributable
+          include Gitlab::Utils::StrongMemoize
 
           # By default the build will finish in a few seconds, not giving the webide
           # enough time to connect to the terminal. This default script provides
@@ -51,21 +52,26 @@ module Gitlab
           private
 
           def to_hash
-            { tag_list: tags || [],
-              yaml_variables: yaml_variables,
+            {
+              tag_list: tags || [],
+              yaml_variables: yaml_variables, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581
+              job_variables: yaml_variables,
               options: {
                 image: image_value,
                 services: services_value,
                 before_script: before_script_value,
                 script: script_value || DEFAULT_SCRIPT
-              }.compact }
+              }.compact
+            }.compact
           end
 
           def yaml_variables
-            return unless variables_value
+            strong_memoize(:yaml_variables) do
+              next unless variables_value
 
-            variables_value.map do |key, value|
-              { key: key.to_s, value: value, public: true }
+              variables_value.map do |key, value|
+                { key: key.to_s, value: value, public: true }
+              end
             end
           end
         end
diff --git a/lib/gitlab/word_diff/chunk_collection.rb b/lib/gitlab/word_diff/chunk_collection.rb
index dd388f75302..d5c3e59d405 100644
--- a/lib/gitlab/word_diff/chunk_collection.rb
+++ b/lib/gitlab/word_diff/chunk_collection.rb
@@ -18,6 +18,27 @@ module Gitlab
       def reset
         @chunks = []
       end
+
+      def marker_ranges
+        start = 0
+
+        @chunks.each_with_object([]) do |element, ranges|
+          mode = mode_for_element(element)
+
+          ranges << Gitlab::MarkerRange.new(start, start + element.length - 1, mode: mode) if mode
+
+          start += element.length
+        end
+      end
+
+      private
+
+      def mode_for_element(element)
+        return Gitlab::MarkerRange::DELETION if element.removed?
+        return Gitlab::MarkerRange::ADDITION if element.added?
+
+        nil
+      end
     end
   end
 end
diff --git a/lib/gitlab/word_diff/parser.rb b/lib/gitlab/word_diff/parser.rb
index 3b6d4d4d384..e611abb5692 100644
--- a/lib/gitlab/word_diff/parser.rb
+++ b/lib/gitlab/word_diff/parser.rb
@@ -31,7 +31,7 @@ module Gitlab
               @chunks.add(segment)
 
             when Segments::Newline
-              yielder << build_line(@chunks.content, nil, parent_file: diff_file)
+              yielder << build_line(@chunks.content, nil, parent_file: diff_file).tap { |line| line.set_marker_ranges(@chunks.marker_ranges) }
 
               @chunks.reset
               counter.increase_pos_num
-- 
cgit v1.2.3