From 311b0269b4eb9839fa63f80c8d7a58f32b8138a0 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 18 Nov 2021 13:16:36 +0000 Subject: Add latest changes from gitlab-org/gitlab@14-5-stable-ee --- lib/after_commit_queue.rb | 2 +- lib/api/api.rb | 4 +- lib/api/boards.rb | 2 +- lib/api/branches.rb | 4 +- lib/api/ci/jobs.rb | 35 +- lib/api/commits.rb | 10 +- .../packages/debian_distribution_endpoints.rb | 55 +- .../concerns/packages/debian_package_endpoints.rb | 10 +- lib/api/concerns/packages/npm_endpoints.rb | 4 +- lib/api/debian_group_packages.rb | 2 +- lib/api/deploy_keys.rb | 5 +- lib/api/discussions.rb | 2 +- lib/api/entities/alert_management/alert.rb | 12 + lib/api/entities/ci/job_request/service.rb | 1 + lib/api/entities/ci/lint/result.rb | 1 + lib/api/entities/ci/runner.rb | 4 +- lib/api/entities/ci/runner_details.rb | 8 +- lib/api/entities/deploy_key.rb | 3 + lib/api/entities/group.rb | 3 + lib/api/entities/project.rb | 1 + lib/api/entities/project_statistics.rb | 4 +- lib/api/entities/projects/topic.rb | 17 + lib/api/entities/todo.rb | 2 +- lib/api/error_tracking/collector.rb | 24 +- lib/api/features.rb | 9 +- lib/api/files.rb | 10 +- lib/api/generic_packages.rb | 9 +- lib/api/github/entities.rb | 4 +- lib/api/group_boards.rb | 2 +- lib/api/group_debian_distributions.rb | 8 - lib/api/group_labels.rb | 2 +- lib/api/group_milestones.rb | 2 +- lib/api/helpers.rb | 15 +- lib/api/helpers/award_emoji.rb | 2 +- lib/api/helpers/discussions_helpers.rb | 2 +- lib/api/helpers/file_upload_helpers.rb | 2 +- lib/api/helpers/integrations_helpers.rb | 41 +- lib/api/helpers/notes_helpers.rb | 2 +- lib/api/helpers/project_snapshots_helpers.rb | 2 + lib/api/helpers/projects_helpers.rb | 13 + lib/api/helpers/resource_label_events_helpers.rb | 2 +- lib/api/integrations.rb | 2 +- lib/api/internal/base.rb | 2 +- lib/api/internal/lfs.rb | 2 +- lib/api/invitations.rb | 2 + lib/api/issue_links.rb | 2 +- lib/api/issues.rb | 2 +- lib/api/labels.rb | 2 +- lib/api/lint.rb | 9 +- lib/api/members.rb | 2 + lib/api/merge_request_approvals.rb | 6 +- lib/api/namespaces.rb | 4 +- lib/api/package_files.rb | 5 +- lib/api/project_debian_distributions.rb | 8 - lib/api/project_milestones.rb | 2 +- lib/api/projects.rb | 12 +- lib/api/protected_branches.rb | 2 +- lib/api/releases.rb | 4 + lib/api/repositories.rb | 41 +- lib/api/resource_milestone_events.rb | 2 +- lib/api/resource_state_events.rb | 2 +- lib/api/snippets.rb | 2 +- lib/api/subscriptions.rb | 6 +- lib/api/tags.rb | 16 +- lib/api/terraform/modules/v1/packages.rb | 3 +- lib/api/todos.rb | 2 +- lib/api/topics.rb | 79 +++ lib/api/users.rb | 1 + lib/api/v3/github.rb | 32 +- lib/api/wikis.rb | 2 +- lib/backup/manager.rb | 15 +- lib/banzai/filter/emoji_filter.rb | 19 +- lib/banzai/filter/footnote_filter.rb | 56 +- lib/banzai/filter/markdown_engines/common_mark.rb | 52 +- lib/banzai/filter/markdown_post_escape_filter.rb | 18 +- lib/banzai/filter/plantuml_filter.rb | 18 +- lib/banzai/filter/sanitization_filter.rb | 9 +- lib/banzai/filter/syntax_highlight_filter.rb | 43 +- lib/banzai/renderer.rb | 42 +- lib/banzai/renderer/common_mark/html.rb | 2 + .../common/extractors/ndjson_extractor.rb | 24 +- .../common/pipelines/milestones_pipeline.rb | 15 + .../common/pipelines/uploads_pipeline.rb | 52 ++ lib/bulk_imports/common/pipelines/wiki_pipeline.rb | 42 ++ .../groups/graphql/get_milestones_query.rb | 56 -- lib/bulk_imports/groups/loaders/group_loader.rb | 27 +- .../groups/pipelines/milestones_pipeline.rb | 15 - lib/bulk_imports/groups/stage.rb | 2 +- lib/bulk_imports/ndjson_pipeline.rb | 9 +- .../pipelines/external_pull_requests_pipeline.rb | 15 + .../projects/pipelines/merge_requests_pipeline.rb | 19 + .../pipelines/protected_branches_pipeline.rb | 15 + .../projects/pipelines/repository_pipeline.rb | 10 +- lib/bulk_imports/projects/stage.rb | 26 +- lib/container_registry/client.rb | 4 +- lib/declarative_enum.rb | 19 +- lib/error_tracking/collector/payload_validator.rb | 13 + .../collector/sentry_request_parser.rb | 10 +- lib/feature.rb | 12 +- lib/feature/gitaly.rb | 2 +- .../gitlab/usage_metric_definition_generator.rb | 27 +- .../post_deployment_migration_generator.rb | 14 +- lib/gitlab/access.rb | 30 +- lib/gitlab/action_cable/config.rb | 4 - .../aggregated/base_query_builder.rb | 114 ++++ .../cycle_analytics/aggregated/data_collector.rb | 54 ++ .../cycle_analytics/aggregated/label_filter.rb | 31 + .../analytics/cycle_analytics/aggregated/median.rb | 36 + .../cycle_analytics/aggregated/records_fetcher.rb | 116 ++++ .../aggregated/stage_query_helpers.rb | 41 ++ .../analytics/cycle_analytics/data_collector.rb | 26 +- .../analytics/cycle_analytics/records_fetcher.rb | 18 +- .../analytics/cycle_analytics/request_params.rb | 3 +- lib/gitlab/application_rate_limiter.rb | 34 +- .../syntax_highlighter/html_pipeline_adapter.rb | 6 +- lib/gitlab/auth/o_auth/provider.rb | 1 + lib/gitlab/background_migration.rb | 102 +-- ...dd_primary_email_to_emails_if_user_confirmed.rb | 58 ++ .../backfill_design_internal_ids.rb | 130 ---- .../backfill_project_repositories.rb | 2 +- .../backfill_user_namespace.rb | 38 ++ ...uest_target_project_to_merge_request_metrics.rb | 25 - .../fix_merge_request_diff_commit_users.rb | 156 +++++ .../fix_orphan_promoted_issues.rb | 13 - .../fix_ruby_object_in_audit_events.rb | 13 - lib/gitlab/background_migration/job_coordinator.rb | 134 ++++ .../migrate_fingerprint_sha256_within_keys.rb | 2 +- .../migrate_issue_trackers_sensitive_data.rb | 146 ----- .../migrate_requirements_to_work_items.rb | 13 + .../migrate_users_bio_to_user_details.rb | 32 - .../populate_issue_email_participants.rb | 2 +- .../populate_user_highest_roles_table.rb | 58 -- .../backfill_project_namespaces.rb | 151 +++++ .../project_namespaces/models/namespace.rb | 17 + .../project_namespaces/models/project.rb | 16 + .../remove_duplicate_vulnerabilities_findings.rb | 20 +- ...remove_undefined_occurrence_confidence_level.rb | 13 - .../remove_undefined_occurrence_severity_level.rb | 13 - ...emove_undefined_vulnerability_severity_level.rb | 13 - .../set_default_iteration_cadences.rb | 60 -- .../set_null_external_diff_store_to_local_value.rb | 24 - ...null_package_files_file_store_to_local_value.rb | 26 - .../update_vulnerabilities_to_dismissed.rb | 13 - .../update_vulnerability_confidence.rb | 13 - .../update_vulnerability_occurrences_location.rb | 14 + lib/gitlab/bitbucket_server_import/importer.rb | 10 +- lib/gitlab/blob_helper.rb | 2 +- lib/gitlab/ci/artifact_file_reader.rb | 27 - lib/gitlab/ci/artifacts/metrics.rb | 15 +- lib/gitlab/ci/build/auto_retry.rb | 4 +- lib/gitlab/ci/build/context/base.rb | 22 + lib/gitlab/ci/build/image.rb | 9 +- lib/gitlab/ci/build/rules/rule/clause/exists.rb | 12 +- lib/gitlab/ci/config.rb | 31 +- lib/gitlab/ci/config/entry/include/rules/rule.rb | 4 +- lib/gitlab/ci/config/entry/job.rb | 8 +- lib/gitlab/ci/config/entry/processable.rb | 1 + lib/gitlab/ci/config/entry/service.rb | 6 +- lib/gitlab/ci/config/external/context.rb | 14 + lib/gitlab/ci/parsers/security/common.rb | 5 +- .../security/validators/schema_validator.rb | 2 +- .../validators/schemas/sast-report-format.json | 706 ++++++++++++++++++++ .../parsers/security/validators/schemas/sast.json | 706 -------------------- .../schemas/secret-detection-report-format.json | 729 +++++++++++++++++++++ .../validators/schemas/secret_detection.json | 729 --------------------- lib/gitlab/ci/pipeline/chain/command.rb | 3 + lib/gitlab/ci/pipeline/chain/config/process.rb | 2 +- .../chain/create_cross_database_associations.rb | 21 + lib/gitlab/ci/pipeline/metrics.rb | 9 + lib/gitlab/ci/pipeline/seed/build.rb | 14 +- lib/gitlab/ci/pipeline/seed/stage.rb | 2 +- lib/gitlab/ci/reports/security/finding.rb | 40 +- lib/gitlab/ci/reports/security/report.rb | 4 + lib/gitlab/ci/reports/security/reports.rb | 19 +- lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml | 8 + lib/gitlab/ci/templates/Django.gitlab-ci.yml | 116 ++-- .../Jobs/Browser-Performance-Testing.gitlab-ci.yml | 2 +- ...rowser-Performance-Testing.latest.gitlab-ci.yml | 2 +- .../ci/templates/Jobs/Build.latest.gitlab-ci.yml | 5 +- .../ci/templates/Jobs/CF-Provision.gitlab-ci.yml | 2 +- .../Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml | 8 +- lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml | 24 +- .../ci/templates/Jobs/Deploy.latest.gitlab-ci.yml | 16 +- .../ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml | 4 +- .../ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml | 12 +- .../ci/templates/Jobs/Helm-2to3.gitlab-ci.yml | 12 +- .../Jobs/Load-Performance-Testing.gitlab-ci.yml | 2 +- .../templates/Jobs/SAST-IaC.latest.gitlab-ci.yml | 34 + lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml | 47 ++ .../Security/API-Fuzzing.latest.gitlab-ci.yml | 2 +- .../Security/Cluster-Image-Scanning.gitlab-ci.yml | 2 +- .../ci/templates/Security/DAST.gitlab-ci.yml | 2 +- .../templates/Security/DAST.latest.gitlab-ci.yml | 4 +- .../Security/Dependency-Scanning.gitlab-ci.yml | 6 + .../Security/SAST-IaC.latest.gitlab-ci.yml | 2 + .../ci/templates/Terraform.latest.gitlab-ci.yml | 7 +- .../templates/Terraform/Base.latest.gitlab-ci.yml | 16 +- .../templates/Verify/Accessibility.gitlab-ci.yml | 2 +- lib/gitlab/ci/trace.rb | 17 +- lib/gitlab/ci/trace/archive.rb | 2 +- lib/gitlab/ci/trace/metrics.rb | 10 +- lib/gitlab/ci/variables/builder.rb | 49 ++ lib/gitlab/ci/variables/collection.rb | 4 +- lib/gitlab/ci/yaml_processor/result.rb | 1 - .../config_checker/external_database_checker.rb | 4 +- lib/gitlab/container_repository/tags/cache.rb | 72 ++ .../content_security_policy/config_loader.rb | 49 +- lib/gitlab/content_security_policy/directives.rb | 21 + lib/gitlab/contributions_calendar.rb | 17 +- lib/gitlab/current_settings.rb | 2 +- lib/gitlab/cycle_analytics/stage_summary.rb | 3 +- lib/gitlab/cycle_analytics/summary/base.rb | 4 + .../summary/deployment_frequency.rb | 9 +- lib/gitlab/database.rb | 57 +- lib/gitlab/database/as_with_materialized.rb | 2 +- lib/gitlab/database/async_indexes/index_creator.rb | 2 +- .../database/async_indexes/postgres_async_index.rb | 2 +- lib/gitlab/database/background_migration_job.rb | 1 + lib/gitlab/database/batch_counter.rb | 6 +- lib/gitlab/database/connection.rb | 260 -------- lib/gitlab/database/each_database.rb | 39 ++ lib/gitlab/database/gitlab_schema.rb | 96 +++ lib/gitlab/database/gitlab_schemas.yml | 543 +++++++++++++++ lib/gitlab/database/load_balancing.rb | 2 +- .../database/load_balancing/configuration.rb | 50 +- .../database/load_balancing/connection_proxy.rb | 7 + .../database/load_balancing/load_balancer.rb | 63 +- lib/gitlab/database/load_balancing/primary_host.rb | 5 + .../database/load_balancing/rack_middleware.rb | 10 +- lib/gitlab/database/load_balancing/setup.rb | 87 ++- .../load_balancing/sidekiq_server_middleware.rb | 10 +- lib/gitlab/database/load_balancing/sticking.rb | 5 +- lib/gitlab/database/migration_helpers.rb | 75 ++- .../cascading_namespace_settings.rb | 4 +- lib/gitlab/database/migrations/observation.rb | 3 + lib/gitlab/database/migrations/observers.rb | 3 +- .../migrations/observers/transaction_duration.rb | 42 ++ lib/gitlab/database/partitioning.rb | 83 ++- .../partitioning/detached_partition_dropper.rb | 96 ++- .../database/partitioning/monthly_strategy.rb | 4 - .../multi_database_partition_dropper.rb | 35 - .../multi_database_partition_manager.rb | 37 -- .../database/partitioning/partition_monitoring.rb | 18 +- lib/gitlab/database/partitioning/replace_table.rb | 7 +- lib/gitlab/database/partitioning/time_partition.rb | 2 +- .../table_management_helpers.rb | 4 +- lib/gitlab/database/postgres_foreign_key.rb | 6 + .../postgres_hll/batch_distinct_counter.rb | 6 +- lib/gitlab/database/postgres_index.rb | 3 +- .../database/postgres_index_bloat_estimate.rb | 2 +- lib/gitlab/database/query_analyzer.rb | 129 ++++ lib/gitlab/database/query_analyzers/base.rb | 53 ++ .../query_analyzers/gitlab_schemas_metrics.rb | 46 ++ .../prevent_cross_database_modification.rb | 119 ++++ lib/gitlab/database/reflection.rb | 115 ++++ lib/gitlab/database/reindexing.rb | 41 +- lib/gitlab/database/reindexing/index_selection.rb | 6 +- lib/gitlab/database/reindexing/queued_action.rb | 21 + lib/gitlab/database/reindexing/reindex_action.rb | 2 +- .../database/reindexing/reindex_concurrently.rb | 10 +- lib/gitlab/database/shared_model.rb | 8 +- lib/gitlab/database/unidirectional_copy_trigger.rb | 2 +- lib/gitlab/diff/file.rb | 29 + lib/gitlab/diff/highlight.rb | 3 + lib/gitlab/diff/highlight_cache.rb | 2 +- lib/gitlab/diff/position_tracer/line_strategy.rb | 8 +- lib/gitlab/email/handler/service_desk_handler.rb | 36 +- lib/gitlab/email/message/in_product_marketing.rb | 3 +- .../message/in_product_marketing/admin_verify.rb | 4 + .../email/message/in_product_marketing/base.rb | 37 +- .../email/message/in_product_marketing/create.rb | 4 + .../message/in_product_marketing/experience.rb | 16 +- .../email/message/in_product_marketing/helper.rb | 9 + .../message/in_product_marketing/invite_team.rb | 53 ++ .../email/message/in_product_marketing/verify.rb | 4 + lib/gitlab/email/receiver.rb | 8 +- lib/gitlab/email/reply_parser.rb | 12 +- lib/gitlab/emoji.rb | 68 +- lib/gitlab/etag_caching/router/restful.rb | 4 +- lib/gitlab/git.rb | 1 + lib/gitlab/git/blob.rb | 3 +- lib/gitlab/git/commit.rb | 14 +- lib/gitlab/git/repository.rb | 24 +- lib/gitlab/gitaly_client.rb | 25 +- lib/gitlab/gitaly_client/commit_service.rb | 2 + lib/gitlab/gitaly_client/ref_service.rb | 25 +- lib/gitlab/github_import/bulk_importing.rb | 2 +- .../github_import/importer/diff_note_importer.rb | 138 +++- .../github_import/importer/issue_importer.rb | 2 +- .../github_import/importer/label_links_importer.rb | 2 +- lib/gitlab/github_import/importer/note_importer.rb | 2 +- .../importer/pull_requests_merged_by_importer.rb | 15 +- .../importer/pull_requests_reviews_importer.rb | 2 +- .../github_import/representation/diff_note.rb | 104 ++- .../diff_notes/suggestion_formatter.rb | 34 +- lib/gitlab/gon_helper.rb | 4 +- lib/gitlab/gpg/commit.rb | 32 +- lib/gitlab/grape_logging/loggers/urgency_logger.rb | 19 + lib/gitlab/graphql/known_operations.rb | 45 ++ .../graphql/loaders/full_path_model_loader.rb | 5 +- .../pagination/keyset/generic_keyset_pagination.rb | 2 +- .../graphql/query_analyzers/logger_analyzer.rb | 50 +- .../graphql/tracers/application_context_tracer.rb | 40 ++ lib/gitlab/graphql/tracers/logger_tracer.rb | 58 ++ lib/gitlab/graphql/tracers/metrics_tracer.rb | 48 ++ lib/gitlab/graphql/tracers/timer_tracer.rb | 31 + lib/gitlab/graphql/variables.rb | 7 +- lib/gitlab/health_checks/metric.rb | 3 + lib/gitlab/health_checks/probes/status.rb | 3 + lib/gitlab/health_checks/redis/cache_check.rb | 26 +- lib/gitlab/health_checks/redis/queues_check.rb | 26 +- .../health_checks/redis/rate_limiting_check.rb | 26 +- .../health_checks/redis/redis_abstract_check.rb | 41 ++ lib/gitlab/health_checks/redis/redis_check.rb | 20 +- lib/gitlab/health_checks/redis/sessions_check.rb | 26 +- .../health_checks/redis/shared_state_check.rb | 26 +- .../health_checks/redis/trace_chunks_check.rb | 26 +- lib/gitlab/health_checks/result.rb | 3 + lib/gitlab/http.rb | 2 +- lib/gitlab/i18n.rb | 18 +- lib/gitlab/import/database_helpers.rb | 4 +- lib/gitlab/import/metrics.rb | 6 +- lib/gitlab/import_export/attributes_permitter.rb | 2 +- lib/gitlab/import_export/base/object_builder.rb | 12 +- .../decompressed_archive_size_validator.rb | 2 +- .../import_export/group/relation_tree_restorer.rb | 274 ++++++++ lib/gitlab/import_export/project/import_export.yml | 434 ++++++++++-- lib/gitlab/import_export/project/object_builder.rb | 40 +- .../import_export/project/relation_factory.rb | 4 +- .../project/relation_tree_restorer.rb | 27 + .../project/sample/relation_tree_restorer.rb | 6 +- lib/gitlab/import_export/project/tree_saver.rb | 36 +- lib/gitlab/import_export/relation_tree_restorer.rb | 280 -------- lib/gitlab/instrumentation/redis_interceptor.rb | 15 - lib/gitlab/instrumentation/uploads.rb | 32 + lib/gitlab/instrumentation_helper.rb | 5 + lib/gitlab/integrations/sti_type.rb | 2 +- lib/gitlab/issues/rebalancing/state.rb | 61 +- lib/gitlab/jira/http_client.rb | 1 - lib/gitlab/language_detection.rb | 2 +- lib/gitlab/lfs_token.rb | 15 +- lib/gitlab/lograge/custom_options.rb | 17 +- lib/gitlab/merge_requests/merge_commit_message.rb | 60 ++ lib/gitlab/metrics.rb | 2 +- lib/gitlab/metrics/background_transaction.rb | 17 +- lib/gitlab/metrics/methods.rb | 8 +- lib/gitlab/metrics/rails_slis.rb | 32 +- lib/gitlab/metrics/requests_rack_middleware.rb | 18 +- .../metrics/samplers/action_cable_sampler.rb | 23 +- lib/gitlab/metrics/subscribers/action_view.rb | 2 +- lib/gitlab/metrics/subscribers/external_http.rb | 2 +- lib/gitlab/metrics/subscribers/rails_cache.rb | 2 +- lib/gitlab/metrics/transaction.rb | 25 - lib/gitlab/metrics/web_transaction.rb | 25 + lib/gitlab/middleware/compressed_json.rb | 66 ++ lib/gitlab/middleware/go.rb | 2 + lib/gitlab/middleware/query_analyzer.rb | 15 + lib/gitlab/middleware/release_env.rb | 3 + lib/gitlab/pagination/gitaly_keyset_pager.rb | 4 + lib/gitlab/patch/sidekiq_client.rb | 22 + lib/gitlab/patch/sidekiq_cron_poller.rb | 17 + lib/gitlab/project_template.rb | 1 + lib/gitlab/prometheus/queries/validate_query.rb | 2 +- lib/gitlab/prometheus_client.rb | 8 +- lib/gitlab/quick_actions/issuable_actions.rb | 7 +- lib/gitlab/quick_actions/issue_actions.rb | 31 +- .../issue_and_merge_request_actions.rb | 14 +- lib/gitlab/redis/hll.rb | 3 + lib/gitlab/redis/multi_store.rb | 215 ++++++ lib/gitlab/redis/wrapper.rb | 6 +- lib/gitlab/runtime.rb | 15 +- lib/gitlab/saas.rb | 4 +- lib/gitlab/search_results.rb | 5 + lib/gitlab/setup_helper.rb | 7 +- lib/gitlab/sidekiq_cluster.rb | 171 ----- lib/gitlab/sidekiq_cluster/cli.rb | 230 ------- lib/gitlab/sidekiq_config.rb | 34 +- lib/gitlab/sidekiq_config/cli_methods.rb | 1 + lib/gitlab/sidekiq_config/worker.rb | 7 +- lib/gitlab/sidekiq_enq.rb | 80 ++- lib/gitlab/sidekiq_logging/deduplication_logger.rb | 10 +- lib/gitlab/sidekiq_logging/json_formatter.rb | 2 +- lib/gitlab/sidekiq_middleware.rb | 1 + .../duplicate_jobs/duplicate_job.rb | 54 +- .../sidekiq_middleware/duplicate_jobs/server.rb | 2 +- .../duplicate_jobs/strategies/base.rb | 4 +- .../strategies/deduplicates_when_scheduling.rb | 15 +- .../duplicate_jobs/strategies/until_executed.rb | 3 + lib/gitlab/sidekiq_middleware/query_analyzer.rb | 11 + .../sidekiq_middleware/size_limiter/validator.rb | 39 +- lib/gitlab/sidekiq_status.rb | 6 +- lib/gitlab/slash_commands/result.rb | 3 + lib/gitlab/spamcheck/client.rb | 18 +- lib/gitlab/subscription_portal.rb | 8 +- lib/gitlab/template_parser/ast.rb | 3 + lib/gitlab/testing/request_inspector_middleware.rb | 4 +- lib/gitlab/tracking.rb | 27 +- .../tracking/destinations/product_analytics.rb | 41 -- lib/gitlab/tracking/destinations/snowplow.rb | 36 +- lib/gitlab/tracking/destinations/snowplow_micro.rb | 48 ++ lib/gitlab/tracking/standard_context.rb | 15 +- lib/gitlab/url_blocker.rb | 16 +- lib/gitlab/usage/metric.rb | 4 + .../usage/metrics/names_suggestions/generator.rb | 8 +- lib/gitlab/usage_data.rb | 109 ++- .../known_events/ci_templates.yml | 20 + .../known_events/code_review_events.yml | 24 + .../known_events/quickactions.yml | 4 + .../vs_code_extension_activity_unique_counter.rb | 28 - .../vscode_extension_activity_unique_counter.rb | 28 + lib/gitlab/usage_data_metrics.rb | 12 +- lib/gitlab/usage_data_non_sql_metrics.rb | 11 +- lib/gitlab/usage_data_queries.rb | 11 +- lib/gitlab/utils/usage_data.rb | 9 +- lib/gitlab/webpack/file_loader.rb | 65 ++ lib/gitlab/webpack/graphql_known_operations.rb | 25 + lib/gitlab/webpack/manifest.rb | 50 +- lib/gitlab/workhorse.rb | 5 +- lib/gitlab/x509/certificate.rb | 28 + lib/gitlab/zentao/client.rb | 19 +- lib/gitlab/zentao/query.rb | 78 +++ lib/object_storage/config.rb | 10 - .../ci_configuration/sast_iac_build_action.rb | 19 + .../groups/menus/customer_relations_menu.rb | 60 ++ .../groups/menus/invite_team_members_menu.rb | 46 ++ .../groups/menus/packages_registries_menu.rb | 2 +- lib/sidebars/groups/panel.rb | 11 + lib/sidebars/panel.rb | 1 + lib/sidebars/projects/menus/confluence_menu.rb | 5 + lib/sidebars/projects/menus/infrastructure_menu.rb | 4 +- .../projects/menus/invite_team_members_menu.rb | 47 ++ lib/sidebars/projects/menus/settings_menu.rb | 4 - lib/sidebars/projects/menus/zentao_menu.rb | 77 +++ lib/sidebars/projects/panel.rb | 9 + lib/support/init.d/gitlab | 8 +- lib/support/systemd/gitlab-gitaly.service | 17 + lib/support/systemd/gitlab-mailroom.service | 19 + lib/support/systemd/gitlab-pages.service | 19 + lib/support/systemd/gitlab-puma.service | 26 + lib/support/systemd/gitlab-sidekiq.service | 22 + lib/support/systemd/gitlab-workhorse.service | 21 + lib/support/systemd/gitlab.slice | 8 + lib/support/systemd/gitlab.target | 6 + lib/system_check/app/init_script_exists_check.rb | 29 - .../app/init_script_up_to_date_check.rb | 47 -- ...ystemd_unit_files_or_init_script_exist_check.rb | 39 ++ ...d_unit_files_or_init_script_up_to_date_check.rb | 80 +++ .../incoming_email/initd_configured_check.rb | 34 - .../incoming_email/mail_room_enabled_check.rb | 42 ++ .../incoming_email/mail_room_running_check.rb | 14 +- lib/system_check/incoming_email_check.rb | 2 +- lib/system_check/init_helpers.rb | 24 + lib/system_check/rake_task/app_task.rb | 4 +- lib/system_check/sidekiq_check.rb | 6 + lib/tasks/gemojione.rake | 245 ------- lib/tasks/gitlab/db.rake | 45 +- lib/tasks/gitlab/docs/compile_deprecations.rake | 2 +- lib/tasks/gitlab/gitaly.rake | 3 +- lib/tasks/gitlab/info.rake | 4 +- lib/tasks/gitlab/sidekiq.rake | 7 +- lib/tasks/gitlab/storage.rake | 2 +- lib/tasks/haml-lint.rake | 11 - lib/tasks/tanuki_emoji.rake | 260 ++++++++ lib/uploaded_file.rb | 16 +- 464 files changed, 9865 insertions(+), 5258 deletions(-) create mode 100644 lib/api/entities/alert_management/alert.rb create mode 100644 lib/api/entities/projects/topic.rb create mode 100644 lib/api/topics.rb create mode 100644 lib/bulk_imports/common/pipelines/milestones_pipeline.rb create mode 100644 lib/bulk_imports/common/pipelines/uploads_pipeline.rb create mode 100644 lib/bulk_imports/common/pipelines/wiki_pipeline.rb delete mode 100644 lib/bulk_imports/groups/graphql/get_milestones_query.rb delete mode 100644 lib/bulk_imports/groups/pipelines/milestones_pipeline.rb create mode 100644 lib/bulk_imports/projects/pipelines/external_pull_requests_pipeline.rb create mode 100644 lib/bulk_imports/projects/pipelines/merge_requests_pipeline.rb create mode 100644 lib/bulk_imports/projects/pipelines/protected_branches_pipeline.rb create mode 100644 lib/error_tracking/collector/payload_validator.rb create mode 100644 lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb create mode 100644 lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb create mode 100644 lib/gitlab/analytics/cycle_analytics/aggregated/label_filter.rb create mode 100644 lib/gitlab/analytics/cycle_analytics/aggregated/median.rb create mode 100644 lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb create mode 100644 lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb create mode 100644 lib/gitlab/background_migration/add_primary_email_to_emails_if_user_confirmed.rb delete mode 100644 lib/gitlab/background_migration/backfill_design_internal_ids.rb create mode 100644 lib/gitlab/background_migration/backfill_user_namespace.rb delete mode 100644 lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb create mode 100644 lib/gitlab/background_migration/fix_merge_request_diff_commit_users.rb delete mode 100644 lib/gitlab/background_migration/fix_orphan_promoted_issues.rb delete mode 100644 lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb create mode 100644 lib/gitlab/background_migration/job_coordinator.rb delete mode 100644 lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb create mode 100644 lib/gitlab/background_migration/migrate_requirements_to_work_items.rb delete mode 100644 lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb delete mode 100644 lib/gitlab/background_migration/populate_user_highest_roles_table.rb create mode 100644 lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb create mode 100644 lib/gitlab/background_migration/project_namespaces/models/namespace.rb create mode 100644 lib/gitlab/background_migration/project_namespaces/models/project.rb delete mode 100644 lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb delete mode 100644 lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb delete mode 100644 lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb delete mode 100644 lib/gitlab/background_migration/set_default_iteration_cadences.rb delete mode 100644 lib/gitlab/background_migration/set_null_external_diff_store_to_local_value.rb delete mode 100644 lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb delete mode 100644 lib/gitlab/background_migration/update_vulnerabilities_to_dismissed.rb delete mode 100644 lib/gitlab/background_migration/update_vulnerability_confidence.rb create mode 100644 lib/gitlab/background_migration/update_vulnerability_occurrences_location.rb create mode 100644 lib/gitlab/ci/parsers/security/validators/schemas/sast-report-format.json delete mode 100644 lib/gitlab/ci/parsers/security/validators/schemas/sast.json create mode 100644 lib/gitlab/ci/parsers/security/validators/schemas/secret-detection-report-format.json delete mode 100644 lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json create mode 100644 lib/gitlab/ci/pipeline/chain/create_cross_database_associations.rb create mode 100644 lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml create mode 100644 lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml create mode 100644 lib/gitlab/ci/templates/Security/SAST-IaC.latest.gitlab-ci.yml create mode 100644 lib/gitlab/ci/variables/builder.rb create mode 100644 lib/gitlab/container_repository/tags/cache.rb create mode 100644 lib/gitlab/content_security_policy/directives.rb delete mode 100644 lib/gitlab/database/connection.rb create mode 100644 lib/gitlab/database/each_database.rb create mode 100644 lib/gitlab/database/gitlab_schema.rb create mode 100644 lib/gitlab/database/gitlab_schemas.yml create mode 100644 lib/gitlab/database/migrations/observers/transaction_duration.rb delete mode 100644 lib/gitlab/database/partitioning/multi_database_partition_dropper.rb delete mode 100644 lib/gitlab/database/partitioning/multi_database_partition_manager.rb create mode 100644 lib/gitlab/database/query_analyzer.rb create mode 100644 lib/gitlab/database/query_analyzers/base.rb create mode 100644 lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb create mode 100644 lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb create mode 100644 lib/gitlab/database/reflection.rb create mode 100644 lib/gitlab/database/reindexing/queued_action.rb create mode 100644 lib/gitlab/email/message/in_product_marketing/invite_team.rb create mode 100644 lib/gitlab/grape_logging/loggers/urgency_logger.rb create mode 100644 lib/gitlab/graphql/known_operations.rb create mode 100644 lib/gitlab/graphql/tracers/application_context_tracer.rb create mode 100644 lib/gitlab/graphql/tracers/logger_tracer.rb create mode 100644 lib/gitlab/graphql/tracers/metrics_tracer.rb create mode 100644 lib/gitlab/graphql/tracers/timer_tracer.rb create mode 100644 lib/gitlab/health_checks/redis/redis_abstract_check.rb create mode 100644 lib/gitlab/import_export/group/relation_tree_restorer.rb create mode 100644 lib/gitlab/import_export/project/relation_tree_restorer.rb delete mode 100644 lib/gitlab/import_export/relation_tree_restorer.rb create mode 100644 lib/gitlab/instrumentation/uploads.rb create mode 100644 lib/gitlab/merge_requests/merge_commit_message.rb create mode 100644 lib/gitlab/middleware/compressed_json.rb create mode 100644 lib/gitlab/middleware/query_analyzer.rb create mode 100644 lib/gitlab/patch/sidekiq_client.rb create mode 100644 lib/gitlab/patch/sidekiq_cron_poller.rb create mode 100644 lib/gitlab/redis/multi_store.rb delete mode 100644 lib/gitlab/sidekiq_cluster.rb delete mode 100644 lib/gitlab/sidekiq_cluster/cli.rb create mode 100644 lib/gitlab/sidekiq_middleware/query_analyzer.rb delete mode 100644 lib/gitlab/tracking/destinations/product_analytics.rb create mode 100644 lib/gitlab/tracking/destinations/snowplow_micro.rb delete mode 100644 lib/gitlab/usage_data_counters/vs_code_extension_activity_unique_counter.rb create mode 100644 lib/gitlab/usage_data_counters/vscode_extension_activity_unique_counter.rb create mode 100644 lib/gitlab/webpack/file_loader.rb create mode 100644 lib/gitlab/webpack/graphql_known_operations.rb create mode 100644 lib/gitlab/zentao/query.rb create mode 100644 lib/security/ci_configuration/sast_iac_build_action.rb create mode 100644 lib/sidebars/groups/menus/customer_relations_menu.rb create mode 100644 lib/sidebars/groups/menus/invite_team_members_menu.rb create mode 100644 lib/sidebars/projects/menus/invite_team_members_menu.rb create mode 100644 lib/sidebars/projects/menus/zentao_menu.rb create mode 100644 lib/support/systemd/gitlab-gitaly.service create mode 100644 lib/support/systemd/gitlab-mailroom.service create mode 100644 lib/support/systemd/gitlab-pages.service create mode 100644 lib/support/systemd/gitlab-puma.service create mode 100644 lib/support/systemd/gitlab-sidekiq.service create mode 100644 lib/support/systemd/gitlab-workhorse.service create mode 100644 lib/support/systemd/gitlab.slice create mode 100644 lib/support/systemd/gitlab.target delete mode 100644 lib/system_check/app/init_script_exists_check.rb delete mode 100644 lib/system_check/app/init_script_up_to_date_check.rb create mode 100644 lib/system_check/app/systemd_unit_files_or_init_script_exist_check.rb create mode 100644 lib/system_check/app/systemd_unit_files_or_init_script_up_to_date_check.rb delete mode 100644 lib/system_check/incoming_email/initd_configured_check.rb create mode 100644 lib/system_check/incoming_email/mail_room_enabled_check.rb create mode 100644 lib/system_check/init_helpers.rb delete mode 100644 lib/tasks/gemojione.rake create mode 100644 lib/tasks/tanuki_emoji.rake (limited to 'lib') diff --git a/lib/after_commit_queue.rb b/lib/after_commit_queue.rb index 2698d7adbd7..cbeaea97951 100644 --- a/lib/after_commit_queue.rb +++ b/lib/after_commit_queue.rb @@ -15,7 +15,7 @@ module AfterCommitQueue end def run_after_commit_or_now(&block) - if Gitlab::Database.main.inside_transaction? + if ApplicationRecord.inside_transaction? if ActiveRecord::Base.connection.current_transaction.records&.include?(self) run_after_commit(&block) else diff --git a/lib/api/api.rb b/lib/api/api.rb index a4d42c735cb..dcecaeae558 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -27,7 +27,8 @@ module API Gitlab::GrapeLogging::Loggers::PerfLogger.new, Gitlab::GrapeLogging::Loggers::CorrelationIdLogger.new, Gitlab::GrapeLogging::Loggers::ContextLogger.new, - Gitlab::GrapeLogging::Loggers::ContentLogger.new + Gitlab::GrapeLogging::Loggers::ContentLogger.new, + Gitlab::GrapeLogging::Loggers::UrgencyLogger.new ] allow_access_with_scope :api @@ -283,6 +284,7 @@ module API mount ::API::Tags mount ::API::Templates mount ::API::Todos + mount ::API::Topics mount ::API::Unleash mount ::API::UsageData mount ::API::UsageDataQueries diff --git a/lib/api/boards.rb b/lib/api/boards.rb index 9e829dd5e05..56633c07774 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -7,7 +7,7 @@ module API prepend_mod_with('API::BoardsResponses') # rubocop: disable Cop/InjectEnterpriseEditionModule - feature_category :boards + feature_category :team_planning before { authenticate! } diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 0db5bb82296..462c4a3de4c 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -41,7 +41,7 @@ module API optional :page_token, type: String, desc: 'Name of branch to start the paginaition from' end - get ':id/repository/branches' do + get ':id/repository/branches', urgency: :low do ff_enabled = Feature.enabled?(:api_caching_rate_limit_branches, user_project, default_enabled: :yaml) cache_action_if(ff_enabled, [user_project, :branches, current_user, declared_params], expires_in: 30.seconds) do @@ -86,7 +86,7 @@ module API head do user_project.repository.branch_exists?(params[:branch]) ? no_content! : not_found! end - get do + get '/', urgency: :low do branch = find_branch!(params[:branch]) present branch, with: Entities::Branch, current_user: current_user, project: user_project diff --git a/lib/api/ci/jobs.rb b/lib/api/ci/jobs.rb index eea1637c32a..30ce1454419 100644 --- a/lib/api/ci/jobs.rb +++ b/lib/api/ci/jobs.rb @@ -177,6 +177,39 @@ module API present current_authenticated_job, with: Entities::Ci::Job end + + desc 'Get current agents' do + detail 'Retrieves a list of agents for the given job token' + end + route_setting :authentication, job_token_allowed: true + get '/allowed_agents', feature_category: :kubernetes_management do + validate_current_authenticated_job + + status 200 + + pipeline = current_authenticated_job.pipeline + project = current_authenticated_job.project + agent_authorizations = Clusters::AgentAuthorizationsFinder.new(project).execute + project_groups = project.group&.self_and_ancestor_ids&.map { |id| { id: id } } || [] + user_access_level = project.team.max_member_access(current_user.id) + roles_in_project = Gitlab::Access.sym_options_with_owner + .select { |_role, role_access_level| role_access_level <= user_access_level } + .map(&:first) + + environment = if environment_slug = current_authenticated_job.deployment&.environment&.slug + { slug: environment_slug } + end + + # See https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/kubernetes_ci_access.md#apiv4joballowed_agents-api + { + allowed_agents: Entities::Clusters::AgentAuthorization.represent(agent_authorizations), + job: { id: current_authenticated_job.id }, + pipeline: { id: pipeline.id }, + project: { id: project.id, groups: project_groups }, + user: { id: current_user.id, username: current_user.username, roles_in_project: roles_in_project }, + environment: environment + }.compact + end end helpers do @@ -202,5 +235,3 @@ module API end end end - -API::Ci::Jobs.prepend_mod_with('API::Ci::Jobs') diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 10dc51556b9..8b8d8192524 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -27,7 +27,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS, urgency: :low do desc 'Get a project repository commits' do success Entities::Commit end @@ -43,7 +43,7 @@ module API optional :trailers, type: Boolean, desc: 'Parse and include Git trailers for every commit', default: false use :pagination end - get ':id/repository/commits' do + get ':id/repository/commits', urgency: :low do path = params[:path] before = params[:until] after = params[:since] @@ -169,7 +169,7 @@ module API requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' use :pagination end - get ':id/repository/commits/:sha/diff', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do + get ':id/repository/commits/:sha/diff', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS, urgency: :low do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit @@ -295,7 +295,7 @@ module API optional :type, type: String, values: %w[branch tag all], default: 'all', desc: 'Scope' use :pagination end - get ':id/repository/commits/:sha/refs', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do + get ':id/repository/commits/:sha/refs', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS, urgency: :low do commit = user_project.commit(params[:sha]) not_found!('Commit') unless commit @@ -363,7 +363,7 @@ module API requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag on which to find Merge Requests' use :pagination end - get ':id/repository/commits/:sha/merge_requests', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do + get ':id/repository/commits/:sha/merge_requests', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS, urgency: :low do authorize! :read_merge_request, user_project commit = user_project.commit(params[:sha]) diff --git a/lib/api/concerns/packages/debian_distribution_endpoints.rb b/lib/api/concerns/packages/debian_distribution_endpoints.rb index 798e583b87a..ddc83d0f747 100644 --- a/lib/api/concerns/packages/debian_distribution_endpoints.rb +++ b/lib/api/concerns/packages/debian_distribution_endpoints.rb @@ -15,6 +15,12 @@ module API helpers ::API::Helpers::Packages::BasicAuthHelpers include ::API::Helpers::Authentication + helpers do + def distribution + ::Packages::Debian::DistributionsFinder.new(project_or_group, codename: params[:codename]).execute.last || not_found!('Distribution') + end + end + namespace 'debian_distributions' do helpers do params :optional_distribution_params do @@ -36,9 +42,18 @@ module API end end + rescue_from ArgumentError do |e| + render_api_error!(e.message, 400) + end + + rescue_from ActiveRecord::RecordInvalid do |e| + render_api_error!(e.message, 400) + end + authenticate_with do |accept| - accept.token_types(:personal_access_token, :deploy_token, :job_token) - .sent_through(:http_basic_auth) + accept.token_types(:personal_access_token).sent_through(:http_private_token_header) + accept.token_types(:deploy_token).sent_through(:http_deploy_token_header) + accept.token_types(:job_token).sent_through(:http_job_token_header) end content_type :json, 'application/json' @@ -59,12 +74,12 @@ module API distribution_params = declared_params(include_missing: false) result = ::Packages::Debian::CreateDistributionService.new(project_or_group, current_user, distribution_params).execute - distribution = result.payload[:distribution] + created_distribution = result.payload[:distribution] if result.success? - present distribution, with: ::API::Entities::Packages::Debian::Distribution + present created_distribution, with: ::API::Entities::Packages::Debian::Distribution else - render_validation_error!(distribution) + render_validation_error!(created_distribution) end end @@ -100,11 +115,28 @@ module API get '/:codename' do authorize_read_package!(project_or_group) - distribution = ::Packages::Debian::DistributionsFinder.new(project_or_group, codename: params[:codename]).execute.last! - present distribution, with: ::API::Entities::Packages::Debian::Distribution end + # GET {projects|groups}/:id/debian_distributions/:codename/key + desc 'Get a Debian Distribution Key' do + detail 'This feature was introduced in 14.4' + success ::API::Entities::Packages::Debian::Distribution + end + + params do + requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename' + end + get '/:codename/key.asc' do + authorize_read_package!(project_or_group) + + content_type 'text/plain' + env['api.format'] = :binary + header 'Content-Disposition', "attachment; filename*=UTF-8''#{CGI.escape(params[:codename])}.asc" + + distribution.key&.public_key || not_found!('Distribution key') + end + # PUT {projects|groups}/:id/debian_distributions/:codename desc 'Update a Debian Distribution' do detail 'This feature was introduced in 14.0' @@ -118,15 +150,14 @@ module API put '/:codename' do authorize_create_package!(project_or_group) - distribution = ::Packages::Debian::DistributionsFinder.new(project_or_group, codename: params[:codename]).execute.last! distribution_params = declared_params(include_missing: false).except(:codename) result = ::Packages::Debian::UpdateDistributionService.new(distribution, distribution_params).execute - distribution = result.payload[:distribution] + updated_distribution = result.payload[:distribution] if result.success? - present distribution, with: ::API::Entities::Packages::Debian::Distribution + present updated_distribution, with: ::API::Entities::Packages::Debian::Distribution else - render_validation_error!(distribution) + render_validation_error!(updated_distribution) end end @@ -142,8 +173,6 @@ module API delete '/:codename' do authorize_destroy_package!(project_or_group) - distribution = ::Packages::Debian::DistributionsFinder.new(project_or_group, codename: params[:codename]).execute.last! - accepted! if distribution.destroy render_api_error!('Failed to delete distribution', 400) diff --git a/lib/api/concerns/packages/debian_package_endpoints.rb b/lib/api/concerns/packages/debian_package_endpoints.rb index 0acc015f366..d083643f3d0 100644 --- a/lib/api/concerns/packages/debian_package_endpoints.rb +++ b/lib/api/concerns/packages/debian_package_endpoints.rb @@ -43,11 +43,6 @@ module API end end - authenticate_with do |accept| - accept.token_types(:personal_access_token, :deploy_token, :job_token) - .sent_through(:http_basic_auth) - end - rescue_from ArgumentError do |e| render_api_error!(e.message, 400) end @@ -56,6 +51,11 @@ module API render_api_error!(e.message, 400) end + authenticate_with do |accept| + accept.token_types(:personal_access_token, :deploy_token, :job_token) + .sent_through(:http_basic_auth) + end + format :txt content_type :txt, 'text/plain' diff --git a/lib/api/concerns/packages/npm_endpoints.rb b/lib/api/concerns/packages/npm_endpoints.rb index d6e006df976..7a657be5bf3 100644 --- a/lib/api/concerns/packages/npm_endpoints.rb +++ b/lib/api/concerns/packages/npm_endpoints.rb @@ -121,7 +121,9 @@ module API not_found!('Packages') if packages.empty? - present ::Packages::Npm::PackagePresenter.new(package_name, packages), + include_metadata = Feature.enabled?(:packages_npm_abbreviated_metadata, project, default_enabled: :yaml) + + present ::Packages::Npm::PackagePresenter.new(package_name, packages, include_metadata: include_metadata), with: ::API::Entities::NpmPackage end end diff --git a/lib/api/debian_group_packages.rb b/lib/api/debian_group_packages.rb index 29f5047230a..1f640cc17d0 100644 --- a/lib/api/debian_group_packages.rb +++ b/lib/api/debian_group_packages.rb @@ -32,7 +32,7 @@ module API namespace ':id/-/packages/debian' do include ::API::Concerns::Packages::DebianPackageEndpoints - # GET groups/:id/packages/debian/pool/:distribution/:project_id/:letter/:package_name/:package_version/:file_name + # GET groups/:id/-/packages/debian/pool/:distribution/:project_id/:letter/:package_name/:package_version/:file_name params do requires :project_id, type: Integer, desc: 'The Project Id' use :shared_package_file_params diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 9f0f569b711..0ab9fe6644c 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -23,11 +23,14 @@ module API desc 'Return all deploy keys' params do use :pagination + optional :public, type: Boolean, default: false, desc: "Only return deploy keys that are public" end get "deploy_keys" do authenticated_as_admin! - present paginate(DeployKey.all), with: Entities::DeployKey + deploy_keys = params[:public] ? DeployKey.are_public : DeployKey.all + + present paginate(deploy_keys.including_projects_with_write_access), with: Entities::DeployKey, include_projects_with_write_access: true end params do diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index 580d546b360..cf4b2348458 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -239,7 +239,7 @@ module API # rubocop: disable CodeReuse/ActiveRecord def readable_discussion_notes(noteable, discussion_ids) notes = noteable.notes - .where(discussion_id: discussion_ids) + .with_discussion_ids(discussion_ids) .inc_relations_for_view .includes(:noteable) .fresh diff --git a/lib/api/entities/alert_management/alert.rb b/lib/api/entities/alert_management/alert.rb new file mode 100644 index 00000000000..664cd53293e --- /dev/null +++ b/lib/api/entities/alert_management/alert.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module API + module Entities + module AlertManagement + class Alert < Grape::Entity + expose :iid + expose :title + end + end + end +end diff --git a/lib/api/entities/ci/job_request/service.rb b/lib/api/entities/ci/job_request/service.rb index f89b95c1d5c..0dae5d5a933 100644 --- a/lib/api/entities/ci/job_request/service.rb +++ b/lib/api/entities/ci/job_request/service.rb @@ -6,6 +6,7 @@ module API module JobRequest class Service < Entities::Ci::JobRequest::Image expose :alias, :command + expose :variables end end end diff --git a/lib/api/entities/ci/lint/result.rb b/lib/api/entities/ci/lint/result.rb index 0e4aa238ba2..39039868bba 100644 --- a/lib/api/entities/ci/lint/result.rb +++ b/lib/api/entities/ci/lint/result.rb @@ -9,6 +9,7 @@ module API expose :errors expose :warnings expose :merged_yaml + expose :jobs, if: -> (result, options) { options[:include_jobs] } end end end diff --git a/lib/api/entities/ci/runner.rb b/lib/api/entities/ci/runner.rb index ede698696de..60193fe1df4 100644 --- a/lib/api/entities/ci/runner.rb +++ b/lib/api/entities/ci/runner.rb @@ -12,7 +12,9 @@ module API expose :runner_type expose :name expose :online?, as: :online - expose :status + # DEPRECATED + # TODO Remove in %15.0 in favor of `status` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/344648 + expose :status, as: :deprecated_rest_status end end end diff --git a/lib/api/entities/ci/runner_details.rb b/lib/api/entities/ci/runner_details.rb index 9d44da7e5b3..6ded1296f2a 100644 --- a/lib/api/entities/ci/runner_details.rb +++ b/lib/api/entities/ci/runner_details.rb @@ -15,18 +15,18 @@ module API # rubocop: disable CodeReuse/ActiveRecord expose :projects, with: Entities::BasicProjectDetails do |runner, options| if options[:current_user].admin? # rubocop: disable Cop/UserAdmin - runner.projects + runner.projects.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338659') else - options[:current_user].authorized_projects.where(id: runner.projects) + options[:current_user].authorized_projects.where(id: runner.projects).allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338659') end end # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord expose :groups, with: Entities::BasicGroupDetails do |runner, options| if options[:current_user].admin? # rubocop: disable Cop/UserAdmin - runner.groups + runner.groups.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338659') else - options[:current_user].authorized_groups.where(id: runner.groups) + options[:current_user].authorized_groups.where(id: runner.groups).allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338659') end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/entities/deploy_key.rb b/lib/api/entities/deploy_key.rb index ed922c24eda..e8537c4c677 100644 --- a/lib/api/entities/deploy_key.rb +++ b/lib/api/entities/deploy_key.rb @@ -4,6 +4,9 @@ module API module Entities class DeployKey < Entities::SSHKey expose :key + expose :fingerprint + + expose :projects_with_write_access, using: Entities::ProjectIdentity, if: -> (_, options) { options[:include_projects_with_write_access] } end end end diff --git a/lib/api/entities/group.rb b/lib/api/entities/group.rb index 048b7a3c15a..246fb819890 100644 --- a/lib/api/entities/group.rb +++ b/lib/api/entities/group.rb @@ -31,7 +31,10 @@ module API expose :wiki_size expose :lfs_objects_size expose :build_artifacts_size, as: :job_artifacts_size + expose :pipeline_artifacts_size + expose :packages_size expose :snippets_size + expose :uploads_size end end end diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index 41320d184f9..e3f1e90b80f 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -114,6 +114,7 @@ module API expose :merge_method expose :squash_option expose :suggestion_commit_message + expose :merge_commit_template expose :statistics, using: 'API::Entities::ProjectStatistics', if: -> (project, options) { options[:statistics] && Ability.allowed?(options[:current_user], :read_statistics, project) } diff --git a/lib/api/entities/project_statistics.rb b/lib/api/entities/project_statistics.rb index 70980e670b0..6544e8bc8ff 100644 --- a/lib/api/entities/project_statistics.rb +++ b/lib/api/entities/project_statistics.rb @@ -9,8 +9,10 @@ module API expose :wiki_size expose :lfs_objects_size expose :build_artifacts_size, as: :job_artifacts_size - expose :snippets_size + expose :pipeline_artifacts_size expose :packages_size + expose :snippets_size + expose :uploads_size end end end diff --git a/lib/api/entities/projects/topic.rb b/lib/api/entities/projects/topic.rb new file mode 100644 index 00000000000..d3d1cbec81c --- /dev/null +++ b/lib/api/entities/projects/topic.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module API + module Entities + module Projects + class Topic < Grape::Entity + expose :id + expose :name + expose :description + expose :total_projects_count + expose :avatar_url do |topic, options| + topic.avatar_url(only_path: false) + end + end + end + end +end diff --git a/lib/api/entities/todo.rb b/lib/api/entities/todo.rb index 8d222db488a..5bbbb59f565 100644 --- a/lib/api/entities/todo.rb +++ b/lib/api/entities/todo.rb @@ -33,7 +33,7 @@ module API def todo_target_url(todo) return design_todo_target_url(todo) if todo.for_design? - target_type = todo.target_type.underscore + target_type = todo.target_type.gsub('::', '_').underscore target_url = "#{todo.resource_parent.class.to_s.underscore}_#{target_type}_url" Gitlab::Routing diff --git a/lib/api/error_tracking/collector.rb b/lib/api/error_tracking/collector.rb index 22fbd3a1118..13fda356257 100644 --- a/lib/api/error_tracking/collector.rb +++ b/lib/api/error_tracking/collector.rb @@ -12,6 +12,10 @@ module API content_type :txt, 'text/plain' default_format :envelope + rescue_from ActiveRecord::RecordInvalid do |e| + render_api_error!(e.message, 400) + end + before do not_found!('Project') unless project not_found! unless feature_enabled? @@ -50,6 +54,12 @@ module API bad_request!('Failed to parse sentry request') end end + + def validate_payload(payload) + unless ::ErrorTracking::Collector::PayloadValidator.new.valid?(payload) + bad_request!('Unsupported sentry payload') + end + end end desc 'Submit error tracking event to the project as envelope' do @@ -88,6 +98,8 @@ module API # We don't have use for transaction request yet, # so we record only event one. if type == 'event' + validate_payload(parsed_request[:event]) + ::ErrorTracking::CollectErrorService .new(project, nil, event: parsed_request[:event]) .execute @@ -96,7 +108,10 @@ module API # Collector should never return any information back. # Because DSN and public key are designed for public use, # it is safe only for submission of new events. - no_content! + # + # Some clients sdk require status 200 OK to work correctly. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/343531. + status 200 end desc 'Submit error tracking event to the project' do @@ -122,6 +137,8 @@ module API bad_request!('Failed to parse sentry request') end + validate_payload(parsed_body) + ::ErrorTracking::CollectErrorService .new(project, nil, event: parsed_body) .execute @@ -129,7 +146,10 @@ module API # Collector should never return any information back. # Because DSN and public key are designed for public use, # it is safe only for submission of new events. - no_content! + # + # Some clients sdk require status 200 OK to work correctly. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/343531. + status 200 end end end diff --git a/lib/api/features.rb b/lib/api/features.rb index 2ce2f7c518f..398e57794c8 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -14,7 +14,12 @@ module API when '0', 'false' false else - params[:value].to_i + # https://github.com/jnunemaker/flipper/blob/master/lib/flipper/typecast.rb#L47 + if params[:value].to_s.include?('.') + params[:value].to_f + else + params[:value].to_i + end end end @@ -59,7 +64,7 @@ module API success Entities::Feature end params do - requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time' + requires :value, type: String, desc: '`true` or `false` to enable/disable, a float for percentage of time' optional :key, type: String, desc: '`percentage_of_actors` or the default `percentage_of_time`' optional :feature_group, type: String, desc: 'A Feature group name' optional :user, type: String, desc: 'A GitLab username' diff --git a/lib/api/files.rb b/lib/api/files.rb index 9d2b7cce837..39b3904ec90 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -122,7 +122,7 @@ module API requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' optional :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false end - head ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do + head ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS, urgency: :low do assign_file_vars! set_http_headers(blob_data) @@ -133,7 +133,7 @@ module API requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' optional :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false end - get ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do + get ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS, urgency: :low do assign_file_vars! no_cache_headers @@ -147,7 +147,7 @@ module API requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false end - head ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do + head ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS, urgency: :low do assign_file_vars! set_http_headers(blob_data) @@ -174,7 +174,7 @@ module API params do use :extended_file_params end - post ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do + post ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS, urgency: :low do authorize! :push_code, user_project file_params = declared_params(include_missing: false) @@ -192,7 +192,7 @@ module API params do use :extended_file_params end - put ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do + put ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS, urgency: :low do authorize! :push_code, user_project file_params = declared_params(include_missing: false) diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index 5e184d35255..8cca3378eec 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -54,6 +54,7 @@ module API requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.generic_package_file_name_regex, file_path: true optional :status, type: String, values: ALLOWED_STATUSES, desc: 'Package status' requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + optional :select, type: String, values: %w[package_file] end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true @@ -65,11 +66,15 @@ module API track_package_event('push_package', :generic, project: project, user: current_user, namespace: project.namespace) create_package_file_params = declared_params.merge(build: current_authenticated_job) - ::Packages::Generic::CreatePackageFileService + package_file = ::Packages::Generic::CreatePackageFileService .new(project, current_user, create_package_file_params) .execute - created! + if params[:select] == 'package_file' + present package_file + else + created! + end rescue ObjectStorage::RemoteStoreError => e Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: project.id }) diff --git a/lib/api/github/entities.rb b/lib/api/github/entities.rb index fe228c9a2d2..125985f0e23 100644 --- a/lib/api/github/entities.rb +++ b/lib/api/github/entities.rb @@ -59,8 +59,8 @@ module API expose :parents do |commit| commit.parent_ids.map { |id| { sha: id } } end - expose :files do |commit| - commit.diffs.diff_files.flat_map do |diff| + expose :files do |_commit, options| + options[:diff_files].flat_map do |diff| additions = diff.added_lines deletions = diff.removed_lines diff --git a/lib/api/group_boards.rb b/lib/api/group_boards.rb index 92869f8fbba..e9350da555c 100644 --- a/lib/api/group_boards.rb +++ b/lib/api/group_boards.rb @@ -7,7 +7,7 @@ module API prepend_mod_with('API::BoardsResponses') # rubocop: disable Cop/InjectEnterpriseEditionModule - feature_category :boards + feature_category :team_planning before { authenticate! } diff --git a/lib/api/group_debian_distributions.rb b/lib/api/group_debian_distributions.rb index 01a8774bd97..f0376fe2c9c 100644 --- a/lib/api/group_debian_distributions.rb +++ b/lib/api/group_debian_distributions.rb @@ -7,14 +7,6 @@ module API end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - rescue_from ArgumentError do |e| - render_api_error!(e.message, 400) - end - - rescue_from ActiveRecord::RecordInvalid do |e| - render_api_error!(e.message, 400) - end - after_validation do require_packages_enabled! diff --git a/lib/api/group_labels.rb b/lib/api/group_labels.rb index bea538441ee..7c1f23be828 100644 --- a/lib/api/group_labels.rb +++ b/lib/api/group_labels.rb @@ -7,7 +7,7 @@ module API before { authenticate! } - feature_category :issue_tracking + feature_category :team_planning params do requires :id, type: String, desc: 'The ID of a group' diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb index 061d0410a9c..b097022e9c1 100644 --- a/lib/api/group_milestones.rb +++ b/lib/api/group_milestones.rb @@ -7,7 +7,7 @@ module API before { authenticate! } - feature_category :issue_tracking + feature_category :team_planning params do requires :id, type: String, desc: 'The ID of a group' diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index f9ba5ba8186..76840091112 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -174,9 +174,9 @@ module API # rubocop: disable CodeReuse/ActiveRecord def find_namespace(id) if id.to_s =~ /^\d+$/ - Namespace.find_by(id: id) + Namespace.without_project_namespaces.find_by(id: id) else - Namespace.find_by_full_path(id) + find_namespace_by_path(id) end end # rubocop: enable CodeReuse/ActiveRecord @@ -186,7 +186,7 @@ module API end def find_namespace_by_path(path) - Namespace.find_by_full_path(path) + Namespace.without_project_namespaces.find_by_full_path(path) end def find_namespace_by_path!(path) @@ -488,7 +488,7 @@ module API def handle_api_exception(exception) if report_exception?(exception) define_params_for_grape_middleware - Gitlab::ApplicationContext.push(user: current_user) + Gitlab::ApplicationContext.push(user: current_user, remote_ip: request.ip) Gitlab::ErrorTracking.track_exception(exception) end @@ -681,20 +681,27 @@ module API def send_git_blob(repository, blob) env['api.format'] = :txt content_type 'text/plain' + header['Content-Disposition'] = ActionDispatch::Http::ContentDisposition.format(disposition: 'inline', filename: blob.name) # Let Workhorse examine the content and determine the better content disposition header[Gitlab::Workhorse::DETECT_HEADER] = "true" header(*Gitlab::Workhorse.send_git_blob(repository, blob)) + + body '' end def send_git_archive(repository, **kwargs) header(*Gitlab::Workhorse.send_git_archive(repository, **kwargs)) + + body '' end def send_artifacts_entry(file, entry) header(*Gitlab::Workhorse.send_artifacts_entry(file, entry)) + + body '' end # The Grape Error Middleware only has access to `env` but not `params` nor diff --git a/lib/api/helpers/award_emoji.rb b/lib/api/helpers/award_emoji.rb index 5b659c4dde7..3ea35381c97 100644 --- a/lib/api/helpers/award_emoji.rb +++ b/lib/api/helpers/award_emoji.rb @@ -5,7 +5,7 @@ module API module AwardEmoji def self.awardables [ - { type: 'issue', resource: :projects, find_by: :iid, feature_category: :issue_tracking }, + { type: 'issue', resource: :projects, find_by: :iid, feature_category: :team_planning }, { type: 'merge_request', resource: :projects, find_by: :iid, feature_category: :code_review }, { type: 'snippet', resource: :projects, find_by: :id, feature_category: :snippets } ] diff --git a/lib/api/helpers/discussions_helpers.rb b/lib/api/helpers/discussions_helpers.rb index cb2feeda1e1..c94199b17bc 100644 --- a/lib/api/helpers/discussions_helpers.rb +++ b/lib/api/helpers/discussions_helpers.rb @@ -7,7 +7,7 @@ module API # This is a method instead of a constant, allowing EE to more easily # extend it. { - Issue => :issue_tracking, + Issue => :team_planning, Snippet => :snippets, MergeRequest => :code_review, Commit => :code_review diff --git a/lib/api/helpers/file_upload_helpers.rb b/lib/api/helpers/file_upload_helpers.rb index dd551ec2976..751972b44f0 100644 --- a/lib/api/helpers/file_upload_helpers.rb +++ b/lib/api/helpers/file_upload_helpers.rb @@ -5,7 +5,7 @@ module API module FileUploadHelpers def file_is_valid? filename = params[:file]&.original_filename - filename && ImportExportUploader::EXTENSION_WHITELIST.include?(File.extname(filename).delete('.')) + filename && ImportExportUploader::EXTENSION_ALLOWLIST.include?(File.extname(filename).delete('.')) end def validate_file! diff --git a/lib/api/helpers/integrations_helpers.rb b/lib/api/helpers/integrations_helpers.rb index e0ef9099104..e7fdb6645a5 100644 --- a/lib/api/helpers/integrations_helpers.rb +++ b/lib/api/helpers/integrations_helpers.rb @@ -254,7 +254,7 @@ module API type: Boolean, desc: 'DEPRECATED: This parameter has no effect since SSL verification will always be enabled' } - ], + ], 'campfire' => [ { required: true, @@ -530,6 +530,14 @@ module API desc: 'The Mattermost token' } ], + 'shimo' => [ + { + required: true, + name: :external_wiki_url, + type: String, + desc: 'Shimo workspace URL' + } + ], 'slack-slash-commands' => [ { required: true, @@ -768,7 +776,33 @@ module API desc: 'The Webex Teams webhook. For example, https://api.ciscospark.com/v1/webhooks/incoming/...' }, chat_notification_events - ].flatten + ].flatten, + 'zentao' => [ + { + required: true, + name: :url, + type: String, + desc: 'The base URL to the ZenTao instance web interface which is being linked to this GitLab project. For example, https://www.zentao.net' + }, + { + required: false, + name: :api_url, + type: String, + desc: 'The base URL to the ZenTao instance API. Web URL value will be used if not set. For example, https://www.zentao.net' + }, + { + required: true, + name: :api_token, + type: String, + desc: 'The API token created from ZenTao dashboard' + }, + { + required: true, + name: :zentao_product_xid, + type: String, + desc: 'The product ID of ZenTao project' + } + ] } end @@ -805,7 +839,8 @@ module API ::Integrations::Slack, ::Integrations::SlackSlashCommands, ::Integrations::Teamcity, - ::Integrations::Youtrack + ::Integrations::Youtrack, + ::Integrations::Zentao ] end diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index 356e4a98c97..45671b09be9 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -7,7 +7,7 @@ module API def self.feature_category_per_noteable_type { - Issue => :issue_tracking, + Issue => :team_planning, MergeRequest => :code_review, Snippet => :snippets } diff --git a/lib/api/helpers/project_snapshots_helpers.rb b/lib/api/helpers/project_snapshots_helpers.rb index 0b10641571a..4b48661eeca 100644 --- a/lib/api/helpers/project_snapshots_helpers.rb +++ b/lib/api/helpers/project_snapshots_helpers.rb @@ -11,6 +11,8 @@ module API def send_git_snapshot(repository) header(*Gitlab::Workhorse.send_git_snapshot(repository)) + + body '' end def snapshot_project diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 30edbe91125..42d1c40dd11 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -61,6 +61,7 @@ module API optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests' optional :suggestion_commit_message, type: String, desc: 'The commit message used to apply merge request suggestions' + optional :merge_commit_template, type: String, desc: 'Template used to create merge commit message' optional :initialize_with_readme, type: Boolean, desc: "Initialize a project with a README.md" optional :ci_default_git_depth, type: Integer, desc: 'Default number of revisions for shallow cloning' optional :auto_devops_enabled, type: Boolean, desc: 'Flag indication if Auto DevOps is enabled' @@ -160,6 +161,7 @@ module API :wiki_access_level, :avatar, :suggestion_commit_message, + :merge_commit_template, :repository_storage, :compliance_framework_setting, :packages_enabled, @@ -178,6 +180,17 @@ module API def filter_attributes_using_license!(attrs) end + + def validate_git_import_url!(import_url, import_enabled: true) + return if import_url.blank? + return unless import_enabled + + result = Import::ValidateRemoteGitEndpointService.new(url: import_url).execute # network call + + if result.error? + render_api_error!(result.message, 422) + end + end end end end diff --git a/lib/api/helpers/resource_label_events_helpers.rb b/lib/api/helpers/resource_label_events_helpers.rb index 7e641130062..eeb68362c1d 100644 --- a/lib/api/helpers/resource_label_events_helpers.rb +++ b/lib/api/helpers/resource_label_events_helpers.rb @@ -7,7 +7,7 @@ module API # This is a method instead of a constant, allowing EE to more easily # extend it. { - Issue => :issue_tracking, + Issue => :team_planning, MergeRequest => :code_review } end diff --git a/lib/api/integrations.rb b/lib/api/integrations.rb index 926cde340a0..bab8e556a73 100644 --- a/lib/api/integrations.rb +++ b/lib/api/integrations.rb @@ -153,7 +153,7 @@ module API requires setting[:name], type: setting[:type], desc: setting[:desc] end end - post "#{path}/#{integration_slug.underscore}/trigger" do + post "#{path}/#{integration_slug.underscore}/trigger", urgency: :low do project = find_project(params[:id]) # This is not accurate, but done to prevent leakage of the project names diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index dc9257ebd62..d8e39d089e4 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -145,7 +145,7 @@ module API check_allowed(params) end - post "/lfs_authenticate", feature_category: :source_code_management do + post "/lfs_authenticate", feature_category: :source_code_management, urgency: :high do not_found! unless container&.lfs_enabled? status 200 diff --git a/lib/api/internal/lfs.rb b/lib/api/internal/lfs.rb index 66baa4f1034..e94da8d34e0 100644 --- a/lib/api/internal/lfs.rb +++ b/lib/api/internal/lfs.rb @@ -24,7 +24,7 @@ module API requires :oid, type: String, desc: 'The object ID to query' requires :gl_repository, type: String, desc: "Project identifier (e.g. project-1)" end - get "/" do + get "/", urgency: :high do lfs_object = find_lfs_object(params[:oid]) not_found! unless lfs_object diff --git a/lib/api/invitations.rb b/lib/api/invitations.rb index 5cade301d81..f7f5af07378 100644 --- a/lib/api/invitations.rb +++ b/lib/api/invitations.rb @@ -25,6 +25,8 @@ module API optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' optional :invite_source, type: String, desc: 'Source that triggered the member creation process', default: 'invitations-api' optional :areas_of_focus, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Areas the inviter wants the member to focus upon' + optional :tasks_to_be_done, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Tasks the inviter wants the member to do' + optional :tasks_project_id, type: Integer, desc: 'The project ID in which to create the task issues' end post ":id/invitations" do params[:source] = find_source(source_type, params[:id]) diff --git a/lib/api/issue_links.rb b/lib/api/issue_links.rb index 0b4f4e06d0b..98451afb12d 100644 --- a/lib/api/issue_links.rb +++ b/lib/api/issue_links.rb @@ -6,7 +6,7 @@ module API before { authenticate! } - feature_category :issue_tracking + feature_category :team_planning params do requires :id, type: String, desc: 'The ID of a project' diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 43e83bd58fe..9958526fa7f 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -7,7 +7,7 @@ module API before { authenticate_non_get! } - feature_category :issue_tracking + feature_category :team_planning helpers do params :negatable_issue_filter_params do diff --git a/lib/api/labels.rb b/lib/api/labels.rb index aa3746dae42..e3253d15c15 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -7,7 +7,7 @@ module API before { authenticate! } - feature_category :issue_tracking + feature_category :team_planning LABEL_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( name: API::NO_SLASH_URL_PART_REGEX, diff --git a/lib/api/lint.rb b/lib/api/lint.rb index fa871b4bc57..f1e19e9c3c5 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -9,6 +9,7 @@ module API params do requires :content, type: String, desc: 'Content of .gitlab-ci.yml' optional :include_merged_yaml, type: Boolean, desc: 'Whether or not to include merged CI config yaml in the response' + optional :include_jobs, type: Boolean, desc: 'Whether or not to include CI jobs in the response' end post '/lint' do unauthorized! if (Gitlab::CurrentSettings.signup_disabled? || Gitlab::CurrentSettings.signup_limited?) && current_user.nil? @@ -17,7 +18,7 @@ module API .validate(params[:content], dry_run: false) status 200 - Entities::Ci::Lint::Result.represent(result, current_user: current_user).serializable_hash.tap do |presented_result| + Entities::Ci::Lint::Result.represent(result, current_user: current_user, include_jobs: params[:include_jobs]).serializable_hash.tap do |presented_result| presented_result[:status] = presented_result[:valid] ? 'valid' : 'invalid' presented_result.delete(:merged_yaml) unless params[:include_merged_yaml] end @@ -30,6 +31,7 @@ module API end params do optional :dry_run, type: Boolean, default: false, desc: 'Run pipeline creation simulation, or only do static check.' + optional :include_jobs, type: Boolean, desc: 'Whether or not to include CI jobs in the response' end get ':id/ci/lint' do authorize! :download_code, user_project @@ -39,7 +41,7 @@ module API .new(project: user_project, current_user: current_user) .validate(content, dry_run: params[:dry_run]) - present result, with: Entities::Ci::Lint::Result, current_user: current_user + present result, with: Entities::Ci::Lint::Result, current_user: current_user, include_jobs: params[:include_jobs] end end @@ -50,6 +52,7 @@ module API params do requires :content, type: String, desc: 'Content of .gitlab-ci.yml' optional :dry_run, type: Boolean, default: false, desc: 'Run pipeline creation simulation, or only do static check.' + optional :include_jobs, type: Boolean, desc: 'Whether or not to include CI jobs in the response' end post ':id/ci/lint' do authorize! :create_pipeline, user_project @@ -59,7 +62,7 @@ module API .validate(params[:content], dry_run: params[:dry_run]) status 200 - present result, with: Entities::Ci::Lint::Result, current_user: current_user + present result, with: Entities::Ci::Lint::Result, current_user: current_user, include_jobs: params[:include_jobs] end end end diff --git a/lib/api/members.rb b/lib/api/members.rb index 332520ccd26..f488c8c26fc 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -95,6 +95,8 @@ module API optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' optional :invite_source, type: String, desc: 'Source that triggered the member creation process', default: 'members-api' optional :areas_of_focus, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Areas the inviter wants the member to focus upon' + optional :tasks_to_be_done, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Tasks the inviter wants the member to do' + optional :tasks_project_id, type: Integer, desc: 'The project ID in which to create the task issues' end post ":id/members" do diff --git a/lib/api/merge_request_approvals.rb b/lib/api/merge_request_approvals.rb index 83150bb51ca..dd49624c74f 100644 --- a/lib/api/merge_request_approvals.rb +++ b/lib/api/merge_request_approvals.rb @@ -25,7 +25,7 @@ module API # Examples: # GET /projects/:id/merge_requests/:merge_request_iid/approvals desc 'List approvals for merge request' - get 'approvals' do + get 'approvals', urgency: :low do not_found!("Merge Request") unless can?(current_user, :read_merge_request, user_project) merge_request = find_merge_request_with_access(params[:merge_request_iid]) @@ -47,7 +47,7 @@ module API use :ee_approval_params end - post 'approve' do + post 'approve', urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid], :approve_merge_request) check_sha_param!(params, merge_request) @@ -63,7 +63,7 @@ module API end desc 'Remove an approval from a merge request' - post 'unapprove' do + post 'unapprove', urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid], :approve_merge_request) success = ::MergeRequests::RemoveApprovalService diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index c2d839571a6..d2468fb1c2e 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -37,7 +37,7 @@ module API namespaces = current_user.admin ? Namespace.all : current_user.namespaces(owned_only: owned_only) - namespaces = namespaces.include_route + namespaces = namespaces.without_project_namespaces.include_route namespaces = namespaces.include_gitlab_subscription_with_hosted_plan if Gitlab.ee? @@ -70,7 +70,7 @@ module API get ':namespace/exists', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do namespace_path = params[:namespace] - exists = Namespace.by_parent(params[:parent_id]).filter_by_path(namespace_path).exists? + exists = Namespace.without_project_namespaces.by_parent(params[:parent_id]).filter_by_path(namespace_path).exists? suggestions = exists ? [Namespace.clean_path(namespace_path)] : [] present :exists, exists diff --git a/lib/api/package_files.rb b/lib/api/package_files.rb index 6d0c1f44a36..79ebf18ff27 100644 --- a/lib/api/package_files.rb +++ b/lib/api/package_files.rb @@ -28,7 +28,10 @@ module API package = ::Packages::PackageFinder .new(user_project, params[:package_id]).execute - present paginate(package.package_files), with: ::API::Entities::PackageFile + files = package.package_files + .preload_pipelines + + present paginate(files), with: ::API::Entities::PackageFile end desc 'Remove a package file' do diff --git a/lib/api/project_debian_distributions.rb b/lib/api/project_debian_distributions.rb index f057251fb6b..2ba1ff85adb 100644 --- a/lib/api/project_debian_distributions.rb +++ b/lib/api/project_debian_distributions.rb @@ -7,14 +7,6 @@ module API end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - rescue_from ArgumentError do |e| - render_api_error!(e.message, 400) - end - - rescue_from ActiveRecord::RecordInvalid do |e| - render_api_error!(e.message, 400) - end - after_validation do require_packages_enabled! diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb index 107311ea446..435e4bed776 100644 --- a/lib/api/project_milestones.rb +++ b/lib/api/project_milestones.rb @@ -7,7 +7,7 @@ module API before { authenticate! } - feature_category :issue_tracking + feature_category :team_planning params do requires :id, type: String, desc: 'The ID of a project' diff --git a/lib/api/projects.rb b/lib/api/projects.rb index bb74849a98a..9f0077d23d8 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -91,7 +91,7 @@ module API end def check_import_by_url_is_enabled - forbidden! unless Gitlab::CurrentSettings.import_sources&.include?('git') + Gitlab::CurrentSettings.import_sources&.include?('git') || forbidden! end end @@ -269,7 +269,9 @@ module API attrs = declared_params(include_missing: false) attrs = translate_params_for_compatibility(attrs) filter_attributes_using_license!(attrs) - check_import_by_url_is_enabled if params[:import_url].present? + + validate_git_import_url!(params[:import_url], import_enabled: check_import_by_url_is_enabled) + project = ::Projects::CreateService.new(current_user, attrs).execute if project.saved? @@ -307,6 +309,8 @@ module API attrs = declared_params(include_missing: false) attrs = translate_params_for_compatibility(attrs) filter_attributes_using_license!(attrs) + validate_git_import_url!(params[:import_url]) + project = ::Projects::CreateService.new(user, attrs).execute if project.saved? @@ -400,7 +404,7 @@ module API use :collection_params use :with_custom_attributes end - get ':id/forks', feature_category: :source_code_management do + get ':id/forks', feature_category: :source_code_management, urgency: :low do forks = ForkProjectsFinder.new(user_project, params: project_finder_params, current_user: current_user).execute present_projects forks, request_scope: user_project @@ -510,7 +514,7 @@ module API end desc 'Get languages in project repository' - get ':id/languages', feature_category: :source_code_management do + get ':id/languages', feature_category: :source_code_management, urgency: :medium do ::Projects::RepositoryLanguagesService .new(user_project, current_user) .execute.to_h { |lang| [lang.name, lang.share] } diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index 3cebc308f51..a4f5dfefae6 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -91,7 +91,7 @@ module API requires :name, type: String, desc: 'The name of the protected branch' end # rubocop: disable CodeReuse/ActiveRecord - delete ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do + delete ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS, urgency: :low do protected_branch = user_project.protected_branches.find_by!(name: params[:name]) destroy_conditionally!(protected_branch) do diff --git a/lib/api/releases.rb b/lib/api/releases.rb index 3b7e2b4bd27..7b89a177fd9 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -32,6 +32,7 @@ module API optional :include_html_description, type: Boolean, desc: 'If `true`, a response includes HTML rendered markdown of the release description.' end + route_setting :authentication, job_token_allowed: true get ':id/releases' do releases = ::ReleasesFinder.new(user_project, current_user, declared_params.slice(:order_by, :sort)).execute @@ -59,6 +60,7 @@ module API optional :include_html_description, type: Boolean, desc: 'If `true`, a response includes HTML rendered markdown of the release description.' end + route_setting :authentication, job_token_allowed: true get ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMENTS do authorize_download_code! @@ -117,6 +119,7 @@ module API optional :released_at, type: DateTime, desc: 'The date when the release will be/was ready.' optional :milestones, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The titles of the related milestones' end + route_setting :authentication, job_token_allowed: true put ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMENTS do authorize_update_release! @@ -142,6 +145,7 @@ module API params do requires :tag_name, type: String, desc: 'The name of the tag', as: :tag end + route_setting :authentication, job_token_allowed: true delete ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMENTS do authorize_destroy_release! diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 1aa76906b3d..2dd0e40afba 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -42,6 +42,26 @@ module API not_found! 'Blob' unless @blob end + + def fetch_target_project(current_user, user_project, params) + return user_project unless params[:from_project_id].present? + + MergeRequestTargetProjectFinder + .new(current_user: current_user, source_project: user_project, project_feature: :repository) + .execute(include_routes: true).find_by_id(params[:from_project_id]) + end + + def compare_cache_key(current_user, user_project, target_project, params) + [ + user_project, + target_project, + current_user, + :repository_compare, + target_project.repository.commit(params[:from]), + user_project.repository.commit(params[:to]), + params + ] + end end desc 'Get a project repository tree' do @@ -59,7 +79,7 @@ module API optional :page_token, type: String, desc: 'Record from which to start the keyset pagination' end end - get ':id/repository/tree' do + get ':id/repository/tree', urgency: :low do tree_finder = ::Repositories::TreeFinder.new(user_project, declared_params(include_missing: false)) not_found!("Tree") unless tree_finder.commit_exists? @@ -124,22 +144,17 @@ module API optional :from_project_id, type: String, desc: 'The project to compare from' optional :straight, type: Boolean, desc: 'Comparison method, `true` for direct comparison between `from` and `to` (`from`..`to`), `false` to compare using merge base (`from`...`to`)', default: false end - get ':id/repository/compare' do + get ':id/repository/compare', urgency: :low do ff_enabled = Feature.enabled?(:api_caching_rate_limit_repository_compare, user_project, default_enabled: :yaml) + target_project = fetch_target_project(current_user, user_project, params) - cache_action_if(ff_enabled, [user_project, :repository_compare, current_user, declared_params], expires_in: 1.minute) do - if params[:from_project_id].present? - target_project = MergeRequestTargetProjectFinder - .new(current_user: current_user, source_project: user_project, project_feature: :repository) - .execute(include_routes: true).find_by_id(params[:from_project_id]) + if target_project.blank? + render_api_error!("Target project id:#{params[:from_project_id]} is not a fork of project id:#{params[:id]}", 400) + end - if target_project.blank? - render_api_error!("Target project id:#{params[:from_project_id]} is not a fork of project id:#{params[:id]}", 400) - end - else - target_project = user_project - end + cache_key = compare_cache_key(current_user, user_project, target_project, declared_params) + cache_action_if(ff_enabled, cache_key, expires_in: 1.minute) do compare = CompareService.new(user_project, params[:to]).execute(target_project, params[:from], straight: params[:straight]) if compare diff --git a/lib/api/resource_milestone_events.rb b/lib/api/resource_milestone_events.rb index aeedd7ad109..c0483ca59c2 100644 --- a/lib/api/resource_milestone_events.rb +++ b/lib/api/resource_milestone_events.rb @@ -8,7 +8,7 @@ module API before { authenticate! } { - Issue => :issue_tracking, + Issue => :team_planning, MergeRequest => :code_review }.each do |eventable_type, feature_category| parent_type = eventable_type.parent_class.to_s.underscore diff --git a/lib/api/resource_state_events.rb b/lib/api/resource_state_events.rb index 3460aa2c00e..9b6f6a954b4 100644 --- a/lib/api/resource_state_events.rb +++ b/lib/api/resource_state_events.rb @@ -8,7 +8,7 @@ module API before { authenticate! } { - Issue => :issue_tracking, + Issue => :team_planning, MergeRequest => :code_review }.each do |eventable_class, feature_category| eventable_name = eventable_class.to_s.underscore diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index f1ec1024492..c4b17a62b59 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -43,7 +43,7 @@ module API params do use :pagination end - get 'public' do + get 'public', urgency: :low do authenticate! present paginate(public_snippets), with: Entities::PersonalSnippet, current_user: current_user diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index 87dc1358a51..cda30dc957f 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -22,21 +22,21 @@ module API entity: Entities::Issue, source: Project, finder: ->(id) { find_project_issue(id) }, - feature_category: :issue_tracking + feature_category: :team_planning }, { type: 'labels', entity: Entities::ProjectLabel, source: Project, finder: ->(id) { find_label(user_project, id) }, - feature_category: :issue_tracking + feature_category: :team_planning }, { type: 'labels', entity: Entities::GroupLabel, source: Group, finder: ->(id) { find_label(user_group, id) }, - feature_category: :issue_tracking + feature_category: :team_planning } ] diff --git a/lib/api/tags.rb b/lib/api/tags.rb index f018b421edd..1b37d38ef06 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -21,20 +21,28 @@ module API optional :order_by, type: String, values: %w[name updated], default: 'updated', desc: 'Return tags ordered by `name` or `updated` fields.' optional :search, type: String, desc: 'Return list of tags matching the search criteria' + optional :page_token, type: String, desc: 'Name of tag to start the paginaition from' use :pagination end - get ':id/repository/tags', feature_category: :source_code_management do - tags, _ = ::TagsFinder.new(user_project.repository, + get ':id/repository/tags', feature_category: :source_code_management, urgency: :low do + tags_finder = ::TagsFinder.new(user_project.repository, sort: "#{params[:order_by]}_#{params[:sort]}", - search: params[:search]).execute + search: params[:search], + page_token: params[:page_token], + per_page: params[:per_page]) - paginated_tags = paginate(::Kaminari.paginate_array(tags)) + paginated_tags = Gitlab::Pagination::GitalyKeysetPager.new(self, user_project).paginate(tags_finder) if Feature.enabled?(:api_caching_tags, user_project, type: :development) present_cached paginated_tags, with: Entities::Tag, project: user_project, cache_context: -> (_tag) { user_project.cache_key } else present paginated_tags, with: Entities::Tag, project: user_project end + + rescue Gitlab::Git::InvalidPageToken => e + unprocessable_entity!(e.message) + rescue Gitlab::Git::CommandError + service_unavailable! end desc 'Get a single repository tag' do diff --git a/lib/api/terraform/modules/v1/packages.rb b/lib/api/terraform/modules/v1/packages.rb index aa59b6a4fee..ad5a4ae7ea6 100644 --- a/lib/api/terraform/modules/v1/packages.rb +++ b/lib/api/terraform/modules/v1/packages.rb @@ -46,7 +46,8 @@ module API def finder_params { package_type: :terraform_module, - package_name: "#{params[:module_name]}/#{params[:module_system]}" + package_name: "#{params[:module_name]}/#{params[:module_system]}", + exact_name: true }.tap do |finder_params| finder_params[:package_version] = params[:module_version] if params.has_key?(:module_version) end diff --git a/lib/api/todos.rb b/lib/api/todos.rb index e0e5ca615ac..57a6ee0bebb 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -6,7 +6,7 @@ module API before { authenticate! } - feature_category :issue_tracking + feature_category :team_planning ISSUABLE_TYPES = { 'merge_requests' => ->(iid) { find_merge_request_with_access(iid) }, diff --git a/lib/api/topics.rb b/lib/api/topics.rb new file mode 100644 index 00000000000..bd28ebe58a9 --- /dev/null +++ b/lib/api/topics.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module API + class Topics < ::API::Base + include PaginationParams + + feature_category :projects + + desc 'Get topics' do + detail 'This feature was introduced in GitLab 14.5.' + success Entities::Projects::Topic + end + params do + optional :search, type: String, desc: 'Return list of topics matching the search criteria' + use :pagination + end + get 'topics' do + topics = ::Projects::TopicsFinder.new(params: declared_params(include_missing: false)).execute + + present paginate(topics), with: Entities::Projects::Topic + end + + desc 'Get topic' do + detail 'This feature was introduced in GitLab 14.5.' + success Entities::Projects::Topic + end + params do + requires :id, type: Integer, desc: 'ID of project topic' + end + get 'topics/:id' do + topic = ::Projects::Topic.find(params[:id]) + + present topic, with: Entities::Projects::Topic + end + + desc 'Create a topic' do + detail 'This feature was introduced in GitLab 14.5.' + success Entities::Projects::Topic + end + params do + requires :name, type: String, desc: 'Name' + optional :description, type: String, desc: 'Description' + optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for topic' + end + post 'topics' do + authenticated_as_admin! + + topic = ::Projects::Topic.new(declared_params(include_missing: false)) + + if topic.save + present topic, with: Entities::Projects::Topic + else + render_validation_error!(topic) + end + end + + desc 'Update a topic' do + detail 'This feature was introduced in GitLab 14.5.' + success Entities::Projects::Topic + end + params do + requires :id, type: Integer, desc: 'ID of project topic' + optional :name, type: String, desc: 'Name' + optional :description, type: String, desc: 'Description' + optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for topic' + end + put 'topics/:id' do + authenticated_as_admin! + + topic = ::Projects::Topic.find(params[:id]) + + if topic.update(declared_params(include_missing: false)) + present topic, with: Entities::Projects::Topic + else + render_validation_error!(topic) + end + end + end +end diff --git a/lib/api/users.rb b/lib/api/users.rb index f16e1148618..ce0a0e9b502 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -1062,6 +1062,7 @@ module API requires :credit_card_expiration_year, type: Integer, desc: 'The year the credit card expires' requires :credit_card_holder_name, type: String, desc: 'The credit card holder name' requires :credit_card_mask_number, type: String, desc: 'The last 4 digits of credit card number' + requires :credit_card_type, type: String, desc: 'The credit card network name' end put ":user_id/credit_card_validation", feature_category: :users do authenticated_as_admin! diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb index 310054c298a..677d0840208 100644 --- a/lib/api/v3/github.rb +++ b/lib/api/v3/github.rb @@ -20,6 +20,9 @@ module API # Jira Server user agent format: Jira DVCS Connector/version JIRA_DVCS_CLOUD_USER_AGENT = 'Jira DVCS Connector Vertigo' + GITALY_TIMEOUT_CACHE_KEY = 'api:v3:Gitaly-timeout-cache-key' + GITALY_TIMEOUT_CACHE_EXPIRY = 1.day + include PaginationParams feature_category :integrations @@ -93,6 +96,32 @@ module API notes.select { |n| n.readable_by?(current_user) } end # rubocop: enable CodeReuse/ActiveRecord + + # Returns an empty Array instead of the Commit diff files for a period + # of time after a Gitaly timeout, to mitigate frequent Gitaly timeouts + # for some Commit diffs. + def diff_files(commit) + return commit.diffs.diff_files unless Feature.enabled?(:api_v3_commits_skip_diff_files, commit.project, default_enabled: :yaml) + + cache_key = [ + GITALY_TIMEOUT_CACHE_KEY, + commit.project.id, + commit.cache_key + ].join(':') + + return [] if Rails.cache.read(cache_key).present? + + begin + commit.diffs.diff_files + rescue GRPC::DeadlineExceeded => error + # Gitaly fails to load diffs consistently for some commits. The other information + # is still valuable for Jira. So we skip the loading and respond with a 200 excluding diffs + # Remove this when https://gitlab.com/gitlab-org/gitaly/-/issues/3741 is fixed. + Rails.cache.write(cache_key, 1, expires_in: GITALY_TIMEOUT_CACHE_EXPIRY) + Gitlab::ErrorTracking.track_exception(error) + [] + end + end end resource :orgs do @@ -228,10 +257,9 @@ module API user_project = find_project_with_access(params) commit = user_project.commit(params[:sha]) - not_found! 'Commit' unless commit - present commit, with: ::API::Github::Entities::RepoCommit + present commit, with: ::API::Github::Entities::RepoCommit, diff_files: diff_files(commit) end end end diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index 8441aeb10ab..fdce3c5ce18 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -32,7 +32,7 @@ module API params do optional :with_content, type: Boolean, default: false, desc: "Include pages' content" end - get ':id/wikis' do + get ':id/wikis', urgency: :low do authorize! :read_wiki, container entity = params[:with_content] ? Entities::WikiPage : Entities::WikiPageBasic diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 6c5350082e8..1bdc4965e5d 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -99,9 +99,10 @@ module Backup # - 1495527122_gitlab_backup.tar # - 1495527068_2017_05_23_gitlab_backup.tar # - 1495527097_2017_05_23_9.3.0-pre_gitlab_backup.tar - next unless file =~ /^(\d{10})(?:_\d{4}_\d{2}_\d{2}(_\d+\.\d+\.\d+((-|\.)(pre|rc\d))?(-ee)?)?)?_gitlab_backup\.tar$/ + matched = backup_file?(file) + next unless matched - timestamp = Regexp.last_match(1).to_i + timestamp = matched[1].to_i if Time.at(timestamp) < (Time.now - keep_time) begin @@ -192,6 +193,10 @@ module Backup private + def backup_file?(file) + file.match(/^(\d{10})(?:_\d{4}_\d{2}_\d{2}(_\d+\.\d+\.\d+((-|\.)(pre|rc\d))?(-ee)?)?)?_gitlab_backup\.tar$/) + end + def non_tarred_backup? File.exist?(File.join(backup_path, 'backup_information.yml')) end @@ -210,9 +215,7 @@ module Backup def object_storage_config @object_storage_config ||= begin - config = ObjectStorage::Config.new(Gitlab.config.backup.upload) - config.load_provider - config + ObjectStorage::Config.new(Gitlab.config.backup.upload) end end @@ -316,3 +319,5 @@ module Backup end end end + +Backup::Manager.prepend_mod diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index 9d24bf028b6..d8c9fd0a7f0 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -8,7 +8,6 @@ module Banzai # Based on HTML::Pipeline::EmojiFilter class EmojiFilter < HTML::Pipeline::Filter IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set - IGNORE_UNICODE_EMOJIS = %w(™ © ®).freeze def call doc.xpath('descendant-or-self::text()').each do |node| @@ -35,7 +34,8 @@ module Banzai def emoji_name_element_unicode_filter(text) text.gsub(emoji_pattern) do |match| name = Regexp.last_match(1) - Gitlab::Emoji.gl_emoji_tag(name) + emoji = TanukiEmoji.find_by_alpha_code(name) + Gitlab::Emoji.gl_emoji_tag(emoji) end end @@ -46,26 +46,19 @@ module Banzai # Returns a String with unicode emoji replaced with gl-emoji unicode. def emoji_unicode_element_unicode_filter(text) text.gsub(emoji_unicode_pattern) do |moji| - emoji_info = Gitlab::Emoji.emojis_by_moji[moji] - Gitlab::Emoji.gl_emoji_tag(emoji_info['name']) + emoji = TanukiEmoji.find_by_codepoints(moji) + Gitlab::Emoji.gl_emoji_tag(emoji) end end # Build a regexp that matches all valid :emoji: names. def self.emoji_pattern - @emoji_pattern ||= - %r{(?<=[^[:alnum:]:]|\n|^) - :(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}): - (?=[^[:alnum:]:]|$)}x + @emoji_pattern ||= TanukiEmoji.index.alpha_code_pattern end # Build a regexp that matches all valid unicode emojis names. def self.emoji_unicode_pattern - @emoji_unicode_pattern ||= - begin - filtered_emojis = Gitlab::Emoji.emojis_unicodes - IGNORE_UNICODE_EMOJIS - /(#{filtered_emojis.map { |moji| Regexp.escape(moji) }.join('|')})/ - end + @emoji_unicode_pattern ||= TanukiEmoji.index.codepoints_pattern end private diff --git a/lib/banzai/filter/footnote_filter.rb b/lib/banzai/filter/footnote_filter.rb index 0f856dc0eb9..39c42ceaf9b 100644 --- a/lib/banzai/filter/footnote_filter.rb +++ b/lib/banzai/filter/footnote_filter.rb @@ -16,37 +16,60 @@ module Banzai # can be used for a single render). So you get `id=fn1-4335` and `id=fn2-4335`. # class FootnoteFilter < HTML::Pipeline::Filter - INTEGER_PATTERN = /\A\d+\z/.freeze - FOOTNOTE_ID_PREFIX = 'fn' - FOOTNOTE_LINK_ID_PREFIX = 'fnref' - FOOTNOTE_LI_REFERENCE_PATTERN = /\A#{FOOTNOTE_ID_PREFIX}\d+\z/.freeze - FOOTNOTE_LINK_REFERENCE_PATTERN = /\A#{FOOTNOTE_LINK_ID_PREFIX}\d+\z/.freeze - FOOTNOTE_START_NUMBER = 1 - - CSS_SECTION = "ol > li[id=#{FOOTNOTE_ID_PREFIX}#{FOOTNOTE_START_NUMBER}]" + FOOTNOTE_ID_PREFIX = 'fn-' + FOOTNOTE_LINK_ID_PREFIX = 'fnref-' + FOOTNOTE_LI_REFERENCE_PATTERN = /\A#{FOOTNOTE_ID_PREFIX}.+\z/.freeze + FOOTNOTE_LINK_REFERENCE_PATTERN = /\A#{FOOTNOTE_LINK_ID_PREFIX}.+\z/.freeze + + CSS_SECTION = "ol > li a[href^=\"\##{FOOTNOTE_LINK_ID_PREFIX}\"]" XPATH_SECTION = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_SECTION).freeze CSS_FOOTNOTE = 'sup > a[id]' XPATH_FOOTNOTE = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_FOOTNOTE).freeze + # only needed when feature flag use_cmark_renderer is turned off + INTEGER_PATTERN = /\A\d+\z/.freeze + FOOTNOTE_ID_PREFIX_OLD = 'fn' + FOOTNOTE_LINK_ID_PREFIX_OLD = 'fnref' + FOOTNOTE_LI_REFERENCE_PATTERN_OLD = /\A#{FOOTNOTE_ID_PREFIX_OLD}\d+\z/.freeze + FOOTNOTE_LINK_REFERENCE_PATTERN_OLD = /\A#{FOOTNOTE_LINK_ID_PREFIX_OLD}\d+\z/.freeze + FOOTNOTE_START_NUMBER = 1 + CSS_SECTION_OLD = "ol > li[id=#{FOOTNOTE_ID_PREFIX_OLD}#{FOOTNOTE_START_NUMBER}]" + XPATH_SECTION_OLD = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_SECTION_OLD).freeze + def call - return doc unless first_footnote = doc.at_xpath(XPATH_SECTION) + xpath_section = Feature.enabled?(:use_cmark_renderer) ? XPATH_SECTION : XPATH_SECTION_OLD + return doc unless first_footnote = doc.at_xpath(xpath_section) # Sanitization stripped off the section wrapper - add it back in - first_footnote.parent.wrap('
') + if Feature.enabled?(:use_cmark_renderer) + first_footnote.parent.parent.parent.wrap('
') + else + first_footnote.parent.wrap('
') + end + rand_suffix = "-#{random_number}" modified_footnotes = {} doc.xpath(XPATH_FOOTNOTE).each do |link_node| - ref_num = link_node[:id].delete_prefix(FOOTNOTE_LINK_ID_PREFIX) - node_xpath = Gitlab::Utils::Nokogiri.css_to_xpath("li[id=#{fn_id(ref_num)}]") + if Feature.enabled?(:use_cmark_renderer) + ref_num = link_node[:id].delete_prefix(FOOTNOTE_LINK_ID_PREFIX) + ref_num.gsub!(/[[:punct:]]/, '\\\\\&') + else + ref_num = link_node[:id].delete_prefix(FOOTNOTE_LINK_ID_PREFIX_OLD) + end + + node_xpath = Gitlab::Utils::Nokogiri.css_to_xpath("li[id=#{fn_id(ref_num)}]") footnote_node = doc.at_xpath(node_xpath) - if INTEGER_PATTERN.match?(ref_num) && (footnote_node || modified_footnotes[ref_num]) + if footnote_node || modified_footnotes[ref_num] + next if Feature.disabled?(:use_cmark_renderer) && !INTEGER_PATTERN.match?(ref_num) + link_node[:href] += rand_suffix link_node[:id] += rand_suffix # Sanitization stripped off class - add it back in link_node.parent.append_class('footnote-ref') + link_node['data-footnote-ref'] = nil if Feature.enabled?(:use_cmark_renderer) unless modified_footnotes[ref_num] footnote_node[:id] += rand_suffix @@ -55,6 +78,7 @@ module Banzai if backref_node backref_node[:href] += rand_suffix backref_node.append_class('footnote-backref') + backref_node['data-footnote-backref'] = nil if Feature.enabled?(:use_cmark_renderer) end modified_footnotes[ref_num] = true @@ -72,11 +96,13 @@ module Banzai end def fn_id(num) - "#{FOOTNOTE_ID_PREFIX}#{num}" + prefix = Feature.enabled?(:use_cmark_renderer) ? FOOTNOTE_ID_PREFIX : FOOTNOTE_ID_PREFIX_OLD + "#{prefix}#{num}" end def fnref_id(num) - "#{FOOTNOTE_LINK_ID_PREFIX}#{num}" + prefix = Feature.enabled?(:use_cmark_renderer) ? FOOTNOTE_LINK_ID_PREFIX : FOOTNOTE_LINK_ID_PREFIX_OLD + "#{prefix}#{num}" end end end diff --git a/lib/banzai/filter/markdown_engines/common_mark.rb b/lib/banzai/filter/markdown_engines/common_mark.rb index 7be52fc497f..a25ebedf029 100644 --- a/lib/banzai/filter/markdown_engines/common_mark.rb +++ b/lib/banzai/filter/markdown_engines/common_mark.rb @@ -13,8 +13,7 @@ module Banzai EXTENSIONS = [ :autolink, # provides support for automatically converting URLs to anchor tags. :strikethrough, # provides support for strikethroughs. - :table, # provides support for tables. - :tagfilter # strips out several "unsafe" HTML tags from being used: https://github.github.com/gfm/#disallowed-raw-html-extension- + :table # provides support for tables. ].freeze PARSE_OPTIONS = [ @@ -23,36 +22,63 @@ module Banzai :VALIDATE_UTF8 # replace illegal sequences with the replacement character U+FFFD. ].freeze + RENDER_OPTIONS_C = [ + :GITHUB_PRE_LANG, # use GitHub-style 
 for fenced code blocks.
+          :FOOTNOTES,        # render footnotes.
+          :FULL_INFO_STRING, # include full info strings of code blocks in separate attribute.
+          :UNSAFE            # allow raw/custom HTML and unsafe links.
+        ].freeze
+
         # The `:GITHUB_PRE_LANG` option is not used intentionally because
         # it renders a fence block with language as `
some code\n
`
         # while GitLab's syntax is `
some code\n
`.
         # If in the future the syntax is about to be made GitHub-compatible, please, add `:GITHUB_PRE_LANG` render option below
         # and remove `code_block` method from `lib/banzai/renderer/common_mark/html.rb`.
-        RENDER_OPTIONS = [
+        RENDER_OPTIONS_RUBY = [
           # as of commonmarker 0.18.0, we need to use :UNSAFE to get the same as the original :DEFAULT
           # https://github.com/gjtorikian/commonmarker/pull/81
-          :UNSAFE
-        ].freeze
-
-        RENDER_OPTIONS_SOURCEPOS = RENDER_OPTIONS + [
-          :SOURCEPOS # enable embedding of source position information
+          :UNSAFE # allow raw/custom HTML and unsafe links.
         ].freeze
 
         def initialize(context)
-          @context  = context
-          @renderer = Banzai::Renderer::CommonMark::HTML.new(options: render_options)
+          @context = context
+          @renderer = Banzai::Renderer::CommonMark::HTML.new(options: render_options) if Feature.disabled?(:use_cmark_renderer)
         end
 
         def render(text)
-          doc = CommonMarker.render_doc(text, PARSE_OPTIONS, EXTENSIONS)
+          if Feature.enabled?(:use_cmark_renderer)
+            CommonMarker.render_html(text, render_options, extensions)
+          else
+            doc = CommonMarker.render_doc(text, PARSE_OPTIONS, extensions)
 
-          @renderer.render(doc)
+            @renderer.render(doc)
+          end
         end
 
         private
 
+        def extensions
+          if Feature.enabled?(:use_cmark_renderer)
+            EXTENSIONS
+          else
+            EXTENSIONS + [
+              :tagfilter # strips out several "unsafe" HTML tags from being used: https://github.github.com/gfm/#disallowed-raw-html-extension-
+            ].freeze
+          end
+        end
+
         def render_options
-          @context[:no_sourcepos] ? RENDER_OPTIONS : RENDER_OPTIONS_SOURCEPOS
+          @context[:no_sourcepos] ? render_options_no_sourcepos : render_options_sourcepos
+        end
+
+        def render_options_no_sourcepos
+          Feature.enabled?(:use_cmark_renderer) ? RENDER_OPTIONS_C : RENDER_OPTIONS_RUBY
+        end
+
+        def render_options_sourcepos
+          render_options_no_sourcepos + [
+            :SOURCEPOS # enable embedding of source position information
+          ].freeze
         end
       end
     end
diff --git a/lib/banzai/filter/markdown_post_escape_filter.rb b/lib/banzai/filter/markdown_post_escape_filter.rb
index b69afdcfebe..ccffe1bfbb1 100644
--- a/lib/banzai/filter/markdown_post_escape_filter.rb
+++ b/lib/banzai/filter/markdown_post_escape_filter.rb
@@ -8,10 +8,8 @@ module Banzai
       NOT_LITERAL_REGEX = %r{#{LITERAL_KEYWORD}-((%5C|\\).+?)-#{LITERAL_KEYWORD}}.freeze
       SPAN_REGEX        = %r{(.*?)}.freeze
 
-      CSS_A      = 'a'
-      XPATH_A    = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_A).freeze
-      CSS_CODE   = 'code'
-      XPATH_CODE = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_CODE).freeze
+      CSS_A   = 'a'
+      XPATH_A = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_A).freeze
 
       def call
         return doc unless result[:escaped_literals]
@@ -34,12 +32,22 @@ module Banzai
           node.attributes['title'].value = node.attributes['title'].value.gsub(SPAN_REGEX, '\1') if node.attributes['title']
         end
 
-        doc.xpath(XPATH_CODE).each do |node|
+        doc.xpath(lang_tag).each do |node|
           node.attributes['lang'].value  = node.attributes['lang'].value.gsub(SPAN_REGEX, '\1') if node.attributes['lang']
         end
 
         doc
       end
+
+      private
+
+      def lang_tag
+        if Feature.enabled?(:use_cmark_renderer)
+          Gitlab::Utils::Nokogiri.css_to_xpath('pre')
+        else
+          Gitlab::Utils::Nokogiri.css_to_xpath('code')
+        end
+      end
     end
   end
 end
diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb
index 93370178a61..e67cdc7df12 100644
--- a/lib/banzai/filter/plantuml_filter.rb
+++ b/lib/banzai/filter/plantuml_filter.rb
@@ -5,18 +5,15 @@ require "asciidoctor_plantuml/plantuml"
 
 module Banzai
   module Filter
-    # HTML that replaces all `code plantuml` tags with PlantUML img tags.
+    # HTML that replaces all `lang plantuml` tags with PlantUML img tags.
     #
     class PlantumlFilter < HTML::Pipeline::Filter
-      CSS   = 'pre > code[lang="plantuml"]'
-      XPATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS).freeze
-
       def call
-        return doc unless settings.plantuml_enabled? && doc.at_xpath(XPATH)
+        return doc unless settings.plantuml_enabled? && doc.at_xpath(lang_tag)
 
         plantuml_setup
 
-        doc.xpath(XPATH).each do |node|
+        doc.xpath(lang_tag).each do |node|
           img_tag = Nokogiri::HTML::DocumentFragment.parse(
             Asciidoctor::PlantUml::Processor.plantuml_content(node.content, {}))
           node.parent.replace(img_tag)
@@ -27,6 +24,15 @@ module Banzai
 
       private
 
+      def lang_tag
+        @lang_tag ||=
+          if Feature.enabled?(:use_cmark_renderer)
+            Gitlab::Utils::Nokogiri.css_to_xpath('pre[lang="plantuml"] > code').freeze
+          else
+            Gitlab::Utils::Nokogiri.css_to_xpath('pre > code[lang="plantuml"]').freeze
+          end
+      end
+
       def settings
         Gitlab::CurrentSettings.current_application_settings
       end
diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb
index 1e84e7e8af3..7afbc1a1c9c 100644
--- a/lib/banzai/filter/sanitization_filter.rb
+++ b/lib/banzai/filter/sanitization_filter.rb
@@ -54,8 +54,13 @@ module Banzai
             return unless node.name == 'a' || node.name == 'li'
             return unless node.has_attribute?('id')
 
-            return if node.name == 'a' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LINK_REFERENCE_PATTERN
-            return if node.name == 'li' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LI_REFERENCE_PATTERN
+            if Feature.enabled?(:use_cmark_renderer)
+              return if node.name == 'a' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LINK_REFERENCE_PATTERN
+              return if node.name == 'li' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LI_REFERENCE_PATTERN
+            else
+              return if node.name == 'a' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LINK_REFERENCE_PATTERN_OLD
+              return if node.name == 'li' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LI_REFERENCE_PATTERN_OLD
+            end
 
             node.remove_attribute('id')
           end
diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb
index 8d869cd63d3..66bd86c5bb4 100644
--- a/lib/banzai/filter/syntax_highlight_filter.rb
+++ b/lib/banzai/filter/syntax_highlight_filter.rb
@@ -11,7 +11,7 @@ module Banzai
     class SyntaxHighlightFilter < HTML::Pipeline::Filter
       include OutputSafety
 
-      PARAMS_DELIMITER = ':'
+      LANG_PARAMS_DELIMITER = ':'
       LANG_PARAMS_ATTR = 'data-lang-params'
 
       CSS   = 'pre:not([data-math-style]):not([data-mermaid-style]):not([data-kroki-style]) > code'
@@ -27,7 +27,7 @@ module Banzai
 
       def highlight_node(node)
         css_classes = +'code highlight js-syntax-highlight'
-        lang, lang_params = parse_lang_params(node.attr('lang'))
+        lang, lang_params = parse_lang_params(node)
         sourcepos = node.parent.attr('data-sourcepos')
         retried = false
 
@@ -56,7 +56,7 @@ module Banzai
           retry
         end
 
-        sourcepos_attr = sourcepos ? "data-sourcepos=\"#{sourcepos}\"" : ""
+        sourcepos_attr = sourcepos ? "data-sourcepos=\"#{sourcepos}\"" : ''
 
         highlighted = %(
 '
'.
+        # "```ruby:with:options```" -> '
'.
+        #
+        # New ones:
+        # "```ruby with options```" -> '
'.
+        # "```ruby:with:options```" -> '
'.
+
+        language = node.attr('lang')
+
         return unless language
 
-        lang, params = language.split(PARAMS_DELIMITER, 2)
-        formatted_params = %(#{LANG_PARAMS_ATTR}="#{escape_once(params)}") if params
+        language, language_params = language.split(LANG_PARAMS_DELIMITER, 2)
+
+        if Feature.enabled?(:use_cmark_renderer)
+          language_params = [node.attr('data-meta'), language_params].compact.join(' ')
+        end
+
+        formatted_language_params = format_language_params(language_params)
 
-        [lang, formatted_params]
+        [language, formatted_language_params]
       end
 
       # Separate method so it can be instrumented.
@@ -95,6 +118,12 @@ module Banzai
       def use_rouge?(language)
         (%w(math suggestion) + ::AsciidoctorExtensions::Kroki::SUPPORTED_DIAGRAM_NAMES).exclude?(language)
       end
+
+      def format_language_params(language_params)
+        return if language_params.blank?
+
+        %(#{LANG_PARAMS_ATTR}="#{escape_once(language_params)}")
+      end
     end
   end
 end
diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb
index fbbd6135959..b16af78841a 100644
--- a/lib/banzai/renderer.rb
+++ b/lib/banzai/renderer.rb
@@ -160,16 +160,40 @@ module Banzai
     def self.cacheless_render(text, context = {})
       return text.to_s unless text.present?
 
-      Gitlab::Metrics.measure(:banzai_cacheless_render) do
-        result = render_result(text, context)
+      real_start = Gitlab::Metrics::System.monotonic_time
+      cpu_start = Gitlab::Metrics::System.cpu_time
 
-        output = result[:output]
-        if output.respond_to?(:to_html)
-          output.to_html
-        else
-          output.to_s
-        end
-      end
+      result = render_result(text, context)
+
+      output = result[:output]
+      rendered = if output.respond_to?(:to_html)
+                   output.to_html
+                 else
+                   output.to_s
+                 end
+
+      cpu_duration_histogram.observe({}, Gitlab::Metrics::System.cpu_time - cpu_start)
+      real_duration_histogram.observe({}, Gitlab::Metrics::System.monotonic_time - real_start)
+
+      rendered
+    end
+
+    def self.real_duration_histogram
+      Gitlab::Metrics.histogram(
+        :gitlab_banzai_cacheless_render_real_duration_seconds,
+        'Duration of Banzai pipeline rendering in real time',
+        {},
+        [0.01, 0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10.0, 50, 100]
+      )
+    end
+
+    def self.cpu_duration_histogram
+      Gitlab::Metrics.histogram(
+        :gitlab_banzai_cacheless_render_cpu_duration_seconds,
+        'Duration of Banzai pipeline rendering in cpu time',
+        {},
+        Gitlab::Metrics::EXECUTION_MEASUREMENT_BUCKETS
+      )
     end
 
     def self.full_cache_key(cache_key, pipeline_name)
diff --git a/lib/banzai/renderer/common_mark/html.rb b/lib/banzai/renderer/common_mark/html.rb
index 837665451a1..d9a2d9a9564 100644
--- a/lib/banzai/renderer/common_mark/html.rb
+++ b/lib/banzai/renderer/common_mark/html.rb
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+# Remove this entire file when removing `use_cmark_renderer` feature flag and switching to the CMARK html renderer.
+# https://gitlab.com/gitlab-org/gitlab/-/issues/345744
 module Banzai
   module Renderer
     module CommonMark
diff --git a/lib/bulk_imports/common/extractors/ndjson_extractor.rb b/lib/bulk_imports/common/extractors/ndjson_extractor.rb
index 6b4acd45ea9..ecd7c08bd25 100644
--- a/lib/bulk_imports/common/extractors/ndjson_extractor.rb
+++ b/lib/bulk_imports/common/extractors/ndjson_extractor.rb
@@ -7,10 +7,6 @@ module BulkImports
         include Gitlab::ImportExport::CommandLineUtil
         include Gitlab::Utils::StrongMemoize
 
-        FILE_SIZE_LIMIT = 5.gigabytes
-        ALLOWED_CONTENT_TYPES = %w(application/gzip application/octet-stream).freeze
-        EXPORT_DOWNLOAD_URL_PATH = "/%{resource}/%{full_path}/export_relations/download?relation=%{relation}"
-
         def initialize(relation:)
           @relation = relation
           @tmp_dir = Dir.mktmpdir
@@ -39,33 +35,19 @@ module BulkImports
         def download_service(tmp_dir, context)
           @download_service ||= BulkImports::FileDownloadService.new(
             configuration: context.configuration,
-            relative_url: relative_resource_url(context),
+            relative_url: context.entity.relation_download_url_path(relation),
             dir: tmp_dir,
-            filename: filename,
-            file_size_limit: FILE_SIZE_LIMIT,
-            allowed_content_types: ALLOWED_CONTENT_TYPES
+            filename: filename
           )
         end
 
         def decompression_service(tmp_dir)
-          @decompression_service ||= BulkImports::FileDecompressionService.new(
-            dir: tmp_dir,
-            filename: filename
-          )
+          @decompression_service ||= BulkImports::FileDecompressionService.new(dir: tmp_dir, filename: filename)
         end
 
         def ndjson_reader(tmp_dir)
           @ndjson_reader ||= Gitlab::ImportExport::Json::NdjsonReader.new(tmp_dir)
         end
-
-        def relative_resource_url(context)
-          strong_memoize(:relative_resource_url) do
-            resource = context.entity.pluralized_name
-            encoded_full_path = context.entity.encoded_source_full_path
-
-            EXPORT_DOWNLOAD_URL_PATH % { resource: resource, full_path: encoded_full_path, relation: relation }
-          end
-        end
       end
     end
   end
diff --git a/lib/bulk_imports/common/pipelines/milestones_pipeline.rb b/lib/bulk_imports/common/pipelines/milestones_pipeline.rb
new file mode 100644
index 00000000000..aea2a04c1c7
--- /dev/null
+++ b/lib/bulk_imports/common/pipelines/milestones_pipeline.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Common
+    module Pipelines
+      class MilestonesPipeline
+        include NdjsonPipeline
+
+        relation_name 'milestones'
+
+        extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/common/pipelines/uploads_pipeline.rb b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb
new file mode 100644
index 00000000000..15e126e1646
--- /dev/null
+++ b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Common
+    module Pipelines
+      class UploadsPipeline
+        include Pipeline
+        include Gitlab::ImportExport::CommandLineUtil
+
+        FILENAME = 'uploads.tar.gz'
+
+        def extract(context)
+          download_service(tmp_dir, context).execute
+          untar_zxf(archive: File.join(tmp_dir, FILENAME), dir: tmp_dir)
+          upload_file_paths = Dir.glob(File.join(tmp_dir, '**', '*'))
+
+          BulkImports::Pipeline::ExtractedData.new(data: upload_file_paths)
+        end
+
+        def load(context, file_path)
+          dynamic_path = FileUploader.extract_dynamic_path(file_path)
+
+          return unless dynamic_path
+          return if File.directory?(file_path)
+
+          named_captures = dynamic_path.named_captures.symbolize_keys
+
+          UploadService.new(context.portable, File.open(file_path, 'r'), FileUploader, **named_captures).execute
+        end
+
+        def after_run(_)
+          FileUtils.remove_entry(tmp_dir)
+        end
+
+        private
+
+        def download_service(tmp_dir, context)
+          BulkImports::FileDownloadService.new(
+            configuration: context.configuration,
+            relative_url: context.entity.relation_download_url_path('uploads'),
+            dir: tmp_dir,
+            filename: FILENAME
+          )
+        end
+
+        def tmp_dir
+          @tmp_dir ||= Dir.mktmpdir('bulk_imports')
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/common/pipelines/wiki_pipeline.rb b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb
new file mode 100644
index 00000000000..ccab0b979b2
--- /dev/null
+++ b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Common
+    module Pipelines
+      class WikiPipeline
+        include Pipeline
+
+        def extract(*)
+          BulkImports::Pipeline::ExtractedData.new(data: { url: url_from_parent_path(context.entity.source_full_path) })
+        end
+
+        def transform(_, data)
+          data&.slice(:url)
+        end
+
+        def load(context, data)
+          return unless context.portable.wiki
+
+          url = data[:url].sub("://", "://oauth2:#{context.configuration.access_token}@")
+
+          Gitlab::UrlBlocker.validate!(url, allow_local_network: allow_local_requests?, allow_localhost: allow_local_requests?)
+
+          context.portable.wiki.ensure_repository
+          context.portable.wiki.repository.fetch_as_mirror(url)
+        end
+
+        private
+
+        def url_from_parent_path(parent_path)
+          wiki_path = parent_path + ".wiki.git"
+          root = context.configuration.url
+          Gitlab::Utils.append_path(root, wiki_path)
+        end
+
+        def allow_local_requests?
+          Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services?
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/groups/graphql/get_milestones_query.rb b/lib/bulk_imports/groups/graphql/get_milestones_query.rb
deleted file mode 100644
index 5dd5b31cf0e..00000000000
--- a/lib/bulk_imports/groups/graphql/get_milestones_query.rb
+++ /dev/null
@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-module BulkImports
-  module Groups
-    module Graphql
-      module GetMilestonesQuery
-        extend self
-
-        def to_s
-          <<-'GRAPHQL'
-          query ($full_path: ID!, $cursor: String, $per_page: Int) {
-            group(fullPath: $full_path) {
-              milestones(first: $per_page, after: $cursor, includeDescendants: false) {
-                page_info: pageInfo {
-                  next_page: endCursor
-                  has_next_page: hasNextPage
-                }
-                nodes {
-                  iid
-                  title
-                  description
-                  state
-                  start_date: startDate
-                  due_date: dueDate
-                  created_at: createdAt
-                  updated_at: updatedAt
-                }
-              }
-            }
-          }
-          GRAPHQL
-        end
-
-        def variables(context)
-          {
-            full_path: context.entity.source_full_path,
-            cursor: context.tracker.next_page,
-            per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE
-          }
-        end
-
-        def base_path
-          %w[data group milestones]
-        end
-
-        def data_path
-          base_path << 'nodes'
-        end
-
-        def page_info_path
-          base_path << 'page_info'
-        end
-      end
-    end
-  end
-end
diff --git a/lib/bulk_imports/groups/loaders/group_loader.rb b/lib/bulk_imports/groups/loaders/group_loader.rb
index a631685c2ad..5f5307123a5 100644
--- a/lib/bulk_imports/groups/loaders/group_loader.rb
+++ b/lib/bulk_imports/groups/loaders/group_loader.rb
@@ -4,10 +4,21 @@ module BulkImports
   module Groups
     module Loaders
       class GroupLoader
+        GroupCreationError = Class.new(StandardError)
+
         def load(context, data)
-          return unless user_can_create_group?(context.current_user, data)
+          path = data['path']
+          current_user = context.current_user
+          destination_namespace = context.entity.destination_namespace
+
+          raise(GroupCreationError, 'Path is missing') unless path.present?
+          raise(GroupCreationError, 'Destination is not a group') if user_namespace_destination?(destination_namespace)
+          raise(GroupCreationError, 'User not allowed to create group') unless user_can_create_group?(current_user, data)
+          raise(GroupCreationError, 'Group exists') if group_exists?(destination_namespace, path)
+
+          group = ::Groups::CreateService.new(current_user, data).execute
 
-          group = ::Groups::CreateService.new(context.current_user, data).execute
+          raise(GroupCreationError, group.errors.full_messages.to_sentence) if group.errors.any?
 
           context.entity.update!(group: group)
 
@@ -25,6 +36,18 @@ module BulkImports
             Ability.allowed?(current_user, :create_group)
           end
         end
+
+        def group_exists?(destination_namespace, path)
+          full_path = destination_namespace.present? ? File.join(destination_namespace, path) : path
+
+          Group.find_by_full_path(full_path).present?
+        end
+
+        def user_namespace_destination?(destination_namespace)
+          return false unless destination_namespace.present?
+
+          Namespace.find_by_full_path(destination_namespace)&.user_namespace?
+        end
       end
     end
   end
diff --git a/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb b/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb
deleted file mode 100644
index b2bd14952e7..00000000000
--- a/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module BulkImports
-  module Groups
-    module Pipelines
-      class MilestonesPipeline
-        include NdjsonPipeline
-
-        relation_name 'milestones'
-
-        extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation
-      end
-    end
-  end
-end
diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb
index a1869b4cb0e..241dd428dd5 100644
--- a/lib/bulk_imports/groups/stage.rb
+++ b/lib/bulk_imports/groups/stage.rb
@@ -28,7 +28,7 @@ module BulkImports
             stage: 1
           },
           milestones: {
-            pipeline: BulkImports::Groups::Pipelines::MilestonesPipeline,
+            pipeline: BulkImports::Common::Pipelines::MilestonesPipeline,
             stage: 1
           },
           badges: {
diff --git a/lib/bulk_imports/ndjson_pipeline.rb b/lib/bulk_imports/ndjson_pipeline.rb
index f01ce22a46d..6cc29d63919 100644
--- a/lib/bulk_imports/ndjson_pipeline.rb
+++ b/lib/bulk_imports/ndjson_pipeline.rb
@@ -13,7 +13,7 @@ module BulkImports
         relation_hash, relation_index = data
         relation_definition = import_export_config.top_relation_tree(relation)
 
-        deep_transform_relation!(relation_hash, relation, relation_definition) do |key, hash|
+        relation_object = deep_transform_relation!(relation_hash, relation, relation_definition) do |key, hash|
           relation_factory.create(
             relation_index: relation_index,
             relation_sym: key.to_sym,
@@ -25,6 +25,9 @@ module BulkImports
             excluded_keys: import_export_config.relation_excluded_keys(key)
           )
         end
+
+        relation_object.assign_attributes(portable_class_sym => portable)
+        relation_object
       end
 
       def load(_, object)
@@ -94,6 +97,10 @@ module BulkImports
       def members_mapper
         @members_mapper ||= BulkImports::UsersMapper.new(context: context)
       end
+
+      def portable_class_sym
+        portable.class.to_s.downcase.to_sym
+      end
     end
   end
 end
diff --git a/lib/bulk_imports/projects/pipelines/external_pull_requests_pipeline.rb b/lib/bulk_imports/projects/pipelines/external_pull_requests_pipeline.rb
new file mode 100644
index 00000000000..1f720596c8f
--- /dev/null
+++ b/lib/bulk_imports/projects/pipelines/external_pull_requests_pipeline.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Projects
+    module Pipelines
+      class ExternalPullRequestsPipeline
+        include NdjsonPipeline
+
+        relation_name 'external_pull_requests'
+
+        extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/projects/pipelines/merge_requests_pipeline.rb b/lib/bulk_imports/projects/pipelines/merge_requests_pipeline.rb
new file mode 100644
index 00000000000..264bda6e654
--- /dev/null
+++ b/lib/bulk_imports/projects/pipelines/merge_requests_pipeline.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Projects
+    module Pipelines
+      class MergeRequestsPipeline
+        include NdjsonPipeline
+
+        relation_name 'merge_requests'
+
+        extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation
+
+        def after_run(_)
+          context.portable.merge_requests.set_latest_merge_request_diff_ids!
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/projects/pipelines/protected_branches_pipeline.rb b/lib/bulk_imports/projects/pipelines/protected_branches_pipeline.rb
new file mode 100644
index 00000000000..a570143227d
--- /dev/null
+++ b/lib/bulk_imports/projects/pipelines/protected_branches_pipeline.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Projects
+    module Pipelines
+      class ProtectedBranchesPipeline
+        include NdjsonPipeline
+
+        relation_name 'protected_branches'
+
+        extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/projects/pipelines/repository_pipeline.rb b/lib/bulk_imports/projects/pipelines/repository_pipeline.rb
index 86e696f87a4..6bbd4d0688b 100644
--- a/lib/bulk_imports/projects/pipelines/repository_pipeline.rb
+++ b/lib/bulk_imports/projects/pipelines/repository_pipeline.rb
@@ -17,10 +17,18 @@ module BulkImports
         def load(context, data)
           url = data['httpUrlToRepo']
           url = url.sub("://", "://oauth2:#{context.configuration.access_token}@")
+          project = context.portable
 
           Gitlab::UrlBlocker.validate!(url, allow_local_network: allow_local_requests?, allow_localhost: allow_local_requests?)
 
-          context.portable.repository.import_repository(url)
+          project.ensure_repository
+          project.repository.fetch_as_mirror(url)
+        end
+
+        # The initial fetch can bring in lots of loose refs and objects.
+        # Running a `git gc` will make importing merge requests faster.
+        def after_run(_)
+          ::Repositories::HousekeepingService.new(context.portable, :gc).execute
         end
 
         private
diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb
index 3ada0f406ca..9ccc9efff1d 100644
--- a/lib/bulk_imports/projects/stage.rb
+++ b/lib/bulk_imports/projects/stage.rb
@@ -19,6 +19,10 @@ module BulkImports
             pipeline: BulkImports::Common::Pipelines::LabelsPipeline,
             stage: 2
           },
+          milestones: {
+            pipeline: BulkImports::Common::Pipelines::MilestonesPipeline,
+            stage: 2
+          },
           issues: {
             pipeline: BulkImports::Projects::Pipelines::IssuesPipeline,
             stage: 3
@@ -27,9 +31,29 @@ module BulkImports
             pipeline: BulkImports::Common::Pipelines::BoardsPipeline,
             stage: 4
           },
+          merge_requests: {
+            pipeline: BulkImports::Projects::Pipelines::MergeRequestsPipeline,
+            stage: 4
+          },
+          external_pull_requests: {
+            pipeline: BulkImports::Projects::Pipelines::ExternalPullRequestsPipeline,
+            stage: 4
+          },
+          protected_branches: {
+            pipeline: BulkImports::Projects::Pipelines::ProtectedBranchesPipeline,
+            stage: 4
+          },
+          wiki: {
+            pipeline: BulkImports::Common::Pipelines::WikiPipeline,
+            stage: 5
+          },
+          uploads: {
+            pipeline: BulkImports::Common::Pipelines::UploadsPipeline,
+            stage: 5
+          },
           finisher: {
             pipeline: BulkImports::Common::Pipelines::EntityFinisher,
-            stage: 5
+            stage: 6
           }
         }
       end
diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb
index 46399224a5d..c2ad9e6ae89 100644
--- a/lib/container_registry/client.rb
+++ b/lib/container_registry/client.rb
@@ -90,7 +90,7 @@ module ContainerRegistry
 
     def repository_tag_digest(name, reference)
       response = faraday.head("/v2/#{name}/manifests/#{reference}")
-      response.headers['docker-content-digest'] if response.success?
+      response.headers[DependencyProxy::Manifest::DIGEST_HEADER] if response.success?
     end
 
     def delete_repository_tag_by_digest(name, reference)
@@ -171,7 +171,7 @@ module ContainerRegistry
         req.body = Gitlab::Json.pretty_generate(manifest)
       end
 
-      response.headers['docker-content-digest'] if response.success?
+      response.headers[DependencyProxy::Manifest::DIGEST_HEADER] if response.success?
     end
 
     private
diff --git a/lib/declarative_enum.rb b/lib/declarative_enum.rb
index 8dea9d6130b..7875e0ba4f3 100644
--- a/lib/declarative_enum.rb
+++ b/lib/declarative_enum.rb
@@ -15,9 +15,9 @@
 #     TEXT
 #
 #     define do
-#       acceptable_risk value: 0, description: 'The vulnerability is known but is considered to be an acceptable business risk.'
-#       false_positive value: 1, description: 'An error in reporting the presence of a vulnerability in a system when the vulnerability is not present.'
-#       used_in_tests value: 2, description: 'The finding is not a vulnerability because it is part of a test or is test data.'
+#       acceptable_risk value: 0, description: N_('The vulnerability is known but is considered to be an acceptable business risk.')
+#       false_positive value: 1, description: N_('An error in reporting the presence of a vulnerability in a system when the vulnerability is not present.')
+#       used_in_tests value: 2, description: N_('The finding is not a vulnerability because it is part of a test or is test data.')
 #     end
 #
 # Then we can use this module to register enums for our Active Record models like so,
@@ -63,6 +63,19 @@ module DeclarativeEnum
     @description
   end
 
+  def values
+    definition.transform_values { |definition| definition[:value] }
+  end
+
+  # Return list of dynamically translated descriptions.
+  #
+  # It is required to define descriptions with `N_(...)`.
+  #
+  # See https://github.com/grosser/fast_gettext#n_-and-nn_-make-dynamic-translations-available-to-the-parser
+  def translated_descriptions
+    definition.transform_values { |definition| _(definition[:description]) }
+  end
+
   def define(&block)
     raise LocalJumpError, 'No block given' unless block
 
diff --git a/lib/error_tracking/collector/payload_validator.rb b/lib/error_tracking/collector/payload_validator.rb
new file mode 100644
index 00000000000..aae19a3635a
--- /dev/null
+++ b/lib/error_tracking/collector/payload_validator.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module ErrorTracking
+  module Collector
+    class PayloadValidator
+      PAYLOAD_SCHEMA_PATH = Rails.root.join('app', 'validators', 'json_schemas', 'error_tracking_event_payload.json').to_s
+
+      def valid?(payload)
+        JSONSchemer.schema(Pathname.new(PAYLOAD_SCHEMA_PATH)).valid?(payload)
+      end
+    end
+  end
+end
diff --git a/lib/error_tracking/collector/sentry_request_parser.rb b/lib/error_tracking/collector/sentry_request_parser.rb
index 29e4cc8976f..ae632ebd518 100644
--- a/lib/error_tracking/collector/sentry_request_parser.rb
+++ b/lib/error_tracking/collector/sentry_request_parser.rb
@@ -4,15 +4,7 @@ module ErrorTracking
   module Collector
     class SentryRequestParser
       def self.parse(request)
-        # Request body can be "" or "gzip".
-        # If later then body was compressed with Zlib.gzip
-        encoding = request.headers['Content-Encoding']
-
-        body = if encoding == 'gzip'
-                 Zlib.gunzip(request.body.read)
-               else
-                 request.body.read
-               end
+        body = request.body.read
 
         # Request body contains 3 json objects merged together in one StringIO.
         # We need to separate and parse them into array of hash objects.
diff --git a/lib/feature.rb b/lib/feature.rb
index f8d34e9c386..8186fbc40fa 100644
--- a/lib/feature.rb
+++ b/lib/feature.rb
@@ -6,6 +6,8 @@ require 'flipper/adapters/active_support_cache_store'
 class Feature
   # Classes to override flipper table names
   class FlipperFeature < Flipper::Adapters::ActiveRecord::Feature
+    include DatabaseReflection
+
     # Using `self.table_name` won't work. ActiveRecord bug?
     superclass.table_name = 'features'
 
@@ -36,7 +38,7 @@ class Feature
     end
 
     def persisted_names
-      return [] unless Gitlab::Database.main.exists?
+      return [] unless ApplicationRecord.database.exists?
 
       # This loads names of all stored feature flags
       # and returns a stable Set in the following order:
@@ -73,7 +75,7 @@ class Feature
 
       # During setup the database does not exist yet. So we haven't stored a value
       # for the feature yet and return the default.
-      return default_enabled unless Gitlab::Database.main.exists?
+      return default_enabled unless ApplicationRecord.database.exists?
 
       feature = get(key)
 
@@ -155,13 +157,13 @@ class Feature
 
     def flipper
       if Gitlab::SafeRequestStore.active?
-        Gitlab::SafeRequestStore[:flipper] ||= build_flipper_instance
+        Gitlab::SafeRequestStore[:flipper] ||= build_flipper_instance(memoize: true)
       else
         @flipper ||= build_flipper_instance
       end
     end
 
-    def build_flipper_instance
+    def build_flipper_instance(memoize: false)
       active_record_adapter = Flipper::Adapters::ActiveRecord.new(
         feature_class: FlipperFeature,
         gate_class: FlipperGate)
@@ -182,7 +184,7 @@ class Feature
         expires_in: 1.minute)
 
       Flipper.new(flipper_adapter).tap do |flip|
-        flip.memoize = true
+        flip.memoize = memoize
       end
     end
 
diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb
index a061a83e79c..a1f7dc0ee39 100644
--- a/lib/feature/gitaly.rb
+++ b/lib/feature/gitaly.rb
@@ -15,7 +15,7 @@ class Feature
 
       def server_feature_flags(project = nil)
         # We need to check that both the DB connection and table exists
-        return {} unless ::Gitlab::Database.main.cached_table_exists?(FlipperFeature.table_name)
+        return {} unless FlipperFeature.database.cached_table_exists?
 
         Feature.persisted_names
           .select { |f| f.start_with?(PREFIX) }
diff --git a/lib/generators/gitlab/usage_metric_definition_generator.rb b/lib/generators/gitlab/usage_metric_definition_generator.rb
index 2d65363bf7b..bd34ab0a16f 100644
--- a/lib/generators/gitlab/usage_metric_definition_generator.rb
+++ b/lib/generators/gitlab/usage_metric_definition_generator.rb
@@ -30,18 +30,20 @@ module Gitlab
 
     source_root File.expand_path('../../../generator_templates/usage_metric_definition', __dir__)
 
-    desc 'Generates a metric definition yml file'
+    desc 'Generates metric definitions yml files'
 
     class_option :ee, type: :boolean, optional: true, default: false, desc: 'Indicates if metric is for ee'
     class_option :dir,
       type: :string, desc: "Indicates the metric location. It must be one of: #{VALID_INPUT_DIRS.join(', ')}"
 
-    argument :key_path, type: :string, desc: 'Unique JSON key path for the metric'
+    argument :key_paths, type: :array, desc: 'Unique JSON key paths for the metrics'
 
     def create_metric_file
       validate!
 
-      template "metric_definition.yml", file_path
+      key_paths.each do |key_path|
+        template "metric_definition.yml", file_path(key_path), key_path
+      end
     end
 
     def time_frame
@@ -66,12 +68,12 @@ module Gitlab
 
     private
 
-    def metric_name_suggestion
+    def metric_name_suggestion(key_path)
       "\nname: \"#{Usage::Metrics::NamesSuggestions::Generator.generate(key_path)}\""
     end
 
-    def file_path
-      path = File.join(TOP_LEVEL_DIR, 'metrics', directory&.name, "#{file_name}.yml")
+    def file_path(key_path)
+      path = File.join(TOP_LEVEL_DIR, 'metrics', directory&.name, "#{file_name(key_path)}.yml")
       path = File.join(TOP_LEVEL_DIR_EE, path) if ee?
       path
     end
@@ -79,7 +81,10 @@ module Gitlab
     def validate!
       raise "--dir option is required" unless input_dir.present?
       raise "Invalid dir #{input_dir}, allowed options are #{VALID_INPUT_DIRS.join(', ')}" unless directory.present?
-      raise "Metric definition with key path '#{key_path}' already exists" if metric_definition_exists?
+
+      key_paths.each do |key_path|
+        raise "Metric definition with key path '#{key_path}' already exists" if metric_definition_exists?(key_path)
+      end
     end
 
     def ee?
@@ -93,15 +98,15 @@ module Gitlab
     # Example of file name
     #
     # 20210201124931_g_project_management_issue_title_changed_weekly.yml
-    def file_name
-      "#{Time.now.utc.strftime("%Y%m%d%H%M%S")}_#{metric_name}"
+    def file_name(key_path)
+      "#{Time.now.utc.strftime("%Y%m%d%H%M%S")}_#{metric_name(key_path)}"
     end
 
     def directory
       @directory ||= TIME_FRAME_DIRS.find { |d| d.match?(input_dir) }
     end
 
-    def metric_name
+    def metric_name(key_path)
       key_path.split('.').last
     end
 
@@ -109,7 +114,7 @@ module Gitlab
       @definitions ||= Gitlab::Usage::MetricDefinition.definitions(skip_validation: true)
     end
 
-    def metric_definition_exists?
+    def metric_definition_exists?(key_path)
       metric_definitions[key_path].present?
     end
   end
diff --git a/lib/generators/post_deployment_migration/post_deployment_migration_generator.rb b/lib/generators/post_deployment_migration/post_deployment_migration_generator.rb
index 66ee0e2440f..792c49a820d 100644
--- a/lib/generators/post_deployment_migration/post_deployment_migration_generator.rb
+++ b/lib/generators/post_deployment_migration/post_deployment_migration_generator.rb
@@ -1,17 +1,13 @@
 # frozen_string_literal: true
 
 require 'rails/generators'
+require 'rails/generators/active_record'
+require 'rails/generators/active_record/migration/migration_generator'
 
 module PostDeploymentMigration
-  class PostDeploymentMigrationGenerator < Rails::Generators::NamedBase
-    def create_migration_file
-      timestamp = Time.now.utc.strftime('%Y%m%d%H%M%S')
-
-      template "migration.rb", "db/post_migrate/#{timestamp}_#{file_name}.rb"
-    end
-
-    def migration_class_name
-      file_name.camelize
+  class PostDeploymentMigrationGenerator < ActiveRecord::Generators::MigrationGenerator
+    def db_migrate_path
+      super.sub("migrate", "post_migrate")
     end
   end
 end
diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb
index d3c96a0f934..3e09d488bc3 100644
--- a/lib/gitlab/access.rb
+++ b/lib/gitlab/access.rb
@@ -74,16 +74,32 @@ module Gitlab
       end
 
       def protection_options
-        {
-          "Not protected: Both developers and maintainers can push new commits and force push." => PROTECTION_NONE,
-          "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch. Maintainers can push to the branch." => PROTECTION_DEV_CAN_MERGE,
-          "Partially protected: Both developers and maintainers can push new commits, but cannot force push." => PROTECTION_DEV_CAN_PUSH,
-          "Fully protected: Developers cannot push new commits, but maintainers can. No one can force push." => PROTECTION_FULL
-        }
+        [
+          {
+            label: s_('DefaultBranchProtection|Not protected'),
+            help_text: s_('DefaultBranchProtection|Both developers and maintainers can push new commits, force push, or delete the branch.'),
+            value: PROTECTION_NONE
+          },
+          {
+            label: s_('DefaultBranchProtection|Protected against pushes'),
+            help_text: s_('DefaultBranchProtection|Developers cannot push new commits, but are allowed to accept merge requests to the branch. Maintainers can push to the branch.'),
+            value: PROTECTION_DEV_CAN_MERGE
+          },
+          {
+            label: s_('DefaultBranchProtection|Partially protected'),
+            help_text: s_('DefaultBranchProtection|Both developers and maintainers can push new commits, but cannot force push.'),
+            value: PROTECTION_DEV_CAN_PUSH
+          },
+          {
+            label: s_('DefaultBranchProtection|Fully protected'),
+            help_text: s_('DefaultBranchProtection|Developers cannot push new commits, but maintainers can. No one can force push.'),
+            value: PROTECTION_FULL
+          }
+        ]
       end
 
       def protection_values
-        protection_options.values
+        protection_options.map { |option| option[:value] }
       end
 
       def human_access(access)
diff --git a/lib/gitlab/action_cable/config.rb b/lib/gitlab/action_cable/config.rb
index 38e870353eb..77d4ec0733d 100644
--- a/lib/gitlab/action_cable/config.rb
+++ b/lib/gitlab/action_cable/config.rb
@@ -4,10 +4,6 @@ module Gitlab
   module ActionCable
     class Config
       class << self
-        def in_app?
-          Gitlab::Utils.to_boolean(ENV.fetch('ACTION_CABLE_IN_APP', false))
-        end
-
         def worker_pool_size
           ENV.fetch('ACTION_CABLE_WORKER_POOL_SIZE', 4).to_i
         end
diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb
new file mode 100644
index 00000000000..1e50c980a3a
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb
@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module Aggregated
+        # rubocop: disable CodeReuse/ActiveRecord
+        class BaseQueryBuilder
+          include StageQueryHelpers
+
+          MODEL_CLASSES = {
+            MergeRequest.to_s => ::Analytics::CycleAnalytics::MergeRequestStageEvent,
+            Issue.to_s => ::Analytics::CycleAnalytics::IssueStageEvent
+          }.freeze
+
+          # Allowed params:
+          # * from - stage end date filter start date
+          # * to - stage end date filter to date
+          # * author_username
+          # * milestone_title
+          # * label_name (array)
+          # * assignee_username (array)
+          # * project_ids (array)
+          def initialize(stage:, params: {})
+            @stage = stage
+            @params = params
+            @root_ancestor = stage.parent.root_ancestor
+            @stage_event_model = MODEL_CLASSES.fetch(stage.subject_class.to_s)
+          end
+
+          def build
+            query = base_query
+            query = filter_by_stage_parent(query)
+            query = filter_author(query)
+            query = filter_milestone_ids(query)
+            query = filter_label_names(query)
+            filter_assignees(query)
+          end
+
+          def filter_author(query)
+            return query if params[:author_username].blank?
+
+            user = User.by_username(params[:author_username]).first
+
+            return query.none if user.blank?
+
+            query.authored(user)
+          end
+
+          def filter_milestone_ids(query)
+            return query if params[:milestone_title].blank?
+
+            milestone = MilestonesFinder
+              .new(group_ids: root_ancestor.self_and_descendant_ids, project_ids: root_ancestor.all_projects.select(:id), title: params[:milestone_title])
+              .execute
+              .first
+
+            return query.none if milestone.blank?
+
+            query.with_milestone_id(milestone.id)
+          end
+
+          def filter_label_names(query)
+            return query if params[:label_name].blank?
+
+            LabelFilter.new(
+              stage: stage,
+              params: params,
+              project: nil,
+              group: root_ancestor
+            ).filter(query)
+          end
+
+          def filter_assignees(query)
+            return query if params[:assignee_username].blank?
+
+            Issuables::AssigneeFilter
+              .new(params: { assignee_username: params[:assignee_username] })
+              .filter(query)
+          end
+
+          def filter_by_stage_parent(query)
+            query.by_project_id(stage.parent_id)
+          end
+
+          def base_query
+            query = stage_event_model
+              .by_stage_event_hash_id(stage.stage_event_hash_id)
+
+            from = params[:from] || 30.days.ago
+            if in_progress?
+              query = query
+                .end_event_is_not_happened_yet
+                .opened_state
+                .start_event_timestamp_after(from)
+              query = query.start_event_timestamp_before(params[:to]) if params[:to]
+            else
+              query = query.end_event_timestamp_after(from)
+              query = query.end_event_timestamp_before(params[:to]) if params[:to]
+            end
+
+            query
+          end
+
+          private
+
+          attr_reader :stage, :params, :root_ancestor, :stage_event_model
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+      end
+    end
+  end
+end
+Gitlab::Analytics::CycleAnalytics::Aggregated::BaseQueryBuilder.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::Aggregated::BaseQueryBuilder')
diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb
new file mode 100644
index 00000000000..c8b11ecb4a8
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module Aggregated
+        # Arguments:
+        #   stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::GroupStage
+        #   params:
+        #     current_user: an instance of User
+        #     from: DateTime
+        #     to: DateTime
+        class DataCollector
+          include Gitlab::Utils::StrongMemoize
+
+          MAX_COUNT = 10001
+
+          delegate :serialized_records, to: :records_fetcher
+
+          def initialize(stage:, params: {})
+            @stage = stage
+            @params = params
+          end
+
+          def median
+            strong_memoize(:median) { Median.new(stage: stage, query: query, params: params) }
+          end
+
+          def count
+            strong_memoize(:count) { limit_count }
+          end
+
+          def records_fetcher
+            strong_memoize(:records_fetcher) do
+              RecordsFetcher.new(stage: stage, query: query, params: params)
+            end
+          end
+
+          private
+
+          attr_reader :stage, :params
+
+          def query
+            BaseQueryBuilder.new(stage: stage, params: params).build
+          end
+
+          def limit_count
+            query.limit(MAX_COUNT).count
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/label_filter.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/label_filter.rb
new file mode 100644
index 00000000000..6d87ae91a9c
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/aggregated/label_filter.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module Aggregated
+        # This class makes it possible to add label filters to stage event tables
+        class LabelFilter < Issuables::LabelFilter
+          extend ::Gitlab::Utils::Override
+
+          def initialize(stage:, project:, group:, **kwargs)
+            @stage = stage
+
+            super(project: project, group: group, **kwargs)
+          end
+
+          private
+
+          attr_reader :stage
+
+          override :label_link_query
+          def label_link_query(target_model, label_ids: nil)
+            join_column = target_model.arel_table[target_model.issuable_id_column]
+
+            LabelLink.by_target_for_exists_query(stage.subject_class.name, join_column, label_ids)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/median.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/median.rb
new file mode 100644
index 00000000000..181ee20948b
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/aggregated/median.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module Aggregated
+        class Median
+          include StageQueryHelpers
+
+          def initialize(stage:, query:, params:)
+            @stage = stage
+            @query = query
+            @params = params
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def seconds
+            @query = @query.select(median_duration_in_seconds.as('median')).reorder(nil)
+            result = @query.take || {}
+
+            result['median'] || nil
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+
+          def days
+            seconds ? seconds.fdiv(1.day) : nil
+          end
+
+          private
+
+          attr_reader :stage, :query, :params
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb
new file mode 100644
index 00000000000..7dce757cdc8
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb
@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module Aggregated
+        class RecordsFetcher
+          include Gitlab::Utils::StrongMemoize
+          include StageQueryHelpers
+
+          MAX_RECORDS = 20
+
+          MAPPINGS = {
+            Issue => {
+              serializer_class: AnalyticsIssueSerializer,
+              includes_for_query: { project: { namespace: [:route] }, author: [] },
+              columns_for_select: %I[title iid id created_at author_id project_id]
+            },
+            MergeRequest => {
+              serializer_class: AnalyticsMergeRequestSerializer,
+              includes_for_query: { target_project: [:namespace], author: [] },
+              columns_for_select: %I[title iid id created_at author_id state_id target_project_id]
+            }
+          }.freeze
+
+          def initialize(stage:, query:, params: {})
+            @stage = stage
+            @query = query
+            @params = params
+            @sort = params[:sort] || :end_event
+            @direction = params[:direction] || :desc
+            @page = params[:page] || 1
+            @per_page = MAX_RECORDS
+            @stage_event_model = query.model
+          end
+
+          def serialized_records
+            strong_memoize(:serialized_records) do
+              records = ordered_and_limited_query.select(stage_event_model.arel_table[Arel.star], duration.as('total_time'))
+
+              yield records if block_given?
+              issuables_and_records = load_issuables(records)
+
+              preload_associations(issuables_and_records.map(&:first))
+
+              issuables_and_records.map do |issuable, record|
+                project = issuable.project
+                attributes = issuable.attributes.merge({
+                  project_path: project.path,
+                  namespace_path: project.namespace.route.path,
+                  author: issuable.author,
+                  total_time: record.total_time
+                })
+                serializer.represent(attributes)
+              end
+            end
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def ordered_and_limited_query
+            sorting_options = {
+              end_event: {
+                asc: -> { query.order(end_event_timestamp: :asc) },
+                desc: -> { query.order(end_event_timestamp: :desc) }
+              },
+              duration: {
+                asc: -> { query.order(duration.asc) },
+                desc: -> { query.order(duration.desc) }
+              }
+            }
+
+            sort_lambda = sorting_options.dig(sort, direction) || sorting_options.dig(:end_event, :desc)
+
+            sort_lambda.call
+              .page(page)
+              .per(per_page)
+              .without_count
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+
+          private
+
+          attr_reader :stage, :query, :sort, :direction, :params, :page, :per_page, :stage_event_model
+
+          delegate :subject_class, to: :stage
+
+          def load_issuables(stage_event_records)
+            stage_event_records_by_issuable_id = stage_event_records.index_by(&:issuable_id)
+
+            issuable_model = stage_event_model.issuable_model
+            issuables_by_id = issuable_model.id_in(stage_event_records_by_issuable_id.keys).index_by(&:id)
+
+            stage_event_records_by_issuable_id.map do |issuable_id, record|
+              [issuables_by_id[issuable_id], record] if issuables_by_id[issuable_id]
+            end.compact
+          end
+
+          def serializer
+            MAPPINGS.fetch(subject_class).fetch(:serializer_class).new
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def preload_associations(records)
+            ActiveRecord::Associations::Preloader.new.preload(
+              records,
+              MAPPINGS.fetch(subject_class).fetch(:includes_for_query)
+            )
+
+            records
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb
new file mode 100644
index 00000000000..f23d1832df9
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module Aggregated
+        module StageQueryHelpers
+          def percentile_cont
+            percentile_cont_ordering = Arel::Nodes::UnaryOperation.new(Arel::Nodes::SqlLiteral.new('ORDER BY'), duration)
+            Arel::Nodes::NamedFunction.new(
+              'percentile_cont(0.5) WITHIN GROUP',
+              [percentile_cont_ordering]
+            )
+          end
+
+          def duration
+            if in_progress?
+              Arel::Nodes::Subtraction.new(
+                Arel::Nodes::NamedFunction.new('TO_TIMESTAMP', [Time.current.to_i]),
+                query.model.arel_table[:start_event_timestamp]
+              )
+            else
+              Arel::Nodes::Subtraction.new(
+                query.model.arel_table[:end_event_timestamp],
+                query.model.arel_table[:start_event_timestamp]
+              )
+            end
+          end
+
+          def median_duration_in_seconds
+            Arel::Nodes::Extract.new(percentile_cont, :epoch)
+          end
+
+          def in_progress?
+            params[:end_event_filter] == :in_progress
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/data_collector.rb
index 56179533ffb..a20481dd39e 100644
--- a/lib/gitlab/analytics/cycle_analytics/data_collector.rb
+++ b/lib/gitlab/analytics/cycle_analytics/data_collector.rb
@@ -23,13 +23,21 @@ module Gitlab
 
         def records_fetcher
           strong_memoize(:records_fetcher) do
-            RecordsFetcher.new(stage: stage, query: query, params: params)
+            if use_aggregated_data_collector?
+              aggregated_data_collector.records_fetcher
+            else
+              RecordsFetcher.new(stage: stage, query: query, params: params)
+            end
           end
         end
 
         def median
           strong_memoize(:median) do
-            Median.new(stage: stage, query: query, params: params)
+            if use_aggregated_data_collector?
+              aggregated_data_collector.median
+            else
+              Median.new(stage: stage, query: query, params: params)
+            end
           end
         end
 
@@ -41,7 +49,11 @@ module Gitlab
 
         def count
           strong_memoize(:count) do
-            limit_count
+            if use_aggregated_data_collector?
+              aggregated_data_collector.count
+            else
+              limit_count
+            end
           end
         end
 
@@ -59,6 +71,14 @@ module Gitlab
         def limit_count
           query.limit(MAX_COUNT).count
         end
+
+        def aggregated_data_collector
+          @aggregated_data_collector ||= Aggregated::DataCollector.new(stage: stage, params: params)
+        end
+
+        def use_aggregated_data_collector?
+          params.fetch(:use_aggregated_data_collector, false)
+        end
       end
     end
   end
diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
index f94696e3186..140c4a300ca 100644
--- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
+++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
@@ -8,23 +8,11 @@ module Gitlab
         include StageQueryHelpers
         include Gitlab::CycleAnalytics::MetricsTables
 
-        MAX_RECORDS = 20
-
-        MAPPINGS = {
-          Issue => {
-            serializer_class: AnalyticsIssueSerializer,
-            includes_for_query: { project: { namespace: [:route] }, author: [] },
-            columns_for_select: %I[title iid id created_at author_id project_id]
-          },
-          MergeRequest => {
-            serializer_class: AnalyticsMergeRequestSerializer,
-            includes_for_query: { target_project: [:namespace], author: [] },
-            columns_for_select: %I[title iid id created_at author_id state_id target_project_id]
-          }
-        }.freeze
-
         delegate :subject_class, to: :stage
 
+        MAX_RECORDS = Gitlab::Analytics::CycleAnalytics::Aggregated::RecordsFetcher::MAX_RECORDS
+        MAPPINGS = Gitlab::Analytics::CycleAnalytics::Aggregated::RecordsFetcher::MAPPINGS
+
         def initialize(stage:, query:, params: {})
           @stage = stage
           @query = query
diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb
index 94e20762368..bc9d94ef09c 100644
--- a/lib/gitlab/analytics/cycle_analytics/request_params.rb
+++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb
@@ -79,7 +79,8 @@ module Gitlab
             sort: sort&.to_sym,
             direction: direction&.to_sym,
             page: page,
-            end_event_filter: end_event_filter.to_sym
+            end_event_filter: end_event_filter.to_sym,
+            use_aggregated_data_collector: Feature.enabled?(:use_vsa_aggregated_tables, group || project, default_enabled: :yaml)
           }.merge(attributes.symbolize_keys.slice(*FINDER_PARAM_NAMES))
         end
 
diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb
index 7c37f67b766..3db2f1295f9 100644
--- a/lib/gitlab/application_rate_limiter.rb
+++ b/lib/gitlab/application_rate_limiter.rb
@@ -11,6 +11,8 @@ module Gitlab
   #   redirect_to(edit_project_path(@project), status: :too_many_requests)
   # end
   class ApplicationRateLimiter
+    InvalidKeyError = Class.new(StandardError)
+
     def initialize(key, **options)
       @key = key
       @options = options
@@ -64,39 +66,43 @@ module Gitlab
       # @param key [Symbol] Key attribute registered in `.rate_limits`
       # @option scope [Array] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project)
       # @option threshold [Integer] Optional threshold value to override default one registered in `.rate_limits`
-      # @option interval [Integer] Optional interval value to override default one registered in `.rate_limits`
       # @option users_allowlist [Array] Optional list of usernames to exclude from the limit. This param will only be functional if Scope includes a current user.
       #
       # @return [Boolean] Whether or not a request should be throttled
       def throttled?(key, **options)
-        return unless rate_limits[key]
+        raise InvalidKeyError unless rate_limits[key]
 
         return if scoped_user_in_allowlist?(options)
 
         threshold_value = options[:threshold] || threshold(key)
         threshold_value > 0 &&
-          increment(key, options[:scope], options[:interval]) > threshold_value
+          increment(key, options[:scope]) > threshold_value
       end
 
-      # Increments the given cache key and increments the value by 1 with the
-      # expiration interval defined in `.rate_limits`.
+      # Increments a cache key that is based on the current time and interval.
+      # So that when time passes to the next interval, the key changes and the count starts again from 0.
+      #
+      # Based on https://github.com/rack/rack-attack/blob/886ba3a18d13c6484cd511a4dc9b76c0d14e5e96/lib/rack/attack/cache.rb#L63-L68
       #
       # @param key [Symbol] Key attribute registered in `.rate_limits`
       # @option scope [Array] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project)
-      # @option interval [Integer] Optional interval value to override default one registered in `.rate_limits`
       #
       # @return [Integer] incremented value
-      def increment(key, scope, interval = nil)
-        value = 0
-        interval_value = interval || interval(key)
+      def increment(key, scope)
+        interval_value = interval(key)
+
+        period_key, time_elapsed_in_period = Time.now.to_i.divmod(interval_value)
+
+        cache_key = "#{action_key(key, scope)}:#{period_key}"
+        # We add a 1 second buffer to avoid timing issues when we're at the end of a period
+        expiry = interval_value - time_elapsed_in_period + 1
 
         ::Gitlab::Redis::RateLimiting.with do |redis|
-          cache_key = action_key(key, scope)
-          value     = redis.incr(cache_key)
-          redis.expire(cache_key, interval_value) if value == 1
+          redis.pipelined do
+            redis.incr(cache_key)
+            redis.expire(cache_key, expiry)
+          end.first
         end
-
-        value
       end
 
       # Logs request using provided logger
diff --git a/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb b/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb
index 5fc3323f0fd..6dbe6f691f6 100644
--- a/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb
+++ b/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb
@@ -7,7 +7,11 @@ module Gitlab
         register_for 'gitlab-html-pipeline'
 
         def format(node, lang, opts)
-          %(
#{node.content}
)
+          if Feature.enabled?(:use_cmark_renderer)
+            %(
#{node.content}
)
+          else
+            %(
#{node.content}
)
+          end
         end
       end
     end
diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb
index ab6ac815601..41a8739b0b6 100644
--- a/lib/gitlab/auth/o_auth/provider.rb
+++ b/lib/gitlab/auth/o_auth/provider.rb
@@ -5,6 +5,7 @@ module Gitlab
     module OAuth
       class Provider
         LABELS = {
+          "dingtalk"                 => "DingTalk",
           "github"                   => "GitHub",
           "gitlab"                   => "GitLab.com",
           "google_oauth2"            => "Google",
diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb
index 0826887dd0a..22b4b685f81 100644
--- a/lib/gitlab/background_migration.rb
+++ b/lib/gitlab/background_migration.rb
@@ -2,8 +2,12 @@
 
 module Gitlab
   module BackgroundMigration
-    def self.queue
-      @queue ||= BackgroundMigrationWorker.sidekiq_options['queue']
+    def self.coordinator_for_database(database)
+      JobCoordinator.for_database(database)
+    end
+
+    def self.queue(database: :main)
+      coordinator_for_database(database).queue
     end
 
     # Begins stealing jobs from the background migrations queue, blocking the
@@ -16,35 +20,10 @@ module Gitlab
     # re-raises the exception.
     #
     # steal_class - The name of the class for which to steal jobs.
-    def self.steal(steal_class, retry_dead_jobs: false)
-      queues = [
-        Sidekiq::ScheduledSet.new,
-        Sidekiq::Queue.new(self.queue)
-      ]
-
-      if retry_dead_jobs
-        queues << Sidekiq::RetrySet.new
-        queues << Sidekiq::DeadSet.new
-      end
-
-      queues.each do |queue|
-        queue.each do |job|
-          migration_class, migration_args = job.args
-
-          next unless job.klass == 'BackgroundMigrationWorker'
-          next unless migration_class == steal_class
-          next if block_given? && !(yield job)
-
-          begin
-            perform(migration_class, migration_args) if job.delete
-          rescue Exception # rubocop:disable Lint/RescueException
-            BackgroundMigrationWorker # enqueue this migration again
-              .perform_async(migration_class, migration_args)
-
-            raise
-          end
-        end
-      end
+    # retry_dead_jobs - Flag to control whether jobs in Sidekiq::RetrySet or Sidekiq::DeadSet are retried.
+    # database - tracking database this migration executes against
+    def self.steal(steal_class, retry_dead_jobs: false, database: :main, &block)
+      coordinator_for_database(database).steal(steal_class, retry_dead_jobs: retry_dead_jobs, &block)
     end
 
     ##
@@ -55,64 +34,17 @@ module Gitlab
     #
     # arguments - The arguments to pass to the background migration's "perform"
     #             method.
-    def self.perform(class_name, arguments)
-      migration_class_for(class_name).new.perform(*arguments)
-    end
-
-    def self.remaining
-      enqueued = Sidekiq::Queue.new(self.queue)
-      scheduled = Sidekiq::ScheduledSet.new
-
-      [enqueued, scheduled].sum do |set|
-        set.count do |job|
-          job.klass == 'BackgroundMigrationWorker'
-        end
-      end
-    end
-
-    def self.exists?(migration_class, additional_queues = [])
-      enqueued = Sidekiq::Queue.new(self.queue)
-      scheduled = Sidekiq::ScheduledSet.new
-
-      enqueued_job?([enqueued, scheduled], migration_class)
-    end
-
-    def self.dead_jobs?(migration_class)
-      dead_set = Sidekiq::DeadSet.new
-
-      enqueued_job?([dead_set], migration_class)
+    # database - tracking database this migration executes against
+    def self.perform(class_name, arguments, database: :main)
+      coordinator_for_database(database).perform(class_name, arguments)
     end
 
-    def self.retrying_jobs?(migration_class)
-      retry_set = Sidekiq::RetrySet.new
-
-      enqueued_job?([retry_set], migration_class)
-    end
-
-    def self.migration_class_for(class_name)
-      # We don't pass class name with Gitlab::BackgroundMigration:: prefix anymore
-      # but some jobs could be already spawned so we need to have some backward compatibility period.
-      # Can be removed since 13.x
-      full_class_name_prefix_regexp = /\A(::)?Gitlab::BackgroundMigration::/
-
-      if class_name.match(full_class_name_prefix_regexp)
-        Gitlab::ErrorTracking.track_and_raise_for_dev_exception(
-          StandardError.new("Full class name is used"),
-          class_name: class_name
-        )
-
-        class_name = class_name.sub(full_class_name_prefix_regexp, '')
-      end
-
-      const_get(class_name, false)
+    def self.exists?(migration_class, additional_queues = [], database: :main)
+      coordinator_for_database(database).exists?(migration_class, additional_queues) # rubocop:disable CodeReuse/ActiveRecord
     end
 
-    def self.enqueued_job?(queues, migration_class)
-      queues.any? do |queue|
-        queue.any? do |job|
-          job.klass == 'BackgroundMigrationWorker' && job.args.first == migration_class
-        end
-      end
+    def self.remaining(database: :main)
+      coordinator_for_database(database).remaining
     end
   end
 end
diff --git a/lib/gitlab/background_migration/add_primary_email_to_emails_if_user_confirmed.rb b/lib/gitlab/background_migration/add_primary_email_to_emails_if_user_confirmed.rb
new file mode 100644
index 00000000000..b39c0953fb1
--- /dev/null
+++ b/lib/gitlab/background_migration/add_primary_email_to_emails_if_user_confirmed.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Add user primary email to emails table if confirmed
+    class AddPrimaryEmailToEmailsIfUserConfirmed
+      INNER_BATCH_SIZE = 1_000
+
+      # Stubbed class to access the User table
+      class User < ActiveRecord::Base
+        include ::EachBatch
+
+        self.table_name = 'users'
+        self.inheritance_column = :_type_disabled
+
+        scope :confirmed, -> { where.not(confirmed_at: nil) }
+
+        has_many :emails
+      end
+
+      # Stubbed class to access the Emails table
+      class Email < ActiveRecord::Base
+        self.table_name = 'emails'
+        self.inheritance_column = :_type_disabled
+
+        belongs_to :user
+      end
+
+      def perform(start_id, end_id)
+        User.confirmed.where(id: start_id..end_id).select(:id, :email, :confirmed_at).each_batch(of: INNER_BATCH_SIZE) do |users|
+          current_time = Time.now.utc
+
+          attributes = users.map do |user|
+            {
+              user_id: user.id,
+              email: user.email,
+              confirmed_at: user.confirmed_at,
+              created_at: current_time,
+              updated_at: current_time
+            }
+          end
+
+          Email.insert_all(attributes)
+        end
+        mark_job_as_succeeded(start_id, end_id)
+      end
+
+      private
+
+      def mark_job_as_succeeded(*arguments)
+        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
+          'AddPrimaryEmailToEmailsIfUserConfirmed',
+          arguments
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/backfill_design_internal_ids.rb b/lib/gitlab/background_migration/backfill_design_internal_ids.rb
deleted file mode 100644
index 236c6b6eb9a..00000000000
--- a/lib/gitlab/background_migration/backfill_design_internal_ids.rb
+++ /dev/null
@@ -1,130 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module BackgroundMigration
-    # Backfill design.iid for a range of projects
-    class BackfillDesignInternalIds
-      # See app/models/internal_id
-      # This is a direct copy of the application code with the following changes:
-      # - usage enum is hard-coded to the value for design_management_designs
-      # - init is not passed around, but ignored
-      class InternalId < ActiveRecord::Base
-        def self.track_greatest(subject, scope, new_value)
-          InternalIdGenerator.new(subject, scope).track_greatest(new_value)
-        end
-
-        # Increments #last_value with new_value if it is greater than the current,
-        # and saves the record
-        #
-        # The operation locks the record and gathers a `ROW SHARE` lock (in PostgreSQL).
-        # As such, the increment is atomic and safe to be called concurrently.
-        def track_greatest_and_save!(new_value)
-          update_and_save { self.last_value = [last_value || 0, new_value].max }
-        end
-
-        private
-
-        def update_and_save(&block)
-          lock!
-          yield
-          # update_and_save_counter.increment(usage: usage, changed: last_value_changed?)
-          save!
-          last_value
-        end
-      end
-
-      # See app/models/internal_id
-      class InternalIdGenerator
-        attr_reader :subject, :scope, :scope_attrs
-
-        def initialize(subject, scope)
-          @subject = subject
-          @scope = scope
-
-          raise ArgumentError, 'Scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty?
-        end
-
-        # Create a record in internal_ids if one does not yet exist
-        # and set its new_value if it is higher than the current last_value
-        #
-        # Note this will acquire a ROW SHARE lock on the InternalId record
-        def track_greatest(new_value)
-          subject.transaction do
-            record.track_greatest_and_save!(new_value)
-          end
-        end
-
-        def record
-          @record ||= (lookup || create_record)
-        end
-
-        def lookup
-          InternalId.find_by(**scope, usage: usage_value)
-        end
-
-        def usage_value
-          10 # see Enums::InternalId - this is the value for design_management_designs
-        end
-
-        # Create InternalId record for (scope, usage) combination, if it doesn't exist
-        #
-        # We blindly insert without synchronization. If another process
-        # was faster in doing this, we'll realize once we hit the unique key constraint
-        # violation. We can safely roll-back the nested transaction and perform
-        # a lookup instead to retrieve the record.
-        def create_record
-          subject.transaction(requires_new: true) do # rubocop:disable Performance/ActiveRecordSubtransactions
-            InternalId.create!(
-              **scope,
-              usage: usage_value,
-              last_value: 0
-            )
-          end
-        rescue ActiveRecord::RecordNotUnique
-          lookup
-        end
-      end
-
-      attr_reader :design_class
-
-      def initialize(design_class)
-        @design_class = design_class
-      end
-
-      def perform(relation)
-        start_id, end_id = relation.pluck("min(project_id), max(project_id)").flatten
-        table = 'design_management_designs'
-
-        ActiveRecord::Base.connection.execute <<~SQL
-          WITH
-          starting_iids(project_id, iid) as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported}(
-            SELECT project_id, MAX(COALESCE(iid, 0))
-            FROM #{table}
-            WHERE project_id BETWEEN #{start_id} AND #{end_id}
-            GROUP BY project_id
-          ),
-          with_calculated_iid(id, iid) as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported}(
-            SELECT design.id,
-                   init.iid + ROW_NUMBER() OVER (PARTITION BY design.project_id ORDER BY design.id ASC)
-              FROM #{table} as design, starting_iids as init
-              WHERE design.project_id BETWEEN #{start_id} AND #{end_id}
-                AND design.iid IS NULL
-                AND init.project_id = design.project_id
-          )
-
-          UPDATE #{table}
-             SET iid = with_calculated_iid.iid
-            FROM with_calculated_iid
-           WHERE #{table}.id = with_calculated_iid.id
-        SQL
-
-        # track the new greatest IID value
-        relation.each do |design|
-          current_max = design_class.where(project_id: design.project_id).maximum(:iid)
-          scope = { project_id: design.project_id }
-          InternalId.track_greatest(design, scope, current_max)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb
index a9eaeb0562d..05e2ed72fb3 100644
--- a/lib/gitlab/background_migration/backfill_project_repositories.rb
+++ b/lib/gitlab/background_migration/backfill_project_repositories.rb
@@ -189,7 +189,7 @@ module Gitlab
       end
 
       def perform(start_id, stop_id)
-        Gitlab::Database.main.bulk_insert(:project_repositories, project_repositories(start_id, stop_id)) # rubocop:disable Gitlab/BulkInsert
+        ApplicationRecord.legacy_bulk_insert(:project_repositories, project_repositories(start_id, stop_id)) # rubocop:disable Gitlab/BulkInsert
       end
 
       private
diff --git a/lib/gitlab/background_migration/backfill_user_namespace.rb b/lib/gitlab/background_migration/backfill_user_namespace.rb
new file mode 100644
index 00000000000..f55eaa3b14e
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_user_namespace.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Backfills the `namespaces.type` column, replacing any
+    # instances of `NULL` with `User`
+    class BackfillUserNamespace
+      include Gitlab::Database::DynamicModelHelpers
+
+      def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms)
+        parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id)
+        parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size, order_hint: :type) do |sub_batch|
+          batch_metrics.time_operation(:update_all) do
+            sub_batch.update_all(type: 'User')
+          end
+          pause_ms = 0 if pause_ms < 0
+          sleep(pause_ms * 0.001)
+        end
+      end
+
+      def batch_metrics
+        @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new
+      end
+
+      private
+
+      def connection
+        ActiveRecord::Base.connection
+      end
+
+      def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id)
+        define_batchable_model(source_table)
+          .where(source_key_column => start_id..stop_id)
+          .where(type: nil)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb b/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb
deleted file mode 100644
index 691bdb457d7..00000000000
--- a/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-# rubocop:disable Style/Documentation
-
-module Gitlab
-  module BackgroundMigration
-    class CopyMergeRequestTargetProjectToMergeRequestMetrics
-      extend ::Gitlab::Utils::Override
-
-      def perform(start_id, stop_id)
-        ActiveRecord::Base.connection.execute <<~SQL
-          WITH merge_requests_batch AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
-            SELECT id, target_project_id
-            FROM merge_requests WHERE id BETWEEN #{Integer(start_id)} AND #{Integer(stop_id)}
-          )
-          UPDATE
-            merge_request_metrics
-          SET
-            target_project_id = merge_requests_batch.target_project_id
-          FROM merge_requests_batch
-          WHERE merge_request_metrics.merge_request_id=merge_requests_batch.id
-        SQL
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/background_migration/fix_merge_request_diff_commit_users.rb b/lib/gitlab/background_migration/fix_merge_request_diff_commit_users.rb
new file mode 100644
index 00000000000..ea3e56cb14a
--- /dev/null
+++ b/lib/gitlab/background_migration/fix_merge_request_diff_commit_users.rb
@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Background migration for fixing merge_request_diff_commit rows that don't
+    # have committer/author details due to
+    # https://gitlab.com/gitlab-org/gitlab/-/issues/344080.
+    #
+    # This migration acts on a single project and corrects its data. Because
+    # this process needs Git/Gitaly access, and duplicating all that code is far
+    # too much, this migration relies on global models such as Project,
+    # MergeRequest, etc.
+    # rubocop: disable Metrics/ClassLength
+    class FixMergeRequestDiffCommitUsers
+      BATCH_SIZE = 100
+
+      def initialize
+        @commits = {}
+        @users = {}
+      end
+
+      def perform(project_id)
+        if (project = ::Project.find_by_id(project_id))
+          process(project)
+        end
+
+        ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
+          'FixMergeRequestDiffCommitUsers',
+          [project_id]
+        )
+
+        schedule_next_job
+      end
+
+      def process(project)
+        # Loading everything using one big query may result in timeouts (e.g.
+        # for projects the size of gitlab-org/gitlab). So instead we query
+        # data on a per merge request basis.
+        project.merge_requests.each_batch(column: :iid) do |mrs|
+          mrs.ids.each do |mr_id|
+            each_row_to_check(mr_id) do |commit|
+              update_commit(project, commit)
+            end
+          end
+        end
+      end
+
+      def each_row_to_check(merge_request_id, &block)
+        columns = %w[merge_request_diff_id relative_order].map do |col|
+          Pagination::Keyset::ColumnOrderDefinition.new(
+            attribute_name: col,
+            order_expression: MergeRequestDiffCommit.arel_table[col.to_sym].asc,
+            nullable: :not_nullable,
+            distinct: false
+          )
+        end
+
+        order = Pagination::Keyset::Order.build(columns)
+        scope = MergeRequestDiffCommit
+          .joins(:merge_request_diff)
+          .where(merge_request_diffs: { merge_request_id: merge_request_id })
+          .where('commit_author_id IS NULL OR committer_id IS NULL')
+          .order(order)
+
+        Pagination::Keyset::Iterator
+          .new(scope: scope, use_union_optimization: true)
+          .each_batch(of: BATCH_SIZE) do |rows|
+            rows
+              .select([
+                :merge_request_diff_id,
+                :relative_order,
+                :sha,
+                :committer_id,
+                :commit_author_id
+              ])
+              .each(&block)
+          end
+      end
+
+      # rubocop: disable Metrics/AbcSize
+      def update_commit(project, row)
+        commit = find_commit(project, row.sha)
+        updates = []
+
+        unless row.commit_author_id
+          author_id = find_or_create_user(commit, :author_name, :author_email)
+
+          updates << [arel_table[:commit_author_id], author_id] if author_id
+        end
+
+        unless row.committer_id
+          committer_id =
+            find_or_create_user(commit, :committer_name, :committer_email)
+
+          updates << [arel_table[:committer_id], committer_id] if committer_id
+        end
+
+        return if updates.empty?
+
+        update = Arel::UpdateManager
+          .new
+          .table(MergeRequestDiffCommit.arel_table)
+          .where(matches_row(row))
+          .set(updates)
+          .to_sql
+
+        MergeRequestDiffCommit.connection.execute(update)
+      end
+      # rubocop: enable Metrics/AbcSize
+
+      def schedule_next_job
+        job = Database::BackgroundMigrationJob
+          .for_migration_class('FixMergeRequestDiffCommitUsers')
+          .pending
+          .first
+
+        return unless job
+
+        BackgroundMigrationWorker.perform_in(
+          2.minutes,
+          'FixMergeRequestDiffCommitUsers',
+          job.arguments
+        )
+      end
+
+      def find_commit(project, sha)
+        @commits[sha] ||= (project.commit(sha)&.to_hash || {})
+      end
+
+      def find_or_create_user(commit, name_field, email_field)
+        name = commit[name_field]
+        email = commit[email_field]
+
+        return unless name && email
+
+        @users[[name, email]] ||=
+          MergeRequest::DiffCommitUser.find_or_create(name, email).id
+      end
+
+      def matches_row(row)
+        primary_key = Arel::Nodes::Grouping
+          .new([arel_table[:merge_request_diff_id], arel_table[:relative_order]])
+
+        primary_val = Arel::Nodes::Grouping
+          .new([row.merge_request_diff_id, row.relative_order])
+
+        primary_key.eq(primary_val)
+      end
+
+      def arel_table
+        MergeRequestDiffCommit.arel_table
+      end
+    end
+    # rubocop: enable Metrics/ClassLength
+  end
+end
diff --git a/lib/gitlab/background_migration/fix_orphan_promoted_issues.rb b/lib/gitlab/background_migration/fix_orphan_promoted_issues.rb
deleted file mode 100644
index c50bf430d92..00000000000
--- a/lib/gitlab/background_migration/fix_orphan_promoted_issues.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module BackgroundMigration
-    # No OP for CE
-    class FixOrphanPromotedIssues
-      def perform(note_id)
-      end
-    end
-  end
-end
-
-Gitlab::BackgroundMigration::FixOrphanPromotedIssues.prepend_mod_with('Gitlab::BackgroundMigration::FixOrphanPromotedIssues')
diff --git a/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb b/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb
deleted file mode 100644
index 47a68c61fcc..00000000000
--- a/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module BackgroundMigration
-    # Remove serialized Ruby object in audit_events
-    class FixRubyObjectInAuditEvents
-      def perform(start_id, stop_id)
-      end
-    end
-  end
-end
-
-Gitlab::BackgroundMigration::FixRubyObjectInAuditEvents.prepend_mod_with('Gitlab::BackgroundMigration::FixRubyObjectInAuditEvents')
diff --git a/lib/gitlab/background_migration/job_coordinator.rb b/lib/gitlab/background_migration/job_coordinator.rb
new file mode 100644
index 00000000000..1c8819eaa62
--- /dev/null
+++ b/lib/gitlab/background_migration/job_coordinator.rb
@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Class responsible for executing background migrations based on the given database.
+    #
+    # Chooses the correct worker class when selecting jobs from the queue based on the
+    # convention of how the queues and worker classes are setup for each database.
+    #
+    # Also provides a database connection to the correct tracking database.
+    class JobCoordinator
+      VALID_DATABASES = %i[main].freeze
+      WORKER_CLASS_NAME = 'BackgroundMigrationWorker'
+
+      def self.for_database(database)
+        database = database.to_sym
+
+        unless VALID_DATABASES.include?(database)
+          raise ArgumentError, "database must be one of [#{VALID_DATABASES.join(', ')}], got '#{database}'"
+        end
+
+        namespace = database.to_s.capitalize unless database == :main
+        namespaced_worker_class = [namespace, WORKER_CLASS_NAME].compact.join('::')
+
+        new(database, "::#{namespaced_worker_class}".constantize)
+      end
+
+      attr_reader :database, :worker_class
+
+      def queue
+        @queue ||= worker_class.sidekiq_options['queue']
+      end
+
+      def with_shared_connection(&block)
+        Gitlab::Database::SharedModel.using_connection(connection, &block)
+      end
+
+      def steal(steal_class, retry_dead_jobs: false)
+        with_shared_connection do
+          queues = [
+            Sidekiq::ScheduledSet.new,
+            Sidekiq::Queue.new(self.queue)
+          ]
+
+          if retry_dead_jobs
+            queues << Sidekiq::RetrySet.new
+            queues << Sidekiq::DeadSet.new
+          end
+
+          queues.each do |queue|
+            queue.each do |job|
+              migration_class, migration_args = job.args
+
+              next unless job.klass == worker_class.name
+              next unless migration_class == steal_class
+              next if block_given? && !(yield job)
+
+              begin
+                perform(migration_class, migration_args) if job.delete
+              rescue Exception # rubocop:disable Lint/RescueException
+                worker_class # enqueue this migration again
+                  .perform_async(migration_class, migration_args)
+
+                raise
+              end
+            end
+          end
+        end
+      end
+
+      def perform(class_name, arguments)
+        with_shared_connection do
+          migration_class_for(class_name).new.perform(*arguments)
+        end
+      end
+
+      def remaining
+        enqueued = Sidekiq::Queue.new(self.queue)
+        scheduled = Sidekiq::ScheduledSet.new
+
+        [enqueued, scheduled].sum do |set|
+          set.count do |job|
+            job.klass == worker_class.name
+          end
+        end
+      end
+
+      def exists?(migration_class, additional_queues = [])
+        enqueued = Sidekiq::Queue.new(self.queue)
+        scheduled = Sidekiq::ScheduledSet.new
+
+        enqueued_job?([enqueued, scheduled], migration_class)
+      end
+
+      def dead_jobs?(migration_class)
+        dead_set = Sidekiq::DeadSet.new
+
+        enqueued_job?([dead_set], migration_class)
+      end
+
+      def retrying_jobs?(migration_class)
+        retry_set = Sidekiq::RetrySet.new
+
+        enqueued_job?([retry_set], migration_class)
+      end
+
+      def migration_class_for(class_name)
+        Gitlab::BackgroundMigration.const_get(class_name, false)
+      end
+
+      def enqueued_job?(queues, migration_class)
+        queues.any? do |queue|
+          queue.any? do |job|
+            job.klass == worker_class.name && job.args.first == migration_class
+          end
+        end
+      end
+
+      private
+
+      def initialize(database, worker_class)
+        @database = database
+        @worker_class = worker_class
+      end
+
+      def connection
+        @connection ||= Gitlab::Database
+          .database_base_models
+          .fetch(database, Gitlab::Database::PRIMARY_DATABASE_NAME)
+          .connection
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb
index 1c60473750d..36a339c6b80 100644
--- a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb
+++ b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb
@@ -34,7 +34,7 @@ module Gitlab
             end
           end
 
-          Gitlab::Database.main.bulk_insert(TEMP_TABLE, fingerprints) # rubocop:disable Gitlab/BulkInsert
+          ApplicationRecord.legacy_bulk_insert(TEMP_TABLE, fingerprints) # rubocop:disable Gitlab/BulkInsert
 
           execute("ANALYZE #{TEMP_TABLE}")
 
diff --git a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb
deleted file mode 100644
index 14c72bb4a72..00000000000
--- a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb
+++ /dev/null
@@ -1,146 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module BackgroundMigration
-    # This migration takes all issue trackers
-    # and move data from properties to data field tables (jira_tracker_data and issue_tracker_data)
-    class MigrateIssueTrackersSensitiveData
-      delegate :select_all, :execute, :quote_string, to: :connection
-
-      # we need to define this class and set fields encryption
-      class IssueTrackerData < ApplicationRecord
-        self.table_name = 'issue_tracker_data'
-
-        def self.encryption_options
-          {
-            key: Settings.attr_encrypted_db_key_base_32,
-            encode: true,
-            mode: :per_attribute_iv,
-            algorithm: 'aes-256-gcm'
-          }
-        end
-
-        attr_encrypted :project_url, encryption_options
-        attr_encrypted :issues_url, encryption_options
-        attr_encrypted :new_issue_url, encryption_options
-      end
-
-      # we need to define this class and set fields encryption
-      class JiraTrackerData < ApplicationRecord
-        self.table_name = 'jira_tracker_data'
-
-        def self.encryption_options
-          {
-            key: Settings.attr_encrypted_db_key_base_32,
-            encode: true,
-            mode: :per_attribute_iv,
-            algorithm: 'aes-256-gcm'
-          }
-        end
-
-        attr_encrypted :url, encryption_options
-        attr_encrypted :api_url, encryption_options
-        attr_encrypted :username, encryption_options
-        attr_encrypted :password, encryption_options
-      end
-
-      def perform(start_id, stop_id)
-        columns = 'id, properties, title, description, type'
-        batch_condition = "id >= #{start_id} AND id <= #{stop_id} AND category = 'issue_tracker' \
-          AND properties IS NOT NULL AND properties != '{}' AND properties != ''"
-
-        data_subselect = "SELECT 1 \
-          FROM jira_tracker_data \
-          WHERE jira_tracker_data.service_id = services.id \
-          UNION SELECT 1 \
-          FROM issue_tracker_data \
-          WHERE issue_tracker_data.service_id = services.id"
-
-        query = "SELECT #{columns} FROM services WHERE #{batch_condition} AND NOT EXISTS (#{data_subselect})"
-
-        migrated_ids = []
-        data_to_insert(query).each do |table, data|
-          service_ids = data.map { |s| s['service_id'] }
-
-          next if service_ids.empty?
-
-          migrated_ids += service_ids
-          Gitlab::Database.main.bulk_insert(table, data) # rubocop:disable Gitlab/BulkInsert
-        end
-
-        return if migrated_ids.empty?
-
-        move_title_description(migrated_ids)
-      end
-
-      private
-
-      def data_to_insert(query)
-        data = { 'jira_tracker_data' => [], 'issue_tracker_data' => [] }
-        select_all(query).each do |service|
-          begin
-            properties = Gitlab::Json.parse(service['properties'])
-          rescue JSON::ParserError
-            logger.warn(
-              message: 'Properties data not parsed - invalid json',
-              service_id: service['id'],
-              properties: service['properties']
-            )
-            next
-          end
-
-          if service['type'] == 'JiraService'
-            row = data_row(JiraTrackerData, jira_mapping(properties), service)
-            key = 'jira_tracker_data'
-          else
-            row = data_row(IssueTrackerData, issue_tracker_mapping(properties), service)
-            key = 'issue_tracker_data'
-          end
-
-          data[key] << row if row
-        end
-
-        data
-      end
-
-      def data_row(klass, mapping, service)
-        base_params = { service_id: service['id'], created_at: Time.current, updated_at: Time.current }
-        klass.new(mapping).slice(*klass.column_names).compact.merge(base_params)
-      end
-
-      def move_title_description(service_ids)
-        query = "UPDATE services SET \
-          title = cast(properties as json)->>'title', \
-          description = cast(properties as json)->>'description' \
-          WHERE id IN (#{service_ids.join(',')}) AND title IS NULL AND description IS NULL"
-
-        execute(query)
-      end
-
-      def jira_mapping(properties)
-        {
-          url: properties['url'],
-          api_url: properties['api_url'],
-          username: properties['username'],
-          password: properties['password']
-        }
-      end
-
-      def issue_tracker_mapping(properties)
-        {
-          project_url: properties['project_url'],
-          issues_url: properties['issues_url'],
-          new_issue_url: properties['new_issue_url']
-        }
-      end
-
-      def connection
-        @connection ||= ActiveRecord::Base.connection
-      end
-
-      def logger
-        @logger ||= Gitlab::BackgroundMigration::Logger.build
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/background_migration/migrate_requirements_to_work_items.rb b/lib/gitlab/background_migration/migrate_requirements_to_work_items.rb
new file mode 100644
index 00000000000..017791f197c
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_requirements_to_work_items.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # No op on CE
+    class MigrateRequirementsToWorkItems
+      def perform(start_id, end_id)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::MigrateRequirementsToWorkItems.prepend_mod_with('Gitlab::BackgroundMigration::MigrateRequirementsToWorkItems')
diff --git a/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb b/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb
deleted file mode 100644
index bbe2164ae4e..00000000000
--- a/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb
+++ /dev/null
@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-# rubocop:disable Style/Documentation
-
-module Gitlab
-  module BackgroundMigration
-    class MigrateUsersBioToUserDetails
-      class User < ActiveRecord::Base
-        self.table_name = 'users'
-      end
-
-      class UserDetails < ActiveRecord::Base
-        self.table_name = 'user_details'
-      end
-
-      def perform(start_id, stop_id)
-        relation = User
-          .select("id AS user_id", "substring(COALESCE(bio, '') from 1 for 255) AS bio")
-          .where("(COALESCE(bio, '') IS DISTINCT FROM '')")
-          .where(id: (start_id..stop_id))
-
-        ActiveRecord::Base.connection.execute <<-EOF.strip_heredoc
-          INSERT INTO user_details
-          (user_id, bio)
-          #{relation.to_sql}
-          ON CONFLICT (user_id)
-          DO UPDATE SET
-            "bio" = EXCLUDED."bio";
-        EOF
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/background_migration/populate_issue_email_participants.rb b/lib/gitlab/background_migration/populate_issue_email_participants.rb
index 0a56ac1dae8..2b959b81f45 100644
--- a/lib/gitlab/background_migration/populate_issue_email_participants.rb
+++ b/lib/gitlab/background_migration/populate_issue_email_participants.rb
@@ -21,7 +21,7 @@ module Gitlab
           }
         end
 
-        Gitlab::Database.main.bulk_insert(:issue_email_participants, rows, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert
+        ApplicationRecord.legacy_bulk_insert(:issue_email_participants, rows, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert
       end
     end
   end
diff --git a/lib/gitlab/background_migration/populate_user_highest_roles_table.rb b/lib/gitlab/background_migration/populate_user_highest_roles_table.rb
deleted file mode 100644
index 16386ebf9c3..00000000000
--- a/lib/gitlab/background_migration/populate_user_highest_roles_table.rb
+++ /dev/null
@@ -1,58 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module BackgroundMigration
-    # This background migration creates records on user_highest_roles according to
-    # the given user IDs range. IDs will load users with a left outer joins to
-    # have a record for users without a Group or Project. One INSERT per ID is
-    # issued.
-    class PopulateUserHighestRolesTable
-      BATCH_SIZE = 100
-
-      # rubocop:disable Style/Documentation
-      class User < ActiveRecord::Base
-        self.table_name = 'users'
-
-        scope :active, -> {
-          where(state: 'active', user_type: nil, bot_type: nil)
-            .where('ghost IS NOT TRUE')
-        }
-      end
-
-      def perform(from_id, to_id)
-        return unless User.column_names.include?('bot_type')
-
-        (from_id..to_id).each_slice(BATCH_SIZE) do |ids|
-          execute(
-            <<-EOF
-              INSERT INTO user_highest_roles (updated_at, user_id, highest_access_level)
-              #{select_sql(from_id, to_id)}
-              ON CONFLICT (user_id) DO
-              UPDATE SET highest_access_level = EXCLUDED.highest_access_level
-            EOF
-          )
-        end
-      end
-
-      private
-
-      def select_sql(from_id, to_id)
-        User
-          .select('NOW() as updated_at, users.id, MAX(access_level) AS highest_access_level')
-          .joins('LEFT OUTER JOIN members ON members.user_id = users.id AND members.requested_at IS NULL')
-          .where(users: { id: active_user_ids(from_id, to_id) })
-          .group('users.id')
-          .to_sql
-      end
-
-      def active_user_ids(from_id, to_id)
-        User.active.where(users: { id: from_id..to_id }).pluck(:id)
-      end
-
-      def execute(sql)
-        @connection ||= ActiveRecord::Base.connection
-        @connection.execute(sql)
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb
new file mode 100644
index 00000000000..8e94c16369e
--- /dev/null
+++ b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb
@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module ProjectNamespaces
+      # Back-fill project namespaces for projects that do not yet have a namespace.
+      #
+      # TODO: remove this comment when an actuall backfill migration is added.
+      #
+      # This is first being added without an actual migration as we need to initially test
+      # if backfilling project namespaces affects performance in any significant way.
+      # rubocop: disable Metrics/ClassLength
+      class BackfillProjectNamespaces
+        BATCH_SIZE = 100
+        DELETE_BATCH_SIZE = 10
+        PROJECT_NAMESPACE_STI_NAME = 'Project'
+
+        IsolatedModels = ::Gitlab::BackgroundMigration::ProjectNamespaces::Models
+
+        def perform(start_id, end_id, namespace_id, migration_type = 'up')
+          load_project_ids(start_id, end_id, namespace_id)
+
+          case migration_type
+          when 'up'
+            backfill_project_namespaces(namespace_id)
+            mark_job_as_succeeded(start_id, end_id, namespace_id, 'up')
+          when 'down'
+            cleanup_backfilled_project_namespaces(namespace_id)
+            mark_job_as_succeeded(start_id, end_id, namespace_id, 'down')
+          else
+            raise "Unknown migration type"
+          end
+        end
+
+        private
+
+        attr_accessor :project_ids
+
+        def backfill_project_namespaces(namespace_id)
+          project_ids.each_slice(BATCH_SIZE) do |project_ids|
+            # We need to lock these project records for the period when we create project namespaces
+            # and link them to projects so that if a project is modified in the time between creating
+            # project namespaces `batch_insert_namespaces` and linking them to projects `batch_update_projects`
+            # we do not get them out of sync.
+            #
+            # see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/72527#note_730679469
+            Project.transaction do
+              Project.where(id: project_ids).select(:id).lock!('FOR UPDATE')
+
+              batch_insert_namespaces(project_ids)
+              batch_update_projects(project_ids)
+            end
+
+            batch_update_project_namespaces_traversal_ids(project_ids)
+          end
+        end
+
+        def cleanup_backfilled_project_namespaces(namespace_id)
+          project_ids.each_slice(BATCH_SIZE) do |project_ids|
+            # IMPORTANT: first nullify project_namespace_id in projects table to avoid removing projects when records
+            # from namespaces are deleted due to FK/triggers
+            nullify_project_namespaces_in_projects(project_ids)
+            delete_project_namespace_records(project_ids)
+          end
+        end
+
+        def batch_insert_namespaces(project_ids)
+          projects = IsolatedModels::Project.where(id: project_ids)
+            .select("projects.id, projects.name, projects.path, projects.namespace_id, projects.visibility_level, shared_runners_enabled, '#{PROJECT_NAMESPACE_STI_NAME}', now(), now()")
+
+          ActiveRecord::Base.connection.execute <<~SQL
+            INSERT INTO namespaces (tmp_project_id, name, path, parent_id, visibility_level, shared_runners_enabled, type, created_at, updated_at)
+            #{projects.to_sql}
+            ON CONFLICT DO NOTHING;
+          SQL
+        end
+
+        def batch_update_projects(project_ids)
+          projects = IsolatedModels::Project.where(id: project_ids)
+                       .joins("INNER JOIN namespaces ON projects.id = namespaces.tmp_project_id")
+                       .select("namespaces.id, namespaces.tmp_project_id")
+
+          ActiveRecord::Base.connection.execute <<~SQL
+            WITH cte(project_namespace_id, project_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
+              #{projects.to_sql}
+            )
+            UPDATE projects
+            SET project_namespace_id = cte.project_namespace_id
+            FROM cte
+            WHERE id = cte.project_id AND projects.project_namespace_id IS DISTINCT FROM cte.project_namespace_id
+          SQL
+        end
+
+        def batch_update_project_namespaces_traversal_ids(project_ids)
+          namespaces = Namespace.where(tmp_project_id: project_ids)
+                         .joins("INNER JOIN namespaces n2 ON namespaces.parent_id = n2.id")
+                         .select("namespaces.id as project_namespace_id, n2.traversal_ids")
+
+          ActiveRecord::Base.connection.execute <<~SQL
+            UPDATE namespaces
+            SET traversal_ids = array_append(project_namespaces.traversal_ids, project_namespaces.project_namespace_id)
+            FROM (#{namespaces.to_sql}) as project_namespaces(project_namespace_id, traversal_ids)
+            WHERE id = project_namespaces.project_namespace_id
+          SQL
+        end
+
+        def nullify_project_namespaces_in_projects(project_ids)
+          IsolatedModels::Project.where(id: project_ids).update_all(project_namespace_id: nil)
+        end
+
+        def delete_project_namespace_records(project_ids)
+          project_ids.each_slice(DELETE_BATCH_SIZE) do |p_ids|
+            IsolatedModels::Namespace.where(type: PROJECT_NAMESPACE_STI_NAME).where(tmp_project_id: p_ids).delete_all
+          end
+        end
+
+        def load_project_ids(start_id, end_id, namespace_id)
+          projects = IsolatedModels::Project.arel_table
+          relation = IsolatedModels::Project.where(projects[:id].between(start_id..end_id))
+          relation = relation.where(projects[:namespace_id].in(Arel::Nodes::SqlLiteral.new(hierarchy_cte(namespace_id)))) if namespace_id
+
+          @project_ids = relation.pluck(:id)
+        end
+
+        def mark_job_as_succeeded(*arguments)
+          ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('BackfillProjectNamespaces', arguments)
+        end
+
+        def hierarchy_cte(root_namespace_id)
+          <<-SQL
+              WITH RECURSIVE "base_and_descendants" AS (
+                  (
+                      SELECT "namespaces"."id"
+                      FROM "namespaces"
+                      WHERE "namespaces"."type" = 'Group' AND "namespaces"."id" = #{root_namespace_id.to_i}
+                  )
+                  UNION
+                  (
+                      SELECT "namespaces"."id"
+                      FROM "namespaces", "base_and_descendants"
+                      WHERE "namespaces"."type" = 'Group' AND "namespaces"."parent_id" = "base_and_descendants"."id"
+                  )
+              )
+              SELECT "id" FROM "base_and_descendants" AS "namespaces"
+          SQL
+        end
+      end
+      # rubocop: enable Metrics/ClassLength
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/project_namespaces/models/namespace.rb b/lib/gitlab/background_migration/project_namespaces/models/namespace.rb
new file mode 100644
index 00000000000..5576c34cf65
--- /dev/null
+++ b/lib/gitlab/background_migration/project_namespaces/models/namespace.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module ProjectNamespaces
+      module Models
+        # isolated Namespace model
+        class Namespace < ActiveRecord::Base
+          include EachBatch
+
+          self.table_name = 'namespaces'
+          self.inheritance_column = :_type_disabled
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/project_namespaces/models/project.rb b/lib/gitlab/background_migration/project_namespaces/models/project.rb
new file mode 100644
index 00000000000..4a6a309e289
--- /dev/null
+++ b/lib/gitlab/background_migration/project_namespaces/models/project.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    module ProjectNamespaces
+      module Models
+        # isolated Project model
+        class Project < ActiveRecord::Base
+          include EachBatch
+
+          self.table_name = 'projects'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb b/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb
index ca61118a06c..15799659b55 100644
--- a/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb
+++ b/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb
@@ -2,7 +2,7 @@
 
 # rubocop: disable Style/Documentation
 class Gitlab::BackgroundMigration::RemoveDuplicateVulnerabilitiesFindings
-  DELETE_BATCH_SIZE = 100
+  DELETE_BATCH_SIZE = 50
 
   # rubocop:disable Gitlab/NamespacedClass
   class VulnerabilitiesFinding < ActiveRecord::Base
@@ -10,6 +10,12 @@ class Gitlab::BackgroundMigration::RemoveDuplicateVulnerabilitiesFindings
   end
   # rubocop:enable Gitlab/NamespacedClass
 
+  # rubocop:disable Gitlab/NamespacedClass
+  class Vulnerability < ActiveRecord::Base
+    self.table_name = "vulnerabilities"
+  end
+  # rubocop:enable Gitlab/NamespacedClass
+
   def perform(start_id, end_id)
     batch = VulnerabilitiesFinding.where(id: start_id..end_id)
 
@@ -40,11 +46,19 @@ class Gitlab::BackgroundMigration::RemoveDuplicateVulnerabilitiesFindings
       ids_to_delete.concat(duplicate_ids)
 
       if ids_to_delete.size == DELETE_BATCH_SIZE
-        VulnerabilitiesFinding.where(id: ids_to_delete).delete_all
+        delete_findings_and_vulnerabilities(ids_to_delete)
         ids_to_delete.clear
       end
     end
 
-    VulnerabilitiesFinding.where(id: ids_to_delete).delete_all if ids_to_delete.any?
+    delete_findings_and_vulnerabilities(ids_to_delete) if ids_to_delete.any?
+  end
+
+  private
+
+  def delete_findings_and_vulnerabilities(ids)
+    vulnerability_ids = VulnerabilitiesFinding.where(id: ids).pluck(:vulnerability_id).compact
+    VulnerabilitiesFinding.where(id: ids).delete_all
+    Vulnerability.where(id: vulnerability_ids).delete_all
   end
 end
diff --git a/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb b/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb
deleted file mode 100644
index 540ffc6f548..00000000000
--- a/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-# rubocop:disable Style/Documentation
-
-module Gitlab
-  module BackgroundMigration
-    class RemoveUndefinedOccurrenceConfidenceLevel
-      def perform(start_id, stop_id)
-      end
-    end
-  end
-end
-
-Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel.prepend_mod_with('Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel')
diff --git a/lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb b/lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb
deleted file mode 100644
index cecb385afa0..00000000000
--- a/lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-# rubocop:disable Style/Documentation
-
-module Gitlab
-  module BackgroundMigration
-    class RemoveUndefinedOccurrenceSeverityLevel
-      def perform(start_id, stop_id)
-      end
-    end
-  end
-end
-
-Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceSeverityLevel.prepend_mod_with('Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceSeverityLevel')
diff --git a/lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb b/lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb
deleted file mode 100644
index 1ea483f929f..00000000000
--- a/lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-# rubocop:disable Style/Documentation
-
-module Gitlab
-  module BackgroundMigration
-    class RemoveUndefinedVulnerabilitySeverityLevel
-      def perform(start_id, stop_id)
-      end
-    end
-  end
-end
-
-Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel.prepend_mod_with('Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel')
diff --git a/lib/gitlab/background_migration/set_default_iteration_cadences.rb b/lib/gitlab/background_migration/set_default_iteration_cadences.rb
deleted file mode 100644
index 42f9d33ab71..00000000000
--- a/lib/gitlab/background_migration/set_default_iteration_cadences.rb
+++ /dev/null
@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module BackgroundMigration
-    # rubocop:disable Style/Documentation
-    class SetDefaultIterationCadences
-      class Iteration < ApplicationRecord
-        self.table_name = 'sprints'
-      end
-
-      class IterationCadence < ApplicationRecord
-        self.table_name = 'iterations_cadences'
-
-        include BulkInsertSafe
-      end
-
-      class Group < ApplicationRecord
-        self.table_name = 'namespaces'
-
-        self.inheritance_column = :_type_disabled
-      end
-
-      def perform(*group_ids)
-        create_iterations_cadences(group_ids)
-        assign_iterations_cadences(group_ids)
-      end
-
-      private
-
-      def create_iterations_cadences(group_ids)
-        groups_with_cadence = IterationCadence.select(:group_id)
-
-        new_cadences = Group.where(id: group_ids).where.not(id: groups_with_cadence).map do |group|
-          last_iteration = Iteration.where(group_id: group.id).order(:start_date)&.last
-
-          next unless last_iteration
-
-          time = Time.now
-          IterationCadence.new(
-            group_id: group.id,
-            title: "#{group.name} Iterations",
-            start_date: last_iteration.start_date,
-            last_run_date: last_iteration.start_date,
-            automatic: false,
-            created_at: time,
-            updated_at: time
-          )
-        end
-
-        IterationCadence.bulk_insert!(new_cadences.compact, skip_duplicates: true)
-      end
-
-      def assign_iterations_cadences(group_ids)
-        IterationCadence.where(group_id: group_ids).each do |cadence|
-          Iteration.where(iterations_cadence_id: nil).where(group_id: cadence.group_id).update_all(iterations_cadence_id: cadence.id)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/background_migration/set_null_external_diff_store_to_local_value.rb b/lib/gitlab/background_migration/set_null_external_diff_store_to_local_value.rb
deleted file mode 100644
index 71f3483987e..00000000000
--- a/lib/gitlab/background_migration/set_null_external_diff_store_to_local_value.rb
+++ /dev/null
@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module BackgroundMigration
-    # This class is responsible for migrating a range of merge request diffs
-    # with external_diff_store == NULL to 1.
-    #
-    # The index `index_merge_request_diffs_external_diff_store_is_null` is
-    # expected to be used to find the rows here and in the migration scheduling
-    # the jobs that run this class.
-    class SetNullExternalDiffStoreToLocalValue
-      LOCAL_STORE = 1 # equal to ObjectStorage::Store::LOCAL
-
-      # Temporary AR class for merge request diffs
-      class MergeRequestDiff < ActiveRecord::Base
-        self.table_name = 'merge_request_diffs'
-      end
-
-      def perform(start_id, stop_id)
-        MergeRequestDiff.where(external_diff_store: nil, id: start_id..stop_id).update_all(external_diff_store: LOCAL_STORE)
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb b/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb
deleted file mode 100644
index c485c23f3be..00000000000
--- a/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb
+++ /dev/null
@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module BackgroundMigration
-    # This class is responsible for migrating a range of package files
-    # with file_store == NULL to 1.
-    #
-    # The index `index_packages_package_files_file_store_is_null` is
-    # expected to be used to find the rows here and in the migration scheduling
-    # the jobs that run this class.
-    class SetNullPackageFilesFileStoreToLocalValue
-      LOCAL_STORE = 1 # equal to ObjectStorage::Store::LOCAL
-
-      module Packages
-        # Temporary AR class for package files
-        class PackageFile < ActiveRecord::Base
-          self.table_name = 'packages_package_files'
-        end
-      end
-
-      def perform(start_id, stop_id)
-        Packages::PackageFile.where(file_store: nil, id: start_id..stop_id).update_all(file_store: LOCAL_STORE)
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/background_migration/update_vulnerabilities_to_dismissed.rb b/lib/gitlab/background_migration/update_vulnerabilities_to_dismissed.rb
deleted file mode 100644
index 60adb6b7e3e..00000000000
--- a/lib/gitlab/background_migration/update_vulnerabilities_to_dismissed.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module BackgroundMigration
-    # rubocop: disable Style/Documentation
-    class UpdateVulnerabilitiesToDismissed
-      def perform(project_id)
-      end
-    end
-  end
-end
-
-Gitlab::BackgroundMigration::UpdateVulnerabilitiesToDismissed.prepend_mod_with('Gitlab::BackgroundMigration::UpdateVulnerabilitiesToDismissed')
diff --git a/lib/gitlab/background_migration/update_vulnerability_confidence.rb b/lib/gitlab/background_migration/update_vulnerability_confidence.rb
deleted file mode 100644
index 40d29978dd4..00000000000
--- a/lib/gitlab/background_migration/update_vulnerability_confidence.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module BackgroundMigration
-    # rubocop: disable Style/Documentation
-    class UpdateVulnerabilityConfidence
-      def perform(start_id, stop_id)
-      end
-    end
-  end
-end
-
-Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence.prepend_mod_with('Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence')
diff --git a/lib/gitlab/background_migration/update_vulnerability_occurrences_location.rb b/lib/gitlab/background_migration/update_vulnerability_occurrences_location.rb
new file mode 100644
index 00000000000..458e0537f1c
--- /dev/null
+++ b/lib/gitlab/background_migration/update_vulnerability_occurrences_location.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # rubocop: disable Style/Documentation
+    class UpdateVulnerabilityOccurrencesLocation
+      def perform(start_id, stop_id)
+      end
+    end
+    # rubocop: enable Style/Documentation
+  end
+end
+
+Gitlab::BackgroundMigration::UpdateVulnerabilityOccurrencesLocation.prepend_mod_with('Gitlab::BackgroundMigration::UpdateVulnerabilityOccurrencesLocation')
diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb
index e0eee64dc58..899e2e6c1c5 100644
--- a/lib/gitlab/bitbucket_server_import/importer.rb
+++ b/lib/gitlab/bitbucket_server_import/importer.rb
@@ -461,10 +461,14 @@ module Gitlab
       end
 
       def uid(rep_object)
-        find_user_id(by: :email, value: rep_object.author_email) unless Feature.enabled?(:bitbucket_server_user_mapping_by_username)
-
-        find_user_id(by: :username, value: rep_object.author_username) ||
+        # We want this explicit to only be username on the FF
+        # Otherwise, match email.
+        # There should be no default fall-through on username. Fall-through to import user
+        if Feature.enabled?(:bitbucket_server_user_mapping_by_username)
+          find_user_id(by: :username, value: rep_object.author_username)
+        else
           find_user_id(by: :email, value: rep_object.author_email)
+        end
       end
     end
   end
diff --git a/lib/gitlab/blob_helper.rb b/lib/gitlab/blob_helper.rb
index c5b183d113d..9e4ea934edb 100644
--- a/lib/gitlab/blob_helper.rb
+++ b/lib/gitlab/blob_helper.rb
@@ -47,7 +47,7 @@ module Gitlab
     end
 
     def image?
-      ['.png', '.jpg', '.jpeg', '.gif', '.svg'].include?(extname.downcase)
+      ['.png', '.jpg', '.jpeg', '.gif', '.svg', '.webp'].include?(extname.downcase)
     end
 
     # Internal: Lookup mime type for extension.
diff --git a/lib/gitlab/ci/artifact_file_reader.rb b/lib/gitlab/ci/artifact_file_reader.rb
index 3cfed8e5e2c..b0fad026ec5 100644
--- a/lib/gitlab/ci/artifact_file_reader.rb
+++ b/lib/gitlab/ci/artifact_file_reader.rb
@@ -45,14 +45,6 @@ module Gitlab
       end
 
       def read_zip_file!(file_path)
-        if ::Feature.enabled?(:ci_new_artifact_file_reader, job.project, default_enabled: :yaml)
-          read_with_new_artifact_file_reader(file_path)
-        else
-          read_with_legacy_artifact_file_reader(file_path)
-        end
-      end
-
-      def read_with_new_artifact_file_reader(file_path)
         job.artifacts_file.use_open_file do |file|
           zip_file = Zip::File.new(file, false, true)
           entry = zip_file.find_entry(file_path)
@@ -69,25 +61,6 @@ module Gitlab
         end
       end
 
-      def read_with_legacy_artifact_file_reader(file_path)
-        job.artifacts_file.use_file do |archive_path|
-          Zip::File.open(archive_path) do |zip_file|
-            entry = zip_file.find_entry(file_path)
-            unless entry
-              raise Error, "Path `#{file_path}` does not exist inside the `#{job.name}` artifacts archive!"
-            end
-
-            if entry.name_is_directory?
-              raise Error, "Path `#{file_path}` was expected to be a file but it was a directory!"
-            end
-
-            zip_file.get_input_stream(entry) do |is|
-              is.read
-            end
-          end
-        end
-      end
-
       def max_archive_size_in_mb
         ActiveSupport::NumberHelper.number_to_human_size(MAX_ARCHIVE_SIZE)
       end
diff --git a/lib/gitlab/ci/artifacts/metrics.rb b/lib/gitlab/ci/artifacts/metrics.rb
index 656f4d2cc13..03459c4bf36 100644
--- a/lib/gitlab/ci/artifacts/metrics.rb
+++ b/lib/gitlab/ci/artifacts/metrics.rb
@@ -6,10 +6,14 @@ module Gitlab
       class Metrics
         include Gitlab::Utils::StrongMemoize
 
-        def increment_destroyed_artifacts(size)
+        def increment_destroyed_artifacts_count(size)
           destroyed_artifacts_counter.increment({}, size.to_i)
         end
 
+        def increment_destroyed_artifacts_bytes(bytes)
+          destroyed_artifacts_bytes_counter.increment({}, bytes)
+        end
+
         private
 
         def destroyed_artifacts_counter
@@ -20,6 +24,15 @@ module Gitlab
             ::Gitlab::Metrics.counter(name, comment)
           end
         end
+
+        def destroyed_artifacts_bytes_counter
+          strong_memoize(:destroyed_artifacts_bytes_counter) do
+            name = :destroyed_job_artifacts_bytes_total
+            comment = 'Counter of bytes of destroyed expired job artifacts'
+
+            ::Gitlab::Metrics.counter(name, comment)
+          end
+        end
       end
     end
   end
diff --git a/lib/gitlab/ci/build/auto_retry.rb b/lib/gitlab/ci/build/auto_retry.rb
index 6ab567dff7c..4950a7616c8 100644
--- a/lib/gitlab/ci/build/auto_retry.rb
+++ b/lib/gitlab/ci/build/auto_retry.rb
@@ -10,7 +10,9 @@ class Gitlab::Ci::Build::AutoRetry
   RETRY_OVERRIDES = {
     ci_quota_exceeded: 0,
     no_matching_runner: 0,
-    missing_dependency_failure: 0
+    missing_dependency_failure: 0,
+    forward_deployment_failure: 0,
+    environment_creation_failure: 0
   }.freeze
 
   def initialize(build)
diff --git a/lib/gitlab/ci/build/context/base.rb b/lib/gitlab/ci/build/context/base.rb
index 02b97ea76e9..c7ea7c78e2f 100644
--- a/lib/gitlab/ci/build/context/base.rb
+++ b/lib/gitlab/ci/build/context/base.rb
@@ -5,6 +5,8 @@ module Gitlab
     module Build
       module Context
         class Base
+          include Gitlab::Utils::StrongMemoize
+
           attr_reader :pipeline
 
           def initialize(pipeline)
@@ -15,6 +17,26 @@ module Gitlab
             raise NotImplementedError
           end
 
+          def project
+            pipeline.project
+          end
+
+          def sha
+            pipeline.sha
+          end
+
+          def top_level_worktree_paths
+            strong_memoize(:top_level_worktree_paths) do
+              project.repository.tree(sha).blobs.map(&:path)
+            end
+          end
+
+          def all_worktree_paths
+            strong_memoize(:all_worktree_paths) do
+              project.repository.ls_files(sha)
+            end
+          end
+
           protected
 
           def pipeline_attributes
diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb
index 1d7bfba75cd..8ddcf1d523e 100644
--- a/lib/gitlab/ci/build/image.rb
+++ b/lib/gitlab/ci/build/image.rb
@@ -4,7 +4,7 @@ module Gitlab
   module Ci
     module Build
       class Image
-        attr_reader :alias, :command, :entrypoint, :name, :ports
+        attr_reader :alias, :command, :entrypoint, :name, :ports, :variables
 
         class << self
           def from_image(job)
@@ -33,6 +33,7 @@ module Gitlab
             @entrypoint = image[:entrypoint]
             @name = image[:name]
             @ports = build_ports(image).select(&:valid?)
+            @variables = build_variables(image)
           end
         end
 
@@ -45,6 +46,12 @@ module Gitlab
         def build_ports(image)
           image[:ports].to_a.map { |port| ::Gitlab::Ci::Build::Port.new(port) }
         end
+
+        def build_variables(image)
+          image[:variables].to_a.map do |key, value|
+            { key: key, value: value.to_s }
+          end
+        end
       end
     end
   end
diff --git a/lib/gitlab/ci/build/rules/rule/clause/exists.rb b/lib/gitlab/ci/build/rules/rule/clause/exists.rb
index 85e77438f51..e2b54797dc8 100644
--- a/lib/gitlab/ci/build/rules/rule/clause/exists.rb
+++ b/lib/gitlab/ci/build/rules/rule/clause/exists.rb
@@ -15,19 +15,21 @@ module Gitlab
           @exact_globs, @pattern_globs = globs.partition(&method(:exact_glob?))
         end
 
-        def satisfied_by?(pipeline, context)
-          paths = worktree_paths(pipeline)
+        def satisfied_by?(_pipeline, context)
+          paths = worktree_paths(context)
 
           exact_matches?(paths) || pattern_matches?(paths)
         end
 
         private
 
-        def worktree_paths(pipeline)
+        def worktree_paths(context)
+          return unless context.project
+
           if @top_level_only
-            pipeline.top_level_worktree_paths
+            context.top_level_worktree_paths
           else
-            pipeline.all_worktree_paths
+            context.all_worktree_paths
           end
         end
 
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index aceaf012f7e..6f149385969 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -19,11 +19,12 @@ module Gitlab
 
       attr_reader :root, :context, :source_ref_path, :source
 
-      def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, source_ref_path: nil, source: nil)
-        @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline, ref: source_ref_path)
+      def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil)
+        @source_ref_path = pipeline&.source_ref_path
+
+        @context = build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline)
         @context.set_deadline(TIMEOUT_SECONDS)
 
-        @source_ref_path = source_ref_path
         @source = source
 
         @config = expand_config(config)
@@ -108,16 +109,16 @@ module Gitlab
         end
       end
 
-      def build_context(project:, sha:, user:, parent_pipeline:, ref:)
+      def build_context(project:, pipeline:, sha:, user:, parent_pipeline:)
         Config::External::Context.new(
           project: project,
           sha: sha || find_sha(project),
           user: user,
           parent_pipeline: parent_pipeline,
-          variables: build_variables(project: project, ref: ref))
+          variables: build_variables(project: project, pipeline: pipeline))
       end
 
-      def build_variables(project:, ref:)
+      def build_variables(project:, pipeline:)
         Gitlab::Ci::Variables::Collection.new.tap do |variables|
           break variables unless project
 
@@ -126,18 +127,12 @@ module Gitlab
           #
           # See more detail in the docs: https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
           variables.concat(project.predefined_variables)
-          variables.concat(pipeline_predefined_variables(ref: ref))
-          variables.concat(project.ci_instance_variables_for(ref: ref))
-          variables.concat(project.group.ci_variables_for(ref, project)) if project.group
-          variables.concat(project.ci_variables_for(ref: ref))
-        end
-      end
-
-      # https://gitlab.com/gitlab-org/gitlab/-/issues/337633 aims to add all predefined variables
-      # to this list, but only CI_COMMIT_REF_NAME is available right now to support compliance pipelines.
-      def pipeline_predefined_variables(ref:)
-        Gitlab::Ci::Variables::Collection.new.tap do |v|
-          v.append(key: 'CI_COMMIT_REF_NAME', value: ref)
+          variables.concat(pipeline.predefined_variables) if pipeline
+          variables.concat(project.ci_instance_variables_for(ref: source_ref_path))
+          variables.concat(project.group.ci_variables_for(source_ref_path, project)) if project.group
+          variables.concat(project.ci_variables_for(ref: source_ref_path))
+          variables.concat(pipeline.variables) if pipeline
+          variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline&.pipeline_schedule
         end
       end
 
diff --git a/lib/gitlab/ci/config/entry/include/rules/rule.rb b/lib/gitlab/ci/config/entry/include/rules/rule.rb
index d3d0f098814..fa99a7204d6 100644
--- a/lib/gitlab/ci/config/entry/include/rules/rule.rb
+++ b/lib/gitlab/ci/config/entry/include/rules/rule.rb
@@ -9,9 +9,9 @@ module Gitlab
             include ::Gitlab::Config::Entry::Validatable
             include ::Gitlab::Config::Entry::Attributable
 
-            ALLOWED_KEYS = %i[if].freeze
+            ALLOWED_KEYS = %i[if exists].freeze
 
-            attributes :if
+            attributes :if, :exists
 
             validations do
               validates :config, presence: true
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index f867189d521..75bbe2ccb1b 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -14,10 +14,10 @@ module Gitlab
           ALLOWED_KEYS = %i[tags script type image services start_in artifacts
                             cache dependencies before_script after_script
                             environment coverage retry parallel interruptible timeout
-                            release dast_configuration secrets].freeze
+                            release].freeze
 
           validations do
-            validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS
+            validates :config, allowed_keys: Gitlab::Ci::Config::Entry::Job.allowed_keys + PROCESSABLE_ALLOWED_KEYS
             validates :script, presence: true
 
             with_options allow_nil: true do
@@ -178,6 +178,10 @@ module Gitlab
             allow_failure_defined? ? static_allow_failure : manual_action?
           end
 
+          def self.allowed_keys
+            ALLOWED_KEYS
+          end
+
           private
 
           def allow_failure_criteria
diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb
index 2549c35ebd6..520b1ce6119 100644
--- a/lib/gitlab/ci/config/entry/processable.rb
+++ b/lib/gitlab/ci/config/entry/processable.rb
@@ -23,6 +23,7 @@ module Gitlab
               validates :config, presence: true
               validates :name, presence: true
               validates :name, type: Symbol
+              validates :name, length: { maximum: 255 }, if: -> { ::Feature.enabled?(:ci_validate_job_length, default_enabled: :yaml) }
 
               validates :config, disallowed_keys: {
                   in: %i[only except when start_in],
diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb
index 247bf930d3b..f27dca4986e 100644
--- a/lib/gitlab/ci/config/entry/service.rb
+++ b/lib/gitlab/ci/config/entry/service.rb
@@ -15,7 +15,7 @@ module Gitlab
           include ::Gitlab::Config::Entry::Attributable
           include ::Gitlab::Config::Entry::Configurable
 
-          ALLOWED_KEYS = %i[name entrypoint command alias ports].freeze
+          ALLOWED_KEYS = %i[name entrypoint command alias ports variables].freeze
 
           validations do
             validates :config, hash_or_string: true
@@ -32,6 +32,10 @@ module Gitlab
           entry :ports, Entry::Ports,
             description: 'Ports used to expose the service'
 
+          entry :variables, ::Gitlab::Ci::Config::Entry::Variables,
+            description: 'Environment variables available for this service.',
+            inherit: false
+
           attributes :ports
 
           def alias
diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb
index e0adb1b19c2..51624dc30ea 100644
--- a/lib/gitlab/ci/config/external/context.rb
+++ b/lib/gitlab/ci/config/external/context.rb
@@ -5,6 +5,8 @@ module Gitlab
     class Config
       module External
         class Context
+          include Gitlab::Utils::StrongMemoize
+
           TimeoutError = Class.new(StandardError)
 
           attr_reader :project, :sha, :user, :parent_pipeline, :variables
@@ -22,6 +24,18 @@ module Gitlab
             yield self if block_given?
           end
 
+          def top_level_worktree_paths
+            strong_memoize(:top_level_worktree_paths) do
+              project.repository.tree(sha).blobs.map(&:path)
+            end
+          end
+
+          def all_worktree_paths
+            strong_memoize(:all_worktree_paths) do
+              project.repository.ls_files(sha)
+            end
+          end
+
           def mutate(attrs = {})
             self.class.new(**attrs) do |ctx|
               ctx.expandset = expandset
diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb
index 1cf4f252ab9..0c969daf7fd 100644
--- a/lib/gitlab/ci/parsers/security/common.rb
+++ b/lib/gitlab/ci/parsers/security/common.rb
@@ -33,8 +33,7 @@ module Gitlab
             report_data
           rescue JSON::ParserError
             raise SecurityReportParserError, 'JSON parsing failed'
-          rescue StandardError => e
-            Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
+          rescue StandardError
             raise SecurityReportParserError, "#{report.type} security report parsing failed"
           end
 
@@ -115,7 +114,7 @@ module Gitlab
                 flags: flags,
                 links: links,
                 remediations: remediations,
-                raw_metadata: data.to_json,
+                original_data: data,
                 metadata_version: report_version,
                 details: data['details'] || {},
                 signatures: signatures,
diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
index 143b930c669..73cfa02ce4b 100644
--- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
+++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
@@ -34,7 +34,7 @@ module Gitlab
               end
 
               def file_name
-                "#{report_type}.json"
+                "#{report_type.to_s.dasherize}-report-format.json"
               end
             end
 
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/sast-report-format.json
new file mode 100644
index 00000000000..a7159be0190
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/sast-report-format.json
@@ -0,0 +1,706 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Report format for GitLab SAST",
+  "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).",
+  "definitions": {
+    "detail_type": {
+      "oneOf": [
+        {
+          "$ref": "#/definitions/named_list"
+        },
+        {
+          "$ref": "#/definitions/list"
+        },
+        {
+          "$ref": "#/definitions/table"
+        },
+        {
+          "$ref": "#/definitions/text"
+        },
+        {
+          "$ref": "#/definitions/url"
+        },
+        {
+          "$ref": "#/definitions/code"
+        },
+        {
+          "$ref": "#/definitions/value"
+        },
+        {
+          "$ref": "#/definitions/diff"
+        },
+        {
+          "$ref": "#/definitions/markdown"
+        },
+        {
+          "$ref": "#/definitions/commit"
+        },
+        {
+          "$ref": "#/definitions/file_location"
+        },
+        {
+          "$ref": "#/definitions/module_location"
+        }
+      ]
+    },
+    "text_value": {
+      "type": "string"
+    },
+    "named_field": {
+      "type": "object",
+      "required": [
+        "name"
+      ],
+      "properties": {
+        "name": {
+          "$ref": "#/definitions/text_value",
+          "minLength": 1
+        },
+        "description": {
+          "$ref": "#/definitions/text_value"
+        }
+      }
+    },
+    "named_list": {
+      "type": "object",
+      "description": "An object with named and typed fields",
+      "required": [
+        "type",
+        "items"
+      ],
+      "properties": {
+        "type": {
+          "const": "named-list"
+        },
+        "items": {
+          "type": "object",
+          "patternProperties": {
+            "^.*$": {
+              "allOf": [
+                {
+                  "$ref": "#/definitions/named_field"
+                },
+                {
+                  "$ref": "#/definitions/detail_type"
+                }
+              ]
+            }
+          }
+        }
+      }
+    },
+    "list": {
+      "type": "object",
+      "description": "A list of typed fields",
+      "required": [
+        "type",
+        "items"
+      ],
+      "properties": {
+        "type": {
+          "const": "list"
+        },
+        "items": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/detail_type"
+          }
+        }
+      }
+    },
+    "table": {
+      "type": "object",
+      "description": "A table of typed fields",
+      "required": [
+        "type",
+        "rows"
+      ],
+      "properties": {
+        "type": {
+          "const": "table"
+        },
+        "header": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/detail_type"
+          }
+        },
+        "rows": {
+          "type": "array",
+          "items": {
+            "type": "array",
+            "items": {
+              "$ref": "#/definitions/detail_type"
+            }
+          }
+        }
+      }
+    },
+    "text": {
+      "type": "object",
+      "description": "Raw text",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "text"
+        },
+        "value": {
+          "$ref": "#/definitions/text_value"
+        }
+      }
+    },
+    "url": {
+      "type": "object",
+      "description": "A single URL",
+      "required": [
+        "type",
+        "href"
+      ],
+      "properties": {
+        "type": {
+          "const": "url"
+        },
+        "text": {
+          "$ref": "#/definitions/text_value"
+        },
+        "href": {
+          "type": "string",
+          "minLength": 1,
+          "examples": [
+            "http://mysite.com"
+          ]
+        }
+      }
+    },
+    "code": {
+      "type": "object",
+      "description": "A codeblock",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "code"
+        },
+        "value": {
+          "type": "string"
+        },
+        "lang": {
+          "type": "string",
+          "description": "A programming language"
+        }
+      }
+    },
+    "value": {
+      "type": "object",
+      "description": "A field that can store a range of types of value",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "value"
+        },
+        "value": {
+          "type": [
+            "number",
+            "string",
+            "boolean"
+          ]
+        }
+      }
+    },
+    "diff": {
+      "type": "object",
+      "description": "A diff",
+      "required": [
+        "type",
+        "before",
+        "after"
+      ],
+      "properties": {
+        "type": {
+          "const": "diff"
+        },
+        "before": {
+          "type": "string"
+        },
+        "after": {
+          "type": "string"
+        }
+      }
+    },
+    "markdown": {
+      "type": "object",
+      "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "markdown"
+        },
+        "value": {
+          "$ref": "#/definitions/text_value",
+          "examples": [
+            "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
+          ]
+        }
+      }
+    },
+    "commit": {
+      "type": "object",
+      "description": "A commit/tag/branch within the GitLab project",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "commit"
+        },
+        "value": {
+          "type": "string",
+          "description": "The commit SHA",
+          "minLength": 1
+        }
+      }
+    },
+    "file_location": {
+      "type": "object",
+      "description": "A location within a file in the project",
+      "required": [
+        "type",
+        "file_name",
+        "line_start"
+      ],
+      "properties": {
+        "type": {
+          "const": "file-location"
+        },
+        "file_name": {
+          "type": "string",
+          "minLength": 1
+        },
+        "line_start": {
+          "type": "integer"
+        },
+        "line_end": {
+          "type": "integer"
+        }
+      }
+    },
+    "module_location": {
+      "type": "object",
+      "description": "A location within a binary module of the form module+relative_offset",
+      "required": [
+        "type",
+        "module_name",
+        "offset"
+      ],
+      "properties": {
+        "type": {
+          "const": "module-location"
+        },
+        "module_name": {
+          "type": "string",
+          "minLength": 1,
+          "examples": [
+            "compiled_binary"
+          ]
+        },
+        "offset": {
+          "type": "integer",
+          "examples": [
+            100
+          ]
+        }
+      }
+    }
+  },
+  "self": {
+    "version": "14.0.0"
+  },
+  "required": [
+    "version",
+    "vulnerabilities"
+  ],
+  "additionalProperties": true,
+  "properties": {
+    "scan": {
+      "type": "object",
+      "required": [
+        "end_time",
+        "scanner",
+        "start_time",
+        "status",
+        "type"
+      ],
+      "properties": {
+        "end_time": {
+          "type": "string",
+          "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
+          "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+          "examples": [
+            "2020-01-28T03:26:02"
+          ]
+        },
+        "messages": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "description": "Communication intended for the initiator of a scan.",
+            "required": [
+              "level",
+              "value"
+            ],
+            "properties": {
+              "level": {
+                "type": "string",
+                "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
+                "enum": [
+                  "info",
+                  "warn",
+                  "fatal"
+                ],
+                "examples": [
+                  "info"
+                ]
+              },
+              "value": {
+                "type": "string",
+                "description": "The message to communicate.",
+                "minLength": 1,
+                "examples": [
+                  "Permission denied, scanning aborted"
+                ]
+              }
+            }
+          }
+        },
+        "scanner": {
+          "type": "object",
+          "description": "Object defining the scanner used to perform the scan.",
+          "required": [
+            "id",
+            "name",
+            "version",
+            "vendor"
+          ],
+          "properties": {
+            "id": {
+              "type": "string",
+              "description": "Unique id that identifies the scanner.",
+              "minLength": 1,
+              "examples": [
+                "my-sast-scanner"
+              ]
+            },
+            "name": {
+              "type": "string",
+              "description": "A human readable value that identifies the scanner, not required to be unique.",
+              "minLength": 1,
+              "examples": [
+                "My SAST Scanner"
+              ]
+            },
+            "url": {
+              "type": "string",
+              "description": "A link to more information about the scanner.",
+              "examples": [
+                "https://scanner.url"
+              ]
+            },
+            "version": {
+              "type": "string",
+              "description": "The version of the scanner.",
+              "minLength": 1,
+              "examples": [
+                "1.0.2"
+              ]
+            },
+            "vendor": {
+              "type": "object",
+              "description": "The vendor/maintainer of the scanner.",
+              "required": [
+                "name"
+              ],
+              "properties": {
+                "name": {
+                  "type": "string",
+                  "description": "The name of the vendor.",
+                  "minLength": 1,
+                  "examples": [
+                    "GitLab"
+                  ]
+                }
+              }
+            }
+          }
+        },
+        "start_time": {
+          "type": "string",
+          "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
+          "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+          "examples": [
+            "2020-02-14T16:01:59"
+          ]
+        },
+        "status": {
+          "type": "string",
+          "description": "Result of the scan.",
+          "enum": [
+            "success",
+            "failure"
+          ]
+        },
+        "type": {
+          "type": "string",
+          "description": "Type of the scan.",
+          "enum": [
+            "sast"
+          ]
+        }
+      }
+    },
+    "schema": {
+      "type": "string",
+      "description": "URI pointing to the validating security report schema.",
+      "format": "uri"
+    },
+    "version": {
+      "type": "string",
+      "description": "The version of the schema to which the JSON report conforms.",
+      "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
+    },
+    "vulnerabilities": {
+      "type": "array",
+      "description": "Array of vulnerability objects.",
+      "items": {
+        "type": "object",
+        "description": "Describes the vulnerability.",
+        "required": [
+          "category",
+          "cve",
+          "identifiers",
+          "location",
+          "scanner"
+        ],
+        "properties": {
+          "id": {
+            "type": "string",
+            "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+            "examples": [
+              "642735a5-1425-428d-8d4e-3c854885a3c9"
+            ]
+          },
+          "category": {
+            "type": "string",
+            "minLength": 1,
+            "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
+          },
+          "name": {
+            "type": "string",
+            "description": "The name of the vulnerability. This must not include the finding's specific information."
+          },
+          "message": {
+            "type": "string",
+            "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
+          },
+          "description": {
+            "type": "string",
+            "description": "A long text section describing the vulnerability more fully."
+          },
+          "cve": {
+            "type": "string",
+            "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
+          },
+          "severity": {
+            "type": "string",
+            "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
+            "enum": [
+              "Info",
+              "Unknown",
+              "Low",
+              "Medium",
+              "High",
+              "Critical"
+            ]
+          },
+          "confidence": {
+            "type": "string",
+            "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
+            "enum": [
+              "Ignore",
+              "Unknown",
+              "Experimental",
+              "Low",
+              "Medium",
+              "High",
+              "Confirmed"
+            ]
+          },
+          "solution": {
+            "type": "string",
+            "description": "Explanation of how to fix the vulnerability."
+          },
+          "scanner": {
+            "description": "Describes the scanner used to find this vulnerability.",
+            "type": "object",
+            "required": [
+              "id",
+              "name"
+            ],
+            "properties": {
+              "id": {
+                "type": "string",
+                "minLength": 1,
+                "description": "The scanner's ID, as a snake_case string."
+              },
+              "name": {
+                "type": "string",
+                "minLength": 1,
+                "description": "Human-readable name of the scanner."
+              }
+            }
+          },
+          "identifiers": {
+            "type": "array",
+            "minItems": 1,
+            "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
+            "items": {
+              "type": "object",
+              "required": [
+                "type",
+                "name",
+                "value"
+              ],
+              "properties": {
+                "type": {
+                  "type": "string",
+                  "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+                  "minLength": 1
+                },
+                "name": {
+                  "type": "string",
+                  "description": "Human-readable name of the identifier.",
+                  "minLength": 1
+                },
+                "url": {
+                  "type": "string",
+                  "description": "URL of the identifier's documentation.",
+                  "format": "uri"
+                },
+                "value": {
+                  "type": "string",
+                  "description": "Value of the identifier, for matching purpose.",
+                  "minLength": 1
+                }
+              }
+            }
+          },
+          "links": {
+            "type": "array",
+            "description": "An array of references to external documentation or articles that describe the vulnerability.",
+            "items": {
+              "type": "object",
+              "required": [
+                "url"
+              ],
+              "properties": {
+                "name": {
+                  "type": "string",
+                  "description": "Name of the vulnerability details link."
+                },
+                "url": {
+                  "type": "string",
+                  "description": "URL of the vulnerability details document.",
+                  "format": "uri"
+                }
+              }
+            }
+          },
+          "details": {
+            "$ref": "#/definitions/named_list/properties/items"
+          },
+          "location": {
+            "type": "object",
+            "description": "Identifies the vulnerability's location.",
+            "properties": {
+              "file": {
+                "type": "string",
+                "description": "Path to the file where the vulnerability is located."
+              },
+              "start_line": {
+                "type": "number",
+                "description": "The first line of the code affected by the vulnerability."
+              },
+              "end_line": {
+                "type": "number",
+                "description": "The last line of the code affected by the vulnerability."
+              },
+              "class": {
+                "type": "string",
+                "description": "Provides the name of the class where the vulnerability is located."
+              },
+              "method": {
+                "type": "string",
+                "description": "Provides the name of the method where the vulnerability is located."
+              }
+            }
+          },
+          "raw_source_code_extract": {
+            "type": "string",
+            "description": "Provides an unsanitized excerpt of the affected source code."
+          }
+        }
+      }
+    },
+    "remediations": {
+      "type": "array",
+      "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
+      "items": {
+        "type": "object",
+        "required": [
+          "fixes",
+          "summary",
+          "diff"
+        ],
+        "properties": {
+          "fixes": {
+            "type": "array",
+            "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
+            "items": {
+              "type": "object",
+              "required": [
+                "cve"
+              ],
+              "properties": {
+                "cve": {
+                  "type": "string",
+                  "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
+                }
+              }
+            }
+          },
+          "summary": {
+            "type": "string",
+            "minLength": 1,
+            "description": "An overview of how the vulnerabilities were fixed."
+          },
+          "diff": {
+            "type": "string",
+            "minLength": 1,
+            "description": "A base64-encoded remediation code diff, compatible with git apply."
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/sast.json b/lib/gitlab/ci/parsers/security/validators/schemas/sast.json
deleted file mode 100644
index a7159be0190..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/sast.json
+++ /dev/null
@@ -1,706 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "Report format for GitLab SAST",
-  "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).",
-  "definitions": {
-    "detail_type": {
-      "oneOf": [
-        {
-          "$ref": "#/definitions/named_list"
-        },
-        {
-          "$ref": "#/definitions/list"
-        },
-        {
-          "$ref": "#/definitions/table"
-        },
-        {
-          "$ref": "#/definitions/text"
-        },
-        {
-          "$ref": "#/definitions/url"
-        },
-        {
-          "$ref": "#/definitions/code"
-        },
-        {
-          "$ref": "#/definitions/value"
-        },
-        {
-          "$ref": "#/definitions/diff"
-        },
-        {
-          "$ref": "#/definitions/markdown"
-        },
-        {
-          "$ref": "#/definitions/commit"
-        },
-        {
-          "$ref": "#/definitions/file_location"
-        },
-        {
-          "$ref": "#/definitions/module_location"
-        }
-      ]
-    },
-    "text_value": {
-      "type": "string"
-    },
-    "named_field": {
-      "type": "object",
-      "required": [
-        "name"
-      ],
-      "properties": {
-        "name": {
-          "$ref": "#/definitions/text_value",
-          "minLength": 1
-        },
-        "description": {
-          "$ref": "#/definitions/text_value"
-        }
-      }
-    },
-    "named_list": {
-      "type": "object",
-      "description": "An object with named and typed fields",
-      "required": [
-        "type",
-        "items"
-      ],
-      "properties": {
-        "type": {
-          "const": "named-list"
-        },
-        "items": {
-          "type": "object",
-          "patternProperties": {
-            "^.*$": {
-              "allOf": [
-                {
-                  "$ref": "#/definitions/named_field"
-                },
-                {
-                  "$ref": "#/definitions/detail_type"
-                }
-              ]
-            }
-          }
-        }
-      }
-    },
-    "list": {
-      "type": "object",
-      "description": "A list of typed fields",
-      "required": [
-        "type",
-        "items"
-      ],
-      "properties": {
-        "type": {
-          "const": "list"
-        },
-        "items": {
-          "type": "array",
-          "items": {
-            "$ref": "#/definitions/detail_type"
-          }
-        }
-      }
-    },
-    "table": {
-      "type": "object",
-      "description": "A table of typed fields",
-      "required": [
-        "type",
-        "rows"
-      ],
-      "properties": {
-        "type": {
-          "const": "table"
-        },
-        "header": {
-          "type": "array",
-          "items": {
-            "$ref": "#/definitions/detail_type"
-          }
-        },
-        "rows": {
-          "type": "array",
-          "items": {
-            "type": "array",
-            "items": {
-              "$ref": "#/definitions/detail_type"
-            }
-          }
-        }
-      }
-    },
-    "text": {
-      "type": "object",
-      "description": "Raw text",
-      "required": [
-        "type",
-        "value"
-      ],
-      "properties": {
-        "type": {
-          "const": "text"
-        },
-        "value": {
-          "$ref": "#/definitions/text_value"
-        }
-      }
-    },
-    "url": {
-      "type": "object",
-      "description": "A single URL",
-      "required": [
-        "type",
-        "href"
-      ],
-      "properties": {
-        "type": {
-          "const": "url"
-        },
-        "text": {
-          "$ref": "#/definitions/text_value"
-        },
-        "href": {
-          "type": "string",
-          "minLength": 1,
-          "examples": [
-            "http://mysite.com"
-          ]
-        }
-      }
-    },
-    "code": {
-      "type": "object",
-      "description": "A codeblock",
-      "required": [
-        "type",
-        "value"
-      ],
-      "properties": {
-        "type": {
-          "const": "code"
-        },
-        "value": {
-          "type": "string"
-        },
-        "lang": {
-          "type": "string",
-          "description": "A programming language"
-        }
-      }
-    },
-    "value": {
-      "type": "object",
-      "description": "A field that can store a range of types of value",
-      "required": [
-        "type",
-        "value"
-      ],
-      "properties": {
-        "type": {
-          "const": "value"
-        },
-        "value": {
-          "type": [
-            "number",
-            "string",
-            "boolean"
-          ]
-        }
-      }
-    },
-    "diff": {
-      "type": "object",
-      "description": "A diff",
-      "required": [
-        "type",
-        "before",
-        "after"
-      ],
-      "properties": {
-        "type": {
-          "const": "diff"
-        },
-        "before": {
-          "type": "string"
-        },
-        "after": {
-          "type": "string"
-        }
-      }
-    },
-    "markdown": {
-      "type": "object",
-      "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
-      "required": [
-        "type",
-        "value"
-      ],
-      "properties": {
-        "type": {
-          "const": "markdown"
-        },
-        "value": {
-          "$ref": "#/definitions/text_value",
-          "examples": [
-            "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
-          ]
-        }
-      }
-    },
-    "commit": {
-      "type": "object",
-      "description": "A commit/tag/branch within the GitLab project",
-      "required": [
-        "type",
-        "value"
-      ],
-      "properties": {
-        "type": {
-          "const": "commit"
-        },
-        "value": {
-          "type": "string",
-          "description": "The commit SHA",
-          "minLength": 1
-        }
-      }
-    },
-    "file_location": {
-      "type": "object",
-      "description": "A location within a file in the project",
-      "required": [
-        "type",
-        "file_name",
-        "line_start"
-      ],
-      "properties": {
-        "type": {
-          "const": "file-location"
-        },
-        "file_name": {
-          "type": "string",
-          "minLength": 1
-        },
-        "line_start": {
-          "type": "integer"
-        },
-        "line_end": {
-          "type": "integer"
-        }
-      }
-    },
-    "module_location": {
-      "type": "object",
-      "description": "A location within a binary module of the form module+relative_offset",
-      "required": [
-        "type",
-        "module_name",
-        "offset"
-      ],
-      "properties": {
-        "type": {
-          "const": "module-location"
-        },
-        "module_name": {
-          "type": "string",
-          "minLength": 1,
-          "examples": [
-            "compiled_binary"
-          ]
-        },
-        "offset": {
-          "type": "integer",
-          "examples": [
-            100
-          ]
-        }
-      }
-    }
-  },
-  "self": {
-    "version": "14.0.0"
-  },
-  "required": [
-    "version",
-    "vulnerabilities"
-  ],
-  "additionalProperties": true,
-  "properties": {
-    "scan": {
-      "type": "object",
-      "required": [
-        "end_time",
-        "scanner",
-        "start_time",
-        "status",
-        "type"
-      ],
-      "properties": {
-        "end_time": {
-          "type": "string",
-          "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
-          "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
-          "examples": [
-            "2020-01-28T03:26:02"
-          ]
-        },
-        "messages": {
-          "type": "array",
-          "items": {
-            "type": "object",
-            "description": "Communication intended for the initiator of a scan.",
-            "required": [
-              "level",
-              "value"
-            ],
-            "properties": {
-              "level": {
-                "type": "string",
-                "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
-                "enum": [
-                  "info",
-                  "warn",
-                  "fatal"
-                ],
-                "examples": [
-                  "info"
-                ]
-              },
-              "value": {
-                "type": "string",
-                "description": "The message to communicate.",
-                "minLength": 1,
-                "examples": [
-                  "Permission denied, scanning aborted"
-                ]
-              }
-            }
-          }
-        },
-        "scanner": {
-          "type": "object",
-          "description": "Object defining the scanner used to perform the scan.",
-          "required": [
-            "id",
-            "name",
-            "version",
-            "vendor"
-          ],
-          "properties": {
-            "id": {
-              "type": "string",
-              "description": "Unique id that identifies the scanner.",
-              "minLength": 1,
-              "examples": [
-                "my-sast-scanner"
-              ]
-            },
-            "name": {
-              "type": "string",
-              "description": "A human readable value that identifies the scanner, not required to be unique.",
-              "minLength": 1,
-              "examples": [
-                "My SAST Scanner"
-              ]
-            },
-            "url": {
-              "type": "string",
-              "description": "A link to more information about the scanner.",
-              "examples": [
-                "https://scanner.url"
-              ]
-            },
-            "version": {
-              "type": "string",
-              "description": "The version of the scanner.",
-              "minLength": 1,
-              "examples": [
-                "1.0.2"
-              ]
-            },
-            "vendor": {
-              "type": "object",
-              "description": "The vendor/maintainer of the scanner.",
-              "required": [
-                "name"
-              ],
-              "properties": {
-                "name": {
-                  "type": "string",
-                  "description": "The name of the vendor.",
-                  "minLength": 1,
-                  "examples": [
-                    "GitLab"
-                  ]
-                }
-              }
-            }
-          }
-        },
-        "start_time": {
-          "type": "string",
-          "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
-          "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
-          "examples": [
-            "2020-02-14T16:01:59"
-          ]
-        },
-        "status": {
-          "type": "string",
-          "description": "Result of the scan.",
-          "enum": [
-            "success",
-            "failure"
-          ]
-        },
-        "type": {
-          "type": "string",
-          "description": "Type of the scan.",
-          "enum": [
-            "sast"
-          ]
-        }
-      }
-    },
-    "schema": {
-      "type": "string",
-      "description": "URI pointing to the validating security report schema.",
-      "format": "uri"
-    },
-    "version": {
-      "type": "string",
-      "description": "The version of the schema to which the JSON report conforms.",
-      "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
-    },
-    "vulnerabilities": {
-      "type": "array",
-      "description": "Array of vulnerability objects.",
-      "items": {
-        "type": "object",
-        "description": "Describes the vulnerability.",
-        "required": [
-          "category",
-          "cve",
-          "identifiers",
-          "location",
-          "scanner"
-        ],
-        "properties": {
-          "id": {
-            "type": "string",
-            "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
-            "examples": [
-              "642735a5-1425-428d-8d4e-3c854885a3c9"
-            ]
-          },
-          "category": {
-            "type": "string",
-            "minLength": 1,
-            "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
-          },
-          "name": {
-            "type": "string",
-            "description": "The name of the vulnerability. This must not include the finding's specific information."
-          },
-          "message": {
-            "type": "string",
-            "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
-          },
-          "description": {
-            "type": "string",
-            "description": "A long text section describing the vulnerability more fully."
-          },
-          "cve": {
-            "type": "string",
-            "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
-          },
-          "severity": {
-            "type": "string",
-            "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
-            "enum": [
-              "Info",
-              "Unknown",
-              "Low",
-              "Medium",
-              "High",
-              "Critical"
-            ]
-          },
-          "confidence": {
-            "type": "string",
-            "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
-            "enum": [
-              "Ignore",
-              "Unknown",
-              "Experimental",
-              "Low",
-              "Medium",
-              "High",
-              "Confirmed"
-            ]
-          },
-          "solution": {
-            "type": "string",
-            "description": "Explanation of how to fix the vulnerability."
-          },
-          "scanner": {
-            "description": "Describes the scanner used to find this vulnerability.",
-            "type": "object",
-            "required": [
-              "id",
-              "name"
-            ],
-            "properties": {
-              "id": {
-                "type": "string",
-                "minLength": 1,
-                "description": "The scanner's ID, as a snake_case string."
-              },
-              "name": {
-                "type": "string",
-                "minLength": 1,
-                "description": "Human-readable name of the scanner."
-              }
-            }
-          },
-          "identifiers": {
-            "type": "array",
-            "minItems": 1,
-            "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
-            "items": {
-              "type": "object",
-              "required": [
-                "type",
-                "name",
-                "value"
-              ],
-              "properties": {
-                "type": {
-                  "type": "string",
-                  "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
-                  "minLength": 1
-                },
-                "name": {
-                  "type": "string",
-                  "description": "Human-readable name of the identifier.",
-                  "minLength": 1
-                },
-                "url": {
-                  "type": "string",
-                  "description": "URL of the identifier's documentation.",
-                  "format": "uri"
-                },
-                "value": {
-                  "type": "string",
-                  "description": "Value of the identifier, for matching purpose.",
-                  "minLength": 1
-                }
-              }
-            }
-          },
-          "links": {
-            "type": "array",
-            "description": "An array of references to external documentation or articles that describe the vulnerability.",
-            "items": {
-              "type": "object",
-              "required": [
-                "url"
-              ],
-              "properties": {
-                "name": {
-                  "type": "string",
-                  "description": "Name of the vulnerability details link."
-                },
-                "url": {
-                  "type": "string",
-                  "description": "URL of the vulnerability details document.",
-                  "format": "uri"
-                }
-              }
-            }
-          },
-          "details": {
-            "$ref": "#/definitions/named_list/properties/items"
-          },
-          "location": {
-            "type": "object",
-            "description": "Identifies the vulnerability's location.",
-            "properties": {
-              "file": {
-                "type": "string",
-                "description": "Path to the file where the vulnerability is located."
-              },
-              "start_line": {
-                "type": "number",
-                "description": "The first line of the code affected by the vulnerability."
-              },
-              "end_line": {
-                "type": "number",
-                "description": "The last line of the code affected by the vulnerability."
-              },
-              "class": {
-                "type": "string",
-                "description": "Provides the name of the class where the vulnerability is located."
-              },
-              "method": {
-                "type": "string",
-                "description": "Provides the name of the method where the vulnerability is located."
-              }
-            }
-          },
-          "raw_source_code_extract": {
-            "type": "string",
-            "description": "Provides an unsanitized excerpt of the affected source code."
-          }
-        }
-      }
-    },
-    "remediations": {
-      "type": "array",
-      "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
-      "items": {
-        "type": "object",
-        "required": [
-          "fixes",
-          "summary",
-          "diff"
-        ],
-        "properties": {
-          "fixes": {
-            "type": "array",
-            "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
-            "items": {
-              "type": "object",
-              "required": [
-                "cve"
-              ],
-              "properties": {
-                "cve": {
-                  "type": "string",
-                  "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
-                }
-              }
-            }
-          },
-          "summary": {
-            "type": "string",
-            "minLength": 1,
-            "description": "An overview of how the vulnerabilities were fixed."
-          },
-          "diff": {
-            "type": "string",
-            "minLength": 1,
-            "description": "A base64-encoded remediation code diff, compatible with git apply."
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/secret-detection-report-format.json
new file mode 100644
index 00000000000..462e23a151c
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/secret-detection-report-format.json
@@ -0,0 +1,729 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Report format for GitLab Secret Detection",
+  "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)",
+  "definitions": {
+    "detail_type": {
+      "oneOf": [
+        {
+          "$ref": "#/definitions/named_list"
+        },
+        {
+          "$ref": "#/definitions/list"
+        },
+        {
+          "$ref": "#/definitions/table"
+        },
+        {
+          "$ref": "#/definitions/text"
+        },
+        {
+          "$ref": "#/definitions/url"
+        },
+        {
+          "$ref": "#/definitions/code"
+        },
+        {
+          "$ref": "#/definitions/value"
+        },
+        {
+          "$ref": "#/definitions/diff"
+        },
+        {
+          "$ref": "#/definitions/markdown"
+        },
+        {
+          "$ref": "#/definitions/commit"
+        },
+        {
+          "$ref": "#/definitions/file_location"
+        },
+        {
+          "$ref": "#/definitions/module_location"
+        }
+      ]
+    },
+    "text_value": {
+      "type": "string"
+    },
+    "named_field": {
+      "type": "object",
+      "required": [
+        "name"
+      ],
+      "properties": {
+        "name": {
+          "$ref": "#/definitions/text_value",
+          "minLength": 1
+        },
+        "description": {
+          "$ref": "#/definitions/text_value"
+        }
+      }
+    },
+    "named_list": {
+      "type": "object",
+      "description": "An object with named and typed fields",
+      "required": [
+        "type",
+        "items"
+      ],
+      "properties": {
+        "type": {
+          "const": "named-list"
+        },
+        "items": {
+          "type": "object",
+          "patternProperties": {
+            "^.*$": {
+              "allOf": [
+                {
+                  "$ref": "#/definitions/named_field"
+                },
+                {
+                  "$ref": "#/definitions/detail_type"
+                }
+              ]
+            }
+          }
+        }
+      }
+    },
+    "list": {
+      "type": "object",
+      "description": "A list of typed fields",
+      "required": [
+        "type",
+        "items"
+      ],
+      "properties": {
+        "type": {
+          "const": "list"
+        },
+        "items": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/detail_type"
+          }
+        }
+      }
+    },
+    "table": {
+      "type": "object",
+      "description": "A table of typed fields",
+      "required": [
+        "type",
+        "rows"
+      ],
+      "properties": {
+        "type": {
+          "const": "table"
+        },
+        "header": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/detail_type"
+          }
+        },
+        "rows": {
+          "type": "array",
+          "items": {
+            "type": "array",
+            "items": {
+              "$ref": "#/definitions/detail_type"
+            }
+          }
+        }
+      }
+    },
+    "text": {
+      "type": "object",
+      "description": "Raw text",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "text"
+        },
+        "value": {
+          "$ref": "#/definitions/text_value"
+        }
+      }
+    },
+    "url": {
+      "type": "object",
+      "description": "A single URL",
+      "required": [
+        "type",
+        "href"
+      ],
+      "properties": {
+        "type": {
+          "const": "url"
+        },
+        "text": {
+          "$ref": "#/definitions/text_value"
+        },
+        "href": {
+          "type": "string",
+          "minLength": 1,
+          "examples": [
+            "http://mysite.com"
+          ]
+        }
+      }
+    },
+    "code": {
+      "type": "object",
+      "description": "A codeblock",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "code"
+        },
+        "value": {
+          "type": "string"
+        },
+        "lang": {
+          "type": "string",
+          "description": "A programming language"
+        }
+      }
+    },
+    "value": {
+      "type": "object",
+      "description": "A field that can store a range of types of value",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "value"
+        },
+        "value": {
+          "type": [
+            "number",
+            "string",
+            "boolean"
+          ]
+        }
+      }
+    },
+    "diff": {
+      "type": "object",
+      "description": "A diff",
+      "required": [
+        "type",
+        "before",
+        "after"
+      ],
+      "properties": {
+        "type": {
+          "const": "diff"
+        },
+        "before": {
+          "type": "string"
+        },
+        "after": {
+          "type": "string"
+        }
+      }
+    },
+    "markdown": {
+      "type": "object",
+      "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "markdown"
+        },
+        "value": {
+          "$ref": "#/definitions/text_value",
+          "examples": [
+            "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
+          ]
+        }
+      }
+    },
+    "commit": {
+      "type": "object",
+      "description": "A commit/tag/branch within the GitLab project",
+      "required": [
+        "type",
+        "value"
+      ],
+      "properties": {
+        "type": {
+          "const": "commit"
+        },
+        "value": {
+          "type": "string",
+          "description": "The commit SHA",
+          "minLength": 1
+        }
+      }
+    },
+    "file_location": {
+      "type": "object",
+      "description": "A location within a file in the project",
+      "required": [
+        "type",
+        "file_name",
+        "line_start"
+      ],
+      "properties": {
+        "type": {
+          "const": "file-location"
+        },
+        "file_name": {
+          "type": "string",
+          "minLength": 1
+        },
+        "line_start": {
+          "type": "integer"
+        },
+        "line_end": {
+          "type": "integer"
+        }
+      }
+    },
+    "module_location": {
+      "type": "object",
+      "description": "A location within a binary module of the form module+relative_offset",
+      "required": [
+        "type",
+        "module_name",
+        "offset"
+      ],
+      "properties": {
+        "type": {
+          "const": "module-location"
+        },
+        "module_name": {
+          "type": "string",
+          "minLength": 1,
+          "examples": [
+            "compiled_binary"
+          ]
+        },
+        "offset": {
+          "type": "integer",
+          "examples": [
+            100
+          ]
+        }
+      }
+    }
+  },
+  "self": {
+    "version": "14.0.0"
+  },
+  "required": [
+    "version",
+    "vulnerabilities"
+  ],
+  "additionalProperties": true,
+  "properties": {
+    "scan": {
+      "type": "object",
+      "required": [
+        "end_time",
+        "scanner",
+        "start_time",
+        "status",
+        "type"
+      ],
+      "properties": {
+        "end_time": {
+          "type": "string",
+          "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
+          "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+          "examples": [
+            "2020-01-28T03:26:02"
+          ]
+        },
+        "messages": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "description": "Communication intended for the initiator of a scan.",
+            "required": [
+              "level",
+              "value"
+            ],
+            "properties": {
+              "level": {
+                "type": "string",
+                "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
+                "enum": [
+                  "info",
+                  "warn",
+                  "fatal"
+                ],
+                "examples": [
+                  "info"
+                ]
+              },
+              "value": {
+                "type": "string",
+                "description": "The message to communicate.",
+                "minLength": 1,
+                "examples": [
+                  "Permission denied, scanning aborted"
+                ]
+              }
+            }
+          }
+        },
+        "scanner": {
+          "type": "object",
+          "description": "Object defining the scanner used to perform the scan.",
+          "required": [
+            "id",
+            "name",
+            "version",
+            "vendor"
+          ],
+          "properties": {
+            "id": {
+              "type": "string",
+              "description": "Unique id that identifies the scanner.",
+              "minLength": 1,
+              "examples": [
+                "my-sast-scanner"
+              ]
+            },
+            "name": {
+              "type": "string",
+              "description": "A human readable value that identifies the scanner, not required to be unique.",
+              "minLength": 1,
+              "examples": [
+                "My SAST Scanner"
+              ]
+            },
+            "url": {
+              "type": "string",
+              "description": "A link to more information about the scanner.",
+              "examples": [
+                "https://scanner.url"
+              ]
+            },
+            "version": {
+              "type": "string",
+              "description": "The version of the scanner.",
+              "minLength": 1,
+              "examples": [
+                "1.0.2"
+              ]
+            },
+            "vendor": {
+              "type": "object",
+              "description": "The vendor/maintainer of the scanner.",
+              "required": [
+                "name"
+              ],
+              "properties": {
+                "name": {
+                  "type": "string",
+                  "description": "The name of the vendor.",
+                  "minLength": 1,
+                  "examples": [
+                    "GitLab"
+                  ]
+                }
+              }
+            }
+          }
+        },
+        "start_time": {
+          "type": "string",
+          "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
+          "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+          "examples": [
+            "2020-02-14T16:01:59"
+          ]
+        },
+        "status": {
+          "type": "string",
+          "description": "Result of the scan.",
+          "enum": [
+            "success",
+            "failure"
+          ]
+        },
+        "type": {
+          "type": "string",
+          "description": "Type of the scan.",
+          "enum": [
+            "secret_detection"
+          ]
+        }
+      }
+    },
+    "schema": {
+      "type": "string",
+      "description": "URI pointing to the validating security report schema.",
+      "format": "uri"
+    },
+    "version": {
+      "type": "string",
+      "description": "The version of the schema to which the JSON report conforms.",
+      "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
+    },
+    "vulnerabilities": {
+      "type": "array",
+      "description": "Array of vulnerability objects.",
+      "items": {
+        "type": "object",
+        "description": "Describes the vulnerability.",
+        "required": [
+          "category",
+          "cve",
+          "identifiers",
+          "location",
+          "scanner"
+        ],
+        "properties": {
+          "id": {
+            "type": "string",
+            "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+            "examples": [
+              "642735a5-1425-428d-8d4e-3c854885a3c9"
+            ]
+          },
+          "category": {
+            "type": "string",
+            "minLength": 1,
+            "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
+          },
+          "name": {
+            "type": "string",
+            "description": "The name of the vulnerability. This must not include the finding's specific information."
+          },
+          "message": {
+            "type": "string",
+            "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
+          },
+          "description": {
+            "type": "string",
+            "description": "A long text section describing the vulnerability more fully."
+          },
+          "cve": {
+            "type": "string",
+            "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
+          },
+          "severity": {
+            "type": "string",
+            "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
+            "enum": [
+              "Info",
+              "Unknown",
+              "Low",
+              "Medium",
+              "High",
+              "Critical"
+            ]
+          },
+          "confidence": {
+            "type": "string",
+            "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
+            "enum": [
+              "Ignore",
+              "Unknown",
+              "Experimental",
+              "Low",
+              "Medium",
+              "High",
+              "Confirmed"
+            ]
+          },
+          "solution": {
+            "type": "string",
+            "description": "Explanation of how to fix the vulnerability."
+          },
+          "scanner": {
+            "description": "Describes the scanner used to find this vulnerability.",
+            "type": "object",
+            "required": [
+              "id",
+              "name"
+            ],
+            "properties": {
+              "id": {
+                "type": "string",
+                "minLength": 1,
+                "description": "The scanner's ID, as a snake_case string."
+              },
+              "name": {
+                "type": "string",
+                "minLength": 1,
+                "description": "Human-readable name of the scanner."
+              }
+            }
+          },
+          "identifiers": {
+            "type": "array",
+            "minItems": 1,
+            "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
+            "items": {
+              "type": "object",
+              "required": [
+                "type",
+                "name",
+                "value"
+              ],
+              "properties": {
+                "type": {
+                  "type": "string",
+                  "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+                  "minLength": 1
+                },
+                "name": {
+                  "type": "string",
+                  "description": "Human-readable name of the identifier.",
+                  "minLength": 1
+                },
+                "url": {
+                  "type": "string",
+                  "description": "URL of the identifier's documentation.",
+                  "format": "uri"
+                },
+                "value": {
+                  "type": "string",
+                  "description": "Value of the identifier, for matching purpose.",
+                  "minLength": 1
+                }
+              }
+            }
+          },
+          "links": {
+            "type": "array",
+            "description": "An array of references to external documentation or articles that describe the vulnerability.",
+            "items": {
+              "type": "object",
+              "required": [
+                "url"
+              ],
+              "properties": {
+                "name": {
+                  "type": "string",
+                  "description": "Name of the vulnerability details link."
+                },
+                "url": {
+                  "type": "string",
+                  "description": "URL of the vulnerability details document.",
+                  "format": "uri"
+                }
+              }
+            }
+          },
+          "details": {
+            "$ref": "#/definitions/named_list/properties/items"
+          },
+          "location": {
+            "required": [
+              "commit"
+            ],
+            "properties": {
+              "file": {
+                "type": "string",
+                "description": "Path to the file where the vulnerability is located"
+              },
+              "commit": {
+                "type": "object",
+                "description": "Represents the commit in which the vulnerability was detected",
+                "required": [
+                  "sha"
+                ],
+                "properties": {
+                  "author": {
+                    "type": "string"
+                  },
+                  "date": {
+                    "type": "string"
+                  },
+                  "message": {
+                    "type": "string"
+                  },
+                  "sha": {
+                    "type": "string",
+                    "minLength": 1
+                  }
+                }
+              },
+              "start_line": {
+                "type": "number",
+                "description": "The first line of the code affected by the vulnerability"
+              },
+              "end_line": {
+                "type": "number",
+                "description": "The last line of the code affected by the vulnerability"
+              },
+              "class": {
+                "type": "string",
+                "description": "Provides the name of the class where the vulnerability is located"
+              },
+              "method": {
+                "type": "string",
+                "description": "Provides the name of the method where the vulnerability is located"
+              }
+            }
+          },
+          "raw_source_code_extract": {
+            "type": "string",
+            "description": "Provides an unsanitized excerpt of the affected source code."
+          }
+        }
+      }
+    },
+    "remediations": {
+      "type": "array",
+      "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
+      "items": {
+        "type": "object",
+        "required": [
+          "fixes",
+          "summary",
+          "diff"
+        ],
+        "properties": {
+          "fixes": {
+            "type": "array",
+            "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
+            "items": {
+              "type": "object",
+              "required": [
+                "cve"
+              ],
+              "properties": {
+                "cve": {
+                  "type": "string",
+                  "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
+                }
+              }
+            }
+          },
+          "summary": {
+            "type": "string",
+            "minLength": 1,
+            "description": "An overview of how the vulnerabilities were fixed."
+          },
+          "diff": {
+            "type": "string",
+            "minLength": 1,
+            "description": "A base64-encoded remediation code diff, compatible with git apply."
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json b/lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json
deleted file mode 100644
index 462e23a151c..00000000000
--- a/lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json
+++ /dev/null
@@ -1,729 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "Report format for GitLab Secret Detection",
-  "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)",
-  "definitions": {
-    "detail_type": {
-      "oneOf": [
-        {
-          "$ref": "#/definitions/named_list"
-        },
-        {
-          "$ref": "#/definitions/list"
-        },
-        {
-          "$ref": "#/definitions/table"
-        },
-        {
-          "$ref": "#/definitions/text"
-        },
-        {
-          "$ref": "#/definitions/url"
-        },
-        {
-          "$ref": "#/definitions/code"
-        },
-        {
-          "$ref": "#/definitions/value"
-        },
-        {
-          "$ref": "#/definitions/diff"
-        },
-        {
-          "$ref": "#/definitions/markdown"
-        },
-        {
-          "$ref": "#/definitions/commit"
-        },
-        {
-          "$ref": "#/definitions/file_location"
-        },
-        {
-          "$ref": "#/definitions/module_location"
-        }
-      ]
-    },
-    "text_value": {
-      "type": "string"
-    },
-    "named_field": {
-      "type": "object",
-      "required": [
-        "name"
-      ],
-      "properties": {
-        "name": {
-          "$ref": "#/definitions/text_value",
-          "minLength": 1
-        },
-        "description": {
-          "$ref": "#/definitions/text_value"
-        }
-      }
-    },
-    "named_list": {
-      "type": "object",
-      "description": "An object with named and typed fields",
-      "required": [
-        "type",
-        "items"
-      ],
-      "properties": {
-        "type": {
-          "const": "named-list"
-        },
-        "items": {
-          "type": "object",
-          "patternProperties": {
-            "^.*$": {
-              "allOf": [
-                {
-                  "$ref": "#/definitions/named_field"
-                },
-                {
-                  "$ref": "#/definitions/detail_type"
-                }
-              ]
-            }
-          }
-        }
-      }
-    },
-    "list": {
-      "type": "object",
-      "description": "A list of typed fields",
-      "required": [
-        "type",
-        "items"
-      ],
-      "properties": {
-        "type": {
-          "const": "list"
-        },
-        "items": {
-          "type": "array",
-          "items": {
-            "$ref": "#/definitions/detail_type"
-          }
-        }
-      }
-    },
-    "table": {
-      "type": "object",
-      "description": "A table of typed fields",
-      "required": [
-        "type",
-        "rows"
-      ],
-      "properties": {
-        "type": {
-          "const": "table"
-        },
-        "header": {
-          "type": "array",
-          "items": {
-            "$ref": "#/definitions/detail_type"
-          }
-        },
-        "rows": {
-          "type": "array",
-          "items": {
-            "type": "array",
-            "items": {
-              "$ref": "#/definitions/detail_type"
-            }
-          }
-        }
-      }
-    },
-    "text": {
-      "type": "object",
-      "description": "Raw text",
-      "required": [
-        "type",
-        "value"
-      ],
-      "properties": {
-        "type": {
-          "const": "text"
-        },
-        "value": {
-          "$ref": "#/definitions/text_value"
-        }
-      }
-    },
-    "url": {
-      "type": "object",
-      "description": "A single URL",
-      "required": [
-        "type",
-        "href"
-      ],
-      "properties": {
-        "type": {
-          "const": "url"
-        },
-        "text": {
-          "$ref": "#/definitions/text_value"
-        },
-        "href": {
-          "type": "string",
-          "minLength": 1,
-          "examples": [
-            "http://mysite.com"
-          ]
-        }
-      }
-    },
-    "code": {
-      "type": "object",
-      "description": "A codeblock",
-      "required": [
-        "type",
-        "value"
-      ],
-      "properties": {
-        "type": {
-          "const": "code"
-        },
-        "value": {
-          "type": "string"
-        },
-        "lang": {
-          "type": "string",
-          "description": "A programming language"
-        }
-      }
-    },
-    "value": {
-      "type": "object",
-      "description": "A field that can store a range of types of value",
-      "required": [
-        "type",
-        "value"
-      ],
-      "properties": {
-        "type": {
-          "const": "value"
-        },
-        "value": {
-          "type": [
-            "number",
-            "string",
-            "boolean"
-          ]
-        }
-      }
-    },
-    "diff": {
-      "type": "object",
-      "description": "A diff",
-      "required": [
-        "type",
-        "before",
-        "after"
-      ],
-      "properties": {
-        "type": {
-          "const": "diff"
-        },
-        "before": {
-          "type": "string"
-        },
-        "after": {
-          "type": "string"
-        }
-      }
-    },
-    "markdown": {
-      "type": "object",
-      "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
-      "required": [
-        "type",
-        "value"
-      ],
-      "properties": {
-        "type": {
-          "const": "markdown"
-        },
-        "value": {
-          "$ref": "#/definitions/text_value",
-          "examples": [
-            "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
-          ]
-        }
-      }
-    },
-    "commit": {
-      "type": "object",
-      "description": "A commit/tag/branch within the GitLab project",
-      "required": [
-        "type",
-        "value"
-      ],
-      "properties": {
-        "type": {
-          "const": "commit"
-        },
-        "value": {
-          "type": "string",
-          "description": "The commit SHA",
-          "minLength": 1
-        }
-      }
-    },
-    "file_location": {
-      "type": "object",
-      "description": "A location within a file in the project",
-      "required": [
-        "type",
-        "file_name",
-        "line_start"
-      ],
-      "properties": {
-        "type": {
-          "const": "file-location"
-        },
-        "file_name": {
-          "type": "string",
-          "minLength": 1
-        },
-        "line_start": {
-          "type": "integer"
-        },
-        "line_end": {
-          "type": "integer"
-        }
-      }
-    },
-    "module_location": {
-      "type": "object",
-      "description": "A location within a binary module of the form module+relative_offset",
-      "required": [
-        "type",
-        "module_name",
-        "offset"
-      ],
-      "properties": {
-        "type": {
-          "const": "module-location"
-        },
-        "module_name": {
-          "type": "string",
-          "minLength": 1,
-          "examples": [
-            "compiled_binary"
-          ]
-        },
-        "offset": {
-          "type": "integer",
-          "examples": [
-            100
-          ]
-        }
-      }
-    }
-  },
-  "self": {
-    "version": "14.0.0"
-  },
-  "required": [
-    "version",
-    "vulnerabilities"
-  ],
-  "additionalProperties": true,
-  "properties": {
-    "scan": {
-      "type": "object",
-      "required": [
-        "end_time",
-        "scanner",
-        "start_time",
-        "status",
-        "type"
-      ],
-      "properties": {
-        "end_time": {
-          "type": "string",
-          "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
-          "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
-          "examples": [
-            "2020-01-28T03:26:02"
-          ]
-        },
-        "messages": {
-          "type": "array",
-          "items": {
-            "type": "object",
-            "description": "Communication intended for the initiator of a scan.",
-            "required": [
-              "level",
-              "value"
-            ],
-            "properties": {
-              "level": {
-                "type": "string",
-                "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
-                "enum": [
-                  "info",
-                  "warn",
-                  "fatal"
-                ],
-                "examples": [
-                  "info"
-                ]
-              },
-              "value": {
-                "type": "string",
-                "description": "The message to communicate.",
-                "minLength": 1,
-                "examples": [
-                  "Permission denied, scanning aborted"
-                ]
-              }
-            }
-          }
-        },
-        "scanner": {
-          "type": "object",
-          "description": "Object defining the scanner used to perform the scan.",
-          "required": [
-            "id",
-            "name",
-            "version",
-            "vendor"
-          ],
-          "properties": {
-            "id": {
-              "type": "string",
-              "description": "Unique id that identifies the scanner.",
-              "minLength": 1,
-              "examples": [
-                "my-sast-scanner"
-              ]
-            },
-            "name": {
-              "type": "string",
-              "description": "A human readable value that identifies the scanner, not required to be unique.",
-              "minLength": 1,
-              "examples": [
-                "My SAST Scanner"
-              ]
-            },
-            "url": {
-              "type": "string",
-              "description": "A link to more information about the scanner.",
-              "examples": [
-                "https://scanner.url"
-              ]
-            },
-            "version": {
-              "type": "string",
-              "description": "The version of the scanner.",
-              "minLength": 1,
-              "examples": [
-                "1.0.2"
-              ]
-            },
-            "vendor": {
-              "type": "object",
-              "description": "The vendor/maintainer of the scanner.",
-              "required": [
-                "name"
-              ],
-              "properties": {
-                "name": {
-                  "type": "string",
-                  "description": "The name of the vendor.",
-                  "minLength": 1,
-                  "examples": [
-                    "GitLab"
-                  ]
-                }
-              }
-            }
-          }
-        },
-        "start_time": {
-          "type": "string",
-          "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
-          "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
-          "examples": [
-            "2020-02-14T16:01:59"
-          ]
-        },
-        "status": {
-          "type": "string",
-          "description": "Result of the scan.",
-          "enum": [
-            "success",
-            "failure"
-          ]
-        },
-        "type": {
-          "type": "string",
-          "description": "Type of the scan.",
-          "enum": [
-            "secret_detection"
-          ]
-        }
-      }
-    },
-    "schema": {
-      "type": "string",
-      "description": "URI pointing to the validating security report schema.",
-      "format": "uri"
-    },
-    "version": {
-      "type": "string",
-      "description": "The version of the schema to which the JSON report conforms.",
-      "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
-    },
-    "vulnerabilities": {
-      "type": "array",
-      "description": "Array of vulnerability objects.",
-      "items": {
-        "type": "object",
-        "description": "Describes the vulnerability.",
-        "required": [
-          "category",
-          "cve",
-          "identifiers",
-          "location",
-          "scanner"
-        ],
-        "properties": {
-          "id": {
-            "type": "string",
-            "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
-            "examples": [
-              "642735a5-1425-428d-8d4e-3c854885a3c9"
-            ]
-          },
-          "category": {
-            "type": "string",
-            "minLength": 1,
-            "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
-          },
-          "name": {
-            "type": "string",
-            "description": "The name of the vulnerability. This must not include the finding's specific information."
-          },
-          "message": {
-            "type": "string",
-            "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
-          },
-          "description": {
-            "type": "string",
-            "description": "A long text section describing the vulnerability more fully."
-          },
-          "cve": {
-            "type": "string",
-            "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
-          },
-          "severity": {
-            "type": "string",
-            "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
-            "enum": [
-              "Info",
-              "Unknown",
-              "Low",
-              "Medium",
-              "High",
-              "Critical"
-            ]
-          },
-          "confidence": {
-            "type": "string",
-            "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
-            "enum": [
-              "Ignore",
-              "Unknown",
-              "Experimental",
-              "Low",
-              "Medium",
-              "High",
-              "Confirmed"
-            ]
-          },
-          "solution": {
-            "type": "string",
-            "description": "Explanation of how to fix the vulnerability."
-          },
-          "scanner": {
-            "description": "Describes the scanner used to find this vulnerability.",
-            "type": "object",
-            "required": [
-              "id",
-              "name"
-            ],
-            "properties": {
-              "id": {
-                "type": "string",
-                "minLength": 1,
-                "description": "The scanner's ID, as a snake_case string."
-              },
-              "name": {
-                "type": "string",
-                "minLength": 1,
-                "description": "Human-readable name of the scanner."
-              }
-            }
-          },
-          "identifiers": {
-            "type": "array",
-            "minItems": 1,
-            "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
-            "items": {
-              "type": "object",
-              "required": [
-                "type",
-                "name",
-                "value"
-              ],
-              "properties": {
-                "type": {
-                  "type": "string",
-                  "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
-                  "minLength": 1
-                },
-                "name": {
-                  "type": "string",
-                  "description": "Human-readable name of the identifier.",
-                  "minLength": 1
-                },
-                "url": {
-                  "type": "string",
-                  "description": "URL of the identifier's documentation.",
-                  "format": "uri"
-                },
-                "value": {
-                  "type": "string",
-                  "description": "Value of the identifier, for matching purpose.",
-                  "minLength": 1
-                }
-              }
-            }
-          },
-          "links": {
-            "type": "array",
-            "description": "An array of references to external documentation or articles that describe the vulnerability.",
-            "items": {
-              "type": "object",
-              "required": [
-                "url"
-              ],
-              "properties": {
-                "name": {
-                  "type": "string",
-                  "description": "Name of the vulnerability details link."
-                },
-                "url": {
-                  "type": "string",
-                  "description": "URL of the vulnerability details document.",
-                  "format": "uri"
-                }
-              }
-            }
-          },
-          "details": {
-            "$ref": "#/definitions/named_list/properties/items"
-          },
-          "location": {
-            "required": [
-              "commit"
-            ],
-            "properties": {
-              "file": {
-                "type": "string",
-                "description": "Path to the file where the vulnerability is located"
-              },
-              "commit": {
-                "type": "object",
-                "description": "Represents the commit in which the vulnerability was detected",
-                "required": [
-                  "sha"
-                ],
-                "properties": {
-                  "author": {
-                    "type": "string"
-                  },
-                  "date": {
-                    "type": "string"
-                  },
-                  "message": {
-                    "type": "string"
-                  },
-                  "sha": {
-                    "type": "string",
-                    "minLength": 1
-                  }
-                }
-              },
-              "start_line": {
-                "type": "number",
-                "description": "The first line of the code affected by the vulnerability"
-              },
-              "end_line": {
-                "type": "number",
-                "description": "The last line of the code affected by the vulnerability"
-              },
-              "class": {
-                "type": "string",
-                "description": "Provides the name of the class where the vulnerability is located"
-              },
-              "method": {
-                "type": "string",
-                "description": "Provides the name of the method where the vulnerability is located"
-              }
-            }
-          },
-          "raw_source_code_extract": {
-            "type": "string",
-            "description": "Provides an unsanitized excerpt of the affected source code."
-          }
-        }
-      }
-    },
-    "remediations": {
-      "type": "array",
-      "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
-      "items": {
-        "type": "object",
-        "required": [
-          "fixes",
-          "summary",
-          "diff"
-        ],
-        "properties": {
-          "fixes": {
-            "type": "array",
-            "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
-            "items": {
-              "type": "object",
-              "required": [
-                "cve"
-              ],
-              "properties": {
-                "cve": {
-                  "type": "string",
-                  "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
-                }
-              }
-            }
-          },
-          "summary": {
-            "type": "string",
-            "minLength": 1,
-            "description": "An overview of how the vulnerabilities were fixed."
-          },
-          "diff": {
-            "type": "string",
-            "minLength": 1,
-            "description": "A base64-encoded remediation code diff, compatible with git apply."
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index c9bc4ec411d..beb8801096b 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Naming/FileName
 # frozen_string_literal: true
 
 module Gitlab
@@ -144,3 +145,5 @@ module Gitlab
     end
   end
 end
+
+# rubocop:enable Naming/FileName
diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb
index 5251dd3d40a..f3c937ddd28 100644
--- a/lib/gitlab/ci/pipeline/chain/config/process.rb
+++ b/lib/gitlab/ci/pipeline/chain/config/process.rb
@@ -14,7 +14,7 @@ module Gitlab
               result = ::Gitlab::Ci::YamlProcessor.new(
                 @command.config_content, {
                   project: project,
-                  source_ref_path: @pipeline.source_ref_path,
+                  pipeline: @pipeline,
                   sha: @pipeline.sha,
                   source: @pipeline.source,
                   user: current_user,
diff --git a/lib/gitlab/ci/pipeline/chain/create_cross_database_associations.rb b/lib/gitlab/ci/pipeline/chain/create_cross_database_associations.rb
new file mode 100644
index 00000000000..bb5b4e722b7
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/create_cross_database_associations.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class CreateCrossDatabaseAssociations < Chain::Base
+          def perform!
+            # to be overridden in EE
+          end
+
+          def break?
+            false # to be overridden in EE
+          end
+        end
+      end
+    end
+  end
+end
+
+Gitlab::Ci::Pipeline::Chain::CreateCrossDatabaseAssociations.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::CreateCrossDatabaseAssociations')
diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb
index 321efa7854f..b5e48f210ad 100644
--- a/lib/gitlab/ci/pipeline/metrics.rb
+++ b/lib/gitlab/ci/pipeline/metrics.rb
@@ -51,6 +51,15 @@ module Gitlab
           ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
         end
 
+        def self.pipeline_builder_scoped_variables_histogram
+          name = :gitlab_ci_pipeline_builder_scoped_variables_duration
+          comment = 'Pipeline variables builder scoped_variables duration'
+          labels = {}
+          buckets = [0.01, 0.05, 0.1, 0.3, 0.5, 1, 2, 5, 10, 30, 60, 120]
+
+          ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
+        end
+
         def self.pipeline_processing_events_counter
           name = :gitlab_ci_pipeline_processing_events_total
           comment = 'Total amount of pipeline processing events'
diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index 9ad5d6538b7..72837b8ec22 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -11,11 +11,11 @@ module Gitlab
 
           delegate :dig, to: :@seed_attributes
 
-          def initialize(context, attributes, previous_stages, current_stage)
+          def initialize(context, attributes, stages_for_needs_lookup = [])
             @context = context
             @pipeline = context.pipeline
             @seed_attributes = attributes
-            @stages_for_needs_lookup = (previous_stages + [current_stage]).compact
+            @stages_for_needs_lookup = stages_for_needs_lookup.compact
             @needs_attributes = dig(:needs_attributes)
             @resource_group_key = attributes.delete(:resource_group_key)
             @job_variables = @seed_attributes.delete(:job_variables)
@@ -90,7 +90,7 @@ module Gitlab
               ::Ci::Bridge.new(attributes)
             else
               ::Ci::Build.new(attributes).tap do |build|
-                build.assign_attributes(self.class.environment_attributes_for(build))
+                build.assign_attributes(self.class.deployment_attributes_for(build))
               end
             end
           end
@@ -101,10 +101,10 @@ module Gitlab
                                               .to_resource
           end
 
-          def self.environment_attributes_for(build)
+          def self.deployment_attributes_for(build, environment = nil)
             return {} unless build.has_environment?
 
-            environment = Seed::Environment.new(build).to_resource
+            environment = Seed::Environment.new(build).to_resource if environment.nil?
 
             unless environment.persisted?
               if Feature.enabled?(:surface_environment_creation_failure, build.project, default_enabled: :yaml) &&
@@ -173,7 +173,7 @@ module Gitlab
           end
 
           def variable_expansion_errors
-            expanded_collection = evaluate_context.variables.sort_and_expand_all(@pipeline.project)
+            expanded_collection = evaluate_context.variables.sort_and_expand_all
             errors = expanded_collection.errors
             ["#{name}: #{errors}"] if errors
           end
@@ -244,5 +244,3 @@ module Gitlab
     end
   end
 end
-
-Gitlab::Ci::Pipeline::Seed::Build.prepend_mod_with('Gitlab::Ci::Pipeline::Seed::Build')
diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb
index 018fb260986..bc56fe9bef9 100644
--- a/lib/gitlab/ci/pipeline/seed/stage.rb
+++ b/lib/gitlab/ci/pipeline/seed/stage.rb
@@ -17,7 +17,7 @@ module Gitlab
             @previous_stages = previous_stages
 
             @builds = attributes.fetch(:builds).map do |attributes|
-              Seed::Build.new(context, attributes, previous_stages, self)
+              Seed::Build.new(context, attributes, previous_stages + [self])
             end
           end
 
diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb
index 39531e12f69..47ec82ac86c 100644
--- a/lib/gitlab/ci/reports/security/finding.rb
+++ b/lib/gitlab/ci/reports/security/finding.rb
@@ -17,7 +17,6 @@ module Gitlab
           attr_reader :name
           attr_reader :old_location
           attr_reader :project_fingerprint
-          attr_reader :raw_metadata
           attr_reader :report_type
           attr_reader :scanner
           attr_reader :scan
@@ -28,10 +27,13 @@ module Gitlab
           attr_reader :details
           attr_reader :signatures
           attr_reader :project_id
+          attr_reader :original_data
 
           delegate :file_path, :start_line, :end_line, to: :location
 
-          def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, metadata_version:, name:, raw_metadata:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists
+          alias_method :cve, :compare_key
+
+          def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists
             @compare_key = compare_key
             @confidence = confidence
             @identifiers = identifiers
@@ -40,7 +42,7 @@ module Gitlab
             @location = location
             @metadata_version = metadata_version
             @name = name
-            @raw_metadata = raw_metadata
+            @original_data = original_data
             @report_type = report_type
             @scanner = scanner
             @scan = scan
@@ -74,6 +76,10 @@ module Gitlab
               uuid
               details
               signatures
+              description
+              message
+              cve
+              solution
             ].each_with_object({}) do |key, hash|
               hash[key] = public_send(key) # rubocop:disable GitlabSecurity/PublicSend
             end
@@ -88,8 +94,8 @@ module Gitlab
             @location = new_location
           end
 
-          def unsafe?(severity_levels)
-            severity.in?(severity_levels)
+          def unsafe?(severity_levels, report_types)
+            severity.to_s.in?(severity_levels) && (report_types.blank? || report_type.to_s.in?(report_types) )
           end
 
           def eql?(other)
@@ -141,6 +147,30 @@ module Gitlab
             scanner <=> other.scanner
           end
 
+          def has_signatures?
+            signatures.present?
+          end
+
+          def raw_metadata
+            @raw_metadata ||= original_data.to_json
+          end
+
+          def description
+            original_data['description']
+          end
+
+          def message
+            original_data['message']
+          end
+
+          def solution
+            original_data['solution']
+          end
+
+          def location_data
+            original_data['location']
+          end
+
           private
 
           def generate_project_fingerprint
diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb
index 1ba2d909d99..417319cb5be 100644
--- a/lib/gitlab/ci/reports/security/report.rb
+++ b/lib/gitlab/ci/reports/security/report.rb
@@ -69,6 +69,10 @@ module Gitlab
 
             primary_scanner <=> other.primary_scanner
           end
+
+          def has_signatures?
+            findings.any?(&:has_signatures?)
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/reports/security/reports.rb b/lib/gitlab/ci/reports/security/reports.rb
index b7a5e36b108..b6372349f68 100644
--- a/lib/gitlab/ci/reports/security/reports.rb
+++ b/lib/gitlab/ci/reports/security/reports.rb
@@ -22,21 +22,24 @@ module Gitlab
             reports.values.flat_map(&:findings)
           end
 
-          def violates_default_policy_against?(target_reports, vulnerabilities_allowed, severity_levels)
-            unsafe_findings_count(target_reports, severity_levels) > vulnerabilities_allowed
+          def violates_default_policy_against?(target_reports, vulnerabilities_allowed, severity_levels, vulnerability_states, report_types = [])
+            unsafe_findings_count(target_reports, severity_levels, vulnerability_states, report_types) > vulnerabilities_allowed
           end
 
-          private
-
-          def findings_diff(target_reports)
-            findings - target_reports&.findings.to_a
+          def unsafe_findings_uuids(severity_levels, report_types)
+            findings.select { |finding| finding.unsafe?(severity_levels, report_types) }.map(&:uuid)
           end
 
-          def unsafe_findings_count(target_reports, severity_levels)
-            findings_diff(target_reports).count {|finding| finding.unsafe?(severity_levels)}
+          private
+
+          def unsafe_findings_count(target_reports, severity_levels, vulnerability_states, report_types)
+            new_uuids = unsafe_findings_uuids(severity_levels, report_types) - target_reports&.unsafe_findings_uuids(severity_levels, report_types).to_a
+            new_uuids.count
           end
         end
       end
     end
   end
 end
+
+Gitlab::Ci::Reports::Security::Reports.prepend_mod_with('Gitlab::Ci::Reports::Security::Reports')
diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
index adb5d430d46..89fd59d98f4 100644
--- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
@@ -179,3 +179,11 @@ include:
   - template: Security/License-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
   - template: Security/SAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
   - template: Security/Secret-Detection.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
+
+# The latest build job generates a dotenv report artifact with a CI_APPLICATION_TAG
+# that also includes the image digest. This configures Auto Deploy to receive
+# this artifact and use the updated CI_APPLICATION_TAG for deployments.
+.auto-deploy:
+  dependencies: [build]
+dast_environment_deploy:
+  dependencies: [build]
diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml
index f147ad9332d..426076c84a1 100644
--- a/lib/gitlab/ci/templates/Django.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml
@@ -1,54 +1,76 @@
-# To contribute improvements to CI/CD templates, please follow the Development guide at:
-# https://docs.gitlab.com/ee/development/cicd/templates.html
-# This specific template is located at:
-# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Django.gitlab-ci.yml
-
-# Official framework image. Look for the different tagged releases at:
-# https://hub.docker.com/r/library/python
-image: python:latest
-
-# Pick zero or more services to be used on all builds.
-# Only needed when using a docker container to run your tests in.
-# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service
-services:
-  - mysql:latest
-  - postgres:latest
+# This example is for testing Django with MySQL.
+#
+# The test CI/CD variables MYSQL_DB, MYSQL_USER and MYSQL_PASS can be set in the project settings at:
+#     Settings --> CI/CD --> Variables
+#
+# The Django settings in settings.py, used in tests, might look similar to:
+#
+#  DATABASES = {
+#      'default': {
+#         'ENGINE': 'django.db.backends.mysql',
+#         'NAME': os.environ.get('MYSQL_DATABASE'),
+#        	'USER':  os.environ.get('MYSQL_USER'),
+#        	'PASSWORD': os.environ.get('MYSQL_PASSWORD'),
+#       	'HOST': 'mysql',
+#     	  'PORT': '3306',
+#         'CONN_MAX_AGE':60,
+#      },
+#  }
+#
+# It is possible to use '--settings' to specify a custom settings file on the command line below or use an environment
+# variable to trigger an include on the bottom of your settings.py:
+#   if os.environ.get('DJANGO_CONFIG')=='test':
+#       from .settings_test import *
+#
+# It is also possible to hardcode the database name and credentials in the settings.py file and in the .gitlab-ci.yml file.
+#
+# The mysql service needs some variables too. See https://hub.docker.com/_/mysql for possible mysql env variables
+# Note that when using a service in GitLab CI/CD that needs environment variables to run, only variables defined in
+# .gitlab-ci.yml are passed to the service and variables defined in the GitLab UI are not.
+# https://gitlab.com/gitlab-org/gitlab/-/issues/30178
 
 variables:
-  POSTGRES_DB: database_name
+  # DJANGO_CONFIG: "test"
+  MYSQL_DATABASE: $MYSQL_DB
+  MYSQL_ROOT_PASSWORD: $MYSQL_PASS
+  MYSQL_USER: $MYSQL_USER
+  MYSQL_PASSWORD: $MYSQL_PASS
 
-# This folder is cached between builds
-# https://docs.gitlab.com/ee/ci/yaml/index.html#cache
-cache:
-  paths:
-    - ~/.cache/pip/
+default:
+  image: ubuntu:20.04
+  #
+  # Pick zero or more services to be used on all builds.
+  # Only needed when using a docker container to run your tests in.
+  # Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service
+  services:
+    - mysql:8.0
+  #
+  # This folder is cached between builds
+  # http://docs.gitlab.com/ee/ci/yaml/README.html#cache
+  cache:
+    paths:
+      - ~/.cache/pip/
+  before_script:
+    - apt -y update
+    - apt -y install apt-utils
+    - apt -y install net-tools python3.8 python3-pip mysql-client libmysqlclient-dev
+    - apt -y upgrade
+    - pip3 install -r requirements.txt
 
-# This is a basic example for a gem or script which doesn't use
-# services such as redis or postgres
-before_script:
-  - python -V  # Print out python version for debugging
-  # Uncomment next line if your Django app needs a JS runtime:
-  # - apt-get update -q && apt-get install nodejs -yqq
-  - pip install -r requirements.txt
 
-# To get Django tests to work you may need to create a settings file using
-# the following DATABASES:
-#
-# DATABASES = {
-#     'default': {
-#        'ENGINE': 'django.db.backends.postgresql_psycopg2',
-#        'NAME': 'ci',
-#        'USER': 'postgres',
-#        'PASSWORD': 'postgres',
-#        'HOST': 'postgres',
-#        'PORT': '5432',
-#    },
-# }
-#
-# and then adding `--settings app.settings.ci` (or similar) to the test command
+migrations:
+  stage: build
+  script:
+    - python3 manage.py makemigrations
+    # - python3 manage.py makemigrations myapp
+    - python3 manage.py migrate
+    - python3 manage.py check
+
 
-test:
-  variables:
-    DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/$POSTGRES_DB"
+django-tests:
+  stage: test
   script:
-    - python manage.py test
+    # The MYSQL user only gets permissions for MYSQL_DB, so Django can't create a test database.
+    - echo "GRANT ALL on *.* to '${MYSQL_USER}';"| mysql -u root --password="${MYSQL_ROOT_PASSWORD}" -h mysql
+    # use python3 explicitly. see https://wiki.ubuntu.com/Python/3
+    - python3 manage.py test
diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml
index 56899614cc6..99fd9870b1d 100644
--- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml
@@ -70,7 +70,7 @@ browser_performance:
     reports:
       browser_performance: browser-performance.json
   rules:
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$BROWSER_PERFORMANCE_DISABLED'
       when: never
diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml
index 56899614cc6..99fd9870b1d 100644
--- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml
@@ -70,7 +70,7 @@ browser_performance:
     reports:
       browser_performance: browser-performance.json
   rules:
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$BROWSER_PERFORMANCE_DISABLED'
       when: never
diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml
index 6a3b0cfa9e7..211adc9bd5b 100644
--- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml
@@ -3,7 +3,7 @@
 #          This template is scheduled for removal when testing is complete: https://gitlab.com/gitlab-org/gitlab/-/issues/337987
 
 variables:
-  AUTO_BUILD_IMAGE_VERSION: 'v1.3.1'
+  AUTO_BUILD_IMAGE_VERSION: 'v1.5.0'
 
 build:
   stage: build
@@ -23,6 +23,9 @@ build:
         export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG}
       fi
     - /build/build.sh
+  artifacts:
+    reports:
+      dotenv: gl-auto-build-variables.env
   rules:
     - if: '$BUILD_DISABLED'
       when: never
diff --git a/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml
index 31ca68c57d7..11f8376f0b4 100644
--- a/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml
@@ -9,6 +9,6 @@ cloud_formation:
   rules:
     - if: '($AUTO_DEVOPS_PLATFORM_TARGET != "EC2") || ($AUTO_DEVOPS_PLATFORM_TARGET != "ECS")'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE'
+    - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG'
       when: never
     - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
index 65a58130962..28ac627f103 100644
--- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
 variables:
-  DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.14.0'
+  DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.17.0'
 
 .dast-auto-deploy:
   image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}"
@@ -10,6 +10,7 @@ dast_environment_deploy:
   script:
     - auto-deploy check_kube_domain
     - auto-deploy download_chart
+    - auto-deploy use_kube_context || true
     - auto-deploy ensure_namespace
     - auto-deploy initialize_tiller
     - auto-deploy create_secret
@@ -29,7 +30,7 @@ dast_environment_deploy:
     - if: $DAST_WEBSITE  # we don't need to create a review app if a URL is already given
       when: never
     - if: $CI_COMMIT_BRANCH &&
-          $CI_KUBERNETES_ACTIVE &&
+          ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) &&
           $GITLAB_FEATURES =~ /\bdast\b/
 
 stop_dast_environment:
@@ -38,6 +39,7 @@ stop_dast_environment:
   variables:
     GIT_STRATEGY: none
   script:
+    - auto-deploy use_kube_context || true
     - auto-deploy initialize_tiller
     - auto-deploy delete
   environment:
@@ -52,6 +54,6 @@ stop_dast_environment:
     - if: $DAST_WEBSITE  # we don't need to create a review app if a URL is already given
       when: never
     - if: $CI_COMMIT_BRANCH &&
-          $CI_KUBERNETES_ACTIVE &&
+          ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) &&
           $GITLAB_FEATURES =~ /\bdast\b/
       when: always
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
index 58f13746a1f..973db26bf2d 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
 variables:
-  AUTO_DEPLOY_IMAGE_VERSION: 'v2.14.0'
+  AUTO_DEPLOY_IMAGE_VERSION: 'v2.17.0'
 
 .auto-deploy:
   image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}"
@@ -11,6 +11,7 @@ review:
   script:
     - auto-deploy check_kube_domain
     - auto-deploy download_chart
+    - auto-deploy use_kube_context || true
     - auto-deploy ensure_namespace
     - auto-deploy initialize_tiller
     - auto-deploy create_secret
@@ -24,7 +25,7 @@ review:
     paths: [environment_url.txt, tiller.log]
     when: always
   rules:
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
       when: never
@@ -38,6 +39,7 @@ stop_review:
   variables:
     GIT_STRATEGY: none
   script:
+    - auto-deploy use_kube_context || true
     - auto-deploy initialize_tiller
     - auto-deploy delete
   environment:
@@ -45,7 +47,7 @@ stop_review:
     action: stop
   allow_failure: true
   rules:
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
       when: never
@@ -66,6 +68,7 @@ staging:
   script:
     - auto-deploy check_kube_domain
     - auto-deploy download_chart
+    - auto-deploy use_kube_context || true
     - auto-deploy ensure_namespace
     - auto-deploy initialize_tiller
     - auto-deploy create_secret
@@ -74,7 +77,7 @@ staging:
     name: staging
     url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN
   rules:
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'
       when: never
@@ -91,6 +94,7 @@ canary:
   script:
     - auto-deploy check_kube_domain
     - auto-deploy download_chart
+    - auto-deploy use_kube_context || true
     - auto-deploy ensure_namespace
     - auto-deploy initialize_tiller
     - auto-deploy create_secret
@@ -101,7 +105,7 @@ canary:
   rules:
     - if: '$CI_DEPLOY_FREEZE != null'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'
       when: never
@@ -114,6 +118,7 @@ canary:
   script:
     - auto-deploy check_kube_domain
     - auto-deploy download_chart
+    - auto-deploy use_kube_context || true
     - auto-deploy ensure_namespace
     - auto-deploy initialize_tiller
     - auto-deploy create_secret
@@ -132,7 +137,7 @@ production:
   rules:
     - if: '$CI_DEPLOY_FREEZE != null'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$STAGING_ENABLED'
       when: never
@@ -150,7 +155,7 @@ production_manual:
   rules:
     - if: '$CI_DEPLOY_FREEZE != null'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$INCREMENTAL_ROLLOUT_ENABLED'
       when: never
@@ -168,6 +173,7 @@ production_manual:
   script:
     - auto-deploy check_kube_domain
     - auto-deploy download_chart
+    - auto-deploy use_kube_context || true
     - auto-deploy ensure_namespace
     - auto-deploy initialize_tiller
     - auto-deploy create_secret
@@ -188,7 +194,7 @@ production_manual:
   rules:
     - if: '$CI_DEPLOY_FREEZE != null'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"'
       when: never
@@ -203,7 +209,7 @@ production_manual:
   rules:
     - if: '$CI_DEPLOY_FREEZE != null'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$INCREMENTAL_ROLLOUT_MODE == "manual"'
       when: never
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
index 530ab1d0f99..248040b8b18 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
@@ -21,7 +21,7 @@ review:
     paths: [environment_url.txt, tiller.log]
     when: always
   rules:
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
       when: never
@@ -42,7 +42,7 @@ stop_review:
     action: stop
   allow_failure: true
   rules:
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
       when: never
@@ -71,7 +71,7 @@ staging:
     name: staging
     url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN
   rules:
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'
       when: never
@@ -96,7 +96,7 @@ canary:
     name: production
     url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN
   rules:
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'
       when: never
@@ -125,7 +125,7 @@ canary:
 production:
   <<: *production_template
   rules:
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$STAGING_ENABLED'
       when: never
@@ -141,7 +141,7 @@ production_manual:
   <<: *production_template
   allow_failure: false
   rules:
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$INCREMENTAL_ROLLOUT_ENABLED'
       when: never
@@ -177,7 +177,7 @@ production_manual:
   resource_group: production
   allow_failure: true
   rules:
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"'
       when: never
@@ -190,7 +190,7 @@ production_manual:
 .timed_rollout_template: &timed_rollout_template
   <<: *rollout_template
   rules:
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$INCREMENTAL_ROLLOUT_MODE == "manual"'
       when: never
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml
index 7efbcab221b..ab3bc511cba 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml
@@ -16,7 +16,7 @@ review_ec2:
   rules:
     - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "EC2"'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE'
+    - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG'
       when: never
     - if: '$REVIEW_DISABLED'
       when: never
@@ -32,7 +32,7 @@ production_ec2:
   rules:
     - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "EC2"'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE'
+    - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG'
       when: never
     - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'
       when: never
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
index 332c58c8695..9bb2ba69d84 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
@@ -42,7 +42,7 @@ review_ecs:
   rules:
     - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "ECS"'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE'
+    - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG'
       when: never
     - if: '$REVIEW_DISABLED'
       when: never
@@ -58,7 +58,7 @@ stop_review_ecs:
   rules:
     - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "ECS"'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE'
+    - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG'
       when: never
     - if: '$REVIEW_DISABLED'
       when: never
@@ -77,7 +77,7 @@ review_fargate:
   rules:
     - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "FARGATE"'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE'
+    - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG'
       when: never
     - if: '$REVIEW_DISABLED'
       when: never
@@ -93,7 +93,7 @@ stop_review_fargate:
   rules:
     - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "FARGATE"'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE'
+    - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG'
       when: never
     - if: '$REVIEW_DISABLED'
       when: never
@@ -107,7 +107,7 @@ production_ecs:
   rules:
     - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "ECS"'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE'
+    - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG'
       when: never
     - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'
       when: never
@@ -118,7 +118,7 @@ production_fargate:
   rules:
     - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "FARGATE"'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE'
+    - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG'
       when: never
     - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'
       when: never
diff --git a/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml
index 1ec1aa60d88..d55c126eeb7 100644
--- a/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml
@@ -72,7 +72,7 @@
   rules:
     - if: '$MIGRATE_HELM_2TO3 != "true"'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
       when: never
@@ -89,7 +89,7 @@ review:helm-2to3:cleanup:
   rules:
     - if: '$MIGRATE_HELM_2TO3 != "true" && $CLEANUP_HELM_2TO3 == null'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
       when: never
@@ -104,7 +104,7 @@ review:helm-2to3:cleanup:
   rules:
     - if: '$MIGRATE_HELM_2TO3 != "true"'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'
       when: never
@@ -119,7 +119,7 @@ staging:helm-2to3:cleanup:
   rules:
     - if: '$MIGRATE_HELM_2TO3 != "true" && $CLEANUP_HELM_2TO3 == null'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'
       when: never
@@ -132,7 +132,7 @@ staging:helm-2to3:cleanup:
   rules:
     - if: '$MIGRATE_HELM_2TO3 != "true"'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
       when: manual
@@ -145,7 +145,7 @@ production:helm-2to3:cleanup:
   rules:
     - if: '$MIGRATE_HELM_2TO3 != "true" && $CLEANUP_HELM_2TO3 == null'
       when: never
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
       when: manual
diff --git a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml
index 9a7c513c25f..8e34388893a 100644
--- a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml
@@ -23,7 +23,7 @@ load_performance:
     reports:
       load_performance: load-performance.json
   rules:
-    - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+    - if: '($CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == "") && ($KUBECONFIG == null || $KUBECONFIG == "")'
       when: never
     - if: '$LOAD_PERFORMANCE_DISABLED'
       when: never
diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml
new file mode 100644
index 00000000000..b763705857e
--- /dev/null
+++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml
@@ -0,0 +1,34 @@
+variables:
+  # Setting this variable will affect all Security templates
+  # (SAST, Dependency Scanning, ...)
+  SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
+  SAST_EXCLUDED_PATHS: "spec, test, tests, tmp"
+
+iac-sast:
+  stage: test
+  artifacts:
+    reports:
+      sast: gl-sast-report.json
+  rules:
+    - when: never
+  # `rules` must be overridden explicitly by each child job
+  # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444
+  variables:
+    SEARCH_MAX_DEPTH: 4
+  allow_failure: true
+  script:
+    - /analyzer run
+
+kics-iac-sast:
+  extends: iac-sast
+  image:
+    name: "$SAST_ANALYZER_IMAGE"
+  variables:
+    SAST_ANALYZER_IMAGE_TAG: 0
+    SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG"
+  rules:
+    - if: $SAST_DISABLED
+      when: never
+    - if: $SAST_EXCLUDED_ANALYZERS =~ /kics/
+      when: never
+    - if: $CI_COMMIT_BRANCH
diff --git a/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml b/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml
new file mode 100644
index 00000000000..f1b1c20b4e0
--- /dev/null
+++ b/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml
@@ -0,0 +1,47 @@
+# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# https://docs.gitlab.com/ee/development/cicd/templates.html
+# This specific template is located at:
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml
+
+# Build and publish a tag/branch to Gitlab Docker Registry using Kaniko and Gitlab Docker executor.
+# Kaniko can build Docker images without using Docker-In-Docker and it's permission
+# drawbacks. No additional configuration required.
+kaniko-build:
+  variables:
+    # Additional options for Kaniko executor.
+    # For more details see https://github.com/GoogleContainerTools/kaniko/blob/master/README.md#additional-flags
+    KANIKO_ARGS: ""
+  stage: build
+  image:
+    # For latest releases see https://github.com/GoogleContainerTools/kaniko/releases
+    # Only debug/*-debug versions of the Kaniko image are known to work within Gitlab CI
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
+  script:
+    # Compose docker tag name
+    # Git Branch/Tag to Docker Image Tag Mapping
+    #   * Default Branch: main -> latest
+    #   * Branch: feature/my-feature -> branch-feature-my-feature
+    #   * Tag: v1.0.0/beta2 -> v1.0.0-beta2
+    - |
+      if [ "$CI_COMMIT_REF_NAME" = $CI_DEFAULT_BRANCH ]; then
+        VERSION="latest"
+      elif [ -n "$CI_COMMIT_TAG" ];then
+        NOSLASH=$(echo "$CI_COMMIT_TAG" | tr -s / - )
+        SANITIZED="${NOSLASH//[^a-zA-Z0-9\-\.]/}"
+        VERSION="$SANITIZED"
+      else \
+        NOSLASH=$(echo "$CI_COMMIT_REF_NAME" | tr -s / - )
+        SANITIZED="${NOSLASH//[^a-zA-Z0-9\-]/}"
+        VERSION="branch-$SANITIZED"
+      fi
+    - echo $VERSION
+    - mkdir -p /kaniko/.docker
+    # Write credentials to access Gitlab Container Registry within the runner/ci
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json
+    # Build and push the container. To disable push add --no-push
+    - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $CI_REGISTRY_IMAGE:$VERSION $KANIKO_ARGS
+  # Run this job in a branch/tag where a Dockerfile exists
+  rules:
+    - exists:
+        - Dockerfile
diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml
index ceeefa8aea6..544774d3b06 100644
--- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml
@@ -1,7 +1,7 @@
 # To contribute improvements to CI/CD templates, please follow the Development guide at:
 # https://docs.gitlab.com/ee/development/cicd/templates.html
 # This specific template is located at:
-# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/API-Fuzzing.lastest.gitlab-ci.yml
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml
 
 # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/
 #
diff --git a/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml
index ed4876c2bcc..6b861510eef 100644
--- a/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml
@@ -12,7 +12,7 @@
 # List of available variables: https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/#available-variables
 
 variables:
-  CIS_ANALYZER_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/cluster-image-scanning:0
+  CIS_ANALYZER_IMAGE: registry.gitlab.com/security-products/cluster-image-scanning:0
 
 cluster_image_scanning:
   image: "$CIS_ANALYZER_IMAGE"
diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
index 0802868d67f..0ecbe5e14b8 100644
--- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
@@ -51,7 +51,7 @@ dast:
           $REVIEW_DISABLED
       when: never
     - if: $CI_COMMIT_BRANCH &&
-          $CI_KUBERNETES_ACTIVE &&
+          ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) &&
           $GITLAB_FEATURES =~ /\bdast\b/
     - if: $CI_COMMIT_BRANCH &&
           $GITLAB_FEATURES =~ /\bdast\b/
diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml
index ac7d87a4cda..3d07674c377 100644
--- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml
@@ -1,7 +1,7 @@
 # To contribute improvements to CI/CD templates, please follow the Development guide at:
 # https://docs.gitlab.com/ee/development/cicd/templates.html
 # This specific template is located at:
-# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST.lastest.gitlab-ci.yml
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml
 
 # To use this template, add the following to your .gitlab-ci.yml file:
 #
@@ -52,7 +52,7 @@ dast:
           $DAST_API_SPECIFICATION == null
       when: never
     - if: $CI_COMMIT_BRANCH &&
-          $CI_KUBERNETES_ACTIVE &&
+          ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) &&
           $GITLAB_FEATURES =~ /\bdast\b/
     - if: $CI_COMMIT_BRANCH &&
           $DAST_WEBSITE
diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
index aa7b394a13c..197ce2438e6 100644
--- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
@@ -74,6 +74,9 @@ gemnasium-maven-dependency_scanning:
     # override the analyzer image with a custom value. This may be subject to change or
     # breakage across GitLab releases.
     DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium-maven:$DS_MAJOR_VERSION"
+    # Stop reporting Gradle as "maven".
+    # See https://gitlab.com/gitlab-org/gitlab/-/issues/338252
+    DS_REPORT_PACKAGE_MANAGER_MAVEN_WHEN_JAVA: "false"
   rules:
     - if: $DEPENDENCY_SCANNING_DISABLED
       when: never
@@ -97,6 +100,9 @@ gemnasium-python-dependency_scanning:
     # override the analyzer image with a custom value. This may be subject to change or
     # breakage across GitLab releases.
     DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium-python:$DS_MAJOR_VERSION"
+    # Stop reporting Pipenv and Setuptools as "pip".
+    # See https://gitlab.com/gitlab-org/gitlab/-/issues/338252
+    DS_REPORT_PACKAGE_MANAGER_PIP_WHEN_PYTHON: "false"
   rules:
     - if: $DEPENDENCY_SCANNING_DISABLED
       when: never
diff --git a/lib/gitlab/ci/templates/Security/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST-IaC.latest.gitlab-ci.yml
new file mode 100644
index 00000000000..8c0d72ff282
--- /dev/null
+++ b/lib/gitlab/ci/templates/Security/SAST-IaC.latest.gitlab-ci.yml
@@ -0,0 +1,2 @@
+include:
+  template: Jobs/SAST-IaC.latest.gitlab-ci.yml
diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
index 081a3a6cc78..e554742735c 100644
--- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
@@ -7,20 +7,17 @@ include:
   - template: Terraform/Base.latest.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
 
 stages:
-  - init
   - validate
   - build
   - deploy
-  - cleanup
-
-init:
-  extends: .terraform:init
 
 fmt:
   extends: .terraform:fmt
+  needs: []
 
 validate:
   extends: .terraform:validate
+  needs: []
 
 build:
   extends: .terraform:build
diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
index 3a70e6bc4b8..a0ec07e61e1 100644
--- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
@@ -21,18 +21,11 @@ cache:
   paths:
     - ${TF_ROOT}/.terraform/
 
-.terraform:init: &terraform_init
-  stage: init
-  script:
-    - cd ${TF_ROOT}
-    - gitlab-terraform init
-
 .terraform:fmt: &terraform_fmt
   stage: validate
-  needs: []
   script:
     - cd ${TF_ROOT}
-    - gitlab-terraform fmt -check -recursive
+    - gitlab-terraform fmt
   allow_failure: true
 
 .terraform:validate: &terraform_validate
@@ -60,10 +53,9 @@ cache:
     - cd ${TF_ROOT}
     - gitlab-terraform apply
   resource_group: ${TF_STATE_NAME}
-  when: manual
-  only:
-    variables:
-      - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      when: manual
 
 .terraform:destroy: &terraform_destroy
   stage: cleanup
diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml
index 22c40d8a8b8..4f63ff93d4d 100644
--- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml
@@ -13,7 +13,7 @@ stages:
 
 a11y:
   stage: accessibility
-  image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:5.3.0-gitlab.3
+  image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:6.0.1
   script: /gitlab-accessibility.sh $a11y_urls
   allow_failure: true
   artifacts:
diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb
index 25075cc8f90..7d08f0230fc 100644
--- a/lib/gitlab/ci/trace.rb
+++ b/lib/gitlab/ci/trace.rb
@@ -78,7 +78,7 @@ module Gitlab
       end
 
       def archived_trace_exist?
-        trace_artifact&.exists?
+        archived?
       end
 
       def live_trace_exist?
@@ -156,7 +156,7 @@ module Gitlab
 
       def read_stream
         stream = Gitlab::Ci::Trace::Stream.new do
-          if trace_artifact
+          if archived?
             trace_artifact.open
           elsif job.trace_chunks.any?
             Gitlab::Ci::Trace::ChunkedIO.new(job)
@@ -174,7 +174,7 @@ module Gitlab
 
       def unsafe_write!(mode, &blk)
         stream = Gitlab::Ci::Trace::Stream.new do
-          if trace_artifact
+          if archived?
             raise AlreadyArchivedError, 'Could not write to the archived trace'
           elsif current_path
             File.open(current_path, mode)
@@ -195,7 +195,7 @@ module Gitlab
       def unsafe_archive!
         raise ArchiveError, 'Job is not finished yet' unless job.complete?
 
-        already_archived?.tap do |archived|
+        archived?.tap do |archived|
           destroy_any_orphan_trace_data!
           raise AlreadyArchivedError, 'Could not archive again' if archived
         end
@@ -218,7 +218,7 @@ module Gitlab
         end
       end
 
-      def already_archived?
+      def archived?
         # TODO check checksum to ensure archive completed successfully
         # See https://gitlab.com/gitlab-org/gitlab/-/issues/259619
         trace_artifact&.archived_trace_exists?
@@ -227,11 +227,12 @@ module Gitlab
       def destroy_any_orphan_trace_data!
         return unless trace_artifact
 
-        if already_archived?
-          # An archive already exists, so make sure to remove the trace chunks
+        if archived?
+          # An archive file exists, so remove the trace chunks
           erase_trace_chunks!
         else
-          # An archive already exists, but its associated file does not, so remove it
+          # A trace artifact record exists with no archive file
+          # but an archive was attempted, so cleanup the associated record
           trace_artifact.destroy!
         end
       end
diff --git a/lib/gitlab/ci/trace/archive.rb b/lib/gitlab/ci/trace/archive.rb
index 5047cf04562..d4a451ca526 100644
--- a/lib/gitlab/ci/trace/archive.rb
+++ b/lib/gitlab/ci/trace/archive.rb
@@ -62,7 +62,7 @@ module Gitlab
           trace_metadata.update!(remote_checksum: remote_checksum)
 
           unless trace_metadata.remote_checksum_valid?
-            metrics.increment_error_counter(type: :archive_invalid_checksum)
+            metrics.increment_error_counter(error_reason: :archive_invalid_checksum)
           end
         end
 
diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb
index 174a5f184ff..f3ded3cda4a 100644
--- a/lib/gitlab/ci/trace/metrics.rb
+++ b/lib/gitlab/ci/trace/metrics.rb
@@ -21,7 +21,7 @@ module Gitlab
           :corrupted  # malformed trace found after comparing CRC32 and size
         ].freeze
 
-        TRACE_ERROR_TYPES = [
+        TRACE_ERROR_REASONS = [
           :chunks_invalid_size,       # used to be :corrupted
           :chunks_invalid_checksum,   # used to be :invalid
           :archive_invalid_checksum   # malformed trace found into object store after comparing MD5
@@ -39,12 +39,12 @@ module Gitlab
           self.class.trace_bytes.increment({}, size.to_i)
         end
 
-        def increment_error_counter(type: :unknown)
-          unless TRACE_ERROR_TYPES.include?(type)
-            raise ArgumentError, "unknown error type: #{type}"
+        def increment_error_counter(error_reason: :unknown)
+          unless TRACE_ERROR_REASONS.include?(error_reason)
+            raise ArgumentError, "unknown error reason: #{error_reason}"
           end
 
-          self.class.trace_errors_counter.increment(type: type)
+          self.class.trace_errors_counter.increment(error_reason: error_reason)
         end
 
         def observe_migration_duration(seconds)
diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb
new file mode 100644
index 00000000000..f4c5a06af97
--- /dev/null
+++ b/lib/gitlab/ci/variables/builder.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Variables
+      class Builder
+        include ::Gitlab::Utils::StrongMemoize
+
+        def initialize(pipeline)
+          @pipeline = pipeline
+        end
+
+        def scoped_variables(job, environment:, dependencies:)
+          Gitlab::Ci::Variables::Collection.new.tap do |variables|
+            variables.concat(predefined_variables(job)) if pipeline.predefined_vars_in_builder_enabled?
+          end
+        end
+
+        private
+
+        attr_reader :pipeline
+
+        def predefined_variables(job)
+          Gitlab::Ci::Variables::Collection.new.tap do |variables|
+            variables.append(key: 'CI_JOB_NAME', value: job.name)
+            variables.append(key: 'CI_JOB_STAGE', value: job.stage)
+            variables.append(key: 'CI_JOB_MANUAL', value: 'true') if job.action?
+            variables.append(key: 'CI_PIPELINE_TRIGGERED', value: 'true') if job.trigger_request
+
+            variables.append(key: 'CI_NODE_INDEX', value: job.options[:instance].to_s) if job.options&.include?(:instance)
+            variables.append(key: 'CI_NODE_TOTAL', value: ci_node_total_value(job).to_s)
+
+            # legacy variables
+            variables.append(key: 'CI_BUILD_NAME', value: job.name)
+            variables.append(key: 'CI_BUILD_STAGE', value: job.stage)
+            variables.append(key: 'CI_BUILD_TRIGGERED', value: 'true') if job.trigger_request
+            variables.append(key: 'CI_BUILD_MANUAL', value: 'true') if job.action?
+          end
+        end
+
+        def ci_node_total_value(job)
+          parallel = job.options&.dig(:parallel)
+          parallel = parallel.dig(:total) if parallel.is_a?(Hash)
+          parallel || 1
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb
index 09c75a2b3f1..a00c1da97ea 100644
--- a/lib/gitlab/ci/variables/collection.rb
+++ b/lib/gitlab/ci/variables/collection.rb
@@ -89,9 +89,7 @@ module Gitlab
           end
         end
 
-        def sort_and_expand_all(project, keep_undefined: false)
-          return self if Feature.disabled?(:variable_inside_variable, project, default_enabled: :yaml)
-
+        def sort_and_expand_all(keep_undefined: false)
           sorted = Sort.new(self)
           return self.class.new(self, sorted.errors) unless sorted.valid?
 
diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb
index a97c7050fbb..6215ba40ebe 100644
--- a/lib/gitlab/ci/yaml_processor/result.rb
+++ b/lib/gitlab/ci/yaml_processor/result.rb
@@ -80,7 +80,6 @@ module Gitlab
             cache: job[:cache],
             resource_group_key: job[:resource_group],
             scheduling_type: job[:scheduling_type],
-            secrets: job[:secrets],
             options: {
               image: job[:image],
               services: job[:services],
diff --git a/lib/gitlab/config_checker/external_database_checker.rb b/lib/gitlab/config_checker/external_database_checker.rb
index a56f2413615..54320b7ff9a 100644
--- a/lib/gitlab/config_checker/external_database_checker.rb
+++ b/lib/gitlab/config_checker/external_database_checker.rb
@@ -6,7 +6,7 @@ module Gitlab
       extend self
 
       def check
-        return [] if Gitlab::Database.main.postgresql_minimum_supported_version?
+        return [] if ApplicationRecord.database.postgresql_minimum_supported_version?
 
         [
           {
@@ -15,7 +15,7 @@ module Gitlab
                        '%{pg_version_minimum} is required for this version of GitLab. ' \
                        'Please upgrade your environment to a supported PostgreSQL version, ' \
                        'see %{pg_requirements_url} for details.') % {
-                                                                      pg_version_current: Gitlab::Database.main.version,
+                                                                      pg_version_current: ApplicationRecord.database.version,
                                                                       pg_version_minimum: Gitlab::Database::MINIMUM_POSTGRES_VERSION,
                                                                       pg_requirements_url: 'database requirements'
                                                                     }
diff --git a/lib/gitlab/container_repository/tags/cache.rb b/lib/gitlab/container_repository/tags/cache.rb
new file mode 100644
index 00000000000..ff457fb9219
--- /dev/null
+++ b/lib/gitlab/container_repository/tags/cache.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ContainerRepository
+    module Tags
+      class Cache
+        def initialize(container_repository)
+          @container_repository = container_repository
+          @cached_tag_names = Set.new
+        end
+
+        def populate(tags)
+          return if tags.empty?
+
+          # This will load all tags in one Redis roundtrip
+          # the maximum number of tags is configurable and is set to 200 by default.
+          # https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/packages/container_registry/index.md#set-cleanup-limits-to-conserve-resources
+          keys = tags.map(&method(:cache_key))
+          cached_tags_count = 0
+
+          ::Gitlab::Redis::Cache.with do |redis|
+            tags.zip(redis.mget(keys)).each do |tag, created_at|
+              next unless created_at
+
+              tag.created_at = DateTime.rfc3339(created_at)
+              @cached_tag_names << tag.name
+              cached_tags_count += 1
+            end
+          end
+
+          cached_tags_count
+        end
+
+        def insert(tags, max_ttl_in_seconds)
+          return unless max_ttl_in_seconds
+          return if tags.empty?
+
+          # tags with nil created_at are not cacheable
+          # tags already cached don't need to be cached again
+          cacheable_tags = tags.select do |tag|
+            tag.created_at.present? && !tag.name.in?(@cached_tag_names)
+          end
+
+          return if cacheable_tags.empty?
+
+          now = Time.zone.now
+
+          ::Gitlab::Redis::Cache.with do |redis|
+            # we use a pipeline instead of a MSET because each tag has
+            # a specific ttl
+            redis.pipelined do
+              cacheable_tags.each do |tag|
+                created_at = tag.created_at
+                # ttl is the max_ttl_in_seconds reduced by the number
+                # of seconds that the tag has already existed
+                ttl = max_ttl_in_seconds - (now - created_at).seconds
+                ttl = ttl.to_i
+                redis.set(cache_key(tag), created_at.rfc3339, ex: ttl) if ttl > 0
+              end
+            end
+          end
+        end
+
+        private
+
+        def cache_key(tag)
+          "container_repository:{#{@container_repository.id}}:tag:#{tag.name}:created_at"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb
index 0e3fa8b8d87..bdae59e7e3c 100644
--- a/lib/gitlab/content_security_policy/config_loader.rb
+++ b/lib/gitlab/content_security_policy/config_loader.rb
@@ -19,30 +19,42 @@ module Gitlab
           'font_src' => "'self'",
           'form_action' => "'self' https: http:",
           'frame_ancestors' => "'self'",
-          'frame_src' => "'self' https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com",
+          'frame_src' => ContentSecurityPolicy::Directives.frame_src,
           'img_src' => "'self' data: blob: http: https:",
           'manifest_src' => "'self'",
           'media_src' => "'self'",
-          'script_src' => "'strict-dynamic' 'self' 'unsafe-inline' 'unsafe-eval' https://www.google.com/recaptcha/ https://www.recaptcha.net https://apis.google.com",
+          'script_src' => ContentSecurityPolicy::Directives.script_src,
           'style_src' => "'self' 'unsafe-inline'",
-          'worker_src' => "'self' blob: data:",
+          'worker_src' => "#{Gitlab::Utils.append_path(Gitlab.config.gitlab.url, 'assets/')} blob: data:",
           'object_src' => "'none'",
           'report_uri' => nil
         }
 
-        # frame-src was deprecated in CSP level 2 in favor of child-src
-        # CSP level 3 "undeprecated" frame-src and browsers fall back on child-src if it's missing
-        # However Safari seems to read child-src first so we'll just keep both equal
-        directives['child_src'] = directives['frame_src']
-
         # connect_src with 'self' includes https/wss variations of the origin,
         # however, safari hasn't covered this yet and we need to explicitly add
         # support for websocket origins until Safari catches up with the specs
+        if Rails.env.development?
+          allow_webpack_dev_server(directives)
+          allow_letter_opener(directives)
+          allow_customersdot(directives) if ENV['CUSTOMER_PORTAL_URL'].present?
+        end
+
         allow_websocket_connections(directives)
-        allow_webpack_dev_server(directives) if Rails.env.development?
         allow_cdn(directives, Settings.gitlab.cdn_host) if Settings.gitlab.cdn_host.present?
-        allow_customersdot(directives) if Rails.env.development? && ENV['CUSTOMER_PORTAL_URL'].present?
         allow_sentry(directives) if Gitlab.config.sentry&.enabled && Gitlab.config.sentry&.clientside_dsn
+        allow_framed_gitlab_paths(directives)
+
+        # The follow section contains workarounds to patch Safari's lack of support for CSP Level 3
+        # See https://gitlab.com/gitlab-org/gitlab/-/issues/343579
+        # frame-src was deprecated in CSP level 2 in favor of child-src
+        # CSP level 3 "undeprecated" frame-src and browsers fall back on child-src if it's missing
+        # However Safari seems to read child-src first so we'll just keep both equal
+        append_to_directive(directives, 'child_src', directives['frame_src'])
+
+        # Safari also doesn't support worker-src and only checks child-src
+        # So for compatibility until it catches up to other browsers we need to
+        # append worker-src's content to child-src
+        append_to_directive(directives, 'child_src', directives['worker_src'])
 
         directives
       end
@@ -100,6 +112,8 @@ module Gitlab
         append_to_directive(directives, 'script_src', cdn_host)
         append_to_directive(directives, 'style_src', cdn_host)
         append_to_directive(directives, 'font_src', cdn_host)
+        append_to_directive(directives, 'worker_src', cdn_host)
+        append_to_directive(directives, 'frame_src', cdn_host)
       end
 
       def self.append_to_directive(directives, directive, text)
@@ -119,6 +133,21 @@ module Gitlab
 
         append_to_directive(directives, 'connect_src', sentry_uri.to_s)
       end
+
+      def self.allow_letter_opener(directives)
+        append_to_directive(directives, 'frame_src', Gitlab::Utils.append_path(Gitlab.config.gitlab.url, '/rails/letter_opener/'))
+      end
+
+      # Using 'self' in the CSP introduces several CSP bypass opportunities
+      # for this reason we list the URLs where GitLab frames itself instead
+      def self.allow_framed_gitlab_paths(directives)
+        # We need the version without trailing / for the sidekiq page itself
+        # and we also need the version with trailing / for "deeper" pages
+        # like /admin/sidekiq/busy
+        ['/admin/sidekiq', '/admin/sidekiq/', '/-/speedscope/index.html'].map do |path|
+          append_to_directive(directives, 'frame_src', Gitlab::Utils.append_path(Gitlab.config.gitlab.url, path))
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/content_security_policy/directives.rb b/lib/gitlab/content_security_policy/directives.rb
new file mode 100644
index 00000000000..30f3c16247d
--- /dev/null
+++ b/lib/gitlab/content_security_policy/directives.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# This module is used to return various SaaS related
+# ContentSecurityPolicy Directives src which may be
+# overridden in other variants of GitLab
+
+module Gitlab
+  module ContentSecurityPolicy
+    module Directives
+      def self.frame_src
+        "https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com"
+      end
+
+      def self.script_src
+        "'strict-dynamic' 'self' 'unsafe-inline' 'unsafe-eval' https://www.google.com/recaptcha/ https://www.recaptcha.net https://apis.google.com"
+      end
+    end
+  end
+end
+
+Gitlab::ContentSecurityPolicy::Directives.prepend_mod
diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb
index 7d7c604d86a..deaaab953aa 100644
--- a/lib/gitlab/contributions_calendar.rb
+++ b/lib/gitlab/contributions_calendar.rb
@@ -2,12 +2,15 @@
 
 module Gitlab
   class ContributionsCalendar
+    include TimeZoneHelper
+
     attr_reader :contributor
     attr_reader :current_user
     attr_reader :projects
 
     def initialize(contributor, current_user = nil)
       @contributor = contributor
+      @contributor_time_instance = local_time_instance(contributor.timezone)
       @current_user = current_user
       @projects = if @contributor.include_private_contributions?
                     ContributedProjectsFinder.new(@contributor).execute(@contributor)
@@ -22,7 +25,7 @@ module Gitlab
 
       # Can't use Event.contributions here because we need to check 3 different
       # project_features for the (currently) 3 different contribution types
-      date_from = 1.year.ago
+      date_from = @contributor_time_instance.now.years_ago(1)
       repo_events = event_counts(date_from, :repository)
         .having(action: :pushed)
       issue_events = event_counts(date_from, :issues)
@@ -47,19 +50,21 @@ module Gitlab
     def events_by_date(date)
       return Event.none unless can_read_cross_project?
 
+      date_in_time_zone = date.in_time_zone(@contributor_time_instance)
+
       Event.contributions.where(author_id: contributor.id)
-        .where(created_at: date.beginning_of_day..date.end_of_day)
+        .where(created_at: date_in_time_zone.beginning_of_day..date_in_time_zone.end_of_day)
         .where(project_id: projects)
         .with_associations
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
     def starting_year
-      1.year.ago.year
+      @contributor_time_instance.now.years_ago(1).year
     end
 
     def starting_month
-      Date.current.month
+      @contributor_time_instance.today.month
     end
 
     private
@@ -82,10 +87,10 @@ module Gitlab
         .select(:id)
 
       conditions = t[:created_at].gteq(date_from.beginning_of_day)
-        .and(t[:created_at].lteq(Date.current.end_of_day))
+        .and(t[:created_at].lteq(@contributor_time_instance.today.end_of_day))
         .and(t[:author_id].eq(contributor.id))
 
-      date_interval = "INTERVAL '#{Time.zone.now.utc_offset} seconds'"
+      date_interval = "INTERVAL '#{@contributor_time_instance.now.utc_offset} seconds'"
 
       Event.reorder(nil)
         .select(t[:project_id], t[:target_type], t[:action], "date(created_at + #{date_interval}) AS date", 'count(id) as total_amount')
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index bfe3f06a56b..b9034cff447 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -85,7 +85,7 @@ module Gitlab
         active_db_connection = ActiveRecord::Base.connection.active? rescue false
 
         active_db_connection &&
-          Gitlab::Database.main.cached_table_exists?('application_settings')
+          ApplicationSetting.database.cached_table_exists?
       rescue ActiveRecord::NoDatabaseError
         false
       end
diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb
index b309802f296..fdbf068303f 100644
--- a/lib/gitlab/cycle_analytics/stage_summary.rb
+++ b/lib/gitlab/cycle_analytics/stage_summary.rb
@@ -38,7 +38,8 @@ module Gitlab
         serialize(
           Summary::DeploymentFrequency.new(
             deployments: deployments_summary.value.raw_value,
-            options: @options),
+            options: @options,
+            project: @project),
           with_unit: true
         )
       end
diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb
index e30e526f017..f2ff86a40a2 100644
--- a/lib/gitlab/cycle_analytics/summary/base.rb
+++ b/lib/gitlab/cycle_analytics/summary/base.rb
@@ -17,6 +17,10 @@ module Gitlab
           raise NotImplementedError, "Expected #{self.name} to implement value"
         end
 
+        def links
+          []
+        end
+
         private
 
         attr_reader :project, :options
diff --git a/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb b/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb
index 1947866d772..2b1529bdc1a 100644
--- a/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb
+++ b/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb
@@ -6,7 +6,7 @@ module Gitlab
       class DeploymentFrequency < Base
         include SummaryHelper
 
-        def initialize(deployments:, options:, project: nil)
+        def initialize(deployments:, options:, project:)
           @deployments = deployments
 
           super(project: project, options: options)
@@ -23,6 +23,13 @@ module Gitlab
         def unit
           _('per day')
         end
+
+        def links
+          [
+            { "name" => _('Deployment frequency'), "url" => Gitlab::Routing.url_helpers.charts_project_pipelines_path(project, chart: 'deployment-frequency'), "label" => s_('ValueStreamAnalytics|Dashboard') },
+            { "name" => _('Deployment frequency'), "url" => Gitlab::Routing.url_helpers.help_page_path('user/analytics/index', anchor: 'definitions'), "docs_link" => true, "label" => s_('ValueStreamAnalytics|Go to docs') }
+          ]
+        end
       end
     end
   end
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index b560d4cbca8..9c74e5d2ca8 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -59,19 +59,8 @@ module Gitlab
         # that inher from ActiveRecord::Base; not just our own models that
         # inherit from ApplicationRecord.
         main: ::ActiveRecord::Base,
-        ci: ::Ci::CiDatabaseRecord.connection_class? ? ::Ci::CiDatabaseRecord : nil
-      }.compact.freeze
-    end
-
-    def self.databases
-      @databases ||= database_base_models
-        .transform_values { |connection_class| Connection.new(connection_class) }
-        .with_indifferent_access
-        .freeze
-    end
-
-    def self.main
-      databases[PRIMARY_DATABASE_NAME]
+        ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil
+      }.compact.with_indifferent_access.freeze
     end
 
     # We configure the database connection pool size automatically based on the
@@ -110,8 +99,10 @@ module Gitlab
     def self.check_postgres_version_and_print_warning
       return if Gitlab::Runtime.rails_runner?
 
-      databases.each do |name, connection|
-        next if connection.postgresql_minimum_supported_version?
+      database_base_models.each do |name, model|
+        database = Gitlab::Database::Reflection.new(model)
+
+        next if database.postgresql_minimum_supported_version?
 
         Kernel.warn ERB.new(Rainbow.new.wrap(<<~EOS).red).result
 
@@ -122,7 +113,7 @@ module Gitlab
                      ███ ███  ██   ██ ██   ██ ██   ████ ██ ██   ████  ██████  
 
           ******************************************************************************
-            You are using PostgreSQL #{connection.version} for the #{name} database, but PostgreSQL >= <%= Gitlab::Database::MINIMUM_POSTGRES_VERSION %>
+            You are using PostgreSQL #{database.version} for the #{name} database, but PostgreSQL >= <%= Gitlab::Database::MINIMUM_POSTGRES_VERSION %>
             is required for this version of GitLab.
             <% if Rails.env.development? || Rails.env.test? %>
             If using gitlab-development-kit, please find the relevant steps here:
@@ -177,18 +168,6 @@ module Gitlab
       yield
     end
 
-    # This method will allow cross database modifications within the block
-    # Example:
-    #
-    # allow_cross_database_modification_within_transaction(url: 'url-to-an-issue') do
-    #   create(:build) # inserts ci_build and project record in one transaction
-    # end
-    def self.allow_cross_database_modification_within_transaction(url:)
-      # this method will be overridden in:
-      # spec/support/database/cross_database_modification_check.rb
-      yield
-    end
-
     def self.add_post_migrate_path_to_rails(force: false)
       return if ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] && !force
 
@@ -263,14 +242,28 @@ module Gitlab
         # A patch over ActiveRecord::Base.transaction that provides
         # observability into transactional methods.
         def transaction(**options, &block)
-          if options[:requires_new] && connection.transaction_open?
-            ::Gitlab::Database::Metrics.subtransactions_increment(self.name)
-          end
+          transaction_type = get_transaction_type(connection.transaction_open?, options[:requires_new])
+
+          ::Gitlab::Database::Metrics.subtransactions_increment(self.name) if transaction_type == :sub_transaction
+
+          payload = { connection: connection, transaction_type: transaction_type }
 
-          ActiveSupport::Notifications.instrument('transaction.active_record', { connection: connection }) do
+          ActiveSupport::Notifications.instrument('transaction.active_record', payload) do
             super(**options, &block)
           end
         end
+
+        private
+
+        def get_transaction_type(transaction_open, requires_new_flag)
+          if transaction_open
+            return :sub_transaction if requires_new_flag
+
+            return :fake_transaction
+          end
+
+          :real_transaction
+        end
       end
     end
 
diff --git a/lib/gitlab/database/as_with_materialized.rb b/lib/gitlab/database/as_with_materialized.rb
index 07809c5b592..a04ea97117d 100644
--- a/lib/gitlab/database/as_with_materialized.rb
+++ b/lib/gitlab/database/as_with_materialized.rb
@@ -19,7 +19,7 @@ module Gitlab
       # Note: to be deleted after the minimum PG version is set to 12.0
       def self.materialized_supported?
         strong_memoize(:materialized_supported) do
-          Gitlab::Database.main.version.match?(/^1[2-9]\./) # version 12.x and above
+          ApplicationRecord.database.version.match?(/^1[2-9]\./) # version 12.x and above
         end
       end
 
diff --git a/lib/gitlab/database/async_indexes/index_creator.rb b/lib/gitlab/database/async_indexes/index_creator.rb
index 00de79ec970..994a1deba57 100644
--- a/lib/gitlab/database/async_indexes/index_creator.rb
+++ b/lib/gitlab/database/async_indexes/index_creator.rb
@@ -40,7 +40,7 @@ module Gitlab
         end
 
         def connection
-          @connection ||= ApplicationRecord.connection
+          @connection ||= async_index.connection
         end
 
         def lease_timeout
diff --git a/lib/gitlab/database/async_indexes/postgres_async_index.rb b/lib/gitlab/database/async_indexes/postgres_async_index.rb
index 236459e6216..6cb40729061 100644
--- a/lib/gitlab/database/async_indexes/postgres_async_index.rb
+++ b/lib/gitlab/database/async_indexes/postgres_async_index.rb
@@ -3,7 +3,7 @@
 module Gitlab
   module Database
     module AsyncIndexes
-      class PostgresAsyncIndex < ApplicationRecord
+      class PostgresAsyncIndex < SharedModel
         self.table_name = 'postgres_async_indexes'
 
         MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH
diff --git a/lib/gitlab/database/background_migration_job.rb b/lib/gitlab/database/background_migration_job.rb
index 1121793917b..c046571a111 100644
--- a/lib/gitlab/database/background_migration_job.rb
+++ b/lib/gitlab/database/background_migration_job.rb
@@ -4,6 +4,7 @@ module Gitlab
   module Database
     class BackgroundMigrationJob < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord
       include EachBatch
+      include BulkInsertSafe
 
       self.table_name = :background_migration_jobs
 
diff --git a/lib/gitlab/database/batch_counter.rb b/lib/gitlab/database/batch_counter.rb
index 7efa5b46ecb..6c0ce9e481a 100644
--- a/lib/gitlab/database/batch_counter.rb
+++ b/lib/gitlab/database/batch_counter.rb
@@ -31,7 +31,7 @@ module Gitlab
       end
 
       def count(batch_size: nil, mode: :itself, start: nil, finish: nil)
-        raise 'BatchCount can not be run inside a transaction' if @relation.connection.transaction_open?
+        raise 'BatchCount can not be run inside a transaction' if transaction_open?
 
         check_mode!(mode)
 
@@ -87,6 +87,10 @@ module Gitlab
         results
       end
 
+      def transaction_open?
+        @relation.connection.transaction_open?
+      end
+
       def merge_results(results, object)
         return object unless results
 
diff --git a/lib/gitlab/database/connection.rb b/lib/gitlab/database/connection.rb
deleted file mode 100644
index cda6220ee6c..00000000000
--- a/lib/gitlab/database/connection.rb
+++ /dev/null
@@ -1,260 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Database
-    # Configuration settings and methods for interacting with a PostgreSQL
-    # database, with support for multiple databases.
-    class Connection
-      attr_reader :scope
-
-      # Initializes a new `Database`.
-      #
-      # The `scope` argument must be an object (such as `ActiveRecord::Base`)
-      # that supports retrieving connections and connection pools.
-      def initialize(scope = ActiveRecord::Base)
-        @config = nil
-        @scope = scope
-        @version = nil
-        @open_transactions_baseline = 0
-      end
-
-      def config
-        # The result of this method must not be cached, as other methods may use
-        # it after making configuration changes and expect those changes to be
-        # present. For example, `disable_prepared_statements` expects the
-        # configuration settings to always be up to date.
-        #
-        # See the following for more information:
-        #
-        # - https://gitlab.com/gitlab-org/release/retrospectives/-/issues/39
-        # - https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5238
-        scope.connection_db_config.configuration_hash.with_indifferent_access
-      end
-
-      def pool_size
-        config[:pool] || Database.default_pool_size
-      end
-
-      def username
-        config[:username] || ENV['USER']
-      end
-
-      def database_name
-        config[:database]
-      end
-
-      def adapter_name
-        config[:adapter]
-      end
-
-      def human_adapter_name
-        if postgresql?
-          'PostgreSQL'
-        else
-          'Unknown'
-        end
-      end
-
-      def postgresql?
-        adapter_name.casecmp('postgresql') == 0
-      end
-
-      def db_config_with_default_pool_size
-        db_config_object = scope.connection_db_config
-        config = db_config_object
-          .configuration_hash
-          .merge(pool: Database.default_pool_size)
-
-        ActiveRecord::DatabaseConfigurations::HashConfig.new(
-          db_config_object.env_name,
-          db_config_object.name,
-          config
-        )
-      end
-
-      # Disables prepared statements for the current database connection.
-      def disable_prepared_statements
-        db_config_object = scope.connection_db_config
-        config = db_config_object.configuration_hash.merge(prepared_statements: false)
-
-        hash_config = ActiveRecord::DatabaseConfigurations::HashConfig.new(
-          db_config_object.env_name,
-          db_config_object.name,
-          config
-        )
-
-        scope.establish_connection(hash_config)
-      end
-
-      # Check whether the underlying database is in read-only mode
-      def db_read_only?
-        pg_is_in_recovery =
-          scope
-            .connection
-            .execute('SELECT pg_is_in_recovery()')
-            .first
-            .fetch('pg_is_in_recovery')
-
-        Gitlab::Utils.to_boolean(pg_is_in_recovery)
-      end
-
-      def db_read_write?
-        !db_read_only?
-      end
-
-      def version
-        @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1]
-      end
-
-      def database_version
-        connection.execute("SELECT VERSION()").first['version']
-      end
-
-      def postgresql_minimum_supported_version?
-        version.to_f >= MINIMUM_POSTGRES_VERSION
-      end
-
-      # Bulk inserts a number of rows into a table, optionally returning their
-      # IDs.
-      #
-      # table - The name of the table to insert the rows into.
-      # rows - An Array of Hash instances, each mapping the columns to their
-      #        values.
-      # return_ids - When set to true the return value will be an Array of IDs of
-      #              the inserted rows
-      # disable_quote - A key or an Array of keys to exclude from quoting (You
-      #                 become responsible for protection from SQL injection for
-      #                 these keys!)
-      # on_conflict - Defines an upsert. Values can be: :disabled (default) or
-      #               :do_nothing
-      def bulk_insert(table, rows, return_ids: false, disable_quote: [], on_conflict: nil)
-        return if rows.empty?
-
-        keys = rows.first.keys
-        columns = keys.map { |key| connection.quote_column_name(key) }
-
-        disable_quote = Array(disable_quote).to_set
-        tuples = rows.map do |row|
-          keys.map do |k|
-            disable_quote.include?(k) ? row[k] : connection.quote(row[k])
-          end
-        end
-
-        sql = <<-EOF
-          INSERT INTO #{table} (#{columns.join(', ')})
-          VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')}
-        EOF
-
-        sql = "#{sql} ON CONFLICT DO NOTHING" if on_conflict == :do_nothing
-
-        sql = "#{sql} RETURNING id" if return_ids
-
-        result = connection.execute(sql)
-
-        if return_ids
-          result.values.map { |tuple| tuple[0].to_i }
-        else
-          []
-        end
-      end
-
-      def cached_column_exists?(table_name, column_name)
-        connection
-          .schema_cache.columns_hash(table_name)
-          .has_key?(column_name.to_s)
-      end
-
-      def cached_table_exists?(table_name)
-        exists? && connection.schema_cache.data_source_exists?(table_name)
-      end
-
-      def exists?
-        # We can't _just_ check if `connection` raises an error, as it will
-        # point to a `ConnectionProxy`, and obtaining those doesn't involve any
-        # database queries. So instead we obtain the database version, which is
-        # cached after the first call.
-        connection.schema_cache.database_version
-        true
-      rescue StandardError
-        false
-      end
-
-      def system_id
-        row = connection
-          .execute('SELECT system_identifier FROM pg_control_system()')
-          .first
-
-        row['system_identifier']
-      end
-
-      def pg_wal_lsn_diff(location1, location2)
-        lsn1 = connection.quote(location1)
-        lsn2 = connection.quote(location2)
-
-        query = <<-SQL.squish
-            SELECT pg_wal_lsn_diff(#{lsn1}, #{lsn2})
-              AS result
-        SQL
-
-        row = connection.select_all(query).first
-        row['result'] if row
-      end
-
-      # @param [ActiveRecord::Connection] ar_connection
-      # @return [String]
-      def get_write_location(ar_connection)
-        use_new_load_balancer_query = Gitlab::Utils
-          .to_boolean(ENV['USE_NEW_LOAD_BALANCER_QUERY'], default: true)
-
-        sql =
-          if use_new_load_balancer_query
-            <<~NEWSQL
-              SELECT CASE
-                  WHEN pg_is_in_recovery() = true AND EXISTS (SELECT 1 FROM pg_stat_get_wal_senders())
-                    THEN pg_last_wal_replay_lsn()::text
-                  WHEN pg_is_in_recovery() = false
-                    THEN pg_current_wal_insert_lsn()::text
-                    ELSE NULL
-                  END AS location;
-            NEWSQL
-          else
-            <<~SQL
-              SELECT pg_current_wal_insert_lsn()::text AS location
-            SQL
-          end
-
-        row = ar_connection.select_all(sql).first
-        row['location'] if row
-      end
-
-      # inside_transaction? will return true if the caller is running within a
-      # transaction. Handles special cases when running inside a test
-      # environment, where tests may be wrapped in transactions
-      def inside_transaction?
-        base = Rails.env.test? ? @open_transactions_baseline : 0
-
-        scope.connection.open_transactions > base
-      end
-
-      # These methods that access @open_transactions_baseline are not
-      # thread-safe.  These are fine though because we only call these in
-      # RSpec's main thread. If we decide to run specs multi-threaded, we would
-      # need to use something like ThreadGroup to keep track of this value
-      def set_open_transactions_baseline
-        @open_transactions_baseline = scope.connection.open_transactions
-      end
-
-      def reset_open_transactions_baseline
-        @open_transactions_baseline = 0
-      end
-
-      private
-
-      def connection
-        scope.connection
-      end
-    end
-  end
-end
-
-Gitlab::Database::Connection.prepend_mod_with('Gitlab::Database::Connection')
diff --git a/lib/gitlab/database/each_database.rb b/lib/gitlab/database/each_database.rb
new file mode 100644
index 00000000000..7c9e65e6691
--- /dev/null
+++ b/lib/gitlab/database/each_database.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module EachDatabase
+      class << self
+        def each_database_connection
+          Gitlab::Database.database_base_models.each_pair do |connection_name, model|
+            connection = model.connection
+
+            with_shared_connection(connection, connection_name) do
+              yield connection, connection_name
+            end
+          end
+        end
+
+        def each_model_connection(models)
+          models.each do |model|
+            connection_name = model.connection.pool.db_config.name
+
+            with_shared_connection(model.connection, connection_name) do
+              yield model, connection_name
+            end
+          end
+        end
+
+        private
+
+        def with_shared_connection(connection, connection_name)
+          Gitlab::Database::SharedModel.using_connection(connection) do
+            Gitlab::AppLogger.debug(message: 'Switched database connection', connection_name: connection_name)
+
+            yield
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb
new file mode 100644
index 00000000000..14807494a79
--- /dev/null
+++ b/lib/gitlab/database/gitlab_schema.rb
@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+# This module gathers information about table to schema mapping
+# to understand table affinity
+#
+# Each table / view needs to have assigned gitlab_schema. Names supported today:
+#
+# - gitlab_shared - defines a set of tables that are found on all databases (data accessed is dependent on connection)
+# - gitlab_main / gitlab_ci - defines a set of tables that can only exist on a given database
+#
+# Tables for the purpose of tests should be prefixed with `_test_my_table_name`
+
+module Gitlab
+  module Database
+    module GitlabSchema
+      # These tables are deleted/renamed, but still referenced by migrations.
+      # This is needed for now, but should be removed in the future
+      DELETED_TABLES = {
+        # main tables
+        'alerts_service_data' => :gitlab_main,
+        'analytics_devops_adoption_segment_selections' => :gitlab_main,
+        'analytics_repository_file_commits' => :gitlab_main,
+        'analytics_repository_file_edits' => :gitlab_main,
+        'analytics_repository_files' => :gitlab_main,
+        'audit_events_archived' => :gitlab_main,
+        'backup_labels' => :gitlab_main,
+        'clusters_applications_fluentd' => :gitlab_main,
+        'forked_project_links' => :gitlab_main,
+        'issue_milestones' => :gitlab_main,
+        'merge_request_milestones' => :gitlab_main,
+        'namespace_onboarding_actions' => :gitlab_main,
+        'services' => :gitlab_main,
+        'terraform_state_registry' => :gitlab_main,
+        'tmp_fingerprint_sha256_migration' => :gitlab_main, # used by lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb
+        'web_hook_logs_archived' => :gitlab_main,
+        'vulnerability_export_registry' => :gitlab_main,
+        'vulnerability_finding_fingerprints' => :gitlab_main,
+        'vulnerability_export_verification_status' => :gitlab_main,
+
+        # CI tables
+        'ci_build_trace_sections' => :gitlab_ci,
+        'ci_build_trace_section_names' => :gitlab_ci,
+        'ci_daily_report_results' => :gitlab_ci,
+        'ci_test_cases' => :gitlab_ci,
+        'ci_test_case_failures' => :gitlab_ci,
+
+        # leftovers from early implementation of partitioning
+        'audit_events_part_5fc467ac26' => :gitlab_main,
+        'web_hook_logs_part_0c5294f417' => :gitlab_main
+      }.freeze
+
+      def self.table_schemas(tables)
+        tables.map { |table| table_schema(table) }.to_set
+      end
+
+      def self.table_schema(name)
+        schema_name, table_name = name.split('.', 2) # Strip schema name like: `public.`
+
+        # Most of names do not have schemas, ensure that this is table
+        unless table_name
+          table_name = schema_name
+          schema_name = nil
+        end
+
+        # strip partition number of a form `loose_foreign_keys_deleted_records_1`
+        table_name.gsub!(/_[0-9]+$/, '')
+
+        # Tables that are properly mapped
+        if gitlab_schema = tables_to_schema[table_name]
+          return gitlab_schema
+        end
+
+        # Tables that are deleted, but we still need to reference them
+        if gitlab_schema = DELETED_TABLES[table_name]
+          return gitlab_schema
+        end
+
+        # All tables from `information_schema.` are `:gitlab_shared`
+        return :gitlab_shared if schema_name == 'information_schema'
+
+        # All tables that start with `_test_` are shared and ignored
+        return :gitlab_shared if table_name.start_with?('_test_')
+
+        # All `pg_` tables are marked as `shared`
+        return :gitlab_shared if table_name.start_with?('pg_')
+
+        # When undefined it's best to return a unique name so that we don't incorrectly assume that 2 undefined schemas belong on the same database
+        :"undefined_#{table_name}"
+      end
+
+      def self.tables_to_schema
+        @tables_to_schema ||= YAML.load_file(Rails.root.join('lib/gitlab/database/gitlab_schemas.yml'))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml
new file mode 100644
index 00000000000..66157e998a0
--- /dev/null
+++ b/lib/gitlab/database/gitlab_schemas.yml
@@ -0,0 +1,543 @@
+abuse_reports: :gitlab_main
+agent_group_authorizations: :gitlab_main
+agent_project_authorizations: :gitlab_main
+alert_management_alert_assignees: :gitlab_main
+alert_management_alerts: :gitlab_main
+alert_management_alert_user_mentions: :gitlab_main
+alert_management_http_integrations: :gitlab_main
+allowed_email_domains: :gitlab_main
+analytics_cycle_analytics_group_stages: :gitlab_main
+analytics_cycle_analytics_group_value_streams: :gitlab_main
+analytics_cycle_analytics_issue_stage_events: :gitlab_main
+analytics_cycle_analytics_merge_request_stage_events: :gitlab_main
+analytics_cycle_analytics_project_stages: :gitlab_main
+analytics_cycle_analytics_project_value_streams: :gitlab_main
+analytics_cycle_analytics_stage_event_hashes: :gitlab_main
+analytics_devops_adoption_segments: :gitlab_main
+analytics_devops_adoption_snapshots: :gitlab_main
+analytics_language_trend_repository_languages: :gitlab_main
+analytics_usage_trends_measurements: :gitlab_main
+appearances: :gitlab_main
+application_settings: :gitlab_main
+application_setting_terms: :gitlab_main
+approval_merge_request_rules_approved_approvers: :gitlab_main
+approval_merge_request_rules: :gitlab_main
+approval_merge_request_rules_groups: :gitlab_main
+approval_merge_request_rule_sources: :gitlab_main
+approval_merge_request_rules_users: :gitlab_main
+approval_project_rules: :gitlab_main
+approval_project_rules_groups: :gitlab_main
+approval_project_rules_protected_branches: :gitlab_main
+approval_project_rules_users: :gitlab_main
+approvals: :gitlab_main
+approver_groups: :gitlab_main
+approvers: :gitlab_main
+ar_internal_metadata: :gitlab_shared
+atlassian_identities: :gitlab_main
+audit_events_external_audit_event_destinations: :gitlab_main
+audit_events: :gitlab_main
+authentication_events: :gitlab_main
+award_emoji: :gitlab_main
+aws_roles: :gitlab_main
+background_migration_jobs: :gitlab_main
+badges: :gitlab_main
+banned_users: :gitlab_main
+batched_background_migration_jobs: :gitlab_main
+batched_background_migrations: :gitlab_main
+board_assignees: :gitlab_main
+board_group_recent_visits: :gitlab_main
+board_labels: :gitlab_main
+board_project_recent_visits: :gitlab_main
+boards_epic_board_labels: :gitlab_main
+boards_epic_board_positions: :gitlab_main
+boards_epic_board_recent_visits: :gitlab_main
+boards_epic_boards: :gitlab_main
+boards_epic_lists: :gitlab_main
+boards_epic_list_user_preferences: :gitlab_main
+boards_epic_user_preferences: :gitlab_main
+boards: :gitlab_main
+board_user_preferences: :gitlab_main
+broadcast_messages: :gitlab_main
+bulk_import_configurations: :gitlab_main
+bulk_import_entities: :gitlab_main
+bulk_import_exports: :gitlab_main
+bulk_import_export_uploads: :gitlab_main
+bulk_import_failures: :gitlab_main
+bulk_imports: :gitlab_main
+bulk_import_trackers: :gitlab_main
+chat_names: :gitlab_main
+chat_teams: :gitlab_main
+ci_build_needs: :gitlab_ci
+ci_build_pending_states: :gitlab_ci
+ci_build_report_results: :gitlab_ci
+ci_builds: :gitlab_ci
+ci_builds_metadata: :gitlab_ci
+ci_builds_runner_session: :gitlab_ci
+ci_build_trace_chunks: :gitlab_ci
+ci_build_trace_metadata: :gitlab_ci
+ci_daily_build_group_report_results: :gitlab_ci
+ci_deleted_objects: :gitlab_ci
+ci_freeze_periods: :gitlab_ci
+ci_group_variables: :gitlab_ci
+ci_instance_variables: :gitlab_ci
+ci_job_artifacts: :gitlab_ci
+ci_job_token_project_scope_links: :gitlab_ci
+ci_job_variables: :gitlab_ci
+ci_minutes_additional_packs: :gitlab_ci
+ci_namespace_monthly_usages: :gitlab_ci
+ci_pending_builds: :gitlab_ci
+ci_pipeline_artifacts: :gitlab_ci
+ci_pipeline_chat_data: :gitlab_ci
+ci_pipeline_messages: :gitlab_ci
+ci_pipeline_schedules: :gitlab_ci
+ci_pipeline_schedule_variables: :gitlab_ci
+ci_pipelines_config: :gitlab_ci
+ci_pipelines: :gitlab_ci
+ci_pipeline_variables: :gitlab_ci
+ci_platform_metrics: :gitlab_ci
+ci_project_monthly_usages: :gitlab_ci
+ci_refs: :gitlab_ci
+ci_resource_groups: :gitlab_ci
+ci_resources: :gitlab_ci
+ci_runner_namespaces: :gitlab_ci
+ci_runner_projects: :gitlab_ci
+ci_runners: :gitlab_ci
+ci_running_builds: :gitlab_ci
+ci_sources_pipelines: :gitlab_ci
+ci_sources_projects: :gitlab_ci
+ci_stages: :gitlab_ci
+ci_subscriptions_projects: :gitlab_ci
+ci_trigger_requests: :gitlab_ci
+ci_triggers: :gitlab_ci
+ci_unit_test_failures: :gitlab_ci
+ci_unit_tests: :gitlab_ci
+ci_variables: :gitlab_ci
+cluster_agents: :gitlab_main
+cluster_agent_tokens: :gitlab_main
+cluster_groups: :gitlab_main
+cluster_platforms_kubernetes: :gitlab_main
+cluster_projects: :gitlab_main
+cluster_providers_aws: :gitlab_main
+cluster_providers_gcp: :gitlab_main
+clusters_applications_cert_managers: :gitlab_main
+clusters_applications_cilium: :gitlab_main
+clusters_applications_crossplane: :gitlab_main
+clusters_applications_elastic_stacks: :gitlab_main
+clusters_applications_helm: :gitlab_main
+clusters_applications_ingress: :gitlab_main
+clusters_applications_jupyter: :gitlab_main
+clusters_applications_knative: :gitlab_main
+clusters_applications_prometheus: :gitlab_main
+clusters_applications_runners: :gitlab_main
+clusters: :gitlab_main
+clusters_integration_elasticstack: :gitlab_main
+clusters_integration_prometheus: :gitlab_main
+clusters_kubernetes_namespaces: :gitlab_main
+commit_user_mentions: :gitlab_main
+compliance_management_frameworks: :gitlab_main
+container_expiration_policies: :gitlab_main
+container_repositories: :gitlab_main
+content_blocked_states: :gitlab_main
+conversational_development_index_metrics: :gitlab_main
+coverage_fuzzing_corpuses: :gitlab_main
+csv_issue_imports: :gitlab_main
+custom_emoji: :gitlab_main
+customer_relations_contacts: :gitlab_main
+customer_relations_organizations: :gitlab_main
+dast_profile_schedules: :gitlab_main
+dast_profiles: :gitlab_main
+dast_profiles_pipelines: :gitlab_main
+dast_scanner_profiles_builds: :gitlab_main
+dast_scanner_profiles: :gitlab_main
+dast_site_profiles_builds: :gitlab_main
+dast_site_profile_secret_variables: :gitlab_main
+dast_site_profiles: :gitlab_main
+dast_site_profiles_pipelines: :gitlab_main
+dast_sites: :gitlab_main
+dast_site_tokens: :gitlab_main
+dast_site_validations: :gitlab_main
+dependency_proxy_blobs: :gitlab_main
+dependency_proxy_group_settings: :gitlab_main
+dependency_proxy_image_ttl_group_policies: :gitlab_main
+dependency_proxy_manifests: :gitlab_main
+deploy_keys_projects: :gitlab_main
+deployment_clusters: :gitlab_main
+deployment_merge_requests: :gitlab_main
+deployments: :gitlab_main
+deploy_tokens: :gitlab_main
+description_versions: :gitlab_main
+design_management_designs: :gitlab_main
+design_management_designs_versions: :gitlab_main
+design_management_versions: :gitlab_main
+design_user_mentions: :gitlab_main
+detached_partitions: :gitlab_shared
+diff_note_positions: :gitlab_main
+dora_daily_metrics: :gitlab_main
+draft_notes: :gitlab_main
+elastic_index_settings: :gitlab_main
+elastic_reindexing_slices: :gitlab_main
+elastic_reindexing_subtasks: :gitlab_main
+elastic_reindexing_tasks: :gitlab_main
+elasticsearch_indexed_namespaces: :gitlab_main
+elasticsearch_indexed_projects: :gitlab_main
+emails: :gitlab_main
+environments: :gitlab_main
+epic_issues: :gitlab_main
+epic_metrics: :gitlab_main
+epics: :gitlab_main
+epic_user_mentions: :gitlab_main
+error_tracking_client_keys: :gitlab_main
+error_tracking_error_events: :gitlab_main
+error_tracking_errors: :gitlab_main
+events: :gitlab_main
+evidences: :gitlab_main
+experiments: :gitlab_main
+experiment_subjects: :gitlab_main
+experiment_users: :gitlab_main
+external_approval_rules: :gitlab_main
+external_approval_rules_protected_branches: :gitlab_main
+external_pull_requests: :gitlab_main
+external_status_checks: :gitlab_main
+external_status_checks_protected_branches: :gitlab_main
+feature_gates: :gitlab_main
+features: :gitlab_main
+fork_network_members: :gitlab_main
+fork_networks: :gitlab_main
+geo_cache_invalidation_events: :gitlab_main
+geo_container_repository_updated_events: :gitlab_main
+geo_event_log: :gitlab_main
+geo_events: :gitlab_main
+geo_hashed_storage_attachments_events: :gitlab_main
+geo_hashed_storage_migrated_events: :gitlab_main
+geo_job_artifact_deleted_events: :gitlab_main
+geo_lfs_object_deleted_events: :gitlab_main
+geo_node_namespace_links: :gitlab_main
+geo_nodes: :gitlab_main
+geo_node_statuses: :gitlab_main
+geo_repositories_changed_events: :gitlab_main
+geo_repository_created_events: :gitlab_main
+geo_repository_deleted_events: :gitlab_main
+geo_repository_renamed_events: :gitlab_main
+geo_repository_updated_events: :gitlab_main
+geo_reset_checksum_events: :gitlab_main
+gitlab_subscription_histories: :gitlab_main
+gitlab_subscriptions: :gitlab_main
+gpg_keys: :gitlab_main
+gpg_key_subkeys: :gitlab_main
+gpg_signatures: :gitlab_main
+grafana_integrations: :gitlab_main
+group_custom_attributes: :gitlab_main
+group_deletion_schedules: :gitlab_main
+group_deploy_keys: :gitlab_main
+group_deploy_keys_groups: :gitlab_main
+group_deploy_tokens: :gitlab_main
+group_group_links: :gitlab_main
+group_import_states: :gitlab_main
+group_merge_request_approval_settings: :gitlab_main
+group_repository_storage_moves: :gitlab_main
+group_wiki_repositories: :gitlab_main
+historical_data: :gitlab_main
+identities: :gitlab_main
+import_export_uploads: :gitlab_main
+import_failures: :gitlab_main
+incident_management_escalation_policies: :gitlab_main
+incident_management_escalation_rules: :gitlab_main
+incident_management_issuable_escalation_statuses: :gitlab_main
+incident_management_oncall_participants: :gitlab_main
+incident_management_oncall_rotations: :gitlab_main
+incident_management_oncall_schedules: :gitlab_main
+incident_management_oncall_shifts: :gitlab_main
+incident_management_pending_alert_escalations: :gitlab_main
+incident_management_pending_issue_escalations: :gitlab_main
+index_statuses: :gitlab_main
+in_product_marketing_emails: :gitlab_main
+insights: :gitlab_main
+integrations: :gitlab_main
+internal_ids: :gitlab_main
+ip_restrictions: :gitlab_main
+issuable_metric_images: :gitlab_main
+issuable_severities: :gitlab_main
+issuable_slas: :gitlab_main
+issue_assignees: :gitlab_main
+issue_customer_relations_contacts: :gitlab_main
+issue_email_participants: :gitlab_main
+issue_links: :gitlab_main
+issue_metrics: :gitlab_main
+issues: :gitlab_main
+issues_prometheus_alert_events: :gitlab_main
+issues_self_managed_prometheus_alert_events: :gitlab_main
+issue_tracker_data: :gitlab_main
+issue_user_mentions: :gitlab_main
+iterations_cadences: :gitlab_main
+jira_connect_installations: :gitlab_main
+jira_connect_subscriptions: :gitlab_main
+jira_imports: :gitlab_main
+jira_tracker_data: :gitlab_main
+keys: :gitlab_main
+label_links: :gitlab_main
+label_priorities: :gitlab_main
+labels: :gitlab_main
+ldap_group_links: :gitlab_main
+lfs_file_locks: :gitlab_main
+lfs_objects: :gitlab_main
+lfs_objects_projects: :gitlab_main
+licenses: :gitlab_main
+lists: :gitlab_main
+list_user_preferences: :gitlab_main
+loose_foreign_keys_deleted_records: :gitlab_shared
+member_tasks: :gitlab_main
+members: :gitlab_main
+merge_request_assignees: :gitlab_main
+merge_request_blocks: :gitlab_main
+merge_request_cleanup_schedules: :gitlab_main
+merge_request_context_commit_diff_files: :gitlab_main
+merge_request_context_commits: :gitlab_main
+merge_request_diff_commits: :gitlab_main
+merge_request_diff_commit_users: :gitlab_main
+merge_request_diff_details: :gitlab_main
+merge_request_diff_files: :gitlab_main
+merge_request_diffs: :gitlab_main
+merge_request_metrics: :gitlab_main
+merge_request_reviewers: :gitlab_main
+merge_requests_closing_issues: :gitlab_main
+merge_requests: :gitlab_main
+merge_request_user_mentions: :gitlab_main
+merge_trains: :gitlab_main
+metrics_dashboard_annotations: :gitlab_main
+metrics_users_starred_dashboards: :gitlab_main
+milestone_releases: :gitlab_main
+milestones: :gitlab_main
+namespace_admin_notes: :gitlab_main
+namespace_aggregation_schedules: :gitlab_main
+namespace_limits: :gitlab_main
+namespace_package_settings: :gitlab_main
+namespace_root_storage_statistics: :gitlab_main
+namespace_settings: :gitlab_main
+namespaces: :gitlab_main
+namespace_statistics: :gitlab_main
+note_diff_files: :gitlab_main
+notes: :gitlab_main
+notification_settings: :gitlab_main
+oauth_access_grants: :gitlab_main
+oauth_access_tokens: :gitlab_main
+oauth_applications: :gitlab_main
+oauth_openid_requests: :gitlab_main
+onboarding_progresses: :gitlab_main
+operations_feature_flags_clients: :gitlab_main
+operations_feature_flag_scopes: :gitlab_main
+operations_feature_flags: :gitlab_main
+operations_feature_flags_issues: :gitlab_main
+operations_scopes: :gitlab_main
+operations_strategies: :gitlab_main
+operations_strategies_user_lists: :gitlab_main
+operations_user_lists: :gitlab_main
+packages_build_infos: :gitlab_main
+packages_composer_cache_files: :gitlab_main
+packages_composer_metadata: :gitlab_main
+packages_conan_file_metadata: :gitlab_main
+packages_conan_metadata: :gitlab_main
+packages_debian_file_metadata: :gitlab_main
+packages_debian_group_architectures: :gitlab_main
+packages_debian_group_component_files: :gitlab_main
+packages_debian_group_components: :gitlab_main
+packages_debian_group_distribution_keys: :gitlab_main
+packages_debian_group_distributions: :gitlab_main
+packages_debian_project_architectures: :gitlab_main
+packages_debian_project_component_files: :gitlab_main
+packages_debian_project_components: :gitlab_main
+packages_debian_project_distribution_keys: :gitlab_main
+packages_debian_project_distributions: :gitlab_main
+packages_debian_publications: :gitlab_main
+packages_dependencies: :gitlab_main
+packages_dependency_links: :gitlab_main
+packages_events: :gitlab_main
+packages_helm_file_metadata: :gitlab_main
+packages_maven_metadata: :gitlab_main
+packages_npm_metadata: :gitlab_main
+packages_nuget_dependency_link_metadata: :gitlab_main
+packages_nuget_metadata: :gitlab_main
+packages_package_file_build_infos: :gitlab_main
+packages_package_files: :gitlab_main
+packages_packages: :gitlab_main
+packages_pypi_metadata: :gitlab_main
+packages_rubygems_metadata: :gitlab_main
+packages_tags: :gitlab_main
+pages_deployments: :gitlab_main
+pages_domain_acme_orders: :gitlab_main
+pages_domains: :gitlab_main
+partitioned_foreign_keys: :gitlab_main
+path_locks: :gitlab_main
+personal_access_tokens: :gitlab_main
+plan_limits: :gitlab_main
+plans: :gitlab_main
+pool_repositories: :gitlab_main
+postgres_async_indexes: :gitlab_shared
+postgres_foreign_keys: :gitlab_shared
+postgres_index_bloat_estimates: :gitlab_shared
+postgres_indexes: :gitlab_shared
+postgres_partitioned_tables: :gitlab_shared
+postgres_partitions: :gitlab_shared
+postgres_reindex_actions: :gitlab_shared
+postgres_reindex_queued_actions: :gitlab_main
+product_analytics_events_experimental: :gitlab_main
+programming_languages: :gitlab_main
+project_access_tokens: :gitlab_main
+project_alerting_settings: :gitlab_main
+project_aliases: :gitlab_main
+project_authorizations: :gitlab_main
+project_auto_devops: :gitlab_main
+project_ci_cd_settings: :gitlab_main
+project_ci_feature_usages: :gitlab_main
+project_compliance_framework_settings: :gitlab_main
+project_custom_attributes: :gitlab_main
+project_daily_statistics: :gitlab_main
+project_deploy_tokens: :gitlab_main
+project_error_tracking_settings: :gitlab_main
+project_export_jobs: :gitlab_main
+project_features: :gitlab_main
+project_feature_usages: :gitlab_main
+project_group_links: :gitlab_main
+project_import_data: :gitlab_main
+project_incident_management_settings: :gitlab_main
+project_metrics_settings: :gitlab_main
+project_mirror_data: :gitlab_main
+project_pages_metadata: :gitlab_main
+project_repositories: :gitlab_main
+project_repository_states: :gitlab_main
+project_repository_storage_moves: :gitlab_main
+project_security_settings: :gitlab_main
+project_settings: :gitlab_main
+projects: :gitlab_main
+project_statistics: :gitlab_main
+project_topics: :gitlab_main
+project_tracing_settings: :gitlab_main
+prometheus_alert_events: :gitlab_main
+prometheus_alerts: :gitlab_main
+prometheus_metrics: :gitlab_main
+protected_branches: :gitlab_main
+protected_branch_merge_access_levels: :gitlab_main
+protected_branch_push_access_levels: :gitlab_main
+protected_branch_unprotect_access_levels: :gitlab_main
+protected_environment_deploy_access_levels: :gitlab_main
+protected_environments: :gitlab_main
+protected_tag_create_access_levels: :gitlab_main
+protected_tags: :gitlab_main
+push_event_payloads: :gitlab_main
+push_rules: :gitlab_main
+raw_usage_data: :gitlab_main
+redirect_routes: :gitlab_main
+release_links: :gitlab_main
+releases: :gitlab_main
+remote_mirrors: :gitlab_main
+repository_languages: :gitlab_main
+required_code_owners_sections: :gitlab_main
+requirements: :gitlab_main
+requirements_management_test_reports: :gitlab_main
+resource_iteration_events: :gitlab_main
+resource_label_events: :gitlab_main
+resource_milestone_events: :gitlab_main
+resource_state_events: :gitlab_main
+resource_weight_events: :gitlab_main
+reviews: :gitlab_main
+routes: :gitlab_main
+saml_group_links: :gitlab_main
+saml_providers: :gitlab_main
+schema_migrations: :gitlab_shared
+scim_identities: :gitlab_main
+scim_oauth_access_tokens: :gitlab_main
+security_findings: :gitlab_main
+security_orchestration_policy_configurations: :gitlab_main
+security_orchestration_policy_rule_schedules: :gitlab_main
+security_scans: :gitlab_main
+self_managed_prometheus_alert_events: :gitlab_main
+sent_notifications: :gitlab_main
+sentry_issues: :gitlab_main
+serverless_domain_cluster: :gitlab_main
+service_desk_settings: :gitlab_main
+shards: :gitlab_main
+slack_integrations: :gitlab_main
+smartcard_identities: :gitlab_main
+snippet_repositories: :gitlab_main
+snippet_repository_storage_moves: :gitlab_main
+snippets: :gitlab_main
+snippet_statistics: :gitlab_main
+snippet_user_mentions: :gitlab_main
+software_license_policies: :gitlab_main
+software_licenses: :gitlab_main
+spam_logs: :gitlab_main
+sprints: :gitlab_main
+status_check_responses: :gitlab_main
+status_page_published_incidents: :gitlab_main
+status_page_settings: :gitlab_main
+subscriptions: :gitlab_main
+suggestions: :gitlab_main
+system_note_metadata: :gitlab_main
+taggings: :gitlab_ci
+tags: :gitlab_ci
+term_agreements: :gitlab_main
+terraform_states: :gitlab_main
+terraform_state_versions: :gitlab_main
+timelogs: :gitlab_main
+todos: :gitlab_main
+token_with_ivs: :gitlab_main
+topics: :gitlab_main
+trending_projects: :gitlab_main
+u2f_registrations: :gitlab_main
+upcoming_reconciliations: :gitlab_main
+uploads: :gitlab_main
+user_agent_details: :gitlab_main
+user_callouts: :gitlab_main
+user_canonical_emails: :gitlab_main
+user_credit_card_validations: :gitlab_main
+user_custom_attributes: :gitlab_main
+user_details: :gitlab_main
+user_follow_users: :gitlab_main
+user_group_callouts: :gitlab_main
+user_highest_roles: :gitlab_main
+user_interacted_projects: :gitlab_main
+user_permission_export_uploads: :gitlab_main
+user_preferences: :gitlab_main
+users: :gitlab_main
+users_ops_dashboard_projects: :gitlab_main
+users_security_dashboard_projects: :gitlab_main
+users_star_projects: :gitlab_main
+users_statistics: :gitlab_main
+user_statuses: :gitlab_main
+user_synced_attributes_metadata: :gitlab_main
+verification_codes: :gitlab_main
+vulnerabilities: :gitlab_main
+vulnerability_exports: :gitlab_main
+vulnerability_external_issue_links: :gitlab_main
+vulnerability_feedback: :gitlab_main
+vulnerability_finding_evidence_assets: :gitlab_main
+vulnerability_finding_evidence_headers: :gitlab_main
+vulnerability_finding_evidence_requests: :gitlab_main
+vulnerability_finding_evidence_responses: :gitlab_main
+vulnerability_finding_evidences: :gitlab_main
+vulnerability_finding_evidence_sources: :gitlab_main
+vulnerability_finding_evidence_supporting_messages: :gitlab_main
+vulnerability_finding_links: :gitlab_main
+vulnerability_finding_signatures: :gitlab_main
+vulnerability_findings_remediations: :gitlab_main
+vulnerability_flags: :gitlab_main
+vulnerability_historical_statistics: :gitlab_main
+vulnerability_identifiers: :gitlab_main
+vulnerability_issue_links: :gitlab_main
+vulnerability_occurrence_identifiers: :gitlab_main
+vulnerability_occurrence_pipelines: :gitlab_main
+vulnerability_occurrences: :gitlab_main
+vulnerability_remediations: :gitlab_main
+vulnerability_scanners: :gitlab_main
+vulnerability_statistics: :gitlab_main
+vulnerability_user_mentions: :gitlab_main
+webauthn_registrations: :gitlab_main
+web_hook_logs: :gitlab_main
+web_hooks: :gitlab_main
+wiki_page_meta: :gitlab_main
+wiki_page_slugs: :gitlab_main
+work_item_types: :gitlab_main
+x509_certificates: :gitlab_main
+x509_commit_signatures: :gitlab_main
+x509_issuers: :gitlab_main
+zentao_tracker_data: :gitlab_main
+zoom_meetings: :gitlab_main
diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb
index 3e322e752b7..52eb0764ae3 100644
--- a/lib/gitlab/database/load_balancing.rb
+++ b/lib/gitlab/database/load_balancing.rb
@@ -26,7 +26,7 @@ module Gitlab
         return to_enum(__method__) unless block_given?
 
         base_models.each do |model|
-          yield model.connection.load_balancer
+          yield model.load_balancer
         end
       end
 
diff --git a/lib/gitlab/database/load_balancing/configuration.rb b/lib/gitlab/database/load_balancing/configuration.rb
index 6156515bd73..da313361073 100644
--- a/lib/gitlab/database/load_balancing/configuration.rb
+++ b/lib/gitlab/database/load_balancing/configuration.rb
@@ -7,7 +7,7 @@ module Gitlab
       class Configuration
         attr_accessor :hosts, :max_replication_difference,
                       :max_replication_lag_time, :replica_check_interval,
-                      :service_discovery, :model
+                      :service_discovery
 
         # Creates a configuration object for the given ActiveRecord model.
         def self.for_model(model)
@@ -41,6 +41,8 @@ module Gitlab
             end
           end
 
+          config.reuse_primary_connection!
+
           config
         end
 
@@ -59,6 +61,28 @@ module Gitlab
             disconnect_timeout: 120,
             use_tcp: false
           }
+
+          # Temporary model for GITLAB_LOAD_BALANCING_REUSE_PRIMARY_
+          # To be removed with FF
+          @primary_model = nil
+        end
+
+        def db_config_name
+          @model.connection_db_config.name.to_sym
+        end
+
+        # With connection re-use the primary connection can be overwritten
+        # to be used from different model
+        def primary_connection_specification_name
+          (@primary_model || @model).connection_specification_name
+        end
+
+        def primary_db_config
+          (@primary_model || @model).connection_db_config
+        end
+
+        def replica_db_config
+          @model.connection_db_config
         end
 
         def pool_size
@@ -86,6 +110,30 @@ module Gitlab
         def service_discovery_enabled?
           service_discovery[:record].present?
         end
+
+        # TODO: This is temporary code to allow re-use of primary connection
+        # if the two connections are pointing to the same host. This is needed
+        # to properly support transaction visibility.
+        #
+        # This behavior is required to support [Phase 3](https://gitlab.com/groups/gitlab-org/-/epics/6160#progress).
+        # This method is meant to be removed as soon as it is finished.
+        #
+        # The remapping is done as-is:
+        #   export GITLAB_LOAD_BALANCING_REUSE_PRIMARY_=
+        #
+        # Ex.:
+        #   export GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci=main
+        #
+        def reuse_primary_connection!
+          new_connection = ENV["GITLAB_LOAD_BALANCING_REUSE_PRIMARY_#{db_config_name}"]
+          return unless new_connection.present?
+
+          @primary_model = Gitlab::Database.database_base_models[new_connection.to_sym]
+
+          unless @primary_model
+            raise "Invalid value for 'GITLAB_LOAD_BALANCING_REUSE_PRIMARY_#{db_config_name}=#{new_connection}'"
+          end
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb
index 1be63da8896..a91df2eccdd 100644
--- a/lib/gitlab/database/load_balancing/connection_proxy.rb
+++ b/lib/gitlab/database/load_balancing/connection_proxy.rb
@@ -13,6 +13,13 @@ module Gitlab
         WriteInsideReadOnlyTransactionError = Class.new(StandardError)
         READ_ONLY_TRANSACTION_KEY = :load_balacing_read_only_transaction
 
+        # The load balancer returned by connection might be different
+        # between `model.connection.load_balancer` vs `model.load_balancer`
+        #
+        # The used `model.connection` is dependent on `use_model_load_balancing`.
+        # See more in: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73949.
+        #
+        # Always use `model.load_balancer` or `model.sticking`.
         attr_reader :load_balancer
 
         # These methods perform writes after which we need to stick to the
diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb
index 2be7f0baa60..1e27bcfc55d 100644
--- a/lib/gitlab/database/load_balancing/load_balancer.rb
+++ b/lib/gitlab/database/load_balancing/load_balancer.rb
@@ -12,7 +12,7 @@ module Gitlab
 
         REPLICA_SUFFIX = '_replica'
 
-        attr_reader :name, :host_list, :configuration
+        attr_reader :host_list, :configuration
 
         # configuration - An instance of `LoadBalancing::Configuration` that
         #                 contains the configuration details (such as the hosts)
@@ -26,8 +26,10 @@ module Gitlab
             else
               HostList.new(configuration.hosts.map { |addr| Host.new(addr, self) })
             end
+        end
 
-          @name = @configuration.model.connection_db_config.name.to_sym
+        def name
+          @configuration.db_config_name
         end
 
         def primary_only?
@@ -64,7 +66,7 @@ module Gitlab
                 # times before using the primary instead.
                 will_retry = conflict_retried < @host_list.length * 3
 
-                LoadBalancing::Logger.warn(
+                ::Gitlab::Database::LoadBalancing::Logger.warn(
                   event: :host_query_conflict,
                   message: 'Query conflict on host',
                   conflict_retried: conflict_retried,
@@ -89,7 +91,7 @@ module Gitlab
             end
           end
 
-          LoadBalancing::Logger.warn(
+          ::Gitlab::Database::LoadBalancing::Logger.warn(
             event: :no_secondaries_available,
             message: 'No secondaries were available, using primary instead',
             conflict_retried: conflict_retried,
@@ -136,7 +138,7 @@ module Gitlab
         # Returns the transaction write location of the primary.
         def primary_write_location
           location = read_write do |connection|
-            ::Gitlab::Database.main.get_write_location(connection)
+            get_write_location(connection)
           end
 
           return location if location
@@ -230,7 +232,7 @@ module Gitlab
         # host - An optional host name to use instead of the default one.
         # port - An optional port to connect to.
         def create_replica_connection_pool(pool_size, host = nil, port = nil)
-          db_config = pool.db_config
+          db_config = @configuration.replica_db_config
 
           env_config = db_config.configuration_hash.dup
           env_config[:pool] = pool_size
@@ -255,22 +257,67 @@ module Gitlab
         # leverage that.
         def pool
           ActiveRecord::Base.connection_handler.retrieve_connection_pool(
-            @configuration.model.connection_specification_name,
+            @configuration.primary_connection_specification_name,
             role: ActiveRecord::Base.writing_role,
             shard: ActiveRecord::Base.default_shard
           ) || raise(::ActiveRecord::ConnectionNotEstablished)
         end
 
+        def wal_diff(location1, location2)
+          read_write do |connection|
+            lsn1 = connection.quote(location1)
+            lsn2 = connection.quote(location2)
+
+            query = <<-SQL.squish
+            SELECT pg_wal_lsn_diff(#{lsn1}, #{lsn2})
+              AS result
+            SQL
+
+            row = connection.select_all(query).first
+            row['result'] if row
+          end
+        end
+
         private
 
         def ensure_caching!
-          host.enable_query_cache! unless host.query_cache_enabled
+          return unless Rails.application.executor.active?
+          return if host.query_cache_enabled
+
+          host.enable_query_cache!
         end
 
         def request_cache
           base = SafeRequestStore[:gitlab_load_balancer] ||= {}
           base[self] ||= {}
         end
+
+        # @param [ActiveRecord::Connection] ar_connection
+        # @return [String]
+        def get_write_location(ar_connection)
+          use_new_load_balancer_query = Gitlab::Utils
+            .to_boolean(ENV['USE_NEW_LOAD_BALANCER_QUERY'], default: true)
+
+          sql =
+            if use_new_load_balancer_query
+              <<~NEWSQL
+                SELECT CASE
+                    WHEN pg_is_in_recovery() = true AND EXISTS (SELECT 1 FROM pg_stat_get_wal_senders())
+                      THEN pg_last_wal_replay_lsn()::text
+                    WHEN pg_is_in_recovery() = false
+                      THEN pg_current_wal_insert_lsn()::text
+                      ELSE NULL
+                    END AS location;
+              NEWSQL
+            else
+              <<~SQL
+                SELECT pg_current_wal_insert_lsn()::text AS location
+              SQL
+            end
+
+          row = ar_connection.select_all(sql).first
+          row['location'] if row
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/load_balancing/primary_host.rb b/lib/gitlab/database/load_balancing/primary_host.rb
index 7070cc54d4b..fb52b384ddb 100644
--- a/lib/gitlab/database/load_balancing/primary_host.rb
+++ b/lib/gitlab/database/load_balancing/primary_host.rb
@@ -49,6 +49,11 @@ module Gitlab
         end
 
         def offline!
+          ::Gitlab::Database::LoadBalancing::Logger.warn(
+            event: :host_offline,
+            message: 'Marking primary host as offline'
+          )
+
           nil
         end
 
diff --git a/lib/gitlab/database/load_balancing/rack_middleware.rb b/lib/gitlab/database/load_balancing/rack_middleware.rb
index 7ce7649cc22..99b1c31b04b 100644
--- a/lib/gitlab/database/load_balancing/rack_middleware.rb
+++ b/lib/gitlab/database/load_balancing/rack_middleware.rb
@@ -38,8 +38,8 @@ module Gitlab
         def unstick_or_continue_sticking(env)
           namespaces_and_ids = sticking_namespaces(env)
 
-          namespaces_and_ids.each do |(model, namespace, id)|
-            model.sticking.unstick_or_continue_sticking(namespace, id)
+          namespaces_and_ids.each do |(sticking, namespace, id)|
+            sticking.unstick_or_continue_sticking(namespace, id)
           end
         end
 
@@ -47,8 +47,8 @@ module Gitlab
         def stick_if_necessary(env)
           namespaces_and_ids = sticking_namespaces(env)
 
-          namespaces_and_ids.each do |model, namespace, id|
-            model.sticking.stick_if_necessary(namespace, id)
+          namespaces_and_ids.each do |sticking, namespace, id|
+            sticking.stick_if_necessary(namespace, id)
           end
         end
 
@@ -74,7 +74,7 @@ module Gitlab
             # models that support load balancing. In the future (if we
             # determined this to be OK) we may be able to relax this.
             ::Gitlab::Database::LoadBalancing.base_models.map do |model|
-              [model, :user, warden.user.id]
+              [model.sticking, :user, warden.user.id]
             end
           elsif env[STICK_OBJECT].present?
             env[STICK_OBJECT].to_a
diff --git a/lib/gitlab/database/load_balancing/setup.rb b/lib/gitlab/database/load_balancing/setup.rb
index 3cce839a960..ef38f42f50b 100644
--- a/lib/gitlab/database/load_balancing/setup.rb
+++ b/lib/gitlab/database/load_balancing/setup.rb
@@ -5,7 +5,7 @@ module Gitlab
     module LoadBalancing
       # Class for setting up load balancing of a specific model.
       class Setup
-        attr_reader :configuration
+        attr_reader :model, :configuration
 
         def initialize(model, start_service_discovery: false)
           @model = model
@@ -14,47 +14,102 @@ module Gitlab
         end
 
         def setup
-          disable_prepared_statements
-          setup_load_balancer
+          configure_connection
+          setup_connection_proxy
           setup_service_discovery
+          setup_feature_flag_to_model_load_balancing
         end
 
-        def disable_prepared_statements
+        def configure_connection
           db_config_object = @model.connection_db_config
-          config =
-            db_config_object.configuration_hash.merge(prepared_statements: false)
+
+          hash = db_config_object.configuration_hash.merge(
+            prepared_statements: false,
+            pool: Gitlab::Database.default_pool_size
+          )
 
           hash_config = ActiveRecord::DatabaseConfigurations::HashConfig.new(
             db_config_object.env_name,
             db_config_object.name,
-            config
+            hash
           )
 
           @model.establish_connection(hash_config)
         end
 
-        def setup_load_balancer
-          lb = LoadBalancer.new(configuration)
-
+        def setup_connection_proxy
           # We just use a simple `class_attribute` here so we don't need to
           # inject any modules and/or expose unnecessary methods.
-          @model.class_attribute(:connection)
-          @model.class_attribute(:sticking)
+          setup_class_attribute(:load_balancer, load_balancer)
+          setup_class_attribute(:connection, ConnectionProxy.new(load_balancer))
+          setup_class_attribute(:sticking, Sticking.new(load_balancer))
+        end
+
+        # TODO: This is temporary code to gradually redirect traffic to use
+        # a dedicated DB replicas, or DB primaries (depending on configuration)
+        # This implements a sticky behavior for the current request if enabled.
+        #
+        # This is needed for Phase 3 and Phase 4 of application rollout
+        # https://gitlab.com/groups/gitlab-org/-/epics/6160#progress
+        #
+        # If `GITLAB_USE_MODEL_LOAD_BALANCING` is set, its value is preferred
+        # Otherwise, a `use_model_load_balancing` FF value is used
+        def setup_feature_flag_to_model_load_balancing
+          return if active_record_base?
 
-          @model.connection = ConnectionProxy.new(lb)
-          @model.sticking = Sticking.new(lb)
+          @model.singleton_class.prepend(ModelLoadBalancingFeatureFlagMixin)
         end
 
         def setup_service_discovery
           return unless configuration.service_discovery_enabled?
 
-          lb = @model.connection.load_balancer
-          sv = ServiceDiscovery.new(lb, **configuration.service_discovery)
+          sv = ServiceDiscovery.new(load_balancer, **configuration.service_discovery)
 
           sv.perform_service_discovery
 
           sv.start if @start_service_discovery
         end
+
+        def load_balancer
+          @load_balancer ||= LoadBalancer.new(configuration)
+        end
+
+        private
+
+        def setup_class_attribute(attribute, value)
+          @model.class_attribute(attribute)
+          @model.public_send("#{attribute}=", value) # rubocop:disable GitlabSecurity/PublicSend
+        end
+
+        def active_record_base?
+          @model == ActiveRecord::Base
+        end
+
+        module ModelLoadBalancingFeatureFlagMixin
+          extend ActiveSupport::Concern
+
+          def use_model_load_balancing?
+            # Cache environment variable and return env variable first if defined
+            use_model_load_balancing_env = Gitlab::Utils.to_boolean(ENV["GITLAB_USE_MODEL_LOAD_BALANCING"])
+
+            unless use_model_load_balancing_env.nil?
+              return use_model_load_balancing_env
+            end
+
+            # Check a feature flag using RequestStore (if active)
+            return false unless Gitlab::SafeRequestStore.active?
+
+            Gitlab::SafeRequestStore.fetch(:use_model_load_balancing) do
+              Feature.enabled?(:use_model_load_balancing, default_enabled: :yaml)
+            end
+          end
+
+          # rubocop:disable Database/MultipleDatabases
+          def connection
+            use_model_load_balancing? ? super : ActiveRecord::Base.connection
+          end
+          # rubocop:enable Database/MultipleDatabases
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb
index f0c7016032b..b9acc36b4cc 100644
--- a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb
+++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb
@@ -13,7 +13,7 @@ module Gitlab
           job['load_balancing_strategy'] = strategy.to_s
 
           if use_primary?(strategy)
-            Session.current.use_primary!
+            ::Gitlab::Database::LoadBalancing::Session.current.use_primary!
           elsif strategy == :retry
             raise JobReplicaNotUpToDate, "Sidekiq job #{worker_class} JID-#{job['jid']} couldn't use the replica."\
               "  Replica was not up to date."
@@ -29,8 +29,8 @@ module Gitlab
         private
 
         def clear
-          LoadBalancing.release_hosts
-          Session.clear_session
+          ::Gitlab::Database::LoadBalancing.release_hosts
+          ::Gitlab::Database::LoadBalancing::Session.clear_session
         end
 
         def use_primary?(strategy)
@@ -66,7 +66,7 @@ module Gitlab
         def legacy_wal_location(job)
           wal_location = job['database_write_location'] || job['database_replica_location']
 
-          { Gitlab::Database::MAIN_DATABASE_NAME.to_sym => wal_location } if wal_location
+          { ::Gitlab::Database::MAIN_DATABASE_NAME.to_sym => wal_location } if wal_location
         end
 
         def load_balancing_available?(worker_class)
@@ -90,7 +90,7 @@ module Gitlab
         end
 
         def databases_in_sync?(wal_locations)
-          LoadBalancing.each_load_balancer.all? do |lb|
+          ::Gitlab::Database::LoadBalancing.each_load_balancer.all? do |lb|
             if (location = wal_locations[lb.name])
               lb.select_up_to_date_host(location)
             else
diff --git a/lib/gitlab/database/load_balancing/sticking.rb b/lib/gitlab/database/load_balancing/sticking.rb
index df4ad18581f..834e9c6d3c6 100644
--- a/lib/gitlab/database/load_balancing/sticking.rb
+++ b/lib/gitlab/database/load_balancing/sticking.rb
@@ -12,7 +12,6 @@ module Gitlab
 
         def initialize(load_balancer)
           @load_balancer = load_balancer
-          @model = load_balancer.configuration.model
         end
 
         # Unsticks or continues sticking the current request.
@@ -27,8 +26,8 @@ module Gitlab
         def stick_or_unstick_request(env, namespace, id)
           unstick_or_continue_sticking(namespace, id)
 
-          env[RackMiddleware::STICK_OBJECT] ||= Set.new
-          env[RackMiddleware::STICK_OBJECT] << [@model, namespace, id]
+          env[::Gitlab::Database::LoadBalancing::RackMiddleware::STICK_OBJECT] ||= Set.new
+          env[::Gitlab::Database::LoadBalancing::RackMiddleware::STICK_OBJECT] << [self, namespace, id]
         end
 
         # Sticks to the primary if a write was performed.
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 9968096b1f6..7dce4fa0ce2 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -10,8 +10,6 @@ module Gitlab
 
       # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
       MAX_IDENTIFIER_NAME_LENGTH = 63
-
-      PERMITTED_TIMESTAMP_COLUMNS = %i[created_at updated_at deleted_at].to_set.freeze
       DEFAULT_TIMESTAMP_COLUMNS = %i[created_at updated_at].freeze
 
       # Adds `created_at` and `updated_at` columns with timezone information.
@@ -28,33 +26,23 @@ module Gitlab
       #  :default - The default value for the column.
       #  :null - When set to `true` the column will allow NULL values.
       #        The default is to not allow NULL values.
-      #  :columns - the column names to create. Must be one
-      #             of `Gitlab::Database::MigrationHelpers::PERMITTED_TIMESTAMP_COLUMNS`.
+      #  :columns - the column names to create. Must end with `_at`.
       #             Default value: `DEFAULT_TIMESTAMP_COLUMNS`
       #
       # All options are optional.
       def add_timestamps_with_timezone(table_name, options = {})
-        options[:null] = false if options[:null].nil?
         columns = options.fetch(:columns, DEFAULT_TIMESTAMP_COLUMNS)
-        default_value = options[:default]
-
-        validate_not_in_transaction!(:add_timestamps_with_timezone, 'with default value') if default_value
 
         columns.each do |column_name|
           validate_timestamp_column_name!(column_name)
 
-          # If default value is presented, use `add_column_with_default` method instead.
-          if default_value
-            add_column_with_default(
-              table_name,
-              column_name,
-              :datetime_with_timezone,
-              default: default_value,
-              allow_null: options[:null]
-            )
-          else
-            add_column(table_name, column_name, :datetime_with_timezone, **options)
-          end
+          add_column(
+            table_name,
+            column_name,
+            :datetime_with_timezone,
+            default: options[:default],
+            null: options[:null] || false
+          )
         end
       end
 
@@ -147,8 +135,18 @@ module Gitlab
         options = options.merge({ algorithm: :concurrently })
 
         if index_exists?(table_name, column_name, **options)
-          Gitlab::AppLogger.warn "Index not created because it already exists (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}"
-          return
+          name = options[:name] || index_name(table_name, column_name)
+          _, schema = table_name.to_s.split('.').reverse
+
+          if index_invalid?(name, schema: schema)
+            say "Index being recreated because the existing version was INVALID: table_name: #{table_name}, column_name: #{column_name}"
+
+            remove_concurrent_index_by_name(table_name, name)
+          else
+            say "Index not created because it already exists (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}"
+
+            return
+          end
         end
 
         disable_statement_timeout do
@@ -159,6 +157,23 @@ module Gitlab
         unprepare_async_index(table_name, column_name, **options)
       end
 
+      def index_invalid?(index_name, schema: nil)
+        index_name = connection.quote(index_name)
+        schema = connection.quote(schema) if schema
+        schema ||= 'current_schema()'
+
+        connection.select_value(<<~SQL)
+          select not i.indisvalid
+          from pg_class c
+          inner join pg_index i
+            on c.oid = i.indexrelid
+          inner join pg_namespace n
+            on n.oid = c.relnamespace
+          where n.nspname = #{schema}
+            and c.relname = #{index_name}
+        SQL
+      end
+
       # Removes an existed index, concurrently
       #
       # Example:
@@ -1245,8 +1260,8 @@ module Gitlab
 
       def check_trigger_permissions!(table)
         unless Grant.create_and_execute_trigger?(table)
-          dbname = Database.main.database_name
-          user = Database.main.username
+          dbname = ApplicationRecord.database.database_name
+          user = ApplicationRecord.database.username
 
           raise <<-EOF
 Your database user is not allowed to create, drop, or execute triggers on the
@@ -1568,8 +1583,8 @@ into similar problems in the future (e.g. when new tables are created).
       def create_extension(extension)
         execute('CREATE EXTENSION IF NOT EXISTS %s' % extension)
       rescue ActiveRecord::StatementInvalid => e
-        dbname = Database.main.database_name
-        user = Database.main.username
+        dbname = ApplicationRecord.database.database_name
+        user = ApplicationRecord.database.username
 
         warn(<<~MSG) if e.to_s =~ /permission denied/
           GitLab requires the PostgreSQL extension '#{extension}' installed in database '#{dbname}', but
@@ -1596,8 +1611,8 @@ into similar problems in the future (e.g. when new tables are created).
       def drop_extension(extension)
         execute('DROP EXTENSION IF EXISTS %s' % extension)
       rescue ActiveRecord::StatementInvalid => e
-        dbname = Database.main.database_name
-        user = Database.main.username
+        dbname = ApplicationRecord.database.database_name
+        user = ApplicationRecord.database.username
 
         warn(<<~MSG) if e.to_s =~ /permission denied/
           This migration attempts to drop the PostgreSQL extension '#{extension}'
@@ -1791,11 +1806,11 @@ into similar problems in the future (e.g. when new tables are created).
       end
 
       def validate_timestamp_column_name!(column_name)
-        return if PERMITTED_TIMESTAMP_COLUMNS.member?(column_name)
+        return if column_name.to_s.end_with?('_at')
 
         raise <<~MESSAGE
           Illegal timestamp column name! Got #{column_name}.
-          Must be one of: #{PERMITTED_TIMESTAMP_COLUMNS.to_a}
+          Must end with `_at`}
         MESSAGE
       end
 
diff --git a/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb b/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb
index d9ef5ab462e..8a37e619285 100644
--- a/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb
+++ b/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb
@@ -31,10 +31,10 @@ module Gitlab
 
           namespace_options = options.merge(null: true, default: nil)
 
-          add_column(:namespace_settings, setting_name, type, namespace_options)
+          add_column(:namespace_settings, setting_name, type, **namespace_options)
           add_column(:namespace_settings, lock_column_name, :boolean, default: false, null: false)
 
-          add_column(:application_settings, setting_name, type, options)
+          add_column(:application_settings, setting_name, type, **options)
           add_column(:application_settings, lock_column_name, :boolean, default: false, null: false)
         end
 
diff --git a/lib/gitlab/database/migrations/observation.rb b/lib/gitlab/database/migrations/observation.rb
index 54eedec3c7b..a494c357950 100644
--- a/lib/gitlab/database/migrations/observation.rb
+++ b/lib/gitlab/database/migrations/observation.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Naming/FileName
 # frozen_string_literal: true
 
 module Gitlab
@@ -14,3 +15,5 @@ module Gitlab
     end
   end
 end
+
+# rubocop:enable Naming/FileName
diff --git a/lib/gitlab/database/migrations/observers.rb b/lib/gitlab/database/migrations/observers.rb
index 140b3feed64..b890e62c2d0 100644
--- a/lib/gitlab/database/migrations/observers.rb
+++ b/lib/gitlab/database/migrations/observers.rb
@@ -9,7 +9,8 @@ module Gitlab
             TotalDatabaseSizeChange,
             QueryStatistics,
             QueryLog,
-            QueryDetails
+            QueryDetails,
+            TransactionDuration
           ]
         end
       end
diff --git a/lib/gitlab/database/migrations/observers/transaction_duration.rb b/lib/gitlab/database/migrations/observers/transaction_duration.rb
new file mode 100644
index 00000000000..a96b94334cf
--- /dev/null
+++ b/lib/gitlab/database/migrations/observers/transaction_duration.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module Migrations
+      module Observers
+        class TransactionDuration < MigrationObserver
+          def before
+            file_path = File.join(output_dir, "#{observation.version}_#{observation.name}-transaction-duration.json")
+            @file = File.open(file_path, 'wb')
+            @writer = Oj::StreamWriter.new(@file, {})
+            @writer.push_array
+            @subscriber = ActiveSupport::Notifications.subscribe('transaction.active_record') do |*args|
+              record_sql_event(*args)
+            end
+          end
+
+          def after
+            ActiveSupport::Notifications.unsubscribe(@subscriber)
+            @writer.pop_all
+            @writer.flush
+            @file.close
+          end
+
+          def record
+            # no-op
+          end
+
+          def record_sql_event(_name, started, finished, _unique_id, payload)
+            return if payload[:transaction_type] == :fake_transaction
+
+            @writer.push_value({
+                                 start_time: started.iso8601(6),
+                                 end_time: finished.iso8601(6),
+                                 transaction_type: payload[:transaction_type]
+                               })
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/partitioning.rb b/lib/gitlab/database/partitioning.rb
index 71fb995577a..1343354715a 100644
--- a/lib/gitlab/database/partitioning.rb
+++ b/lib/gitlab/database/partitioning.rb
@@ -3,20 +3,83 @@
 module Gitlab
   module Database
     module Partitioning
-      def self.register_models(models)
-        registered_models.merge(models)
-      end
+      class TableWithoutModel
+        include PartitionedTable::ClassMethods
 
-      def self.registered_models
-        @registered_models ||= Set.new
-      end
+        attr_reader :table_name
+
+        def initialize(table_name:, partitioned_column:, strategy:)
+          @table_name = table_name
+          partitioned_by(partitioned_column, strategy: strategy)
+        end
 
-      def self.sync_partitions(models_to_sync = registered_models)
-        MultiDatabasePartitionManager.new(models_to_sync).sync_partitions
+        def connection
+          Gitlab::Database::SharedModel.connection
+        end
       end
 
-      def self.drop_detached_partitions
-        MultiDatabasePartitionDropper.new.drop_detached_partitions
+      class << self
+        def register_models(models)
+          models.each do |model|
+            raise "#{model} should have partitioning strategy defined" unless model.respond_to?(:partitioning_strategy)
+
+            registered_models << model
+          end
+        end
+
+        def register_tables(tables)
+          registered_tables.merge(tables)
+        end
+
+        def sync_partitions_ignore_db_error
+          sync_partitions unless ENV['DISABLE_POSTGRES_PARTITION_CREATION_ON_STARTUP']
+        rescue ActiveRecord::ActiveRecordError, PG::Error
+          # ignore - happens when Rake tasks yet have to create a database, e.g. for testing
+        end
+
+        def sync_partitions(models_to_sync = registered_for_sync)
+          Gitlab::AppLogger.info(message: 'Syncing dynamic postgres partitions')
+
+          Gitlab::Database::EachDatabase.each_model_connection(models_to_sync) do |model|
+            PartitionManager.new(model).sync_partitions
+          end
+
+          Gitlab::AppLogger.info(message: 'Finished sync of dynamic postgres partitions')
+        end
+
+        def report_metrics(models_to_monitor = registered_models)
+          partition_monitoring = PartitionMonitoring.new
+
+          Gitlab::Database::EachDatabase.each_model_connection(models_to_monitor) do |model|
+            partition_monitoring.report_metrics_for_model(model)
+          end
+        end
+
+        def drop_detached_partitions
+          Gitlab::AppLogger.info(message: 'Dropping detached postgres partitions')
+
+          Gitlab::Database::EachDatabase.each_database_connection do
+            DetachedPartitionDropper.new.perform
+          end
+
+          Gitlab::AppLogger.info(message: 'Finished dropping detached postgres partitions')
+        end
+
+        def registered_models
+          @registered_models ||= Set.new
+        end
+
+        def registered_tables
+          @registered_tables ||= Set.new
+        end
+
+        private
+
+        def registered_for_sync
+          registered_models + registered_tables.map do |table|
+            TableWithoutModel.new(**table)
+          end
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/partitioning/detached_partition_dropper.rb b/lib/gitlab/database/partitioning/detached_partition_dropper.rb
index 3e7ddece20b..593824384b5 100644
--- a/lib/gitlab/database/partitioning/detached_partition_dropper.rb
+++ b/lib/gitlab/database/partitioning/detached_partition_dropper.rb
@@ -9,13 +9,10 @@ module Gitlab
           Gitlab::AppLogger.info(message: "Checking for previously detached partitions to drop")
 
           Postgresql::DetachedPartition.ready_to_drop.find_each do |detached_partition|
-            connection.transaction do
-              # Another process may have already dropped the table and deleted this entry
-              next unless (detached_partition = Postgresql::DetachedPartition.lock.find_by(id: detached_partition.id))
-
-              drop_detached_partition(detached_partition.table_name)
-
-              detached_partition.destroy!
+            if partition_attached?(qualify_partition_name(detached_partition.table_name))
+              unmark_partition(detached_partition)
+            else
+              drop_partition(detached_partition)
             end
           rescue StandardError => e
             Gitlab::AppLogger.error(message: "Failed to drop previously detached partition",
@@ -27,31 +24,100 @@ module Gitlab
 
         private
 
+        def unmark_partition(detached_partition)
+          connection.transaction do
+            # Another process may have already encountered this case and deleted this entry
+            next unless try_lock_detached_partition(detached_partition.id)
+
+            # The current partition was scheduled for deletion incorrectly
+            # Dropping it now could delete in-use data and take locks that interrupt other database activity
+            Gitlab::AppLogger.error(message: "Prevented an attempt to drop an attached database partition", partition_name: detached_partition.table_name)
+            detached_partition.destroy!
+          end
+        end
+
+        def drop_partition(detached_partition)
+          remove_foreign_keys(detached_partition)
+
+          connection.transaction do
+            # Another process may have already dropped the table and deleted this entry
+            next unless try_lock_detached_partition(detached_partition.id)
+
+            drop_detached_partition(detached_partition.table_name)
+
+            detached_partition.destroy!
+          end
+        end
+
+        def remove_foreign_keys(detached_partition)
+          partition_identifier = qualify_partition_name(detached_partition.table_name)
+
+          # We want to load all of these into memory at once to get a consistent view to loop over,
+          # since we'll be deleting from this list as we go
+          fks_to_drop = PostgresForeignKey.by_constrained_table_identifier(partition_identifier).to_a
+          fks_to_drop.each do |foreign_key|
+            drop_foreign_key_if_present(detached_partition, foreign_key)
+          end
+        end
+
+        # Drops the given foreign key for the given detached partition, but only if another process has not already
+        # detached the partition first. This method must be safe to call even if the associated partition table has already
+        # been detached, as it could be called by multiple processes at once.
+        def drop_foreign_key_if_present(detached_partition, foreign_key)
+          # It is important to only drop one foreign key per transaction.
+          # Dropping a foreign key takes an ACCESS EXCLUSIVE lock on both tables participating in the foreign key.
+
+          partition_identifier = qualify_partition_name(detached_partition.table_name)
+          with_lock_retries do
+            connection.transaction(requires_new: false) do
+              next unless try_lock_detached_partition(detached_partition.id)
+
+              # Another process may have already dropped this foreign key
+              next unless PostgresForeignKey.by_constrained_table_identifier(partition_identifier).where(name: foreign_key.name).exists?
+
+              connection.execute("ALTER TABLE #{connection.quote_table_name(partition_identifier)} DROP CONSTRAINT #{connection.quote_table_name(foreign_key.name)}")
+
+              Gitlab::AppLogger.info(message: "Dropped foreign key for previously detached partition",
+                                     partition_name: detached_partition.table_name,
+                                     referenced_table_name: foreign_key.referenced_table_identifier,
+                                     foreign_key_name: foreign_key.name)
+            end
+          end
+        end
+
         def drop_detached_partition(partition_name)
           partition_identifier = qualify_partition_name(partition_name)
 
-          if partition_detached?(partition_identifier)
-            connection.drop_table(partition_identifier, if_exists: true)
+          connection.drop_table(partition_identifier, if_exists: true)
 
-            Gitlab::AppLogger.info(message: "Dropped previously detached partition", partition_name: partition_name)
-          else
-            Gitlab::AppLogger.error(message: "Attempt to drop attached database partition", partition_name: partition_name)
-          end
+          Gitlab::AppLogger.info(message: "Dropped previously detached partition", partition_name: partition_name)
         end
 
         def qualify_partition_name(table_name)
           "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{table_name}"
         end
 
-        def partition_detached?(partition_identifier)
+        def partition_attached?(partition_identifier)
           # PostgresPartition checks the pg_inherits view, so our partition will only show here if it's still attached
           # and thus should not be dropped
-          !Gitlab::Database::PostgresPartition.for_identifier(partition_identifier).exists?
+          Gitlab::Database::PostgresPartition.for_identifier(partition_identifier).exists?
+        end
+
+        def try_lock_detached_partition(id)
+          Postgresql::DetachedPartition.lock.find_by(id: id).present?
         end
 
         def connection
           Postgresql::DetachedPartition.connection
         end
+
+        def with_lock_retries(&block)
+          Gitlab::Database::WithLockRetries.new(
+            klass: self.class,
+            logger: Gitlab::AppLogger,
+            connection: connection
+          ).run(raise_on_exhaustion: true, &block)
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/partitioning/monthly_strategy.rb b/lib/gitlab/database/partitioning/monthly_strategy.rb
index 4cdde5bf2f1..c93e775d7ed 100644
--- a/lib/gitlab/database/partitioning/monthly_strategy.rb
+++ b/lib/gitlab/database/partitioning/monthly_strategy.rb
@@ -96,10 +96,6 @@ module Gitlab
         def oldest_active_date
           (Date.today - retain_for).beginning_of_month
         end
-
-        def connection
-          ActiveRecord::Base.connection
-        end
       end
     end
   end
diff --git a/lib/gitlab/database/partitioning/multi_database_partition_dropper.rb b/lib/gitlab/database/partitioning/multi_database_partition_dropper.rb
deleted file mode 100644
index 769b658bae4..00000000000
--- a/lib/gitlab/database/partitioning/multi_database_partition_dropper.rb
+++ /dev/null
@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Database
-    module Partitioning
-      class MultiDatabasePartitionDropper
-        def drop_detached_partitions
-          Gitlab::AppLogger.info(message: "Dropping detached postgres partitions")
-
-          each_database_connection do |name, connection|
-            Gitlab::Database::SharedModel.using_connection(connection) do
-              Gitlab::AppLogger.debug(message: "Switched database connection", connection_name: name)
-
-              DetachedPartitionDropper.new.perform
-            end
-          end
-
-          Gitlab::AppLogger.info(message: "Finished dropping detached postgres partitions")
-        end
-
-        private
-
-        def each_database_connection
-          databases.each_pair do |name, connection_wrapper|
-            yield name, connection_wrapper.scope.connection
-          end
-        end
-
-        def databases
-          Gitlab::Database.databases
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/database/partitioning/multi_database_partition_manager.rb b/lib/gitlab/database/partitioning/multi_database_partition_manager.rb
deleted file mode 100644
index 5a93e3fb1fb..00000000000
--- a/lib/gitlab/database/partitioning/multi_database_partition_manager.rb
+++ /dev/null
@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Database
-    module Partitioning
-      class MultiDatabasePartitionManager
-        def initialize(models)
-          @models = models
-        end
-
-        def sync_partitions
-          Gitlab::AppLogger.info(message: "Syncing dynamic postgres partitions")
-
-          models.each do |model|
-            Gitlab::Database::SharedModel.using_connection(model.connection) do
-              Gitlab::AppLogger.debug(message: "Switched database connection",
-                                      connection_name: connection_name,
-                                      table_name: model.table_name)
-
-              PartitionManager.new(model).sync_partitions
-            end
-          end
-
-          Gitlab::AppLogger.info(message: "Finished sync of dynamic postgres partitions")
-        end
-
-        private
-
-        attr_reader :models
-
-        def connection_name
-          Gitlab::Database::SharedModel.connection.pool.db_config.name
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/database/partitioning/partition_monitoring.rb b/lib/gitlab/database/partitioning/partition_monitoring.rb
index e5b561fc447..1a23f58285d 100644
--- a/lib/gitlab/database/partitioning/partition_monitoring.rb
+++ b/lib/gitlab/database/partitioning/partition_monitoring.rb
@@ -4,20 +4,12 @@ module Gitlab
   module Database
     module Partitioning
       class PartitionMonitoring
-        attr_reader :models
+        def report_metrics_for_model(model)
+          strategy = model.partitioning_strategy
 
-        def initialize(models = Gitlab::Database::Partitioning.registered_models)
-          @models = models
-        end
-
-        def report_metrics
-          models.each do |model|
-            strategy = model.partitioning_strategy
-
-            gauge_present.set({ table: model.table_name }, strategy.current_partitions.size)
-            gauge_missing.set({ table: model.table_name }, strategy.missing_partitions.size)
-            gauge_extra.set({ table: model.table_name }, strategy.extra_partitions.size)
-          end
+          gauge_present.set({ table: model.table_name }, strategy.current_partitions.size)
+          gauge_missing.set({ table: model.table_name }, strategy.missing_partitions.size)
+          gauge_extra.set({ table: model.table_name }, strategy.extra_partitions.size)
         end
 
         private
diff --git a/lib/gitlab/database/partitioning/replace_table.rb b/lib/gitlab/database/partitioning/replace_table.rb
index 6f6af223fa2..a7686e97553 100644
--- a/lib/gitlab/database/partitioning/replace_table.rb
+++ b/lib/gitlab/database/partitioning/replace_table.rb
@@ -9,7 +9,8 @@ module Gitlab
         attr_reader :original_table, :replacement_table, :replaced_table, :primary_key_column,
           :sequence, :original_primary_key, :replacement_primary_key, :replaced_primary_key
 
-        def initialize(original_table, replacement_table, replaced_table, primary_key_column)
+        def initialize(connection, original_table, replacement_table, replaced_table, primary_key_column)
+          @connection = connection
           @original_table = original_table
           @replacement_table = replacement_table
           @replaced_table = replaced_table
@@ -29,10 +30,8 @@ module Gitlab
 
         private
 
+        attr_reader :connection
         delegate :execute, :quote_table_name, :quote_column_name, to: :connection
-        def connection
-          @connection ||= ActiveRecord::Base.connection
-        end
 
         def default_sequence(table, column)
           "#{table}_#{column}_seq"
diff --git a/lib/gitlab/database/partitioning/time_partition.rb b/lib/gitlab/database/partitioning/time_partition.rb
index e09ca483549..649687bdd12 100644
--- a/lib/gitlab/database/partitioning/time_partition.rb
+++ b/lib/gitlab/database/partitioning/time_partition.rb
@@ -87,7 +87,7 @@ module Gitlab
         end
 
         def conn
-          @conn ||= ActiveRecord::Base.connection
+          @conn ||= Gitlab::Database::SharedModel.connection
         end
       end
     end
diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
index 0dc9f92e4c8..c382d2f0715 100644
--- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
+++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
@@ -428,8 +428,8 @@ module Gitlab
         end
 
         def replace_table(original_table_name, replacement_table_name, replaced_table_name, primary_key_name)
-          replace_table = Gitlab::Database::Partitioning::ReplaceTable.new(original_table_name.to_s,
-              replacement_table_name, replaced_table_name, primary_key_name)
+          replace_table = Gitlab::Database::Partitioning::ReplaceTable.new(connection,
+              original_table_name.to_s, replacement_table_name, replaced_table_name, primary_key_name)
 
           transaction do
             drop_sync_trigger(original_table_name)
diff --git a/lib/gitlab/database/postgres_foreign_key.rb b/lib/gitlab/database/postgres_foreign_key.rb
index 72640f8785d..241b6f009f7 100644
--- a/lib/gitlab/database/postgres_foreign_key.rb
+++ b/lib/gitlab/database/postgres_foreign_key.rb
@@ -10,6 +10,12 @@ module Gitlab
 
         where(referenced_table_identifier: identifier)
       end
+
+      scope :by_constrained_table_identifier, ->(identifier) do
+        raise ArgumentError, "Constrained table name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/
+
+        where(constrained_table_identifier: identifier)
+      end
     end
   end
 end
diff --git a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb
index 2e3f674cf82..4e973efebca 100644
--- a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb
+++ b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb
@@ -57,7 +57,7 @@ module Gitlab
         # @param finish final pkey range
         # @return [Gitlab::Database::PostgresHll::Buckets] HyperLogLog data structure instance that can estimate number of unique elements
         def execute(batch_size: nil, start: nil, finish: nil)
-          raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open? # rubocop: disable Database/MultipleDatabases
+          raise 'BatchCount can not be run inside a transaction' if transaction_open?
 
           batch_size ||= DEFAULT_BATCH_SIZE
           start = actual_start(start)
@@ -79,6 +79,10 @@ module Gitlab
 
         private
 
+        def transaction_open?
+          @relation.connection.transaction_open?
+        end
+
         def unwanted_configuration?(start, finish, batch_size)
           batch_size <= MIN_REQUIRED_BATCH_SIZE ||
             (finish - start) >= MAX_DATA_VOLUME ||
diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb
index 1079bfdeda3..4a9d8728c83 100644
--- a/lib/gitlab/database/postgres_index.rb
+++ b/lib/gitlab/database/postgres_index.rb
@@ -2,7 +2,7 @@
 
 module Gitlab
   module Database
-    class PostgresIndex < ActiveRecord::Base
+    class PostgresIndex < SharedModel
       include Gitlab::Utils::StrongMemoize
 
       self.table_name = 'postgres_indexes'
@@ -11,6 +11,7 @@ module Gitlab
 
       has_one :bloat_estimate, class_name: 'Gitlab::Database::PostgresIndexBloatEstimate', foreign_key: :identifier
       has_many :reindexing_actions, class_name: 'Gitlab::Database::Reindexing::ReindexAction', foreign_key: :index_identifier
+      has_many :queued_reindexing_actions, class_name: 'Gitlab::Database::Reindexing::QueuedAction', foreign_key: :index_identifier
 
       scope :by_identifier, ->(identifier) do
         raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/
diff --git a/lib/gitlab/database/postgres_index_bloat_estimate.rb b/lib/gitlab/database/postgres_index_bloat_estimate.rb
index 379227bf87c..5c9b5777b74 100644
--- a/lib/gitlab/database/postgres_index_bloat_estimate.rb
+++ b/lib/gitlab/database/postgres_index_bloat_estimate.rb
@@ -6,7 +6,7 @@ module Gitlab
     # for all indexes can be expensive in a large database.
     #
     # Best used on a per-index basis.
-    class PostgresIndexBloatEstimate < ActiveRecord::Base
+    class PostgresIndexBloatEstimate < SharedModel
       self.table_name = 'postgres_index_bloat_estimates'
       self.primary_key = 'identifier'
 
diff --git a/lib/gitlab/database/query_analyzer.rb b/lib/gitlab/database/query_analyzer.rb
new file mode 100644
index 00000000000..0f285688876
--- /dev/null
+++ b/lib/gitlab/database/query_analyzer.rb
@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    # The purpose of this class is to implement a various query analyzers based on `pg_query`
+    # And process them all via `Gitlab::Database::QueryAnalyzers::*`
+    #
+    # Sometimes this might cause errors in specs.
+    # This is best to be disable with `describe '...', query_analyzers: false do`
+    class QueryAnalyzer
+      include ::Singleton
+
+      Parsed = Struct.new(
+        :sql, :connection, :pg
+      )
+
+      attr_reader :all_analyzers
+
+      def initialize
+        @all_analyzers = []
+      end
+
+      def hook!
+        @subscriber = ActiveSupport::Notifications.subscribe('sql.active_record') do |event|
+          # In some cases analyzer code might trigger another SQL call
+          # to avoid stack too deep this detects recursive call of subscriber
+          with_ignored_recursive_calls do
+            process_sql(event.payload[:sql], event.payload[:connection])
+          end
+        end
+      end
+
+      def within
+        # Due to singleton nature of analyzers
+        # only an outer invocation of the `.within`
+        # is allowed to initialize them
+        return yield if already_within?
+
+        begin!
+
+        begin
+          yield
+        ensure
+          end!
+        end
+      end
+
+      def already_within?
+        # If analyzers are set they are already configured
+        !enabled_analyzers.nil?
+      end
+
+      def process_sql(sql, connection)
+        analyzers = enabled_analyzers
+        return unless analyzers&.any?
+
+        parsed = parse(sql, connection)
+        return unless parsed
+
+        analyzers.each do |analyzer|
+          next if analyzer.suppressed?
+
+          analyzer.analyze(parsed)
+        rescue StandardError => e
+          # We catch all standard errors to prevent validation errors to introduce fatal errors in production
+          Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
+        end
+      end
+
+      private
+
+      # Enable query analyzers
+      def begin!
+        analyzers = all_analyzers.select do |analyzer|
+          if analyzer.enabled?
+            analyzer.begin!
+
+            true
+          end
+        rescue StandardError => e
+          Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
+
+          false
+        end
+
+        Thread.current[:query_analyzer_enabled_analyzers] = analyzers
+      end
+
+      # Disable enabled query analyzers
+      def end!
+        enabled_analyzers.select do |analyzer|
+          analyzer.end!
+        rescue StandardError => e
+          Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
+        end
+
+        Thread.current[:query_analyzer_enabled_analyzers] = nil
+      end
+
+      def enabled_analyzers
+        Thread.current[:query_analyzer_enabled_analyzers]
+      end
+
+      def parse(sql, connection)
+        parsed = PgQuery.parse(sql)
+        return unless parsed
+
+        normalized = PgQuery.normalize(sql)
+        Parsed.new(normalized, connection, parsed)
+      rescue PgQuery::ParseError => e
+        # Ignore PgQuery parse errors (due to depth limit or other reasons)
+        Gitlab::ErrorTracking.track_exception(e)
+
+        nil
+      end
+
+      def with_ignored_recursive_calls
+        return if Thread.current[:query_analyzer_recursive]
+
+        begin
+          Thread.current[:query_analyzer_recursive] = true
+          yield
+        ensure
+          Thread.current[:query_analyzer_recursive] = nil
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/query_analyzers/base.rb b/lib/gitlab/database/query_analyzers/base.rb
new file mode 100644
index 00000000000..e8066f7a706
--- /dev/null
+++ b/lib/gitlab/database/query_analyzers/base.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module QueryAnalyzers
+      class Base
+        def self.suppressed?
+          Thread.current[self.suppress_key]
+        end
+
+        def self.suppress=(value)
+          Thread.current[self.suppress_key] = value
+        end
+
+        def self.with_suppressed(value = true, &blk)
+          previous = self.suppressed?
+          self.suppress = value
+          yield
+        ensure
+          self.suppress = previous
+        end
+
+        def self.begin!
+          Thread.current[self.context_key] = {}
+        end
+
+        def self.end!
+          Thread.current[self.context_key] = nil
+        end
+
+        def self.context
+          Thread.current[self.context_key]
+        end
+
+        def self.enabled?
+          raise NotImplementedError
+        end
+
+        def self.analyze(parsed)
+          raise NotImplementedError
+        end
+
+        def self.context_key
+          "#{self.class.name}_context"
+        end
+
+        def self.suppress_key
+          "#{self.class.name}_suppressed"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb
new file mode 100644
index 00000000000..06e2b114c91
--- /dev/null
+++ b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module QueryAnalyzers
+      # The purpose of this analyzer is to observe via prometheus metrics
+      # all unique schemas observed on a given connection
+      #
+      # This effectively allows to do sample 1% or 0.01% of queries hitting
+      # system and observe if on a given connection we observe queries that
+      # are misaligned (`ci_replica` sees queries doing accessing only `gitlab_main`)
+      #
+      class GitlabSchemasMetrics < Base
+        class << self
+          def enabled?
+            ::Feature::FlipperFeature.table_exists? &&
+              Feature.enabled?(:query_analyzer_gitlab_schema_metrics)
+          end
+
+          def analyze(parsed)
+            db_config_name = ::Gitlab::Database.db_config_name(parsed.connection)
+            return unless db_config_name
+
+            gitlab_schemas = ::Gitlab::Database::GitlabSchema.table_schemas(parsed.pg.tables)
+            return if gitlab_schemas.empty?
+
+            # to reduce amount of labels sort schemas used
+            gitlab_schemas = gitlab_schemas.to_a.sort.join(",")
+
+            schemas_metrics.increment({
+              gitlab_schemas: gitlab_schemas,
+              db_config_name: db_config_name
+            })
+          end
+
+          def schemas_metrics
+            @schemas_metrics ||= ::Gitlab::Metrics.counter(
+              :gitlab_database_decomposition_gitlab_schemas_used,
+              'The number of observed schemas dependent on connection'
+            )
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb
new file mode 100644
index 00000000000..2233f3c4646
--- /dev/null
+++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb
@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module QueryAnalyzers
+      class PreventCrossDatabaseModification < Database::QueryAnalyzers::Base
+        CrossDatabaseModificationAcrossUnsupportedTablesError = Class.new(StandardError)
+
+        # This method will allow cross database modifications within the block
+        # Example:
+        #
+        # allow_cross_database_modification_within_transaction(url: 'url-to-an-issue') do
+        #   create(:build) # inserts ci_build and project record in one transaction
+        # end
+        def self.allow_cross_database_modification_within_transaction(url:, &blk)
+          self.with_suppressed(true, &blk)
+        end
+
+        # This method will prevent cross database modifications within the block
+        # if it was allowed previously
+        def self.with_cross_database_modification_prevented(&blk)
+          self.with_suppressed(false, &blk)
+        end
+
+        def self.begin!
+          super
+
+          context.merge!({
+            transaction_depth_by_db: Hash.new { |h, k| h[k] = 0 },
+            modified_tables_by_db: Hash.new { |h, k| h[k] = Set.new }
+          })
+        end
+
+        def self.enabled?
+          ::Feature::FlipperFeature.table_exists? &&
+            Feature.enabled?(:detect_cross_database_modification, default_enabled: :yaml)
+        end
+
+        # rubocop:disable Metrics/AbcSize
+        def self.analyze(parsed)
+          return if in_factory_bot_create?
+
+          database = ::Gitlab::Database.db_config_name(parsed.connection)
+          sql = parsed.sql
+
+          # We ignore BEGIN in tests as this is the outer transaction for
+          # DatabaseCleaner
+          if sql.start_with?('SAVEPOINT') || (!Rails.env.test? && sql.start_with?('BEGIN'))
+            context[:transaction_depth_by_db][database] += 1
+
+            return
+          elsif sql.start_with?('RELEASE SAVEPOINT', 'ROLLBACK TO SAVEPOINT') || (!Rails.env.test? && sql.start_with?('ROLLBACK', 'COMMIT'))
+            context[:transaction_depth_by_db][database] -= 1
+            if context[:transaction_depth_by_db][database] <= 0
+              context[:modified_tables_by_db][database].clear
+            end
+
+            return
+          end
+
+          return if context[:transaction_depth_by_db].values.all?(&:zero?)
+
+          # PgQuery might fail in some cases due to limited nesting:
+          # https://github.com/pganalyze/pg_query/issues/209
+          tables = sql.downcase.include?(' for update') ? parsed.pg.tables : parsed.pg.dml_tables
+
+          # We have some code where plans and gitlab_subscriptions are lazily
+          # created and this causes lots of spec failures
+          # https://gitlab.com/gitlab-org/gitlab/-/issues/343394
+          tables -= %w[plans gitlab_subscriptions]
+
+          return if tables.empty?
+
+          # All migrations will write to schema_migrations in the same transaction.
+          # It's safe to ignore this since schema_migrations exists in all
+          # databases
+          return if tables == ['schema_migrations']
+
+          context[:modified_tables_by_db][database].merge(tables)
+          all_tables = context[:modified_tables_by_db].values.map(&:to_a).flatten
+          schemas = ::Gitlab::Database::GitlabSchema.table_schemas(all_tables)
+
+          if schemas.many?
+            message = "Cross-database data modification of '#{schemas.to_a.join(", ")}' were detected within " \
+              "a transaction modifying the '#{all_tables.to_a.join(", ")}' tables." \
+              "Please refer to https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions for details on how to resolve this exception."
+
+            if schemas.any? { |s| s.to_s.start_with?("undefined") }
+              message += " The gitlab_schema was undefined for one or more of the tables in this transaction. Any new tables must be added to lib/gitlab/database/gitlab_schemas.yml ."
+            end
+
+            raise CrossDatabaseModificationAcrossUnsupportedTablesError, message
+          end
+        rescue CrossDatabaseModificationAcrossUnsupportedTablesError => e
+          ::Gitlab::ErrorTracking.track_exception(e, { gitlab_schemas: schemas, tables: all_tables, query: parsed.sql })
+          raise if raise_exception?
+        end
+        # rubocop:enable Metrics/AbcSize
+
+        # We only raise in tests for now otherwise some features will be broken
+        # in development. For now we've mostly only added allowlist based on
+        # spec names. Until we have allowed all the violations inline we don't
+        # want to raise in development.
+        def self.raise_exception?
+          Rails.env.test?
+        end
+
+        # We ignore execution in the #create method from FactoryBot
+        # because it is not representative of real code we run in
+        # production. There are far too many false positives caused
+        # by instantiating objects in different `gitlab_schema` in a
+        # FactoryBot `create`.
+        def self.in_factory_bot_create?
+          Rails.env.test? && caller_locations.any? { |l| l.path.end_with?('lib/factory_bot/evaluation.rb') && l.label == 'create' }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/reflection.rb b/lib/gitlab/database/reflection.rb
new file mode 100644
index 00000000000..48a4de28541
--- /dev/null
+++ b/lib/gitlab/database/reflection.rb
@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    # A class for reflecting upon a database and its settings, such as the
+    # adapter name, PostgreSQL version, and the presence of tables or columns.
+    class Reflection
+      attr_reader :model
+
+      def initialize(model)
+        @model = model
+        @version = nil
+      end
+
+      def config
+        # The result of this method must not be cached, as other methods may use
+        # it after making configuration changes and expect those changes to be
+        # present. For example, `disable_prepared_statements` expects the
+        # configuration settings to always be up to date.
+        #
+        # See the following for more information:
+        #
+        # - https://gitlab.com/gitlab-org/release/retrospectives/-/issues/39
+        # - https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5238
+        model.connection_db_config.configuration_hash.with_indifferent_access
+      end
+
+      def username
+        config[:username] || ENV['USER']
+      end
+
+      def database_name
+        config[:database]
+      end
+
+      def adapter_name
+        config[:adapter]
+      end
+
+      def human_adapter_name
+        if postgresql?
+          'PostgreSQL'
+        else
+          'Unknown'
+        end
+      end
+
+      def postgresql?
+        adapter_name.casecmp('postgresql') == 0
+      end
+
+      # Check whether the underlying database is in read-only mode
+      def db_read_only?
+        pg_is_in_recovery =
+          connection
+            .execute('SELECT pg_is_in_recovery()')
+            .first
+            .fetch('pg_is_in_recovery')
+
+        Gitlab::Utils.to_boolean(pg_is_in_recovery)
+      end
+
+      def db_read_write?
+        !db_read_only?
+      end
+
+      def version
+        @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1]
+      end
+
+      def database_version
+        connection.execute("SELECT VERSION()").first['version']
+      end
+
+      def postgresql_minimum_supported_version?
+        version.to_f >= MINIMUM_POSTGRES_VERSION
+      end
+
+      def cached_column_exists?(column_name)
+        connection
+          .schema_cache.columns_hash(model.table_name)
+          .has_key?(column_name.to_s)
+      end
+
+      def cached_table_exists?
+        exists? && connection.schema_cache.data_source_exists?(model.table_name)
+      end
+
+      def exists?
+        # We can't _just_ check if `connection` raises an error, as it will
+        # point to a `ConnectionProxy`, and obtaining those doesn't involve any
+        # database queries. So instead we obtain the database version, which is
+        # cached after the first call.
+        connection.schema_cache.database_version
+        true
+      rescue StandardError
+        false
+      end
+
+      def system_id
+        row = connection
+          .execute('SELECT system_identifier FROM pg_control_system()')
+          .first
+
+        row['system_identifier']
+      end
+
+      private
+
+      def connection
+        model.connection
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb
index 04b409a9306..7a22e324bdb 100644
--- a/lib/gitlab/database/reindexing.rb
+++ b/lib/gitlab/database/reindexing.rb
@@ -15,25 +15,58 @@ module Gitlab
       # on e.g. vacuum.
       REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30
 
-      # candidate_indexes: Array of Gitlab::Database::PostgresIndex
-      def self.perform(candidate_indexes, how_many: DEFAULT_INDEXES_PER_INVOCATION)
-        IndexSelection.new(candidate_indexes).take(how_many).each do |index|
+      # Performs automatic reindexing for a limited number of indexes per call
+      #  1. Consume from the explicit reindexing queue
+      #  2. Apply bloat heuristic to find most bloated indexes and reindex those
+      def self.automatic_reindexing(maximum_records: DEFAULT_INDEXES_PER_INVOCATION)
+        # Cleanup leftover temporary indexes from previous, possibly aborted runs (if any)
+        cleanup_leftovers!
+
+        # Consume from the explicit reindexing queue first
+        done_counter = perform_from_queue(maximum_records: maximum_records)
+
+        return if done_counter >= maximum_records
+
+        # Execute reindexing based on bloat heuristic
+        perform_with_heuristic(maximum_records: maximum_records - done_counter)
+      end
+
+      # Reindex based on bloat heuristic for a limited number of indexes per call
+      #
+      # We use a bloat heuristic to estimate the index bloat and pick the
+      # most bloated indexes for reindexing.
+      def self.perform_with_heuristic(candidate_indexes = Gitlab::Database::PostgresIndex.reindexing_support, maximum_records: DEFAULT_INDEXES_PER_INVOCATION)
+        IndexSelection.new(candidate_indexes).take(maximum_records).each do |index|
           Coordinator.new(index).perform
         end
       end
 
+      # Reindex indexes that have been explicitly enqueued (for a limited number of indexes per call)
+      def self.perform_from_queue(maximum_records: DEFAULT_INDEXES_PER_INVOCATION)
+        QueuedAction.in_queue_order.limit(maximum_records).each do |queued_entry|
+          Coordinator.new(queued_entry.index).perform
+
+          queued_entry.done!
+        rescue StandardError => e
+          queued_entry.failed!
+
+          Gitlab::AppLogger.error("Failed to perform reindexing action on queued entry #{queued_entry}: #{e}")
+        end.size
+      end
+
       def self.cleanup_leftovers!
         PostgresIndex.reindexing_leftovers.each do |index|
           Gitlab::AppLogger.info("Removing index #{index.identifier} which is a leftover, temporary index from previous reindexing activity")
 
           retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new(
+            connection: index.connection,
             timing_configuration: REMOVE_INDEX_RETRY_CONFIG,
             klass: self.class,
             logger: Gitlab::AppLogger
           )
 
           retries.run(raise_on_exhaustion: false) do
-            ApplicationRecord.connection.tap do |conn|
+            index.connection.tap do |conn|
               conn.execute("DROP INDEX CONCURRENTLY IF EXISTS #{conn.quote_table_name(index.schema)}.#{conn.quote_table_name(index.name)}")
             end
           end
diff --git a/lib/gitlab/database/reindexing/index_selection.rb b/lib/gitlab/database/reindexing/index_selection.rb
index 2186384e7d7..2d384f2f9e2 100644
--- a/lib/gitlab/database/reindexing/index_selection.rb
+++ b/lib/gitlab/database/reindexing/index_selection.rb
@@ -9,8 +9,8 @@ module Gitlab
         # Only reindex indexes with a relative bloat level (bloat estimate / size) higher than this
         MINIMUM_RELATIVE_BLOAT = 0.2
 
-        # Only consider indexes with a total ondisk size in this range (before reindexing)
-        INDEX_SIZE_RANGE = (1.gigabyte..100.gigabyte).freeze
+        # Only consider indexes beyond this size (before reindexing)
+        INDEX_SIZE_MINIMUM = 1.gigabyte
 
         delegate :each, to: :indexes
 
@@ -32,7 +32,7 @@ module Gitlab
 
           @indexes ||= candidates
             .not_recently_reindexed
-            .where(ondisk_size_bytes: INDEX_SIZE_RANGE)
+            .where('ondisk_size_bytes >= ?', INDEX_SIZE_MINIMUM)
             .sort_by(&:relative_bloat_level) # forced N+1
             .reverse
             .select { |candidate| candidate.relative_bloat_level >= MINIMUM_RELATIVE_BLOAT }
diff --git a/lib/gitlab/database/reindexing/queued_action.rb b/lib/gitlab/database/reindexing/queued_action.rb
new file mode 100644
index 00000000000..c2039a289da
--- /dev/null
+++ b/lib/gitlab/database/reindexing/queued_action.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module Reindexing
+      class QueuedAction < SharedModel
+        self.table_name = 'postgres_reindex_queued_actions'
+
+        enum state: { queued: 0, done: 1, failed: 2 }
+
+        belongs_to :index, foreign_key: :index_identifier, class_name: 'Gitlab::Database::PostgresIndex'
+
+        scope :in_queue_order, -> { queued.order(:created_at) }
+
+        def to_s
+          "queued action [ id = #{id}, index: #{index_identifier} ]"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/reindexing/reindex_action.rb b/lib/gitlab/database/reindexing/reindex_action.rb
index ff465fffb74..73424a76cfe 100644
--- a/lib/gitlab/database/reindexing/reindex_action.rb
+++ b/lib/gitlab/database/reindexing/reindex_action.rb
@@ -3,7 +3,7 @@
 module Gitlab
   module Database
     module Reindexing
-      class ReindexAction < ActiveRecord::Base
+      class ReindexAction < SharedModel
         self.table_name = 'postgres_reindex_actions'
 
         belongs_to :index, foreign_key: :index_identifier, class_name: 'Gitlab::Database::PostgresIndex'
diff --git a/lib/gitlab/database/reindexing/reindex_concurrently.rb b/lib/gitlab/database/reindexing/reindex_concurrently.rb
index 7a720f7c539..152935bd734 100644
--- a/lib/gitlab/database/reindexing/reindex_concurrently.rb
+++ b/lib/gitlab/database/reindexing/reindex_concurrently.rb
@@ -8,7 +8,7 @@ module Gitlab
         ReindexError = Class.new(StandardError)
 
         TEMPORARY_INDEX_PATTERN = '\_ccnew[0-9]*'
-        STATEMENT_TIMEOUT = 9.hours
+        STATEMENT_TIMEOUT = 24.hours
         PG_MAX_INDEX_NAME_LENGTH = 63
 
         attr_reader :index, :logger
@@ -99,6 +99,7 @@ module Gitlab
           logger.info("Removing dangling index #{index.identifier}")
 
           retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new(
+            connection: connection,
             timing_configuration: REMOVE_INDEX_RETRY_CONFIG,
             klass: self.class,
             logger: logger
@@ -109,11 +110,6 @@ module Gitlab
           end
         end
 
-        def with_lock_retries(&block)
-          arguments = { klass: self.class, logger: logger }
-          Gitlab::Database::WithLockRetries.new(**arguments).run(raise_on_exhaustion: true, &block)
-        end
-
         def set_statement_timeout
           execute("SET statement_timeout TO '%ds'" % STATEMENT_TIMEOUT)
           yield
@@ -123,7 +119,7 @@ module Gitlab
 
         delegate :execute, :quote_table_name, to: :connection
         def connection
-          @connection ||= ActiveRecord::Base.connection
+          @connection ||= index.connection
         end
       end
     end
diff --git a/lib/gitlab/database/shared_model.rb b/lib/gitlab/database/shared_model.rb
index f304c32d731..f31dbc01907 100644
--- a/lib/gitlab/database/shared_model.rb
+++ b/lib/gitlab/database/shared_model.rb
@@ -8,13 +8,17 @@ module Gitlab
 
       class << self
         def using_connection(connection)
-          raise 'cannot nest connection overrides for shared models' unless overriding_connection.nil?
+          previous_connection = self.overriding_connection
+
+          unless previous_connection.nil? || previous_connection.equal?(connection)
+            raise 'cannot nest connection overrides for shared models with different connections'
+          end
 
           self.overriding_connection = connection
 
           yield
         ensure
-          self.overriding_connection = nil
+          self.overriding_connection = nil unless previous_connection.equal?(self.overriding_connection)
         end
 
         def connection
diff --git a/lib/gitlab/database/unidirectional_copy_trigger.rb b/lib/gitlab/database/unidirectional_copy_trigger.rb
index 029c894a5ff..146b5cacd9e 100644
--- a/lib/gitlab/database/unidirectional_copy_trigger.rb
+++ b/lib/gitlab/database/unidirectional_copy_trigger.rb
@@ -3,7 +3,7 @@
 module Gitlab
   module Database
     class UnidirectionalCopyTrigger
-      def self.on_table(table_name, connection: ActiveRecord::Base.connection)
+      def self.on_table(table_name, connection:)
         new(table_name, connection)
       end
 
diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
index 1e6d80e1100..83f242ff902 100644
--- a/lib/gitlab/diff/file.rb
+++ b/lib/gitlab/diff/file.rb
@@ -43,6 +43,8 @@ module Gitlab
         # Ensure items are collected in the the batch
         new_blob_lazy
         old_blob_lazy
+
+        preprocess_before_diff(diff) if Feature.enabled?(:jupyter_clean_diffs, repository.project, default_enabled: true)
       end
 
       def position(position_marker, position_type: :text)
@@ -448,6 +450,33 @@ module Gitlab
         find_renderable_viewer_class(classes)
       end
 
+      def preprocess_before_diff(diff)
+        return unless diff.new_path.ends_with? '.ipynb'
+
+        from = old_blob_lazy&.data
+        to = new_blob_lazy&.data
+
+        transformed_diff = IpynbDiff.diff(from, to,
+                                          diff_opts: { context: 5, include_diff_info: true },
+                                          transform_options: { cell_decorator: :percent },
+                                          raise_if_invalid_notebook: true)
+        new_diff = strip_diff_frontmatter(transformed_diff)
+
+        if new_diff
+          diff.diff = new_diff
+          new_blob_lazy.transformed_for_diff = true if new_blob_lazy
+          old_blob_lazy.transformed_for_diff = true if old_blob_lazy
+        end
+
+        Gitlab::AppLogger.info({ message: new_diff ? 'IPYNB_DIFF_GENERATED' : 'IPYNB_DIFF_NIL' })
+      rescue IpynbDiff::InvalidNotebookError => e
+        Gitlab::ErrorTracking.log_exception(e)
+      end
+
+      def strip_diff_frontmatter(diff_content)
+        diff_content.scan(/.*\n/)[2..-1]&.join('') if diff_content.present?
+      end
+
       def alternate_viewer_class
         return unless viewer.instance_of?(DiffViewer::Renamed)
 
diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb
index 32ce35110f8..aedcfe3cb40 100644
--- a/lib/gitlab/diff/highlight.rb
+++ b/lib/gitlab/diff/highlight.rb
@@ -152,6 +152,9 @@ module Gitlab
         return [] unless blob
 
         blob.load_all_data!
+
+        return blob.present.highlight_transformed.lines if Feature.enabled?(:jupyter_clean_diffs, @project, default_enabled: true)
+
         blob.present.highlight.lines
       end
 
diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb
index 075027ebdc8..12ed11b0140 100644
--- a/lib/gitlab/diff/highlight_cache.rb
+++ b/lib/gitlab/diff/highlight_cache.rb
@@ -213,7 +213,7 @@ module Gitlab
       end
 
       def current_transaction
-        ::Gitlab::Metrics::Transaction.current
+        ::Gitlab::Metrics::WebTransaction.current
       end
     end
   end
diff --git a/lib/gitlab/diff/position_tracer/line_strategy.rb b/lib/gitlab/diff/position_tracer/line_strategy.rb
index 8bacc781f61..0f0b8f0c4f3 100644
--- a/lib/gitlab/diff/position_tracer/line_strategy.rb
+++ b/lib/gitlab/diff/position_tracer/line_strategy.rb
@@ -104,7 +104,7 @@ module Gitlab
                 # the current state on the CD diff, so we treat it as outdated.
                 ac_diff = ac_diffs.diff_file_with_new_path(c_path, c_mode)
 
-                { position: new_position(ac_diff, nil, c_line), outdated: true }
+                { position: new_position(ac_diff, nil, c_line, position.line_range), outdated: true }
               end
             else
               # If the line is still in D and not in C, it is still added.
@@ -112,7 +112,7 @@ module Gitlab
             end
           else
             # If the line is no longer in D, it has been removed from the MR.
-            { position: new_position(bd_diff, b_line, nil), outdated: true }
+            { position: new_position(bd_diff, b_line, nil, position.line_range), outdated: true }
           end
         end
 
@@ -140,14 +140,14 @@ module Gitlab
               # removed line into an unchanged one.
               bd_diff = bd_diffs.diff_file_with_new_path(d_path, d_mode)
 
-              { position: new_position(bd_diff, nil, d_line), outdated: true }
+              { position: new_position(bd_diff, nil, d_line, position.line_range), outdated: true }
             else
               # If the line is still in C and not in D, it is still removed.
               { position: new_position(cd_diff, c_line, nil, position.line_range), outdated: false }
             end
           else
             # If the line is no longer in C, it has been removed outside of the MR.
-            { position: new_position(ac_diff, a_line, nil), outdated: true }
+            { position: new_position(ac_diff, a_line, nil, position.line_range), outdated: true }
           end
         end
 
diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb
index 74c8d0a1fd7..8d73aa842be 100644
--- a/lib/gitlab/email/handler/service_desk_handler.rb
+++ b/lib/gitlab/email/handler/service_desk_handler.rb
@@ -15,16 +15,14 @@ module Gitlab
         PROJECT_KEY_PATTERN  = /\A(?.+)-(?[a-z0-9_]+)\z/.freeze
 
         def initialize(mail, mail_key, service_desk_key: nil)
-          super(mail, mail_key)
-
-          if service_desk_key.present?
+          if service_desk_key
+            mail_key ||= service_desk_key
             @service_desk_key = service_desk_key
-          elsif !mail_key&.include?('/') && (matched = HANDLER_REGEX.match(mail_key.to_s))
-            @project_slug = matched[:project_slug]
-            @project_id   = matched[:project_id]&.to_i
-          elsif matched = HANDLER_REGEX_LEGACY.match(mail_key.to_s)
-            @project_path = matched[:project_path]
           end
+
+          super(mail, mail_key)
+
+          match_project_slug || match_legacy_project_slug
         end
 
         def can_handle?
@@ -42,15 +40,29 @@ module Gitlab
           end
         end
 
+        def match_project_slug
+          return if mail_key&.include?('/')
+          return unless matched = HANDLER_REGEX.match(mail_key.to_s)
+
+          @project_slug = matched[:project_slug]
+          @project_id   = matched[:project_id]&.to_i
+        end
+
+        def match_legacy_project_slug
+          return unless matched = HANDLER_REGEX_LEGACY.match(mail_key.to_s)
+
+          @project_path = matched[:project_path]
+        end
+
         def metrics_event
           :receive_email_service_desk
         end
 
         def project
           strong_memoize(:project) do
-            @project = service_desk_key ? project_from_key : super
-            @project = nil unless @project&.service_desk_enabled?
-            @project
+            project_record = super
+            project_record ||= project_from_key if service_desk_key
+            project_record&.service_desk_enabled? ? project_record : nil
           end
         end
 
@@ -96,7 +108,7 @@ module Gitlab
         end
 
         def message_including_template
-          description = message_including_reply
+          description = process_message(trim_reply: false, allow_only_quotes: true)
           template_content = service_desk_setting&.issue_template_content
 
           if template_content.present?
diff --git a/lib/gitlab/email/message/in_product_marketing.rb b/lib/gitlab/email/message/in_product_marketing.rb
index fb4315e74b2..ac9585bcd1a 100644
--- a/lib/gitlab/email/message/in_product_marketing.rb
+++ b/lib/gitlab/email/message/in_product_marketing.rb
@@ -7,7 +7,8 @@ module Gitlab
         UnknownTrackError = Class.new(StandardError)
 
         def self.for(track)
-          raise UnknownTrackError unless Namespaces::InProductMarketingEmailsService::TRACKS.key?(track)
+          valid_tracks = [Namespaces::InviteTeamEmailService::TRACK, Namespaces::InProductMarketingEmailsService::TRACKS.keys].flatten
+          raise UnknownTrackError unless valid_tracks.include?(track)
 
           "Gitlab::Email::Message::InProductMarketing::#{track.to_s.classify}".constantize
         end
diff --git a/lib/gitlab/email/message/in_product_marketing/admin_verify.rb b/lib/gitlab/email/message/in_product_marketing/admin_verify.rb
index 234b93594b5..19d9cf99cdb 100644
--- a/lib/gitlab/email/message/in_product_marketing/admin_verify.rb
+++ b/lib/gitlab/email/message/in_product_marketing/admin_verify.rb
@@ -36,6 +36,10 @@ module Gitlab
           def progress
             super(track_name: 'Admin')
           end
+
+          def invite_members?
+            invite_members_for_task_experiment_enabled?
+          end
         end
       end
     end
diff --git a/lib/gitlab/email/message/in_product_marketing/base.rb b/lib/gitlab/email/message/in_product_marketing/base.rb
index c4895d35a14..7cd54390b9f 100644
--- a/lib/gitlab/email/message/in_product_marketing/base.rb
+++ b/lib/gitlab/email/message/in_product_marketing/base.rb
@@ -7,16 +7,17 @@ module Gitlab
         class Base
           include Gitlab::Email::Message::InProductMarketing::Helper
           include Gitlab::Routing
+          include Gitlab::Experiment::Dsl
 
           attr_accessor :format
 
           def initialize(group:, user:, series:, format: :html)
-            raise ArgumentError, "Only #{total_series} series available for this track." unless series.between?(0, total_series - 1)
-
+            @series = series
             @group = group
             @user = user
-            @series = series
             @format = format
+
+            validate_series!
           end
 
           def subject_line
@@ -56,6 +57,18 @@ module Gitlab
             end
           end
 
+          def invite_members?
+            false
+          end
+
+          def invite_text
+            s_('InProductMarketing|Do you have a teammate who would be perfect for this task?')
+          end
+
+          def invite_link
+            action_link(s_('InProductMarketing|Invite them to help out.'), group_url(group, open_modal: 'invite_members_for_task'))
+          end
+
           def unsubscribe
             parts = Gitlab.com? ? unsubscribe_com : unsubscribe_self_managed(track, series)
 
@@ -102,6 +115,10 @@ module Gitlab
             ["mailers/in_product_marketing", "#{track}-#{series}.png"].join('/')
           end
 
+          def series?
+            total_series > 0
+          end
+
           protected
 
           attr_reader :group, :user, :series
@@ -148,6 +165,20 @@ module Gitlab
 
             link(s_('InProductMarketing|update your preferences'), preference_link)
           end
+
+          def invite_members_for_task_experiment_enabled?
+            return unless user.can?(:admin_group_member, group)
+
+            experiment(:invite_members_for_task, namespace: group) do |e|
+              e.candidate { true }
+              e.record!
+              e.run
+            end
+          end
+
+          def validate_series!
+            raise ArgumentError, "Only #{total_series} series available for this track." unless @series.between?(0, total_series - 1)
+          end
         end
       end
     end
diff --git a/lib/gitlab/email/message/in_product_marketing/create.rb b/lib/gitlab/email/message/in_product_marketing/create.rb
index 4b0c4af4911..2c396775374 100644
--- a/lib/gitlab/email/message/in_product_marketing/create.rb
+++ b/lib/gitlab/email/message/in_product_marketing/create.rb
@@ -61,6 +61,10 @@ module Gitlab
             ][series]
           end
 
+          def invite_members?
+            invite_members_for_task_experiment_enabled?
+          end
+
           private
 
           def project_link
diff --git a/lib/gitlab/email/message/in_product_marketing/experience.rb b/lib/gitlab/email/message/in_product_marketing/experience.rb
index 4156a737517..7520de6d2a3 100644
--- a/lib/gitlab/email/message/in_product_marketing/experience.rb
+++ b/lib/gitlab/email/message/in_product_marketing/experience.rb
@@ -43,7 +43,9 @@ module Gitlab
               survey_id: EASE_SCORE_SURVEY_ID
             }
 
-            "#{Gitlab::Saas.com_url}/-/survey_responses?#{params.to_query}"
+            params[:show_incentive] = true if show_incentive?
+
+            "#{gitlab_com_root_url}/-/survey_responses?#{params.to_query}"
           end
 
           def feedback_ratings(rating)
@@ -70,9 +72,19 @@ module Gitlab
 
           def show_invite_link
             strong_memoize(:show_invite_link) do
-              group.member_count > 1 && group.max_member_access_for_user(user) >= GroupMember::DEVELOPER && user.preferred_language == 'en'
+              group.max_member_access_for_user(user) >= GroupMember::DEVELOPER && user.preferred_language == 'en'
             end
           end
+
+          def show_incentive?
+            show_invite_link && group.member_count > 1
+          end
+
+          def gitlab_com_root_url
+            return root_url.chomp('/') if Rails.env.development?
+
+            Gitlab::Saas.com_url
+          end
         end
       end
     end
diff --git a/lib/gitlab/email/message/in_product_marketing/helper.rb b/lib/gitlab/email/message/in_product_marketing/helper.rb
index cec0aad44a6..bffa90ed4ec 100644
--- a/lib/gitlab/email/message/in_product_marketing/helper.rb
+++ b/lib/gitlab/email/message/in_product_marketing/helper.rb
@@ -36,6 +36,15 @@ module Gitlab
               "#{text} (#{link})"
             end
           end
+
+          def action_link(text, link)
+            case format
+            when :html
+              ActionController::Base.helpers.link_to text, link, target: '_blank', rel: 'noopener noreferrer'
+            else
+              [text, link].join(' >> ')
+            end
+          end
         end
       end
     end
diff --git a/lib/gitlab/email/message/in_product_marketing/invite_team.rb b/lib/gitlab/email/message/in_product_marketing/invite_team.rb
new file mode 100644
index 00000000000..e9334b687f4
--- /dev/null
+++ b/lib/gitlab/email/message/in_product_marketing/invite_team.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Email
+    module Message
+      module InProductMarketing
+        class InviteTeam < Base
+          def subject_line
+            s_('InProductMarketing|Invite your teammates to GitLab')
+          end
+
+          def tagline
+            ''
+          end
+
+          def title
+            s_('InProductMarketing|GitLab is better with teammates to help out!')
+          end
+
+          def subtitle
+            ''
+          end
+
+          def body_line1
+            s_('InProductMarketing|Invite your teammates today and build better code together. You can even assign tasks to new teammates such as setting up CI/CD, to help get projects up and running.')
+          end
+
+          def body_line2
+            ''
+          end
+
+          def cta_text
+            s_('InProductMarketing|Invite your teammates to help')
+          end
+
+          def logo_path
+            'mailers/in_product_marketing/team-0.png'
+          end
+
+          def series?
+            false
+          end
+
+          private
+
+          def validate_series!
+            raise ArgumentError, "Only one email is sent for this track. Value of `series` should be 0." unless @series == 0
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/message/in_product_marketing/verify.rb b/lib/gitlab/email/message/in_product_marketing/verify.rb
index e731c65121e..daf0c969f2b 100644
--- a/lib/gitlab/email/message/in_product_marketing/verify.rb
+++ b/lib/gitlab/email/message/in_product_marketing/verify.rb
@@ -65,6 +65,10 @@ module Gitlab
             ][series]
           end
 
+          def invite_members?
+            invite_members_for_task_experiment_enabled?
+          end
+
           private
 
           def ci_link
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index 242def826be..526f1188065 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -44,6 +44,10 @@ module Gitlab
         }
       end
 
+      def mail
+        strong_memoize(:mail) { build_mail }
+      end
+
       private
 
       def handler
@@ -54,10 +58,6 @@ module Gitlab
         Handler.for(mail, mail_key)
       end
 
-      def mail
-        strong_memoize(:mail) { build_mail }
-      end
-
       def build_mail
         Mail::Message.new(@raw)
       rescue Encoding::UndefinedConversionError,
diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb
index 0f0f4800062..d39fa139abb 100644
--- a/lib/gitlab/email/reply_parser.rb
+++ b/lib/gitlab/email/reply_parser.rb
@@ -4,12 +4,13 @@
 module Gitlab
   module Email
     class ReplyParser
-      attr_accessor :message
+      attr_accessor :message, :allow_only_quotes
 
-      def initialize(message, trim_reply: true, append_reply: false)
+      def initialize(message, trim_reply: true, append_reply: false, allow_only_quotes: false)
         @message = message
         @trim_reply = trim_reply
         @append_reply = append_reply
+        @allow_only_quotes = allow_only_quotes
       end
 
       def execute
@@ -25,7 +26,12 @@ module Gitlab
         # NOTE: We currently don't support empty quotes.
         # EmailReplyTrimmer allows this as a special case,
         # so we detect it manually here.
-        return "" if body.lines.all? { |l| l.strip.empty? || l.start_with?('>') }
+        #
+        # If allow_only_quotes is true a message where all lines starts with ">" is allowed.
+        # This could happen if an email has an empty quote, forwarded without any new content.
+        return "" if body.lines.all? do |l|
+          l.strip.empty? || (!allow_only_quotes && l.start_with?('>'))
+        end
 
         encoded_body = body.force_encoding(encoding).encode("UTF-8")
         return encoded_body unless @append_reply
diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb
index 2b5f465d3c5..519b1d94bf5 100644
--- a/lib/gitlab/emoji.rb
+++ b/lib/gitlab/emoji.rb
@@ -4,40 +4,15 @@ module Gitlab
   module Emoji
     extend self
 
-    def emojis
-      Gemojione.index.instance_variable_get(:@emoji_by_name)
-    end
-
-    def emojis_by_moji
-      Gemojione.index.instance_variable_get(:@emoji_by_moji)
-    end
-
-    def emojis_unicodes
-      emojis_by_moji.keys
-    end
-
-    def emojis_names
-      emojis.keys
-    end
-
-    def emojis_aliases
-      @emoji_aliases ||= Gitlab::Json.parse(File.read(Rails.root.join('fixtures', 'emojis', 'aliases.json')))
-    end
-
-    def emoji_filename(name)
-      emojis[name]["unicode"]
-    end
-
-    def emoji_unicode_filename(moji)
-      emojis_by_moji[moji]["unicode"]
-    end
-
-    def emoji_unicode_version(name)
-      emoji_unicode_versions_by_name[name]
-    end
+    # When updating emoji assets increase the version below
+    # and update the version number in `app/assets/javascripts/emoji/index.js`
+    EMOJI_VERSION = 1
 
-    def normalize_emoji_name(name)
-      emojis_aliases[name] || name
+    # Return a Pathname to emoji's current versioned folder
+    #
+    # @return [Pathname] Absolute Path to versioned emojis folder in `public`
+    def emoji_public_absolute_path
+      Rails.root.join("public/-/emojis/#{EMOJI_VERSION}")
     end
 
     def emoji_image_tag(name, src)
@@ -54,23 +29,19 @@ module Gitlab
       ActionController::Base.helpers.tag(:img, image_options)
     end
 
-    def emoji_exists?(name)
-      emojis.has_key?(name)
-    end
-
     # CSS sprite fallback takes precedence over image fallback
-    def gl_emoji_tag(name, options = {})
-      emoji_name = emojis_aliases[name] || name
-      emoji_info = emojis[emoji_name]
-      return unless emoji_info
+    # @param [TanukiEmoji::Character] emoji
+    # @param [Hash] options
+    def gl_emoji_tag(emoji, options = {})
+      return unless emoji
 
       data = {
-        name: emoji_name,
-        unicode_version: emoji_unicode_version(emoji_name)
+        name: emoji.name,
+        unicode_version: emoji.unicode_version
       }
-      options = { title: emoji_info['description'], data: data }.merge(options)
+      options = { title: emoji.description, data: data }.merge(options)
 
-      ActionController::Base.helpers.content_tag('gl-emoji', emoji_info['moji'], options)
+      ActionController::Base.helpers.content_tag('gl-emoji', emoji.codepoints, options)
     end
 
     def custom_emoji_tag(name, image_source)
@@ -82,12 +53,5 @@ module Gitlab
         emoji_image_tag(name, image_source).html_safe
       end
     end
-
-    private
-
-    def emoji_unicode_versions_by_name
-      @emoji_unicode_versions_by_name ||=
-        Gitlab::Json.parse(File.read(Rails.root.join('fixtures', 'emojis', 'emoji-unicode-version-map.json')))
-    end
   end
 end
diff --git a/lib/gitlab/etag_caching/router/restful.rb b/lib/gitlab/etag_caching/router/restful.rb
index 408a901f69d..176676bd6ba 100644
--- a/lib/gitlab/etag_caching/router/restful.rb
+++ b/lib/gitlab/etag_caching/router/restful.rb
@@ -23,7 +23,7 @@ module Gitlab
           [
             %r(#{RESERVED_WORDS_PREFIX}/noteable/issue/\d+/notes\z),
             'issue_notes',
-            'issue_tracking'
+            'team_planning'
           ],
           [
             %r(#{RESERVED_WORDS_PREFIX}/noteable/merge_request/\d+/notes\z),
@@ -33,7 +33,7 @@ module Gitlab
           [
             %r(#{RESERVED_WORDS_PREFIX}/issues/\d+/realtime_changes\z),
             'issue_title',
-            'issue_tracking'
+            'team_planning'
           ],
           [
             %r(#{RESERVED_WORDS_PREFIX}/commit/\S+/pipelines\.json\z),
diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb
index a1855132b0c..2da30b88d55 100644
--- a/lib/gitlab/git.rb
+++ b/lib/gitlab/git.rb
@@ -17,6 +17,7 @@ module Gitlab
     OSError = Class.new(BaseError)
     UnknownRef = Class.new(BaseError)
     CommandTimedOut = Class.new(CommandError)
+    InvalidPageToken = Class.new(BaseError)
 
     class << self
       include Gitlab::EncodingHelper
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index f72217dedde..b0d194f309a 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -24,7 +24,7 @@ module Gitlab
       LFS_POINTER_MIN_SIZE = 120.bytes
       LFS_POINTER_MAX_SIZE = 200.bytes
 
-      attr_accessor :size, :mode, :id, :commit_id, :loaded_size, :binary
+      attr_accessor :size, :mode, :id, :commit_id, :loaded_size, :binary, :transformed_for_diff
       attr_writer :name, :path, :data
 
       def self.gitlab_blob_truncated_true
@@ -127,6 +127,7 @@ module Gitlab
         # Retain the actual size before it is encoded
         @loaded_size = @data.bytesize if @data
         @loaded_all_data = @loaded_size == size
+        @transformed_for_diff = false
 
         record_metric_blob_size
         record_metric_truncated(truncated?)
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index 6605e896ef1..267107e04e6 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -315,10 +315,18 @@ module Gitlab
       #
       def ref_names(repo)
         refs(repo).map do |ref|
-          ref.sub(%r{^refs/(heads|remotes|tags)/}, "")
+          strip_ref_prefix(ref)
         end
       end
 
+      def first_ref_by_oid(repo)
+        ref = repo.refs_by_oid(oid: id, limit: 1)&.first
+
+        return unless ref
+
+        strip_ref_prefix(ref)
+      end
+
       def message
         encode! @message
       end
@@ -466,6 +474,10 @@ module Gitlab
             commit_id.match?(/\s/)
         )
       end
+
+      def strip_ref_prefix(ref)
+        ref.sub(%r{^refs/(heads|remotes|tags)/}, "")
+      end
     end
   end
 end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 473bc04661c..5afdcc0bd4c 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -20,6 +20,7 @@ module Gitlab
       EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000'
 
       NoRepository = Class.new(::Gitlab::Git::BaseError)
+      RepositoryExists = Class.new(::Gitlab::Git::BaseError)
       InvalidRepository = Class.new(::Gitlab::Git::BaseError)
       InvalidBlobName = Class.new(::Gitlab::Git::BaseError)
       InvalidRef = Class.new(::Gitlab::Git::BaseError)
@@ -101,6 +102,8 @@ module Gitlab
       def create_repository
         wrapped_gitaly_errors do
           gitaly_repository_client.create_repository
+        rescue GRPC::AlreadyExists => e
+          raise RepositoryExists, e.message
         end
       end
 
@@ -198,9 +201,9 @@ module Gitlab
 
       # Returns an Array of Tags
       #
-      def tags(sort_by: nil)
+      def tags(sort_by: nil, pagination_params: nil)
         wrapped_gitaly_errors do
-          gitaly_ref_client.tags(sort_by: sort_by)
+          gitaly_ref_client.tags(sort_by: sort_by, pagination_params: pagination_params)
         end
       end
 
@@ -519,6 +522,17 @@ module Gitlab
         @refs_hash
       end
 
+      # Returns matching refs for OID
+      #
+      # Limit of 0 means there is no limit.
+      def refs_by_oid(oid:, limit: 0)
+        wrapped_gitaly_errors do
+          gitaly_ref_client.find_refs_by_oid(oid: oid, limit: limit)
+        end
+      rescue CommandError, TypeError, NoRepository
+        nil
+      end
+
       # Returns url for submodule
       #
       # Ex.
@@ -784,6 +798,12 @@ module Gitlab
         end
       end
 
+      def list_refs
+        wrapped_gitaly_errors do
+          gitaly_ref_client.list_refs
+        end
+      end
+
       # Refactoring aid; allows us to copy code from app/models/repository.rb
       def commit(ref = 'HEAD')
         Gitlab::Git::Commit.find(self, ref)
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 2c26da037da..cc3f20ab774 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -24,7 +24,6 @@ module Gitlab
       end
     end
 
-    PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m.freeze
     SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'
     MAXIMUM_GITALY_CALLS = 30
     CLIENT_NAME = (Gitlab::Runtime.sidekiq? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
@@ -57,33 +56,15 @@ module Gitlab
       # https://gitlab.com/gitlab-org/gitaly/-/blob/bf9f52bc/client/dial.go#L78
       {
         'grpc.keepalive_time_ms': 20000,
-        'grpc.keepalive_permit_without_calls': 1
+        'grpc.keepalive_permit_without_calls': 1,
+        'grpc.http2.max_pings_without_data': 0
       }
     end
     private_class_method :channel_args
 
-    def self.stub_cert_paths
-      cert_paths = Dir["#{OpenSSL::X509::DEFAULT_CERT_DIR}/*"]
-      cert_paths << OpenSSL::X509::DEFAULT_CERT_FILE if File.exist? OpenSSL::X509::DEFAULT_CERT_FILE
-      cert_paths
-    end
-
-    def self.stub_certs
-      return @certs if @certs
-
-      @certs = stub_cert_paths.flat_map do |cert_file|
-        File.read(cert_file).scan(PEM_REGEX).map do |cert|
-          OpenSSL::X509::Certificate.new(cert).to_pem
-        rescue OpenSSL::OpenSSLError => e
-          Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, cert_file: cert_file)
-          nil
-        end.compact
-      end.uniq.join("\n")
-    end
-
     def self.stub_creds(storage)
       if URI(address(storage)).scheme == 'tls'
-        GRPC::Core::ChannelCredentials.new stub_certs
+        GRPC::Core::ChannelCredentials.new ::Gitlab::X509::Certificate.ca_certs_bundle
       else
         :this_channel_is_insecure
       end
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index 75588ad980c..7c688044e9c 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -205,6 +205,8 @@ module Gitlab
       end
 
       def between(from, to)
+        return list_commits(["^" + from, to], reverse: true) if Feature.enabled?(:between_commits_via_list_commits)
+
         request = Gitaly::CommitsBetweenRequest.new(
           repository: @gitaly_repo,
           from: from,
diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb
index 235eef4575e..c064811b1e7 100644
--- a/lib/gitlab/gitaly_client/ref_service.rb
+++ b/lib/gitlab/gitaly_client/ref_service.rb
@@ -77,8 +77,8 @@ module Gitlab
         consume_find_local_branches_response(response)
       end
 
-      def tags(sort_by: nil)
-        request = Gitaly::FindAllTagsRequest.new(repository: @gitaly_repo)
+      def tags(sort_by: nil, pagination_params: nil)
+        request = Gitaly::FindAllTagsRequest.new(repository: @gitaly_repo, pagination_params: pagination_params)
         request.sort_by = sort_tags_by_param(sort_by) if sort_by
 
         response = GitalyClient.call(@storage, :ref_service, :find_all_tags, request, timeout: GitalyClient.medium_timeout)
@@ -194,18 +194,39 @@ module Gitlab
         raise ArgumentError, ex
       end
 
+      def list_refs(patterns = [Gitlab::Git::BRANCH_REF_PREFIX])
+        request = Gitaly::ListRefsRequest.new(
+          repository: @gitaly_repo,
+          patterns: patterns
+        )
+
+        response = GitalyClient.call(@storage, :ref_service, :list_refs, request, timeout: GitalyClient.fast_timeout)
+        consume_list_refs_response(response)
+      end
+
       def pack_refs
         request = Gitaly::PackRefsRequest.new(repository: @gitaly_repo)
 
         GitalyClient.call(@storage, :ref_service, :pack_refs, request, timeout: GitalyClient.long_timeout)
       end
 
+      def find_refs_by_oid(oid:, limit:)
+        request = Gitaly::FindRefsByOIDRequest.new(repository: @gitaly_repo, sort_field: :refname, oid: oid, limit: limit)
+
+        response = GitalyClient.call(@storage, :ref_service, :find_refs_by_oid, request, timeout: GitalyClient.medium_timeout)
+        response&.refs&.to_a
+      end
+
       private
 
       def consume_refs_response(response)
         response.flat_map { |message| message.names.map { |name| yield(name) } }
       end
 
+      def consume_list_refs_response(response)
+        response.flat_map(&:references)
+      end
+
       def sort_local_branches_by_param(sort_by)
         sort_by = 'name' if sort_by == 'name_asc'
 
diff --git a/lib/gitlab/github_import/bulk_importing.rb b/lib/gitlab/github_import/bulk_importing.rb
index 80f8f8bfbe2..28a39128ec9 100644
--- a/lib/gitlab/github_import/bulk_importing.rb
+++ b/lib/gitlab/github_import/bulk_importing.rb
@@ -30,7 +30,7 @@ module Gitlab
       # Bulk inserts the given rows into the database.
       def bulk_insert(model, rows, batch_size: 100)
         rows.each_slice(batch_size) do |slice|
-          Gitlab::Database.main.bulk_insert(model.table_name, slice) # rubocop:disable Gitlab/BulkInsert
+          ApplicationRecord.legacy_bulk_insert(model.table_name, slice) # rubocop:disable Gitlab/BulkInsert
 
           log_and_increment_counter(slice.size, :imported)
         end
diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb
index 4cfc920e2e3..0aa0896aa57 100644
--- a/lib/gitlab/github_import/importer/diff_note_importer.rb
+++ b/lib/gitlab/github_import/importer/diff_note_importer.rb
@@ -4,41 +4,64 @@ module Gitlab
   module GithubImport
     module Importer
       class DiffNoteImporter
-        attr_reader :note, :project, :client, :user_finder
-
-        # note - An instance of `Gitlab::GithubImport::Representation::DiffNote`.
-        # project - An instance of `Project`.
-        # client - An instance of `Gitlab::GithubImport::Client`.
+        # note - An instance of `Gitlab::GithubImport::Representation::DiffNote`
+        # project - An instance of `Project`
+        # client - An instance of `Gitlab::GithubImport::Client`
         def initialize(note, project, client)
           @note = note
           @project = project
           @client = client
-          @user_finder = GithubImport::UserFinder.new(project, client)
         end
 
         def execute
-          return unless (mr_id = find_merge_request_id)
+          return if merge_request_id.blank?
 
-          author_id, author_found = user_finder.author_id_for(note)
+          note.project = project
+          note.merge_request = merge_request
 
-          note_body = MarkdownText.format(note.note, note.author, author_found)
+          build_author_attributes
 
-          attributes = {
-            discussion_id: Discussion.discussion_id(note),
-            noteable_type: 'MergeRequest',
-            noteable_id: mr_id,
-            project_id: project.id,
-            author_id: author_id,
-            note: note_body,
-            system: false,
-            commit_id: note.original_commit_id,
-            line_code: note.line_code,
-            type: 'LegacyDiffNote',
-            created_at: note.created_at,
-            updated_at: note.updated_at,
-            st_diff: note.diff_hash.to_yaml
-          }
+          # Diff notes with suggestions are imported with DiffNote, which is
+          # slower to import than LegacyDiffNote. Importing DiffNote is slower
+          # because it cannot use the BulkImporting strategy, which skips
+          # callbacks and validations. For this reason, notes that don't have
+          # suggestions are still imported with LegacyDiffNote
+          if import_with_diff_note?
+            import_with_diff_note
+          else
+            import_with_legacy_diff_note
+          end
+        rescue ActiveRecord::InvalidForeignKey => e
+          # It's possible the project and the issue have been deleted since
+          # scheduling this job. In this case we'll just skip creating the note
+          Logger.info(
+            message: e.message,
+            github_identifiers: note.github_identifiers
+          )
+        end
 
+        private
+
+        attr_reader :note, :project, :client, :author_id, :author_found
+
+        def import_with_diff_note?
+          note.contains_suggestion? && use_diff_note_with_suggestions_enabled?
+        end
+
+        def use_diff_note_with_suggestions_enabled?
+          Feature.enabled?(
+            :github_importer_use_diff_note_with_suggestions,
+            default_enabled: :yaml
+          )
+        end
+
+        def build_author_attributes
+          @author_id, @author_found = user_finder.author_id_for(note)
+        end
+
+        # rubocop:disable Gitlab/BulkInsert
+        def import_with_legacy_diff_note
+          log_diff_note_creation('LegacyDiffNote')
           # It's possible that during an import we'll insert tens of thousands
           # of diff notes. If we were to use the Note/LegacyDiffNote model here
           # we'd also have to run additional queries for both validations and
@@ -47,15 +70,70 @@ module Gitlab
           # To work around this we're using bulk_insert with a single row. This
           # allows us to efficiently insert data (even if it's just 1 row)
           # without having to use all sorts of hacks to disable callbacks.
-          Gitlab::Database.main.bulk_insert(LegacyDiffNote.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert
-        rescue ActiveRecord::InvalidForeignKey
-          # It's possible the project and the issue have been deleted since
-          # scheduling this job. In this case we'll just skip creating the note.
+          ApplicationRecord.legacy_bulk_insert(LegacyDiffNote.table_name, [{
+            noteable_type: note.noteable_type,
+            system: false,
+            type: 'LegacyDiffNote',
+            discussion_id: note.discussion_id,
+            noteable_id: merge_request_id,
+            project_id: project.id,
+            author_id: author_id,
+            note: note_body,
+            commit_id: note.original_commit_id,
+            line_code: note.line_code,
+            created_at: note.created_at,
+            updated_at: note.updated_at,
+            st_diff: note.diff_hash.to_yaml
+          }])
+        end
+        # rubocop:enabled Gitlab/BulkInsert
+
+        def import_with_diff_note
+          log_diff_note_creation('DiffNote')
+
+          ::Import::Github::Notes::CreateService.new(project, author, {
+            noteable_type: note.noteable_type,
+            system: false,
+            type: 'DiffNote',
+            noteable_id: merge_request_id,
+            project_id: project.id,
+            note: note_body,
+            discussion_id: note.discussion_id,
+            commit_id: note.original_commit_id,
+            created_at: note.created_at,
+            updated_at: note.updated_at,
+            position: note.diff_position
+          }).execute
+        end
+
+        def note_body
+          @note_body ||= MarkdownText.format(note.note, note.author, author_found)
+        end
+
+        def author
+          @author ||= User.find(author_id)
+        end
+
+        def merge_request
+          @merge_request ||= MergeRequest.find(merge_request_id)
         end
 
         # Returns the ID of the merge request this note belongs to.
-        def find_merge_request_id
-          GithubImport::IssuableFinder.new(project, note).database_id
+        def merge_request_id
+          @merge_request_id ||= GithubImport::IssuableFinder.new(project, note).database_id
+        end
+
+        def user_finder
+          @user_finder ||= GithubImport::UserFinder.new(project, client)
+        end
+
+        def log_diff_note_creation(model)
+          Logger.info(
+            project_id: project.id,
+            importer: self.class.name,
+            github_identifiers: note.github_identifiers,
+            model: model
+          )
         end
       end
     end
diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb
index f8665676ccf..7f46615f17e 100644
--- a/lib/gitlab/github_import/importer/issue_importer.rb
+++ b/lib/gitlab/github_import/importer/issue_importer.rb
@@ -75,7 +75,7 @@ module Gitlab
             end
           end
 
-          Gitlab::Database.main.bulk_insert(IssueAssignee.table_name, assignees) # rubocop:disable Gitlab/BulkInsert
+          ApplicationRecord.legacy_bulk_insert(IssueAssignee.table_name, assignees) # rubocop:disable Gitlab/BulkInsert
         end
       end
     end
diff --git a/lib/gitlab/github_import/importer/label_links_importer.rb b/lib/gitlab/github_import/importer/label_links_importer.rb
index b608bb48e38..5e248c7cfc5 100644
--- a/lib/gitlab/github_import/importer/label_links_importer.rb
+++ b/lib/gitlab/github_import/importer/label_links_importer.rb
@@ -40,7 +40,7 @@ module Gitlab
             }
           end
 
-          Gitlab::Database.main.bulk_insert(LabelLink.table_name, rows) # rubocop:disable Gitlab/BulkInsert
+          ApplicationRecord.legacy_bulk_insert(LabelLink.table_name, rows) # rubocop:disable Gitlab/BulkInsert
         end
 
         def find_target_id
diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb
index 1fd42a69fac..2cc3a82dd9b 100644
--- a/lib/gitlab/github_import/importer/note_importer.rb
+++ b/lib/gitlab/github_import/importer/note_importer.rb
@@ -37,7 +37,7 @@ module Gitlab
           # We're using bulk_insert here so we can bypass any validations and
           # callbacks. Running these would result in a lot of unnecessary SQL
           # queries being executed when importing large projects.
-          Gitlab::Database.main.bulk_insert(Note.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert
+          ApplicationRecord.legacy_bulk_insert(Note.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert
         rescue ActiveRecord::InvalidForeignKey
           # It's possible the project and the issue have been deleted since
           # scheduling this job. In this case we'll just skip creating the note.
diff --git a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb
index 287e0ea7f7f..c56b391cbec 100644
--- a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb
+++ b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb
@@ -31,9 +31,7 @@ module Gitlab
         end
 
         def each_object_to_import
-          project.merge_requests.with_state(:merged).find_each do |merge_request|
-            next if already_imported?(merge_request)
-
+          merge_requests_to_import.find_each do |merge_request|
             Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched)
 
             pull_request = client.pull_request(project.import_source, merge_request.iid)
@@ -42,6 +40,17 @@ module Gitlab
             mark_as_imported(merge_request)
           end
         end
+
+        private
+
+        # Returns only the merge requests that still have merged_by to be imported.
+        def merge_requests_to_import
+          project.merge_requests.id_not_in(already_imported_objects).with_state(:merged)
+        end
+
+        def already_imported_objects
+          Gitlab::Cache::Import::Caching.values_from_set(already_imported_cache_key)
+        end
       end
     end
   end
diff --git a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb
index bd65eb5899c..5e55d09fe3d 100644
--- a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb
+++ b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb
@@ -86,7 +86,7 @@ module Gitlab
 
         # Returns only the merge requests that still have reviews to be imported.
         def merge_requests_to_import
-          project.merge_requests.where.not(id: already_imported_merge_requests) # rubocop: disable CodeReuse/ActiveRecord
+          project.merge_requests.id_not_in(already_imported_merge_requests)
         end
 
         def already_imported_merge_requests
diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb
index a3dcd2e380c..fecff0644c2 100644
--- a/lib/gitlab/github_import/representation/diff_note.rb
+++ b/lib/gitlab/github_import/representation/diff_note.rb
@@ -7,13 +7,14 @@ module Gitlab
         include ToHash
         include ExposeAttribute
 
-        attr_reader :attributes
-
-        expose_attribute :noteable_type, :noteable_id, :commit_id, :file_path,
-                         :diff_hunk, :author, :note, :created_at, :updated_at,
-                         :original_commit_id, :note_id
-
+        NOTEABLE_TYPE = 'MergeRequest'
         NOTEABLE_ID_REGEX = %r{/pull/(?\d+)}i.freeze
+        DISCUSSION_CACHE_KEY = 'github-importer/discussion-id-map/%{project_id}/%{noteable_id}/%{original_note_id}'
+
+        expose_attribute :noteable_id, :commit_id, :file_path,
+          :diff_hunk, :author, :created_at, :updated_at,
+          :original_commit_id, :note_id, :end_line, :start_line,
+          :side, :in_reply_to_id
 
         # Builds a diff note from a GitHub API response.
         #
@@ -30,7 +31,6 @@ module Gitlab
 
           user = Representation::User.from_api_response(note.user) if note.user
           hash = {
-            noteable_type: 'MergeRequest',
             noteable_id: matches[:iid].to_i,
             file_path: note.path,
             commit_id: note.commit_id,
@@ -42,7 +42,9 @@ module Gitlab
             updated_at: note.updated_at,
             note_id: note.id,
             end_line: note.line,
-            start_line: note.start_line
+            start_line: note.start_line,
+            side: note.side,
+            in_reply_to_id: note.in_reply_to_id
           }
 
           new(hash)
@@ -56,21 +58,41 @@ module Gitlab
           new(hash)
         end
 
+        attr_accessor :merge_request, :project
+
         # attributes - A Hash containing the raw note details. The keys of this
         #              Hash must be Symbols.
         def initialize(attributes)
           @attributes = attributes
+
+          @note_formatter = DiffNotes::SuggestionFormatter.new(
+            note: attributes[:note],
+            start_line: attributes[:start_line],
+            end_line: attributes[:end_line]
+          )
+        end
+
+        def noteable_type
+          NOTEABLE_TYPE
+        end
+
+        def contains_suggestion?
+          @note_formatter.contains_suggestion?
+        end
+
+        def note
+          @note_formatter.formatted_note
         end
 
         def line_code
           diff_line = Gitlab::Diff::Parser.new.parse(diff_hunk.lines).to_a.last
 
-          Gitlab::Git
-            .diff_line_code(file_path, diff_line.new_pos, diff_line.old_pos)
+          Gitlab::Git.diff_line_code(file_path, diff_line.new_pos, diff_line.old_pos)
         end
 
         # Returns a Hash that can be used to populate `notes.st_diff`, removing
         # the need for requesting Git data for every diff note.
+        # Used when importing with LegacyDiffNote
         def diff_hash
           {
             diff: diff_hunk,
@@ -85,12 +107,15 @@ module Gitlab
           }
         end
 
-        def note
-          @note ||= DiffNotes::SuggestionFormatter.formatted_note_for(
-            note: attributes[:note],
-            start_line: attributes[:start_line],
-            end_line: attributes[:end_line]
-          )
+        # Used when importing with DiffNote
+        def diff_position
+          position_params = {
+            diff_refs: merge_request.diff_refs,
+            old_path: file_path,
+            new_path: file_path
+          }
+
+          Gitlab::Diff::Position.new(position_params.merge(diff_line_params))
         end
 
         def github_identifiers
@@ -100,6 +125,53 @@ module Gitlab
             noteable_type: noteable_type
           }
         end
+
+        def discussion_id
+          if in_reply_to_id.present?
+            current_discussion_id
+          else
+            Discussion.discussion_id(
+              Struct
+              .new(:noteable_id, :noteable_type)
+              .new(merge_request.id, NOTEABLE_TYPE)
+            ).tap do |discussion_id|
+              cache_discussion_id(discussion_id)
+            end
+          end
+        end
+
+        private
+
+        # Required by ExposeAttribute
+        attr_reader :attributes
+
+        def diff_line_params
+          if addition?
+            { new_line: end_line, old_line: nil }
+          else
+            { new_line: nil, old_line: end_line }
+          end
+        end
+
+        def addition?
+          side == 'RIGHT'
+        end
+
+        def cache_discussion_id(discussion_id)
+          Gitlab::Cache::Import::Caching.write(discussion_id_cache_key(note_id), discussion_id)
+        end
+
+        def current_discussion_id
+          Gitlab::Cache::Import::Caching.read(discussion_id_cache_key(in_reply_to_id))
+        end
+
+        def discussion_id_cache_key(id)
+          DISCUSSION_CACHE_KEY % {
+            project_id: project.id,
+            noteable_id: merge_request.id,
+            original_note_id: id
+          }
+        end
       end
     end
   end
diff --git a/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb b/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb
index 4e5855ee4cd..38b15c4b5bb 100644
--- a/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb
+++ b/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb
@@ -10,30 +10,38 @@ module Gitlab
     module Representation
       module DiffNotes
         class SuggestionFormatter
+          include Gitlab::Utils::StrongMemoize
+
           # A github suggestion:
           # - the ```suggestion tag must be the first text of the line
           #   - it might have up to 3 spaces before the ```suggestion tag
           # - extra text on the ```suggestion tag line will be ignored
           GITHUB_SUGGESTION = /^\ {,3}(?```suggestion\b).*(?\R)/.freeze
 
-          def self.formatted_note_for(...)
-            new(...).formatted_note
-          end
-
           def initialize(note:, start_line: nil, end_line: nil)
             @note = note
             @start_line = start_line
             @end_line = end_line
           end
 
+          # Returns a tuple with:
+          #   - a boolean indicating if the note has suggestions
+          #   - the note with the suggestion formatted for Gitlab
           def formatted_note
-            if contains_suggestion?
-              note.gsub(
-                GITHUB_SUGGESTION,
-                "\\k:#{suggestion_range}\\k"
-              )
-            else
-              note
+            @formatted_note ||=
+              if contains_suggestion?
+                note.gsub(
+                  GITHUB_SUGGESTION,
+                  "\\k:#{suggestion_range}\\k"
+                )
+              else
+                note
+              end
+          end
+
+          def contains_suggestion?
+            strong_memoize(:contain_suggestion) do
+              note.to_s.match?(GITHUB_SUGGESTION)
             end
           end
 
@@ -41,10 +49,6 @@ module Gitlab
 
           attr_reader :note, :start_line, :end_line
 
-          def contains_suggestion?
-            note.to_s.match?(GITHUB_SUGGESTION)
-          end
-
           # Github always saves the comment on the _last_ line of the range.
           # Therefore, the diff hunk will always be related to lines before
           # the comment itself.
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index 9f628a10771..9ad902efb3a 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -39,6 +39,7 @@ module Gitlab
       gon.ee                     = Gitlab.ee?
       gon.jh                     = Gitlab.jh?
       gon.dot_com                = Gitlab.com?
+      gon.dev_env_or_com         = Gitlab.dev_env_or_com?
 
       if current_user
         gon.current_user_id = current_user.id
@@ -55,7 +56,8 @@ module Gitlab
       push_frontend_feature_flag(:security_auto_fix, default_enabled: false)
       push_frontend_feature_flag(:improved_emoji_picker, default_enabled: :yaml)
       push_frontend_feature_flag(:new_header_search, default_enabled: :yaml)
-      push_frontend_feature_flag(:suppress_apollo_errors_during_navigation, current_user, default_enabled: :yaml)
+      push_frontend_feature_flag(:configure_iac_scanning_via_mr, current_user, default_enabled: :yaml)
+      push_frontend_feature_flag(:bootstrap_confirmation_modals, default_enabled: :yaml)
     end
 
     # Exposes the state of a feature flag to the frontend code.
diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb
index 1abbd6dc45b..9a6317e2b76 100644
--- a/lib/gitlab/gpg/commit.rb
+++ b/lib/gitlab/gpg/commit.rb
@@ -48,7 +48,7 @@ module Gitlab
 
           if gpg_key
             Gitlab::Gpg::CurrentKeyChain.add(gpg_key.key)
-            clear_memoization(:verified_signature)
+            clear_memoization(:gpg_signatures)
           end
 
           yield gpg_key
@@ -56,16 +56,7 @@ module Gitlab
       end
 
       def verified_signature
-        strong_memoize(:verified_signature) { gpgme_signature }
-      end
-
-      def gpgme_signature
-        GPGME::Crypto.new.verify(signature_text, signed_text: signed_text) do |verified_signature|
-          # Return the first signature for now: https://gitlab.com/gitlab-org/gitlab-foss/issues/54932
-          break verified_signature
-        end
-      rescue GPGME::Error
-        nil
+        gpg_signatures.first
       end
 
       def create_cached_signature!
@@ -77,6 +68,24 @@ module Gitlab
         end
       end
 
+      def gpg_signatures
+        strong_memoize(:gpg_signatures) do
+          signatures = []
+
+          GPGME::Crypto.new.verify(signature_text, signed_text: signed_text) do |verified_signature|
+            signatures << verified_signature
+          end
+
+          signatures
+        rescue GPGME::Error
+          []
+        end
+      end
+
+      def multiple_signatures?
+        gpg_signatures.size > 1
+      end
+
       def attributes(gpg_key)
         user_infos = user_infos(gpg_key)
         verification_status = verification_status(gpg_key)
@@ -93,6 +102,7 @@ module Gitlab
       end
 
       def verification_status(gpg_key)
+        return :multiple_signatures if multiple_signatures? && Feature.enabled?(:multiple_gpg_signatures, @commit.project, default_enabled: :yaml)
         return :unknown_key unless gpg_key
         return :unverified_key unless gpg_key.verified?
         return :unverified unless verified_signature&.valid?
diff --git a/lib/gitlab/grape_logging/loggers/urgency_logger.rb b/lib/gitlab/grape_logging/loggers/urgency_logger.rb
new file mode 100644
index 00000000000..0a503086d05
--- /dev/null
+++ b/lib/gitlab/grape_logging/loggers/urgency_logger.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GrapeLogging
+    module Loggers
+      class UrgencyLogger < ::GrapeLogging::Loggers::Base
+        def parameters(request, _)
+          endpoint = request.env['api.endpoint']
+          return {} unless endpoint
+
+          urgency = endpoint.options[:for].try(:urgency_for_app, endpoint)
+          return {} unless urgency
+
+          { request_urgency: urgency.name, target_duration_s: urgency.duration }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/known_operations.rb b/lib/gitlab/graphql/known_operations.rb
new file mode 100644
index 00000000000..ead52935945
--- /dev/null
+++ b/lib/gitlab/graphql/known_operations.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    class KnownOperations
+      Operation = Struct.new(:name) do
+        def to_caller_id
+          "graphql:#{name}"
+        end
+
+        def query_urgency
+          # We'll be able to actually correlate query_urgency with https://gitlab.com/gitlab-org/gitlab/-/issues/345141
+          ::Gitlab::EndpointAttributes::DEFAULT_URGENCY
+        end
+      end
+
+      ANONYMOUS = Operation.new("anonymous").freeze
+      UNKNOWN = Operation.new("unknown").freeze
+
+      def self.default
+        @default ||= self.new(Gitlab::Webpack::GraphqlKnownOperations.load)
+      end
+
+      def initialize(operation_names)
+        @operation_hash = operation_names
+          .map { |name| Operation.new(name).freeze }
+          .concat([ANONYMOUS, UNKNOWN])
+          .index_by(&:name)
+      end
+
+      # Returns the known operation from the given ::GraphQL::Query object
+      def from_query(query)
+        operation_name = query.selected_operation_name
+
+        return ANONYMOUS unless operation_name
+
+        @operation_hash[operation_name] || UNKNOWN
+      end
+
+      def operations
+        @operation_hash.values
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/loaders/full_path_model_loader.rb b/lib/gitlab/graphql/loaders/full_path_model_loader.rb
index 7f9013c6e4c..2ea3fa71d5e 100644
--- a/lib/gitlab/graphql/loaders/full_path_model_loader.rb
+++ b/lib/gitlab/graphql/loaders/full_path_model_loader.rb
@@ -16,8 +16,11 @@ module Gitlab
 
         def find
           BatchLoader::GraphQL.for(full_path).batch(key: model_class) do |full_paths, loader, args|
+            scope = args[:key]
+            # this logic cannot be placed in the NamespaceResolver due to N+1
+            scope = scope.without_project_namespaces if scope == Namespace
             # `with_route` avoids an N+1 calculating full_path
-            args[:key].where_full_path_in(full_paths).with_route.each do |model_instance|
+            scope.where_full_path_in(full_paths).with_route.each do |model_instance|
               loader.call(model_instance.full_path.downcase, model_instance)
             end
           end
diff --git a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb
index 5a9d21e7469..15f95edd318 100644
--- a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb
+++ b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb
@@ -76,7 +76,7 @@ module Gitlab
 
           def items
             original_items = super
-            return original_items if Gitlab::Pagination::Keyset::Order.keyset_aware?(original_items) || Feature.disabled?(:new_graphql_keyset_pagination)
+            return original_items if Feature.disabled?(:new_graphql_keyset_pagination, default_enabled: :yaml) || Gitlab::Pagination::Keyset::Order.keyset_aware?(original_items)
 
             strong_memoize(:generic_keyset_pagination_items) do
               rebuilt_items_with_keyset_order, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(original_items)
diff --git a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb
index b8d2f5b0f29..207324e73bd 100644
--- a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb
+++ b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb
@@ -10,15 +10,10 @@ module Gitlab
         ALL_ANALYZERS = [COMPLEXITY_ANALYZER, DEPTH_ANALYZER, FIELD_USAGE_ANALYZER].freeze
 
         def initial_value(query)
-          variables = process_variables(query.provided_variables)
-          default_initial_values(query).merge({
-            operation_name: query.operation_name,
-            query_string: query.query_string,
-            variables: variables
-          })
-        rescue StandardError => e
-          Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
-          default_initial_values(query)
+          {
+            time_started: Gitlab::Metrics::System.monotonic_time,
+            query: query
+          }
         end
 
         def call(memo, *)
@@ -28,25 +23,42 @@ module Gitlab
         def final_value(memo)
           return if memo.nil?
 
-          complexity, depth, field_usages = GraphQL::Analysis.analyze_query(memo[:query], ALL_ANALYZERS)
+          query = memo[:query]
+          complexity, depth, field_usages = GraphQL::Analysis.analyze_query(query, ALL_ANALYZERS)
 
           memo[:depth] = depth
           memo[:complexity] = complexity
           # This duration is not the execution time of the
           # query but the execution time of the analyzer.
-          memo[:duration_s] = duration(memo[:time_started]).round(1)
+          memo[:duration_s] = duration(memo[:time_started])
           memo[:used_fields] = field_usages.first
           memo[:used_deprecated_fields] = field_usages.second
 
-          RequestStore.store[:graphql_logs] ||= []
-          RequestStore.store[:graphql_logs] << memo
-          GraphqlLogger.info(memo.except!(:time_started, :query))
+          push_to_request_store(memo)
+
+          # This gl_analysis is included in the tracer log
+          query.context[:gl_analysis] = memo.except!(:time_started, :query)
         rescue StandardError => e
           Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
         end
 
         private
 
+        def push_to_request_store(memo)
+          query = memo[:query]
+
+          # TODO: This RequestStore management is used to handle setting request wide metadata
+          # to improve preexisting logging. We should handle this either with ApplicationContext
+          # or in a separate tracer.
+          # https://gitlab.com/gitlab-org/gitlab/-/issues/343802
+
+          RequestStore.store[:graphql_logs] ||= []
+          RequestStore.store[:graphql_logs] << memo.except(:time_started, :duration_s, :query).merge({
+            variables: process_variables(query.provided_variables),
+            operation_name: query.operation_name
+          })
+        end
+
         def process_variables(variables)
           filtered_variables = filter_sensitive_variables(variables)
 
@@ -66,16 +78,6 @@ module Gitlab
         def duration(time_started)
           Gitlab::Metrics::System.monotonic_time - time_started
         end
-
-        def default_initial_values(query)
-          {
-            time_started: Gitlab::Metrics::System.monotonic_time,
-            query_string: nil,
-            query: query,
-            variables: nil,
-            duration_s: nil
-          }
-        end
       end
     end
   end
diff --git a/lib/gitlab/graphql/tracers/application_context_tracer.rb b/lib/gitlab/graphql/tracers/application_context_tracer.rb
new file mode 100644
index 00000000000..4193c46e321
--- /dev/null
+++ b/lib/gitlab/graphql/tracers/application_context_tracer.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Tracers
+      # This graphql-ruby tracer sets up `ApplicationContext` for certain operations.
+      class ApplicationContextTracer
+        def self.use(schema)
+          schema.tracer(self.new)
+        end
+
+        # See docs on expected interface for trace
+        # https://graphql-ruby.org/api-doc/1.12.17/GraphQL/Tracing
+        def trace(key, data)
+          case key
+          when "execute_query"
+            operation = known_operation(data)
+
+            ::Gitlab::ApplicationContext.with_context(caller_id: operation.to_caller_id) do
+              yield
+            end
+          else
+            yield
+          end
+        end
+
+        private
+
+        def known_operation(data)
+          # The library guarantees that we should have :query for execute_query, but we're being defensive here
+          query = data.fetch(:query, nil)
+
+          return ::Gitlab::Graphql::KnownOperations.UNKNOWN unless query
+
+          ::Gitlab::Graphql::KnownOperations.default.from_query(query)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/tracers/logger_tracer.rb b/lib/gitlab/graphql/tracers/logger_tracer.rb
new file mode 100644
index 00000000000..c7ba56824db
--- /dev/null
+++ b/lib/gitlab/graphql/tracers/logger_tracer.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Tracers
+      # This tracer writes logs for certain trace events.
+      # It reads duration metadata written by TimerTracer.
+      class LoggerTracer
+        def self.use(schema)
+          schema.tracer(self.new)
+        end
+
+        def trace(key, data)
+          result = yield
+
+          case key
+          when "execute_query"
+            log_execute_query(**data)
+          end
+
+          result
+        end
+
+        private
+
+        def log_execute_query(query: nil, duration_s: 0)
+          # execute_query should always have :query, but we're just being defensive
+          return unless query
+
+          analysis_info = query.context[:gl_analysis]&.transform_keys { |key| "query_analysis.#{key}" }
+          info = {
+            trace_type: 'execute_query',
+            query_fingerprint: query.fingerprint,
+            duration_s: duration_s,
+            operation_name: query.operation_name,
+            operation_fingerprint: query.operation_fingerprint,
+            is_mutation: query.mutation?,
+            variables: clean_variables(query.provided_variables),
+            query_string: query.query_string
+          }
+
+          info.merge!(::Gitlab::ApplicationContext.current)
+          info.merge!(analysis_info) if analysis_info
+
+          ::Gitlab::GraphqlLogger.info(info)
+        end
+
+        def clean_variables(variables)
+          filtered = ActiveSupport::ParameterFilter
+            .new(::Rails.application.config.filter_parameters)
+            .filter(variables)
+
+          filtered&.to_s
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/tracers/metrics_tracer.rb b/lib/gitlab/graphql/tracers/metrics_tracer.rb
new file mode 100644
index 00000000000..9fc001c0a6d
--- /dev/null
+++ b/lib/gitlab/graphql/tracers/metrics_tracer.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Tracers
+      class MetricsTracer
+        def self.use(schema)
+          schema.tracer(self.new)
+        end
+
+        # See https://graphql-ruby.org/api-doc/1.12.16/GraphQL/Tracing for full list of events
+        def trace(key, data)
+          result = yield
+
+          case key
+          when "execute_query"
+            increment_query_sli(data)
+          end
+
+          result
+        end
+
+        private
+
+        def increment_query_sli(data)
+          duration_s = data.fetch(:duration_s, nil)
+          query = data.fetch(:query, nil)
+
+          # We're just being defensive here...
+          # duration_s comes from TimerTracer and we should be pretty much guaranteed it exists
+          return unless duration_s && query
+
+          operation = ::Gitlab::Graphql::KnownOperations.default.from_query(query)
+          query_urgency = operation.query_urgency
+
+          Gitlab::Metrics::RailsSlis.graphql_query_apdex.increment(
+            labels: {
+              endpoint_id: ::Gitlab::ApplicationContext.current_context_attribute(:caller_id),
+              feature_category: ::Gitlab::ApplicationContext.current_context_attribute(:feature_category),
+              query_urgency: query_urgency.name
+            },
+            success: duration_s <= query_urgency.duration
+          )
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/tracers/timer_tracer.rb b/lib/gitlab/graphql/tracers/timer_tracer.rb
new file mode 100644
index 00000000000..326620a22bc
--- /dev/null
+++ b/lib/gitlab/graphql/tracers/timer_tracer.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Tracers
+      # This graphql-ruby tracer records duration for trace events and merges
+      # the duration into the trace event's metadata. This way, separate tracers
+      # can all use the same duration information.
+      #
+      # NOTE: TimerTracer should be applied last **after** other tracers, so
+      # that it runs first (similar to function composition)
+      class TimerTracer
+        def self.use(schema)
+          schema.tracer(self.new)
+        end
+
+        def trace(key, data)
+          start_time = Gitlab::Metrics::System.monotonic_time
+
+          result = yield
+
+          duration_s = Gitlab::Metrics::System.monotonic_time - start_time
+
+          data[:duration_s] = duration_s
+
+          result
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/variables.rb b/lib/gitlab/graphql/variables.rb
index e17ca56d022..102a269dd5b 100644
--- a/lib/gitlab/graphql/variables.rb
+++ b/lib/gitlab/graphql/variables.rb
@@ -24,8 +24,13 @@ module Gitlab
           else
             {}
           end
-        when Hash, ActionController::Parameters
+        when Hash
           ambiguous_param
+        when ActionController::Parameters
+          # We can and have to trust the "Parameters" because `graphql-ruby` handles this hash safely
+          # Also, `graphql-ruby` uses hash-specific methods, for example `size`:
+          # https://sourcegraph.com/github.com/rmosolgo/graphql-ruby@61232b03412df6685406fc46c414e11d3f447817/-/blob/lib/graphql/query.rb?L304
+          ambiguous_param.to_unsafe_h
         when nil
           {}
         else
diff --git a/lib/gitlab/health_checks/metric.rb b/lib/gitlab/health_checks/metric.rb
index b697cb0d027..c1e437831d7 100644
--- a/lib/gitlab/health_checks/metric.rb
+++ b/lib/gitlab/health_checks/metric.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Naming/FileName
 # frozen_string_literal: true
 
 module Gitlab
@@ -5,3 +6,5 @@ module Gitlab
     Metric = Struct.new(:name, :value, :labels)
   end
 end
+
+# rubocop:enable Naming/FileName
diff --git a/lib/gitlab/health_checks/probes/status.rb b/lib/gitlab/health_checks/probes/status.rb
index 192e9366001..1c59f18ff7d 100644
--- a/lib/gitlab/health_checks/probes/status.rb
+++ b/lib/gitlab/health_checks/probes/status.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Naming/FileName
 # frozen_string_literal: true
 
 module Gitlab
@@ -12,3 +13,5 @@ module Gitlab
     end
   end
 end
+
+# rubocop:enable Naming/FileName
diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb
index 0c8fe83893b..bd843bdaac4 100644
--- a/lib/gitlab/health_checks/redis/cache_check.rb
+++ b/lib/gitlab/health_checks/redis/cache_check.rb
@@ -4,31 +4,7 @@ module Gitlab
   module HealthChecks
     module Redis
       class CacheCheck
-        extend SimpleAbstractCheck
-
-        class << self
-          def check_up
-            check
-          end
-
-          private
-
-          def metric_prefix
-            'redis_cache_ping'
-          end
-
-          def successful?(result)
-            result == 'PONG'
-          end
-
-          # rubocop: disable CodeReuse/ActiveRecord
-          def check
-            catch_timeout 10.seconds do
-              Gitlab::Redis::Cache.with(&:ping)
-            end
-          end
-          # rubocop: enable CodeReuse/ActiveRecord
-        end
+        extend RedisAbstractCheck
       end
     end
   end
diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb
index b1e33b9f459..fb92db937dc 100644
--- a/lib/gitlab/health_checks/redis/queues_check.rb
+++ b/lib/gitlab/health_checks/redis/queues_check.rb
@@ -4,31 +4,7 @@ module Gitlab
   module HealthChecks
     module Redis
       class QueuesCheck
-        extend SimpleAbstractCheck
-
-        class << self
-          def check_up
-            check
-          end
-
-          private
-
-          def metric_prefix
-            'redis_queues_ping'
-          end
-
-          def successful?(result)
-            result == 'PONG'
-          end
-
-          # rubocop: disable CodeReuse/ActiveRecord
-          def check
-            catch_timeout 10.seconds do
-              Gitlab::Redis::Queues.with(&:ping)
-            end
-          end
-          # rubocop: enable CodeReuse/ActiveRecord
-        end
+        extend RedisAbstractCheck
       end
     end
   end
diff --git a/lib/gitlab/health_checks/redis/rate_limiting_check.rb b/lib/gitlab/health_checks/redis/rate_limiting_check.rb
index 67c14e26361..0e9d94f7dff 100644
--- a/lib/gitlab/health_checks/redis/rate_limiting_check.rb
+++ b/lib/gitlab/health_checks/redis/rate_limiting_check.rb
@@ -4,31 +4,7 @@ module Gitlab
   module HealthChecks
     module Redis
       class RateLimitingCheck
-        extend SimpleAbstractCheck
-
-        class << self
-          def check_up
-            check
-          end
-
-          private
-
-          def metric_prefix
-            'redis_rate_limiting_ping'
-          end
-
-          def successful?(result)
-            result == 'PONG'
-          end
-
-          # rubocop: disable CodeReuse/ActiveRecord
-          def check
-            catch_timeout 10.seconds do
-              Gitlab::Redis::RateLimiting.with(&:ping)
-            end
-          end
-          # rubocop: enable CodeReuse/ActiveRecord
-        end
+        extend RedisAbstractCheck
       end
     end
   end
diff --git a/lib/gitlab/health_checks/redis/redis_abstract_check.rb b/lib/gitlab/health_checks/redis/redis_abstract_check.rb
new file mode 100644
index 00000000000..ecad4b06ea9
--- /dev/null
+++ b/lib/gitlab/health_checks/redis/redis_abstract_check.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module HealthChecks
+    module Redis
+      module RedisAbstractCheck
+        include SimpleAbstractCheck
+
+        def check_up
+          successful?(check)
+        end
+
+        private
+
+        def redis_instance_class_name
+          Gitlab::Redis.const_get(redis_instance_name.camelize, false)
+        end
+
+        def metric_prefix
+          "redis_#{redis_instance_name}_ping"
+        end
+
+        def redis_instance_name
+          name.sub(/_check$/, '')
+        end
+
+        def successful?(result)
+          result == 'PONG'
+        end
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        def check
+          catch_timeout 10.seconds do
+            redis_instance_class_name.with(&:ping)
+          end
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb
index 25879c18f84..c793a939abd 100644
--- a/lib/gitlab/health_checks/redis/redis_check.rb
+++ b/lib/gitlab/health_checks/redis/redis_check.rb
@@ -14,16 +14,22 @@ module Gitlab
           end
 
           def successful?(result)
-            result == 'PONG'
+            result == true
           end
 
           def check
-            ::Gitlab::HealthChecks::Redis::CacheCheck.check_up &&
-              ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up &&
-              ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up &&
-              ::Gitlab::HealthChecks::Redis::TraceChunksCheck.check_up &&
-              ::Gitlab::HealthChecks::Redis::RateLimitingCheck.check_up &&
-              ::Gitlab::HealthChecks::Redis::SessionsCheck.check_up
+            redis_health_checks.all?(&:check_up)
+          end
+
+          def redis_health_checks
+            [
+              Gitlab::HealthChecks::Redis::CacheCheck,
+              Gitlab::HealthChecks::Redis::QueuesCheck,
+              Gitlab::HealthChecks::Redis::SharedStateCheck,
+              Gitlab::HealthChecks::Redis::TraceChunksCheck,
+              Gitlab::HealthChecks::Redis::RateLimitingCheck,
+              Gitlab::HealthChecks::Redis::SessionsCheck
+            ]
           end
         end
       end
diff --git a/lib/gitlab/health_checks/redis/sessions_check.rb b/lib/gitlab/health_checks/redis/sessions_check.rb
index a0c5e177b4e..90a4c868f40 100644
--- a/lib/gitlab/health_checks/redis/sessions_check.rb
+++ b/lib/gitlab/health_checks/redis/sessions_check.rb
@@ -4,31 +4,7 @@ module Gitlab
   module HealthChecks
     module Redis
       class SessionsCheck
-        extend SimpleAbstractCheck
-
-        class << self
-          def check_up
-            check
-          end
-
-          private
-
-          def metric_prefix
-            'redis_sessions_ping'
-          end
-
-          def successful?(result)
-            result == 'PONG'
-          end
-
-          # rubocop: disable CodeReuse/ActiveRecord
-          def check
-            catch_timeout 10.seconds do
-              Gitlab::Redis::Sessions.with(&:ping)
-            end
-          end
-          # rubocop: enable CodeReuse/ActiveRecord
-        end
+        extend RedisAbstractCheck
       end
     end
   end
diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb
index 285ac271929..80f91784b8c 100644
--- a/lib/gitlab/health_checks/redis/shared_state_check.rb
+++ b/lib/gitlab/health_checks/redis/shared_state_check.rb
@@ -4,31 +4,7 @@ module Gitlab
   module HealthChecks
     module Redis
       class SharedStateCheck
-        extend SimpleAbstractCheck
-
-        class << self
-          def check_up
-            check
-          end
-
-          private
-
-          def metric_prefix
-            'redis_shared_state_ping'
-          end
-
-          def successful?(result)
-            result == 'PONG'
-          end
-
-          # rubocop: disable CodeReuse/ActiveRecord
-          def check
-            catch_timeout 10.seconds do
-              Gitlab::Redis::SharedState.with(&:ping)
-            end
-          end
-          # rubocop: enable CodeReuse/ActiveRecord
-        end
+        extend RedisAbstractCheck
       end
     end
   end
diff --git a/lib/gitlab/health_checks/redis/trace_chunks_check.rb b/lib/gitlab/health_checks/redis/trace_chunks_check.rb
index cf9fa700b0a..9a89a1ce51d 100644
--- a/lib/gitlab/health_checks/redis/trace_chunks_check.rb
+++ b/lib/gitlab/health_checks/redis/trace_chunks_check.rb
@@ -4,31 +4,7 @@ module Gitlab
   module HealthChecks
     module Redis
       class TraceChunksCheck
-        extend SimpleAbstractCheck
-
-        class << self
-          def check_up
-            check
-          end
-
-          private
-
-          def metric_prefix
-            'redis_trace_chunks_ping'
-          end
-
-          def successful?(result)
-            result == 'PONG'
-          end
-
-          # rubocop: disable CodeReuse/ActiveRecord
-          def check
-            catch_timeout 10.seconds do
-              Gitlab::Redis::TraceChunks.with(&:ping)
-            end
-          end
-          # rubocop: enable CodeReuse/ActiveRecord
-        end
+        extend RedisAbstractCheck
       end
     end
   end
diff --git a/lib/gitlab/health_checks/result.rb b/lib/gitlab/health_checks/result.rb
index 38a36100ec7..cbb847d2af2 100644
--- a/lib/gitlab/health_checks/result.rb
+++ b/lib/gitlab/health_checks/result.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Naming/FileName
 # frozen_string_literal: true
 
 module Gitlab
@@ -13,3 +14,5 @@ module Gitlab
     end
   end
 end
+
+# rubocop:enable Naming/FileName
diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb
index 8a19f208adf..1b860001ac0 100644
--- a/lib/gitlab/http.rb
+++ b/lib/gitlab/http.rb
@@ -15,7 +15,7 @@ module Gitlab
     ].freeze
     HTTP_ERRORS = HTTP_TIMEOUT_ERRORS + [
       EOFError, SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError,
-      Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH,
+      Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH,
       Gitlab::HTTP::BlockedUrlError, Gitlab::HTTP::RedirectionTooDeep
     ].freeze
 
diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb
index b090d05de19..251bc34d462 100644
--- a/lib/gitlab/i18n.rb
+++ b/lib/gitlab/i18n.rb
@@ -44,26 +44,26 @@ module Gitlab
       'bg' => 0,
       'cs_CZ' => 0,
       'da_DK' => 52,
-      'de' => 16,
+      'de' => 15,
       'en' => 100,
       'eo' => 0,
-      'es' => 41,
+      'es' => 40,
       'fil_PH' => 0,
       'fr' => 11,
       'gl_ES' => 0,
       'id_ID' => 0,
       'it' => 2,
-      'ja' => 37,
+      'ja' => 36,
       'ko' => 11,
-      'nb_NO' => 35,
+      'nb_NO' => 34,
       'nl_NL' => 0,
       'pl_PL' => 5,
-      'pt_BR' => 45,
+      'pt_BR' => 49,
       'ro_RO' => 24,
-      'ru' => 27,
-      'tr_TR' => 16,
-      'uk' => 40,
-      'zh_CN' => 95,
+      'ru' => 26,
+      'tr_TR' => 15,
+      'uk' => 39,
+      'zh_CN' => 97,
       'zh_HK' => 2,
       'zh_TW' => 3
     }.freeze
diff --git a/lib/gitlab/import/database_helpers.rb b/lib/gitlab/import/database_helpers.rb
index e73c3afe9bd..96490db0c07 100644
--- a/lib/gitlab/import/database_helpers.rb
+++ b/lib/gitlab/import/database_helpers.rb
@@ -11,8 +11,8 @@ module Gitlab
         # We use bulk_insert here so we can bypass any queries executed by
         # callbacks or validation rules, as doing this wouldn't scale when
         # importing very large projects.
-        result = Gitlab::Database.main # rubocop:disable Gitlab/BulkInsert
-                 .bulk_insert(relation.table_name, [attributes], return_ids: true)
+        result = ApplicationRecord # rubocop:disable Gitlab/BulkInsert
+                 .legacy_bulk_insert(relation.table_name, [attributes], return_ids: true)
 
         result.first
       end
diff --git a/lib/gitlab/import/metrics.rb b/lib/gitlab/import/metrics.rb
index 5f27d0ab965..7a0cf1682a6 100644
--- a/lib/gitlab/import/metrics.rb
+++ b/lib/gitlab/import/metrics.rb
@@ -69,11 +69,7 @@ module Gitlab
       end
 
       def observe_histogram
-        if project.github_import?
-          duration_histogram.observe({ project: project.full_path }, duration)
-        else
-          duration_histogram.observe({ importer: importer }, duration)
-        end
+        duration_histogram.observe({ importer: importer }, duration)
       end
 
       def track_finish_metric
diff --git a/lib/gitlab/import_export/attributes_permitter.rb b/lib/gitlab/import_export/attributes_permitter.rb
index 2d8e25a9f70..f6f65f85599 100644
--- a/lib/gitlab/import_export/attributes_permitter.rb
+++ b/lib/gitlab/import_export/attributes_permitter.rb
@@ -44,7 +44,7 @@ module Gitlab
 
       # We want to use AttributesCleaner for these relations instead, in the future this should be removed to make sure
       # we are using AttributesPermitter for every imported relation.
-      DISABLED_RELATION_NAMES = %i[user author issuable_sla].freeze
+      DISABLED_RELATION_NAMES = %i[author issuable_sla].freeze
 
       def initialize(config: ImportExport::Config.new.to_h)
         @config = config
diff --git a/lib/gitlab/import_export/base/object_builder.rb b/lib/gitlab/import_export/base/object_builder.rb
index 48836729ff6..5e9c8292c1e 100644
--- a/lib/gitlab/import_export/base/object_builder.rb
+++ b/lib/gitlab/import_export/base/object_builder.rb
@@ -47,15 +47,15 @@ module Gitlab
           attributes
         end
 
-        private
+        def find_with_cache(key = cache_key)
+          return yield unless lru_cache && key
 
-        attr_reader :klass, :attributes, :lru_cache, :cache_key
+          lru_cache[key] ||= yield
+        end
 
-        def find_with_cache
-          return yield unless lru_cache && cache_key
+        private
 
-          lru_cache[cache_key] ||= yield
-        end
+        attr_reader :klass, :attributes, :lru_cache, :cache_key
 
         def cache_from_request_store
           Gitlab::SafeRequestStore[:lru_cache] ||= LruRedux::Cache.new(LRU_CACHE_SIZE)
diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb
index febfe00af0b..61b37256964 100644
--- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb
+++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb
@@ -6,7 +6,7 @@ module Gitlab
       include Gitlab::Utils::StrongMemoize
 
       DEFAULT_MAX_BYTES = 10.gigabytes.freeze
-      TIMEOUT_LIMIT = 60.seconds
+      TIMEOUT_LIMIT = 210.seconds
 
       def initialize(archive_path:, max_bytes: self.class.max_bytes)
         @archive_path = archive_path
diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb
new file mode 100644
index 00000000000..f3c392b8c20
--- /dev/null
+++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb
@@ -0,0 +1,274 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module Group
+      class RelationTreeRestorer
+        def initialize( # rubocop:disable Metrics/ParameterLists
+          user:,
+          shared:,
+          relation_reader:,
+          members_mapper:,
+          object_builder:,
+          relation_factory:,
+          reader:,
+          importable:,
+          importable_attributes:,
+          importable_path:
+        )
+          @user = user
+          @shared = shared
+          @importable = importable
+          @relation_reader = relation_reader
+          @members_mapper = members_mapper
+          @object_builder = object_builder
+          @relation_factory = relation_factory
+          @reader = reader
+          @importable_attributes = importable_attributes
+          @importable_path = importable_path
+        end
+
+        def restore
+          ActiveRecord::Base.uncached do
+            ActiveRecord::Base.no_touching do
+              update_params!
+
+              BulkInsertableAssociations.with_bulk_insert(enabled: bulk_insert_enabled) do
+                fix_ci_pipelines_not_sorted_on_legacy_project_json!
+                create_relations!
+              end
+            end
+          end
+
+          # ensure that we have latest version of the restore
+          @importable.reload # rubocop:disable Cop/ActiveRecordAssociationReload
+
+          true
+        rescue StandardError => e
+          @shared.error(e)
+          false
+        end
+
+        private
+
+        def bulk_insert_enabled
+          false
+        end
+
+        # Loops through the tree of models defined in import_export.yml and
+        # finds them in the imported JSON so they can be instantiated and saved
+        # in the DB. The structure and relationships between models are guessed from
+        # the configuration yaml file too.
+        # Finally, it updates each attribute in the newly imported project/group.
+        def create_relations!
+          relations.each do |relation_key, relation_definition|
+            process_relation!(relation_key, relation_definition)
+          end
+        end
+
+        def process_relation!(relation_key, relation_definition)
+          @relation_reader.consume_relation(@importable_path, relation_key).each do |data_hash, relation_index|
+            process_relation_item!(relation_key, relation_definition, relation_index, data_hash)
+          end
+        end
+
+        def process_relation_item!(relation_key, relation_definition, relation_index, data_hash)
+          relation_object = build_relation(relation_key, relation_definition, relation_index, data_hash)
+          return unless relation_object
+          return if relation_invalid_for_importable?(relation_object)
+
+          relation_object.assign_attributes(importable_class_sym => @importable)
+
+          import_failure_service.with_retry(action: 'relation_object.save!', relation_key: relation_key, relation_index: relation_index) do
+            relation_object.save!
+            log_relation_creation(@importable, relation_key, relation_object)
+          end
+        rescue StandardError => e
+          import_failure_service.log_import_failure(
+            source: 'process_relation_item!',
+            relation_key: relation_key,
+            relation_index: relation_index,
+            exception: e)
+        end
+
+        def import_failure_service
+          @import_failure_service ||= ImportFailureService.new(@importable)
+        end
+
+        def relations
+          @relations ||=
+            @reader
+              .attributes_finder
+              .find_relations_tree(importable_class_sym)
+              .deep_stringify_keys
+        end
+
+        def update_params!
+          params = @importable_attributes.except(*relations.keys.map(&:to_s))
+          params = params.merge(present_override_params)
+
+          # Cleaning all imported and overridden params
+          params = Gitlab::ImportExport::AttributeCleaner.clean(
+            relation_hash: params,
+            relation_class: importable_class,
+            excluded_keys: excluded_keys_for_relation(importable_class_sym))
+
+          @importable.assign_attributes(params)
+
+          modify_attributes
+
+          Gitlab::Timeless.timeless(@importable) do
+            @importable.save!
+          end
+        end
+
+        def present_override_params
+          # we filter out the empty strings from the overrides
+          # keeping the default values configured
+          override_params&.transform_values do |value|
+            value.is_a?(String) ? value.presence : value
+          end&.compact
+        end
+
+        def override_params
+          @importable_override_params ||= importable_override_params
+        end
+
+        def importable_override_params
+          if @importable.respond_to?(:import_data)
+            @importable.import_data&.data&.fetch('override_params', nil) || {}
+          else
+            {}
+          end
+        end
+
+        def modify_attributes
+          # no-op to be overridden on inheritance
+        end
+
+        def build_relations(relation_key, relation_definition, relation_index, data_hashes)
+          data_hashes
+            .map { |data_hash| build_relation(relation_key, relation_definition, relation_index, data_hash) }
+            .tap { |entries| entries.compact! }
+        end
+
+        def build_relation(relation_key, relation_definition, relation_index, data_hash)
+          # TODO: This is hack to not create relation for the author
+          # Rather make `RelationFactory#set_note_author` to take care of that
+          return data_hash if relation_key == 'author' || already_restored?(data_hash)
+
+          # create relation objects recursively for all sub-objects
+          relation_definition.each do |sub_relation_key, sub_relation_definition|
+            transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition, relation_index)
+          end
+
+          relation = @relation_factory.create(**relation_factory_params(relation_key, relation_index, data_hash))
+
+          if relation && !relation.valid?
+            @shared.logger.warn(
+              message: "[Project/Group Import] Invalid object relation built",
+              relation_key: relation_key,
+              relation_index: relation_index,
+              relation_class: relation.class.name,
+              error_messages: relation.errors.full_messages.join(". ")
+            )
+          end
+
+          relation
+        end
+
+        # Since we update the data hash in place as we restore relation items,
+        # and since we also de-duplicate items, we might encounter items that
+        # have already been restored in a previous iteration.
+        def already_restored?(relation_item)
+          !relation_item.is_a?(Hash)
+        end
+
+        def transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition, relation_index)
+          sub_data_hash = data_hash[sub_relation_key]
+          return unless sub_data_hash
+
+          # if object is a hash we can create simple object
+          # as it means that this is 1-to-1 vs 1-to-many
+          current_item =
+            if sub_data_hash.is_a?(Array)
+              build_relations(
+                sub_relation_key,
+                sub_relation_definition,
+                relation_index,
+                sub_data_hash).presence
+            else
+              build_relation(
+                sub_relation_key,
+                sub_relation_definition,
+                relation_index,
+                sub_data_hash)
+            end
+
+          if current_item
+            data_hash[sub_relation_key] = current_item
+          else
+            data_hash.delete(sub_relation_key)
+          end
+        end
+
+        def relation_invalid_for_importable?(_relation_object)
+          false
+        end
+
+        def excluded_keys_for_relation(relation)
+          @reader.attributes_finder.find_excluded_keys(relation)
+        end
+
+        def importable_class
+          @importable.class
+        end
+
+        def importable_class_sym
+          importable_class.to_s.downcase.to_sym
+        end
+
+        def relation_factory_params(relation_key, relation_index, data_hash)
+          {
+            relation_index: relation_index,
+            relation_sym: relation_key.to_sym,
+            relation_hash: data_hash,
+            importable: @importable,
+            members_mapper: @members_mapper,
+            object_builder: @object_builder,
+            user: @user,
+            excluded_keys: excluded_keys_for_relation(relation_key)
+          }
+        end
+
+        # Temporary fix for https://gitlab.com/gitlab-org/gitlab/-/issues/27883 when import from legacy project.json
+        # This should be removed once legacy JSON format is deprecated.
+        # Ndjson export file will fix the order during project export.
+        def fix_ci_pipelines_not_sorted_on_legacy_project_json!
+          return unless @relation_reader.legacy?
+
+          @relation_reader.sort_ci_pipelines_by_id
+        end
+
+        # Enable logging of each top-level relation creation when Importing
+        # into a Group if feature flag is enabled
+        def log_relation_creation(importable, relation_key, relation_object)
+          root_ancestor_group = importable.try(:root_ancestor)
+
+          return unless root_ancestor_group
+          return unless root_ancestor_group.instance_of?(::Group)
+          return unless Feature.enabled?(:log_import_export_relation_creation, root_ancestor_group)
+
+          @shared.logger.info(
+            importable_type: importable.class.to_s,
+            importable_id: importable.id,
+            relation_key: relation_key,
+            relation_id: relation_object.id,
+            author_id: relation_object.try(:author_id),
+            message: '[Project/Group Import] Created new object relation'
+          )
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml
index 618ef9a4f43..d815dd284ba 100644
--- a/lib/gitlab/import_export/project/import_export.yml
+++ b/lib/gitlab/import_export/project/import_export.yml
@@ -178,17 +178,7 @@ included_attributes:
     - :project_id
     - :key
     - :value
-  label:
-    - :title
-    - :color
-    - :project_id
-    - :group_id
-    - :created_at
-    - :updated_at
-    - :template
-    - :description
-    - :priority
-  labels:
+  label: &label_definition
     - :title
     - :color
     - :project_id
@@ -198,23 +188,13 @@ included_attributes:
     - :template
     - :description
     - :priority
+  labels: *label_definition
   priorities:
     - :project_id
     - :priority
     - :created_at
     - :updated_at
-  milestone:
-    - :iid
-    - :title
-    - :project_id
-    - :group_id
-    - :description
-    - :due_date
-    - :created_at
-    - :updated_at
-    - :start_date
-    - :state
-  milestones:
+  milestone: &milestone_definition
     - :iid
     - :title
     - :project_id
@@ -225,6 +205,7 @@ included_attributes:
     - :updated_at
     - :start_date
     - :state
+  milestones: *milestone_definition
   protected_branches:
     - :project_id
     - :name
@@ -272,6 +253,385 @@ included_attributes:
     - :updated_at
     - :filepath
     - :link_type
+  container_expiration_policy:
+    - :created_at
+    - :updated_at
+    - :next_run_at
+    - :project_id
+    - :name_regex
+    - :cadence
+    - :older_than
+    - :keep_n
+    - :enabled
+    - :name_regex_keep
+  project_feature:
+    - :project_id
+    - :merge_requests_access_level
+    - :issues_access_level
+    - :wiki_access_level
+    - :snippets_access_level
+    - :builds_access_level
+    - :created_at
+    - :updated_at
+    - :repository_access_level
+    - :pages_access_level
+    - :forking_access_level
+    - :metrics_dashboard_access_level
+    - :operations_access_level
+    - :analytics_access_level
+    - :security_and_compliance_access_level
+    - :container_registry_access_level
+  prometheus_metrics:
+    - :created_at
+    - :updated_at
+    - :project_id
+    - :y_label
+    - :unit
+    - :legend
+    - :title
+    - :query
+    - :group
+    - :dashboard_path
+  service_desk_setting:
+    - :project_id
+    - :issue_template_key
+    - :project_key
+  snippets:
+    - :title
+    - :content
+    - :author_id
+    - :project_id
+    - :created_at
+    - :updated_at
+    - :file_name
+    - :visibility_level
+    - :description
+  project_members:
+    - :access_level
+    - :source_type
+    - :user_id
+    - :notification_level
+    - :created_at
+    - :updated_at
+    - :created_by_id
+    - :invite_email
+    - :invite_accepted_at
+    - :requested_at
+    - :expires_at
+    - :ldap
+    - :override
+  merge_request: &merge_request_definition
+    - :target_branch
+    - :source_branch
+    - :source_project_id
+    - :author_id
+    - :assignee_id
+    - :title
+    - :created_at
+    - :updated_at
+    - :state
+    - :merge_status
+    - :target_project_id
+    - :iid
+    - :description
+    - :updated_by_id
+    - :merge_error
+    - :merge_params
+    - :merge_when_pipeline_succeeds
+    - :merge_user_id
+    - :merge_commit_sha
+    - :squash_commit_sha
+    - :in_progress_merge_commit_sha
+    - :lock_version
+    - :approvals_before_merge
+    - :rebase_commit_sha
+    - :time_estimate
+    - :squash
+    - :last_edited_at
+    - :last_edited_by_id
+    - :discussion_locked
+    - :allow_maintainer_to_push
+    - :merge_ref_sha
+    - :draft
+    - :diff_head_sha
+    - :source_branch_sha
+    - :target_branch_sha
+  merge_requests: *merge_request_definition
+  award_emoji:
+    - :user_id
+    - :name
+    - :awardable_type
+    - :created_at
+    - :updated_at
+  commit_author:
+    - :name
+    - :email
+  committer:
+    - :name
+    - :email
+  events:
+    - :target_type
+    - :action
+    - :author_id
+    - :fingerprint
+    - :created_at
+    - :updated_at
+  label_links:
+    - :target_type
+    - :created_at
+    - :updated_at
+  merge_request_diff:
+    - :state
+    - :created_at
+    - :updated_at
+    - :base_commit_sha
+    - :real_size
+    - :head_commit_sha
+    - :start_commit_sha
+    - :commits_count
+    - :files_count
+    - :sorted
+    - :diff_type
+  merge_request_diff_commits:
+    - :author_name
+    - :author_email
+    - :committer_name
+    - :committer_email
+    - :relative_order
+    - :sha
+    - :authored_date
+    - :committed_date
+    - :message
+    - :trailers
+  merge_request_diff_files:
+    - :relative_order
+    - :new_file
+    - :renamed_file
+    - :deleted_file
+    - :new_path
+    - :old_path
+    - :a_mode
+    - :b_mode
+    - :too_large
+    - :binary
+    - :diff
+  metrics:
+    - :created_at
+    - :updated_at
+    - :latest_closed_by_id
+    - :latest_closed_at
+    - :merged_by_id
+    - :merged_at
+    - :latest_build_started_at
+    - :latest_build_finished_at
+    - :first_deployed_to_production_at
+    - :first_comment_at
+    - :first_commit_at
+    - :last_commit_at
+    - :diff_size
+    - :modified_paths_size
+    - :commits_count
+    - :first_approved_at
+    - :first_reassigned_at
+    - :added_lines
+    - :target_project_id
+    - :removed_lines
+  notes:
+    - :note
+    - :noteable_type
+    - :author_id
+    - :created_at
+    - :updated_at
+    - :project_id
+    - :attachment
+    - :line_code
+    - :commit_id
+    - :system
+    - :st_diff
+    - :updated_by_id
+    - :type
+    - :position
+    - :original_position
+    - :change_position
+    - :resolved_at
+    - :resolved_by_id
+    - :resolved_by_push
+    - :discussion_id
+    - :confidential
+    - :last_edited_at
+  push_event_payload:
+    - :commit_count
+    - :action
+    - :ref_type
+    - :commit_from
+    - :commit_to
+    - :ref
+    - :commit_title
+    - :ref_count
+  resource_label_events:
+    - :action
+    - :user_id
+    - :created_at
+  suggestions:
+    - :relative_order
+    - :applied
+    - :commit_id
+    - :from_content
+    - :to_content
+    - :outdated
+    - :lines_above
+    - :lines_below
+  system_note_metadata:
+    - :commit_count
+    - :action
+    - :created_at
+    - :updated_at
+  timelogs:
+    - :time_spent
+    - :user_id
+    - :project_id
+    - :spent_at
+    - :created_at
+    - :updated_at
+    - :summary
+  external_pull_request: &external_pull_request_definition
+    - :created_at
+    - :updated_at
+    - :project_id
+    - :pull_request_iid
+    - :status
+    - :source_branch
+    - :target_branch
+    - :source_repository
+    - :target_repository
+    - :source_sha
+    - :target_sha
+  external_pull_requests: *external_pull_request_definition
+  statuses:
+    - :project_id
+    - :status
+    - :finished_at
+    - :created_at
+    - :updated_at
+    - :started_at
+    - :coverage
+    - :commit_id
+    - :name
+    - :options
+    - :allow_failure
+    - :stage
+    - :stage_idx
+    - :tag
+    - :ref
+    - :user_id
+    - :type
+    - :target_url
+    - :description
+    - :erased_at
+    - :artifacts_expire_at
+    - :environment
+    - :yaml_variables
+    - :queued_at
+    - :lock_version
+    - :coverage_regex
+    - :retried
+    - :protected
+    - :failure_reason
+    - :scheduled_at
+    - :scheduling_type
+  ci_pipelines:
+    - :ref
+    - :sha
+    - :before_sha
+    - :created_at
+    - :updated_at
+    - :tag
+    - :yaml_errors
+    - :committed_at
+    - :project_id
+    - :status
+    - :started_at
+    - :finished_at
+    - :duration
+    - :user_id
+    - :lock_version
+    - :source
+    - :protected
+    - :config_source
+    - :failure_reason
+    - :iid
+    - :source_sha
+    - :target_sha
+  stages:
+    - :name
+    - :status
+    - :position
+    - :lock_version
+    - :project_id
+    - :created_at
+    - :updated_at
+  actions:
+    - :event
+    - :image_v432x230
+  design: &design_definition
+    - :iid
+    - :project_id
+    - :filename
+    - :relative_position
+  designs: *design_definition
+  design_versions:
+    - :created_at
+    - :sha
+    - :author_id
+  issue_assignees:
+    - :user_id
+  sentry_issue:
+    - :sentry_issue_identifier
+  zoom_meetings:
+    - :project_id
+    - :issue_status
+    - :url
+    - :created_at
+    - :updated_at
+  issues:
+    - :title
+    - :author_id
+    - :project_id
+    - :created_at
+    - :updated_at
+    - :description
+    - :state
+    - :iid
+    - :updated_by_id
+    - :confidential
+    - :closed_at
+    - :closed_by_id
+    - :due_date
+    - :lock_version
+    - :weight
+    - :time_estimate
+    - :relative_position
+    - :external_author
+    - :last_edited_at
+    - :last_edited_by_id
+    - :discussion_locked
+    - :health_status
+    - :external_key
+    - :issue_type
+  group_members:
+    - :access_level
+    - :source_type
+    - :user_id
+    - :notification_level
+    - :created_at
+    - :updated_at
+    - :created_by_id
+    - :invite_email
+    - :invite_accepted_at
+    - :requested_at
+    - :expires_at
+    - :ldap
+    - :override
 
 # Do not include the following attributes for the models specified.
 excluded_attributes:
@@ -387,16 +747,7 @@ excluded_attributes:
     - :service_desk_reply_to
     - :upvotes_count
     - :work_item_type_id
-  merge_request:
-    - :milestone_id
-    - :sprint_id
-    - :ref_fetched
-    - :merge_jid
-    - :rebase_jid
-    - :latest_merge_request_diff_id
-    - :head_pipeline_id
-    - :state_id
-  merge_requests:
+  merge_request: &merge_request_excluded_definition
     - :milestone_id
     - :sprint_id
     - :ref_fetched
@@ -405,6 +756,7 @@ excluded_attributes:
     - :latest_merge_request_diff_id
     - :head_pipeline_id
     - :state_id
+  merge_requests: *merge_request_excluded_definition
   award_emoji:
     - :awardable_id
   statuses:
@@ -473,10 +825,9 @@ excluded_attributes:
     - :issue_id
   zoom_meetings:
     - :issue_id
-  design:
-    - :issue_id
-  designs:
+  design: &design_excluded_definition
     - :issue_id
+  designs: *design_excluded_definition
   design_versions:
     - :issue_id
   actions:
@@ -660,4 +1011,13 @@ ee:
       - :name
       - :created_at
       - :updated_at
-
+    project_feature:
+      - :requirements_access_level
+    security_setting:
+      - :project_id
+      - :created_at
+      - :updated_at
+      - :auto_fix_container_scanning
+      - :auto_fix_dast
+      - :auto_fix_dependency_scanning
+      - :auto_fix_sast
diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb
index b03dceba303..f7598ba1337 100644
--- a/lib/gitlab/import_export/project/object_builder.rb
+++ b/lib/gitlab/import_export/project/object_builder.rb
@@ -29,6 +29,7 @@ module Gitlab
         def find
           return if epic? && group.nil?
           return find_diff_commit_user if diff_commit_user?
+          return find_diff_commit if diff_commit?
 
           super
         end
@@ -83,9 +84,38 @@ module Gitlab
         end
 
         def find_diff_commit_user
-          find_with_cache do
-            MergeRequest::DiffCommitUser
-              .find_or_create(@attributes['name'], @attributes['email'])
+          find_or_create_diff_commit_user(@attributes['name'], @attributes['email'])
+        end
+
+        def find_diff_commit
+          row = @attributes.dup
+
+          # Diff commits come in two formats:
+          #
+          # 1. The old format where author/committer details are separate fields
+          # 2. The new format where author/committer details are nested objects,
+          #    and pre-processed by `find_diff_commit_user`.
+          #
+          # The code here ensures we support both the old and new format.
+          aname = row.delete('author_name')
+          amail = row.delete('author_email')
+          cname = row.delete('committer_name')
+          cmail = row.delete('committer_email')
+          author = row.delete('commit_author')
+          committer = row.delete('committer')
+
+          row['commit_author'] = author ||
+            find_or_create_diff_commit_user(aname, amail)
+
+          row['committer'] = committer ||
+            find_or_create_diff_commit_user(cname, cmail)
+
+          MergeRequestDiffCommit.new(row)
+        end
+
+        def find_or_create_diff_commit_user(name, email)
+          find_with_cache([MergeRequest::DiffCommitUser, name, email]) do
+            MergeRequest::DiffCommitUser.find_or_create(name, email)
           end
         end
 
@@ -113,6 +143,10 @@ module Gitlab
           klass == MergeRequest::DiffCommitUser
         end
 
+        def diff_commit?
+          klass == MergeRequestDiffCommit
+        end
+
         # If an existing group milestone used the IID
         # claim the IID back and set the group milestone to use one available
         # This is necessary to fix situations like the following:
diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb
index 888a5a10f2c..d84db92fe69 100644
--- a/lib/gitlab/import_export/project/relation_factory.rb
+++ b/lib/gitlab/import_export/project/relation_factory.rb
@@ -33,7 +33,8 @@ module Gitlab
                       links: 'Releases::Link',
                       metrics_setting: 'ProjectMetricsSetting',
                       commit_author: 'MergeRequest::DiffCommitUser',
-                      committer: 'MergeRequest::DiffCommitUser' }.freeze
+                      committer: 'MergeRequest::DiffCommitUser',
+                      merge_request_diff_commits: 'MergeRequestDiffCommit' }.freeze
 
         BUILD_MODELS = %i[Ci::Build commit_status].freeze
 
@@ -59,6 +60,7 @@ module Gitlab
           external_pull_requests
           DesignManagement::Design
           MergeRequest::DiffCommitUser
+          MergeRequestDiffCommit
         ].freeze
 
         def create
diff --git a/lib/gitlab/import_export/project/relation_tree_restorer.rb b/lib/gitlab/import_export/project/relation_tree_restorer.rb
new file mode 100644
index 00000000000..6e9548f393a
--- /dev/null
+++ b/lib/gitlab/import_export/project/relation_tree_restorer.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module Project
+      class RelationTreeRestorer < ImportExport::Group::RelationTreeRestorer
+        # Relations which cannot be saved at project level (and have a group assigned)
+        GROUP_MODELS = [GroupLabel, Milestone, Epic].freeze
+
+        private
+
+        def bulk_insert_enabled
+          true
+        end
+
+        def modify_attributes
+          @importable.reconcile_shared_runners_setting!
+          @importable.drop_visibility_level!
+        end
+
+        def relation_invalid_for_importable?(relation_object)
+          GROUP_MODELS.include?(relation_object.class) && relation_object.group_id
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb
index 4db92b12968..034122a9f14 100644
--- a/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb
+++ b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb
@@ -4,7 +4,7 @@ module Gitlab
   module ImportExport
     module Project
       module Sample
-        class RelationTreeRestorer < ImportExport::RelationTreeRestorer
+        class RelationTreeRestorer < ImportExport::Project::RelationTreeRestorer
           def initialize(...)
             super(...)
 
@@ -18,10 +18,10 @@ module Gitlab
           end
 
           def dates
-            return [] if relation_reader.legacy?
+            return [] if @relation_reader.legacy?
 
             RelationFactory::DATE_MODELS.flat_map do |tag|
-              relation_reader.consume_relation(@importable_path, tag, mark_as_consumed: false).map do |model|
+              @relation_reader.consume_relation(@importable_path, tag, mark_as_consumed: false).map do |model|
                 model.first['due_date']
               end
             end
diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb
index 1f0fa249390..aafed850afa 100644
--- a/lib/gitlab/import_export/project/tree_saver.rb
+++ b/lib/gitlab/import_export/project/tree_saver.rb
@@ -6,20 +6,16 @@ module Gitlab
       class TreeSaver
         attr_reader :full_path
 
-        def initialize(project:, current_user:, shared:, params: {})
+        def initialize(project:, current_user:, shared:, params: {}, logger: Gitlab::Import::Logger)
           @params       = params
           @project      = project
           @current_user = current_user
           @shared       = shared
+          @logger       = logger
         end
 
         def save
-          ImportExport::Json::StreamingSerializer.new(
-            exportable,
-            reader.project_tree,
-            json_writer,
-            exportable_path: "project"
-          ).execute
+          stream_export
 
           true
         rescue StandardError => e
@@ -31,6 +27,32 @@ module Gitlab
 
         private
 
+        def stream_export
+          on_retry = proc do |exception, try, elapsed_time, next_interval|
+            @logger.info(
+              message: "Project export retry triggered from streaming",
+              'error.class': exception.class,
+              'error.message': exception.message,
+              try_count: try,
+              elapsed_time_s: elapsed_time,
+              wait_to_retry_s: next_interval,
+              project_name: @project.name,
+              project_id: @project.id
+            )
+          end
+
+          serializer = ImportExport::Json::StreamingSerializer.new(
+            exportable,
+            reader.project_tree,
+            json_writer,
+            exportable_path: "project"
+          )
+
+          Retriable.retriable(on: Net::OpenTimeout, on_retry: on_retry) do
+            serializer.execute
+          end
+        end
+
         def reader
           @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
         end
diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb
deleted file mode 100644
index 1eeacafef53..00000000000
--- a/lib/gitlab/import_export/relation_tree_restorer.rb
+++ /dev/null
@@ -1,280 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module ImportExport
-    class RelationTreeRestorer
-      # Relations which cannot be saved at project level (and have a group assigned)
-      GROUP_MODELS = [GroupLabel, Milestone, Epic].freeze
-
-      attr_reader :user
-      attr_reader :shared
-      attr_reader :importable
-      attr_reader :relation_reader
-
-      def initialize( # rubocop:disable Metrics/ParameterLists
-        user:, shared:, relation_reader:,
-        members_mapper:, object_builder:,
-        relation_factory:,
-        reader:,
-        importable:,
-        importable_attributes:,
-        importable_path:
-      )
-        @user = user
-        @shared = shared
-        @importable = importable
-        @relation_reader = relation_reader
-        @members_mapper = members_mapper
-        @object_builder = object_builder
-        @relation_factory = relation_factory
-        @reader = reader
-        @importable_attributes = importable_attributes
-        @importable_path = importable_path
-      end
-
-      def restore
-        ActiveRecord::Base.uncached do
-          ActiveRecord::Base.no_touching do
-            update_params!
-
-            BulkInsertableAssociations.with_bulk_insert(enabled: project?) do
-              fix_ci_pipelines_not_sorted_on_legacy_project_json!
-              create_relations!
-            end
-          end
-        end
-
-        # ensure that we have latest version of the restore
-        @importable.reload # rubocop:disable Cop/ActiveRecordAssociationReload
-
-        true
-      rescue StandardError => e
-        @shared.error(e)
-        false
-      end
-
-      private
-
-      def project?
-        @importable.instance_of?(::Project)
-      end
-
-      # Loops through the tree of models defined in import_export.yml and
-      # finds them in the imported JSON so they can be instantiated and saved
-      # in the DB. The structure and relationships between models are guessed from
-      # the configuration yaml file too.
-      # Finally, it updates each attribute in the newly imported project/group.
-      def create_relations!
-        relations.each do |relation_key, relation_definition|
-          process_relation!(relation_key, relation_definition)
-        end
-      end
-
-      def process_relation!(relation_key, relation_definition)
-        @relation_reader.consume_relation(@importable_path, relation_key).each do |data_hash, relation_index|
-          process_relation_item!(relation_key, relation_definition, relation_index, data_hash)
-        end
-      end
-
-      def process_relation_item!(relation_key, relation_definition, relation_index, data_hash)
-        relation_object = build_relation(relation_key, relation_definition, relation_index, data_hash)
-        return unless relation_object
-        return if project? && group_model?(relation_object)
-
-        relation_object.assign_attributes(importable_class_sym => @importable)
-
-        import_failure_service.with_retry(action: 'relation_object.save!', relation_key: relation_key, relation_index: relation_index) do
-          relation_object.save!
-          log_relation_creation(@importable, relation_key, relation_object)
-        end
-      rescue StandardError => e
-        import_failure_service.log_import_failure(
-          source: 'process_relation_item!',
-          relation_key: relation_key,
-          relation_index: relation_index,
-          exception: e)
-      end
-
-      def import_failure_service
-        @import_failure_service ||= ImportFailureService.new(@importable)
-      end
-
-      def relations
-        @relations ||=
-          @reader
-            .attributes_finder
-            .find_relations_tree(importable_class_sym)
-            .deep_stringify_keys
-      end
-
-      def update_params!
-        params = @importable_attributes.except(*relations.keys.map(&:to_s))
-        params = params.merge(present_override_params)
-
-        # Cleaning all imported and overridden params
-        params = Gitlab::ImportExport::AttributeCleaner.clean(
-          relation_hash:  params,
-          relation_class: importable_class,
-          excluded_keys:  excluded_keys_for_relation(importable_class_sym))
-
-        @importable.assign_attributes(params)
-
-        modify_attributes
-
-        Gitlab::Timeless.timeless(@importable) do
-          @importable.save!
-        end
-      end
-
-      def present_override_params
-        # we filter out the empty strings from the overrides
-        # keeping the default values configured
-        override_params&.transform_values do |value|
-          value.is_a?(String) ? value.presence : value
-        end&.compact
-      end
-
-      def override_params
-        @importable_override_params ||= importable_override_params
-      end
-
-      def importable_override_params
-        if @importable.respond_to?(:import_data)
-          @importable.import_data&.data&.fetch('override_params', nil) || {}
-        else
-          {}
-        end
-      end
-
-      def modify_attributes
-        return unless project?
-
-        @importable.reconcile_shared_runners_setting!
-        @importable.drop_visibility_level!
-      end
-
-      def build_relations(relation_key, relation_definition, relation_index, data_hashes)
-        data_hashes
-          .map { |data_hash| build_relation(relation_key, relation_definition, relation_index, data_hash) }
-          .tap { |entries| entries.compact! }
-      end
-
-      def build_relation(relation_key, relation_definition, relation_index, data_hash)
-        # TODO: This is hack to not create relation for the author
-        # Rather make `RelationFactory#set_note_author` to take care of that
-        return data_hash if relation_key == 'author' || already_restored?(data_hash)
-
-        # create relation objects recursively for all sub-objects
-        relation_definition.each do |sub_relation_key, sub_relation_definition|
-          transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition, relation_index)
-        end
-
-        relation = @relation_factory.create(**relation_factory_params(relation_key, relation_index, data_hash))
-
-        if relation && !relation.valid?
-          @shared.logger.warn(
-            message: "[Project/Group Import] Invalid object relation built",
-            relation_key: relation_key,
-            relation_index: relation_index,
-            relation_class: relation.class.name,
-            error_messages: relation.errors.full_messages.join(". ")
-          )
-        end
-
-        relation
-      end
-
-      # Since we update the data hash in place as we restore relation items,
-      # and since we also de-duplicate items, we might encounter items that
-      # have already been restored in a previous iteration.
-      def already_restored?(relation_item)
-        !relation_item.is_a?(Hash)
-      end
-
-      def transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition, relation_index)
-        sub_data_hash = data_hash[sub_relation_key]
-        return unless sub_data_hash
-
-        # if object is a hash we can create simple object
-        # as it means that this is 1-to-1 vs 1-to-many
-        current_item =
-          if sub_data_hash.is_a?(Array)
-            build_relations(
-              sub_relation_key,
-              sub_relation_definition,
-              relation_index,
-              sub_data_hash).presence
-          else
-            build_relation(
-              sub_relation_key,
-              sub_relation_definition,
-              relation_index,
-              sub_data_hash)
-          end
-
-        if current_item
-          data_hash[sub_relation_key] = current_item
-        else
-          data_hash.delete(sub_relation_key)
-        end
-      end
-
-      def group_model?(relation_object)
-        GROUP_MODELS.include?(relation_object.class) && relation_object.group_id
-      end
-
-      def excluded_keys_for_relation(relation)
-        @reader.attributes_finder.find_excluded_keys(relation)
-      end
-
-      def importable_class
-        @importable.class
-      end
-
-      def importable_class_sym
-        importable_class.to_s.downcase.to_sym
-      end
-
-      def relation_factory_params(relation_key, relation_index, data_hash)
-        {
-          relation_index: relation_index,
-          relation_sym: relation_key.to_sym,
-          relation_hash: data_hash,
-          importable: @importable,
-          members_mapper: @members_mapper,
-          object_builder: @object_builder,
-          user: @user,
-          excluded_keys: excluded_keys_for_relation(relation_key)
-        }
-      end
-
-      # Temporary fix for https://gitlab.com/gitlab-org/gitlab/-/issues/27883 when import from legacy project.json
-      # This should be removed once legacy JSON format is deprecated.
-      # Ndjson export file will fix the order during project export.
-      def fix_ci_pipelines_not_sorted_on_legacy_project_json!
-        return unless relation_reader.legacy?
-
-        relation_reader.sort_ci_pipelines_by_id
-      end
-
-      # Enable logging of each top-level relation creation when Importing
-      # into a Group if feature flag is enabled
-      def log_relation_creation(importable, relation_key, relation_object)
-        root_ancestor_group = importable.try(:root_ancestor)
-
-        return unless root_ancestor_group
-        return unless root_ancestor_group.instance_of?(::Group)
-        return unless Feature.enabled?(:log_import_export_relation_creation, root_ancestor_group)
-
-        @shared.logger.info(
-          importable_type: importable.class.to_s,
-          importable_id: importable.id,
-          relation_key: relation_key,
-          relation_id: relation_object.id,
-          author_id: relation_object.try(:author_id),
-          message: '[Project/Group Import] Created new object relation'
-        )
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb
index ba25e54ac9f..14474693ddf 100644
--- a/lib/gitlab/instrumentation/redis_interceptor.rb
+++ b/lib/gitlab/instrumentation/redis_interceptor.rb
@@ -5,10 +5,6 @@ module Gitlab
     module RedisInterceptor
       APDEX_EXCLUDE = %w[brpop blpop brpoplpush bzpopmin bzpopmax xread xreadgroup].freeze
 
-      # These are temporary to help with investigating
-      # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1183
-      DURATION_ERROR_THRESHOLD = 1.25.seconds
-
       class MysteryRedisDurationError < StandardError
         attr_reader :backtrace
 
@@ -19,7 +15,6 @@ module Gitlab
 
       def call(*args, &block)
         start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined
-        start_real_time = Time.now
         instrumentation_class.instance_count_request
         instrumentation_class.redis_cluster_validate!(args.first)
 
@@ -40,16 +35,6 @@ module Gitlab
           instrumentation_class.add_duration(duration)
           instrumentation_class.add_call_details(duration, args)
         end
-
-        if duration > DURATION_ERROR_THRESHOLD &&
-           instrumentation_class == ::Gitlab::Instrumentation::Redis::SharedState &&
-           Feature.enabled?(:report_on_long_redis_durations, default_enabled: :yaml)
-
-          Gitlab::ErrorTracking.track_exception(MysteryRedisDurationError.new(caller),
-                                                command: command_from_args(args),
-                                                duration: duration,
-                                                timestamp: start_real_time.iso8601(5))
-        end
       end
 
       def write(command)
diff --git a/lib/gitlab/instrumentation/uploads.rb b/lib/gitlab/instrumentation/uploads.rb
new file mode 100644
index 00000000000..02e457453cd
--- /dev/null
+++ b/lib/gitlab/instrumentation/uploads.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Instrumentation
+    class Uploads
+      UPLOAD_DURATION = :uploaded_file_upload_duration_s
+      UPLOADED_FILE_SIZE = :uploaded_file_size_bytes
+
+      def self.track(uploaded_file)
+        if ::Gitlab::SafeRequestStore.active?
+          ::Gitlab::SafeRequestStore[UPLOAD_DURATION] = uploaded_file.upload_duration
+          ::Gitlab::SafeRequestStore[UPLOADED_FILE_SIZE] = uploaded_file.size
+        end
+      end
+
+      def self.get_upload_duration
+        ::Gitlab::SafeRequestStore[UPLOAD_DURATION]
+      end
+
+      def self.get_uploaded_file_size
+        ::Gitlab::SafeRequestStore[UPLOADED_FILE_SIZE]
+      end
+
+      def self.payload
+        {
+          UPLOAD_DURATION => get_upload_duration,
+          UPLOADED_FILE_SIZE => get_uploaded_file_size
+        }.compact
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb
index 26e44d7822e..155e365d04c 100644
--- a/lib/gitlab/instrumentation_helper.rb
+++ b/lib/gitlab/instrumentation_helper.rb
@@ -31,6 +31,7 @@ module Gitlab
       instrument_thread_memory_allocations(payload)
       instrument_load_balancing(payload)
       instrument_pid(payload)
+      instrument_uploads(payload)
     end
 
     def instrument_gitaly(payload)
@@ -116,6 +117,10 @@ module Gitlab
       payload.merge!(load_balancing_payload)
     end
 
+    def instrument_uploads(payload)
+      payload.merge! ::Gitlab::Instrumentation::Uploads.payload
+    end
+
     # Returns the queuing duration for a Sidekiq job in seconds, as a float, if the
     # `enqueued_at` field or `created_at` field is available.
     #
diff --git a/lib/gitlab/integrations/sti_type.rb b/lib/gitlab/integrations/sti_type.rb
index 91797a7b99b..1350d75b216 100644
--- a/lib/gitlab/integrations/sti_type.rb
+++ b/lib/gitlab/integrations/sti_type.rb
@@ -7,7 +7,7 @@ module Gitlab
         Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog
         Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Irker Jenkins Jira Mattermost
         MattermostSlashCommands MicrosoftTeams MockCi MockMonitoring Packagist PipelinesEmail Pivotaltracker
-        Prometheus Pushover Redmine Slack SlackSlashCommands Teamcity UnifyCircuit WebexTeams Youtrack Zentao
+        Prometheus Pushover Redmine Shimo Slack SlackSlashCommands Teamcity UnifyCircuit WebexTeams Youtrack Zentao
       )).freeze
 
       def self.namespaced_integrations
diff --git a/lib/gitlab/issues/rebalancing/state.rb b/lib/gitlab/issues/rebalancing/state.rb
index dce165a3489..3d3fd9419b2 100644
--- a/lib/gitlab/issues/rebalancing/state.rb
+++ b/lib/gitlab/issues/rebalancing/state.rb
@@ -4,6 +4,10 @@ module Gitlab
   module Issues
     module Rebalancing
       class State
+        REDIS_KEY_PREFIX = "gitlab:issues-position-rebalances"
+        CONCURRENT_RUNNING_REBALANCES_KEY = "#{REDIS_KEY_PREFIX}:running_rebalances"
+        RECENTLY_FINISHED_REBALANCE_PREFIX = "#{REDIS_KEY_PREFIX}:recently_finished"
+
         REDIS_EXPIRY_TIME = 10.days
         MAX_NUMBER_OF_CONCURRENT_REBALANCES = 5
         NAMESPACE = 1
@@ -21,25 +25,23 @@ module Gitlab
             redis.multi do |multi|
               # we trigger re-balance for namespaces(groups) or specific user project
               value = "#{rebalanced_container_type}/#{rebalanced_container_id}"
-              multi.sadd(concurrent_running_rebalances_key, value)
-              multi.expire(concurrent_running_rebalances_key, REDIS_EXPIRY_TIME)
+              multi.sadd(CONCURRENT_RUNNING_REBALANCES_KEY, value)
+              multi.expire(CONCURRENT_RUNNING_REBALANCES_KEY, REDIS_EXPIRY_TIME)
             end
           end
         end
 
         def concurrent_running_rebalances_count
-          with_redis { |redis| redis.scard(concurrent_running_rebalances_key).to_i }
+          with_redis { |redis| redis.scard(CONCURRENT_RUNNING_REBALANCES_KEY).to_i }
         end
 
         def rebalance_in_progress?
-          all_rebalanced_containers = with_redis { |redis| redis.smembers(concurrent_running_rebalances_key) }
-
           is_running = case rebalanced_container_type
                        when NAMESPACE
-                         namespace_ids = all_rebalanced_containers.map {|string| string.split("#{NAMESPACE}/").second.to_i }.compact
+                         namespace_ids = self.class.current_rebalancing_containers.map {|string| string.split("#{NAMESPACE}/").second.to_i }.compact
                          namespace_ids.include?(root_namespace.id)
                        when PROJECT
-                         project_ids = all_rebalanced_containers.map {|string| string.split("#{PROJECT}/").second.to_i }.compact
+                         project_ids = self.class.current_rebalancing_containers.map {|string| string.split("#{PROJECT}/").second.to_i }.compact
                          project_ids.include?(projects.take.id) # rubocop:disable CodeReuse/ActiveRecord
                        else
                          false
@@ -101,36 +103,63 @@ module Gitlab
               multi.expire(issue_ids_key, REDIS_EXPIRY_TIME)
               multi.expire(current_index_key, REDIS_EXPIRY_TIME)
               multi.expire(current_project_key, REDIS_EXPIRY_TIME)
-              multi.expire(concurrent_running_rebalances_key, REDIS_EXPIRY_TIME)
+              multi.expire(CONCURRENT_RUNNING_REBALANCES_KEY, REDIS_EXPIRY_TIME)
             end
           end
         end
 
         def cleanup_cache
+          value = "#{rebalanced_container_type}/#{rebalanced_container_id}"
+
           with_redis do |redis|
             redis.multi do |multi|
               multi.del(issue_ids_key)
               multi.del(current_index_key)
               multi.del(current_project_key)
-              multi.srem(concurrent_running_rebalances_key, "#{rebalanced_container_type}/#{rebalanced_container_id}")
+              multi.srem(CONCURRENT_RUNNING_REBALANCES_KEY, value)
+              multi.set(self.class.recently_finished_key(rebalanced_container_type, rebalanced_container_id), true, ex: 1.hour)
             end
           end
         end
 
+        def self.rebalance_recently_finished?(project_id, namespace_id)
+          container_id = project_id || namespace_id
+          container_type = project_id.present? ? PROJECT : NAMESPACE
+
+          Gitlab::Redis::SharedState.with { |redis| redis.get(recently_finished_key(container_type, container_id)) }
+        end
+
+        def self.fetch_rebalancing_groups_and_projects
+          namespace_ids = []
+          project_ids = []
+
+          current_rebalancing_containers.each do |string|
+            container_type, container_id = string.split('/', 2).map(&:to_i)
+
+            if container_type == NAMESPACE
+              namespace_ids << container_id
+            elsif container_type == PROJECT
+              project_ids << container_id
+            end
+          end
+
+          [namespace_ids, project_ids]
+        end
+
         private
 
+        def self.current_rebalancing_containers
+          Gitlab::Redis::SharedState.with { |redis| redis.smembers(CONCURRENT_RUNNING_REBALANCES_KEY) }
+        end
+
         attr_accessor :root_namespace, :projects, :rebalanced_container_type, :rebalanced_container_id
 
         def too_many_rebalances_running?
           concurrent_running_rebalances_count <= MAX_NUMBER_OF_CONCURRENT_REBALANCES
         end
 
-        def redis_key_prefix
-          "gitlab:issues-position-rebalances"
-        end
-
         def issue_ids_key
-          "#{redis_key_prefix}:#{root_namespace.id}"
+          "#{REDIS_KEY_PREFIX}:#{root_namespace.id}"
         end
 
         def current_index_key
@@ -141,8 +170,8 @@ module Gitlab
           "#{issue_ids_key}:current_project_id"
         end
 
-        def concurrent_running_rebalances_key
-          "#{redis_key_prefix}:running_rebalances"
+        def self.recently_finished_key(container_type, container_id)
+          "#{RECENTLY_FINISHED_REBALANCE_PREFIX}:#{container_type}:#{container_id}"
         end
 
         def with_redis(&blk)
diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb
index 13d3bb2b8dc..7abfe8e38e8 100644
--- a/lib/gitlab/jira/http_client.rb
+++ b/lib/gitlab/jira/http_client.rb
@@ -32,7 +32,6 @@ module Gitlab
         request_params = { headers: headers }
         request_params[:body] = body if body.present?
         request_params[:headers][:Cookie] = get_cookies if options[:use_cookies]
-        request_params[:timeout] = options[:read_timeout] if options[:read_timeout]
         request_params[:base_uri] = uri.to_s
         request_params.merge!(auth_params)
 
diff --git a/lib/gitlab/language_detection.rb b/lib/gitlab/language_detection.rb
index fc9fb5caa09..6f7fa9fe03b 100644
--- a/lib/gitlab/language_detection.rb
+++ b/lib/gitlab/language_detection.rb
@@ -18,7 +18,7 @@ module Gitlab
     end
 
     # Newly detected languages, returned in a structure accepted by
-    # Gitlab::Database.main.bulk_insert
+    # ApplicationRecord.legacy_bulk_insert
     def insertions(programming_languages)
       lang_to_id = programming_languages.to_h { |p| [p.name, p.id] }
 
diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb
index 2e8564b6e00..03655eb7237 100644
--- a/lib/gitlab/lfs_token.rb
+++ b/lib/gitlab/lfs_token.rb
@@ -96,24 +96,15 @@ module Gitlab
       attr_reader :actor
 
       def secret
-        salt + key
-      end
-
-      def salt
         case actor
         when DeployKey, Key
-          actor.fingerprint.delete(':').first(16)
+          # Since fingerprint is based on the public key, let's take more bytes from attr_encrypted_db_key_base
+          actor.fingerprint.delete(':').first(16) + Settings.attr_encrypted_db_key_base_32
         when User
           # Take the last 16 characters as they're more unique than the first 16
-          actor.id.to_s + actor.encrypted_password.last(16)
+          actor.id.to_s + actor.encrypted_password.last(16) + Settings.attr_encrypted_db_key_base.first(16)
         end
       end
-
-      def key
-        # Take 16 characters of attr_encrypted_db_key_base, as that's what the
-        # cipher needs exactly
-        Settings.attr_encrypted_db_key_base.first(16)
-      end
     end
   end
 end
diff --git a/lib/gitlab/lograge/custom_options.rb b/lib/gitlab/lograge/custom_options.rb
index 83fd74310d0..e6c9ba0773c 100644
--- a/lib/gitlab/lograge/custom_options.rb
+++ b/lib/gitlab/lograge/custom_options.rb
@@ -7,6 +7,8 @@ module Gitlab
 
       LIMITED_ARRAY_SENTINEL = { key: 'truncated', value: '...' }.freeze
       IGNORE_PARAMS = Set.new(%w(controller action format)).freeze
+      KNOWN_PAYLOAD_PARAMS = [:remote_ip, :user_id, :username, :ua, :queue_duration_s,
+                              :etag_route, :request_urgency, :target_duration_s] + CLOUDFLARE_CUSTOM_HEADERS.values
 
       def self.call(event)
         params = event
@@ -14,24 +16,17 @@ module Gitlab
           .each_with_object([]) { |(k, v), array| array << { key: k, value: v } unless IGNORE_PARAMS.include?(k) }
         payload = {
           time: Time.now.utc.iso8601(3),
-          params: Gitlab::Utils::LogLimitedArray.log_limited_array(params, sentinel: LIMITED_ARRAY_SENTINEL),
-          remote_ip: event.payload[:remote_ip],
-          user_id: event.payload[:user_id],
-          username: event.payload[:username],
-          ua: event.payload[:ua]
+          params: Gitlab::Utils::LogLimitedArray.log_limited_array(params, sentinel: LIMITED_ARRAY_SENTINEL)
         }
+
         payload.merge!(event.payload[:metadata]) if event.payload[:metadata]
+        optional_payload_params = event.payload.slice(*KNOWN_PAYLOAD_PARAMS).compact
+        payload.merge!(optional_payload_params)
 
         ::Gitlab::InstrumentationHelper.add_instrumentation_data(payload)
 
-        payload[:queue_duration_s] = event.payload[:queue_duration_s] if event.payload[:queue_duration_s]
-        payload[:etag_route] = event.payload[:etag_route] if event.payload[:etag_route]
         payload[Labkit::Correlation::CorrelationId::LOG_KEY] = event.payload[Labkit::Correlation::CorrelationId::LOG_KEY] || Labkit::Correlation::CorrelationId.current_id
 
-        CLOUDFLARE_CUSTOM_HEADERS.each do |_, value|
-          payload[value] = event.payload[value] if event.payload[value]
-        end
-
         # https://github.com/roidrage/lograge#logging-errors--exceptions
         exception = event.payload[:exception_object]
 
diff --git a/lib/gitlab/merge_requests/merge_commit_message.rb b/lib/gitlab/merge_requests/merge_commit_message.rb
new file mode 100644
index 00000000000..2a6a7859b33
--- /dev/null
+++ b/lib/gitlab/merge_requests/merge_commit_message.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+module Gitlab
+  module MergeRequests
+    class MergeCommitMessage
+      def initialize(merge_request:)
+        @merge_request = merge_request
+      end
+
+      def message
+        return unless @merge_request.target_project.merge_commit_template.present?
+
+        message = @merge_request.target_project.merge_commit_template
+        message = message.delete("\r")
+
+        # Remove placeholders that correspond to empty values and are the last word in the line
+        # along with all whitespace characters preceding them.
+        # This allows us to recreate previous default merge commit message behaviour - we skipped new line character
+        # before empty description and before closed issues when none were present.
+        PLACEHOLDERS.each do |key, value|
+          unless value.call(merge_request).present?
+            message = message.gsub(BLANK_PLACEHOLDERS_REGEXES[key], '')
+          end
+        end
+
+        Gitlab::StringPlaceholderReplacer
+          .replace_string_placeholders(message, PLACEHOLDERS_REGEX) do |key|
+          PLACEHOLDERS[key].call(merge_request)
+        end
+      end
+
+      private
+
+      attr_reader :merge_request
+
+      PLACEHOLDERS = {
+        'source_branch' => ->(merge_request) { merge_request.source_branch.to_s },
+        'target_branch' => ->(merge_request) { merge_request.target_branch.to_s },
+        'title' => ->(merge_request) { merge_request.title },
+        'issues' => ->(merge_request) do
+          return "" if merge_request.visible_closing_issues_for.blank?
+
+          closes_issues_references = merge_request.visible_closing_issues_for.map do |issue|
+            issue.to_reference(merge_request.target_project)
+          end
+          "Closes #{closes_issues_references.to_sentence}"
+        end,
+        'description' => ->(merge_request) { merge_request.description.presence || '' },
+        'reference' => ->(merge_request) { merge_request.to_reference(full: true) }
+      }.freeze
+
+      PLACEHOLDERS_REGEX = Regexp.union(PLACEHOLDERS.keys.map do |key|
+        Regexp.new(Regexp.escape(key))
+      end).freeze
+
+      BLANK_PLACEHOLDERS_REGEXES = (PLACEHOLDERS.map do |key, value|
+        [key, Regexp.new("[\n\r]+%{#{Regexp.escape(key)}}$")]
+      end).to_h.freeze
+    end
+  end
+end
diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 4c4942c12d5..6d7ecb53ec3 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -29,7 +29,7 @@ module Gitlab
 
     # Allow access from other metrics related middlewares
     def self.current_transaction
-      Transaction.current
+      WebTransaction.current || BackgroundTransaction.current
     end
 
     # Returns the prefix to use for the name of a series.
diff --git a/lib/gitlab/metrics/background_transaction.rb b/lib/gitlab/metrics/background_transaction.rb
index a1fabe75a97..54095461dd4 100644
--- a/lib/gitlab/metrics/background_transaction.rb
+++ b/lib/gitlab/metrics/background_transaction.rb
@@ -2,14 +2,17 @@
 
 module Gitlab
   module Metrics
+    # Exclusive transaction-type metrics for background jobs (Sidekiq). One
+    # instance of this class is created for each job going through the Sidekiq
+    # metric middleware. Any metrics dispatched with this instance include
+    # metadata such as endpoint_id, queue, and feature category.
     class BackgroundTransaction < Transaction
-      # Separate web transaction instance and background transaction instance
-      BACKGROUND_THREAD_KEY = :_gitlab_metrics_background_transaction
-      BACKGROUND_BASE_LABEL_KEYS = %i(endpoint_id feature_category).freeze
+      THREAD_KEY = :_gitlab_metrics_background_transaction
+      BASE_LABEL_KEYS = %i(queue endpoint_id feature_category).freeze
 
       class << self
         def current
-          Thread.current[BACKGROUND_THREAD_KEY]
+          Thread.current[THREAD_KEY]
         end
 
         def prometheus_metric(name, type, &block)
@@ -19,17 +22,17 @@ module Gitlab
 
             evaluate(&block)
             # always filter sensitive labels and merge with base ones
-            label_keys BACKGROUND_BASE_LABEL_KEYS | (label_keys - ::Gitlab::Metrics::Transaction::FILTERED_LABEL_KEYS)
+            label_keys BASE_LABEL_KEYS | (label_keys - ::Gitlab::Metrics::Transaction::FILTERED_LABEL_KEYS)
           end
         end
       end
 
       def run
-        Thread.current[BACKGROUND_THREAD_KEY] = self
+        Thread.current[THREAD_KEY] = self
 
         yield
       ensure
-        Thread.current[BACKGROUND_THREAD_KEY] = nil
+        Thread.current[THREAD_KEY] = nil
       end
 
       def labels
diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb
index 8ddd76ad7ae..dc9a7ed1312 100644
--- a/lib/gitlab/metrics/methods.rb
+++ b/lib/gitlab/metrics/methods.rb
@@ -13,8 +13,12 @@ module Gitlab
       end
 
       class_methods do
-        def reload_metric!(name)
-          @@_metrics_provider_cache.delete(name)
+        def reload_metric!(name = nil)
+          if name.nil?
+            @@_metrics_provider_cache = {}
+          else
+            @@_metrics_provider_cache.delete(name)
+          end
         end
 
         private
diff --git a/lib/gitlab/metrics/rails_slis.rb b/lib/gitlab/metrics/rails_slis.rb
index 69e0c1e9fde..8c40c0ad441 100644
--- a/lib/gitlab/metrics/rails_slis.rb
+++ b/lib/gitlab/metrics/rails_slis.rb
@@ -4,23 +4,32 @@ module Gitlab
   module Metrics
     module RailsSlis
       class << self
-        def request_apdex_counters_enabled?
-          Feature.enabled?(:request_apdex_counters)
-        end
-
         def initialize_request_slis_if_needed!
-          return unless request_apdex_counters_enabled?
-          return if Gitlab::Metrics::Sli.initialized?(:rails_request_apdex)
-
-          Gitlab::Metrics::Sli.initialize_sli(:rails_request_apdex, possible_request_labels)
+          Gitlab::Metrics::Sli.initialize_sli(:rails_request_apdex, possible_request_labels) unless Gitlab::Metrics::Sli.initialized?(:rails_request_apdex)
+          Gitlab::Metrics::Sli.initialize_sli(:graphql_query_apdex, possible_graphql_query_labels) unless Gitlab::Metrics::Sli.initialized?(:graphql_query_apdex)
         end
 
         def request_apdex
           Gitlab::Metrics::Sli[:rails_request_apdex]
         end
 
+        def graphql_query_apdex
+          Gitlab::Metrics::Sli[:graphql_query_apdex]
+        end
+
         private
 
+        def possible_graphql_query_labels
+          ::Gitlab::Graphql::KnownOperations.default.operations.map do |op|
+            {
+              endpoint_id: op.to_caller_id,
+              # We'll be able to correlate feature_category with https://gitlab.com/gitlab-org/gitlab/-/issues/328535
+              feature_category: nil,
+              query_urgency: op.query_urgency.name
+            }
+          end
+        end
+
         def possible_request_labels
           possible_controller_labels + possible_api_labels
         end
@@ -30,10 +39,12 @@ module Gitlab
             endpoint_id = API::Base.endpoint_id_for_route(route)
             route_class = route.app.options[:for]
             feature_category = route_class.feature_category_for_app(route.app)
+            request_urgency = route_class.urgency_for_app(route.app)
 
             {
               endpoint_id: endpoint_id,
-              feature_category: feature_category
+              feature_category: feature_category,
+              request_urgency: request_urgency.name
             }
           end
         end
@@ -42,7 +53,8 @@ module Gitlab
           Gitlab::RequestEndpoints.all_controller_actions.map do |controller, action|
             {
               endpoint_id: controller.endpoint_id_for_action(action),
-              feature_category: controller.feature_category_for_action(action)
+              feature_category: controller.feature_category_for_action(action),
+              request_urgency: controller.urgency_for_action(action).name
             }
           end
         end
diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb
index 3a0e34d5615..c143a7f5a1b 100644
--- a/lib/gitlab/metrics/requests_rack_middleware.rb
+++ b/lib/gitlab/metrics/requests_rack_middleware.rb
@@ -79,7 +79,7 @@ module Gitlab
           if !health_endpoint && ::Gitlab::Metrics.record_duration_for_status?(status)
             self.class.http_request_duration_seconds.observe({ method: method }, elapsed)
 
-            record_apdex_if_needed(env, elapsed)
+            record_apdex(env, elapsed)
           end
 
           [status, headers, body]
@@ -113,12 +113,12 @@ module Gitlab
         ::Gitlab::ApplicationContext.current_context_attribute(:caller_id)
       end
 
-      def record_apdex_if_needed(env, elapsed)
-        return unless Gitlab::Metrics::RailsSlis.request_apdex_counters_enabled?
+      def record_apdex(env, elapsed)
+        urgency = urgency_for_env(env)
 
         Gitlab::Metrics::RailsSlis.request_apdex.increment(
-          labels: labels_from_context,
-          success: satisfactory?(env, elapsed)
+          labels: labels_from_context.merge(request_urgency: urgency.name),
+          success: elapsed < urgency.duration
         )
       end
 
@@ -129,17 +129,15 @@ module Gitlab
         }
       end
 
-      def satisfactory?(env, elapsed)
-        target =
+      def urgency_for_env(env)
+        endpoint_urgency =
           if env['api.endpoint'].present?
             env['api.endpoint'].options[:for].try(:urgency_for_app, env['api.endpoint'])
           elsif env['action_controller.instance'].present? && env['action_controller.instance'].respond_to?(:urgency)
             env['action_controller.instance'].urgency
           end
 
-        target ||= Gitlab::EndpointAttributes::DEFAULT_URGENCY
-
-        elapsed < target.duration
+        endpoint_urgency || Gitlab::EndpointAttributes::DEFAULT_URGENCY
       end
     end
   end
diff --git a/lib/gitlab/metrics/samplers/action_cable_sampler.rb b/lib/gitlab/metrics/samplers/action_cable_sampler.rb
index 043d2ae84cc..adce3030d0d 100644
--- a/lib/gitlab/metrics/samplers/action_cable_sampler.rb
+++ b/lib/gitlab/metrics/samplers/action_cable_sampler.rb
@@ -39,23 +39,14 @@ module Gitlab
 
         def sample
           pool = @action_cable.worker_pool.executor
-          labels = {
-            server_mode: server_mode
-          }
-
-          metrics[:active_connections].set(labels, @action_cable.connections.size)
-          metrics[:pool_min_size].set(labels, pool.min_length)
-          metrics[:pool_max_size].set(labels, pool.max_length)
-          metrics[:pool_current_size].set(labels, pool.length)
-          metrics[:pool_largest_size].set(labels, pool.largest_length)
-          metrics[:pool_completed_tasks].set(labels, pool.completed_task_count)
-          metrics[:pool_pending_tasks].set(labels, pool.queue_length)
-        end
-
-        private
 
-        def server_mode
-          Gitlab::ActionCable::Config.in_app? ? 'in-app' : 'standalone'
+          metrics[:active_connections].set({}, @action_cable.connections.size)
+          metrics[:pool_min_size].set({}, pool.min_length)
+          metrics[:pool_max_size].set({}, pool.max_length)
+          metrics[:pool_current_size].set({}, pool.length)
+          metrics[:pool_largest_size].set({}, pool.largest_length)
+          metrics[:pool_completed_tasks].set({}, pool.completed_task_count)
+          metrics[:pool_pending_tasks].set({}, pool.queue_length)
         end
       end
     end
diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb
index fa129025bfe..bc9032a6942 100644
--- a/lib/gitlab/metrics/subscribers/action_view.rb
+++ b/lib/gitlab/metrics/subscribers/action_view.rb
@@ -40,7 +40,7 @@ module Gitlab
         end
 
         def current_transaction
-          ::Gitlab::Metrics::Transaction.current
+          ::Gitlab::Metrics::WebTransaction.current
         end
       end
     end
diff --git a/lib/gitlab/metrics/subscribers/external_http.rb b/lib/gitlab/metrics/subscribers/external_http.rb
index 60a1b084345..ff8654a2cec 100644
--- a/lib/gitlab/metrics/subscribers/external_http.rb
+++ b/lib/gitlab/metrics/subscribers/external_http.rb
@@ -43,7 +43,7 @@ module Gitlab
         private
 
         def current_transaction
-          ::Gitlab::Metrics::Transaction.current
+          ::Gitlab::Metrics::WebTransaction.current || ::Gitlab::Metrics::BackgroundTransaction.current
         end
 
         def add_to_detail_store(start, payload)
diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb
index 45344e79796..b5e087d107b 100644
--- a/lib/gitlab/metrics/subscribers/rails_cache.rb
+++ b/lib/gitlab/metrics/subscribers/rails_cache.rb
@@ -65,7 +65,7 @@ module Gitlab
         private
 
         def current_transaction
-          ::Gitlab::Metrics::Transaction.current
+          ::Gitlab::Metrics::WebTransaction.current
         end
 
         def metric_cache_operation_duration_seconds
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index 97cc8bed564..56a310548a7 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -6,35 +6,14 @@ module Gitlab
     class Transaction
       include Gitlab::Metrics::Methods
 
-      # base label keys shared among all transactions
-      BASE_LABEL_KEYS = %i(controller action feature_category).freeze
       # labels that potentially contain sensitive information and will be filtered
       FILTERED_LABEL_KEYS = %i(branch path).freeze
 
-      THREAD_KEY = :_gitlab_metrics_transaction
-
       # The series to store events (e.g. Git pushes) in.
       EVENT_SERIES = 'events'
 
       attr_reader :method
 
-      class << self
-        def current
-          Thread.current[THREAD_KEY]
-        end
-
-        def prometheus_metric(name, type, &block)
-          fetch_metric(type, name) do
-            # set default metric options
-            docstring "#{name.to_s.humanize} #{type}"
-
-            evaluate(&block)
-            # always filter sensitive labels and merge with base ones
-            label_keys BASE_LABEL_KEYS | (label_keys - FILTERED_LABEL_KEYS)
-          end
-        end
-      end
-
       def initialize
         @methods = {}
       end
@@ -126,10 +105,6 @@ module Gitlab
         histogram.observe(filter_labels(labels), value)
       end
 
-      def labels
-        BASE_LABEL_KEYS.product([nil]).to_h
-      end
-
       def filter_labels(labels)
         labels.empty? ? self.labels : labels.without(*FILTERED_LABEL_KEYS).merge(self.labels)
       end
diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb
index 544c142f7bb..fcfa86734e8 100644
--- a/lib/gitlab/metrics/web_transaction.rb
+++ b/lib/gitlab/metrics/web_transaction.rb
@@ -2,12 +2,37 @@
 
 module Gitlab
   module Metrics
+    # Exclusive transaction-type metrics for web servers (including Web/Api/Git
+    # fleet). One instance of this class is created for each request going
+    # through the Rack metric middleware. Any metrics dispatched with this
+    # instance include metadata such as controller, action, feature category,
+    # etc.
     class WebTransaction < Transaction
+      THREAD_KEY = :_gitlab_metrics_transaction
+      BASE_LABEL_KEYS = %i(controller action feature_category).freeze
+
       CONTROLLER_KEY = 'action_controller.instance'
       ENDPOINT_KEY = 'api.endpoint'
       ALLOWED_SUFFIXES = Set.new(%w[json js atom rss xml zip])
       SMALL_BUCKETS = [0.1, 0.25, 0.5, 1.0, 2.5, 5.0].freeze
 
+      class << self
+        def current
+          Thread.current[THREAD_KEY]
+        end
+
+        def prometheus_metric(name, type, &block)
+          fetch_metric(type, name) do
+            # set default metric options
+            docstring "#{name.to_s.humanize} #{type}"
+
+            evaluate(&block)
+            # always filter sensitive labels and merge with base ones
+            label_keys BASE_LABEL_KEYS | (label_keys - ::Gitlab::Metrics::Transaction::FILTERED_LABEL_KEYS)
+          end
+        end
+      end
+
       def initialize(env)
         super()
         @env = env
diff --git a/lib/gitlab/middleware/compressed_json.rb b/lib/gitlab/middleware/compressed_json.rb
new file mode 100644
index 00000000000..ef6e0db5673
--- /dev/null
+++ b/lib/gitlab/middleware/compressed_json.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Middleware
+    class CompressedJson
+      COLLECTOR_PATH = '/api/v4/error_tracking/collector'
+      MAXIMUM_BODY_SIZE = 200.kilobytes.to_i
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        if compressed_et_request?(env)
+          input = extract(env['rack.input'])
+
+          if input.length > MAXIMUM_BODY_SIZE
+            return too_large
+          end
+
+          env.delete('HTTP_CONTENT_ENCODING')
+          env['CONTENT_LENGTH'] = input.length
+          env['rack.input'] = StringIO.new(input)
+        end
+
+        @app.call(env)
+      end
+
+      def compressed_et_request?(env)
+        post_request?(env) &&
+          gzip_encoding?(env) &&
+          match_content_type?(env) &&
+          match_path?(env)
+      end
+
+      def too_large
+        [413, { 'Content-Type' => 'text/plain' }, ['Payload Too Large']]
+      end
+
+      def relative_url
+        File.join('', Gitlab.config.gitlab.relative_url_root).chomp('/')
+      end
+
+      def extract(input)
+        Zlib::GzipReader.new(input).read(MAXIMUM_BODY_SIZE + 1)
+      end
+
+      def post_request?(env)
+        env['REQUEST_METHOD'] == 'POST'
+      end
+
+      def gzip_encoding?(env)
+        env['HTTP_CONTENT_ENCODING'] == 'gzip'
+      end
+
+      def match_content_type?(env)
+        env['CONTENT_TYPE'] == 'application/json' ||
+          env['CONTENT_TYPE'] == 'application/x-sentry-envelope'
+      end
+
+      def match_path?(env)
+        env['PATH_INFO'].start_with?((File.join(relative_url, COLLECTOR_PATH)))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb
index a1a0356ff58..bfa4e4cf5f8 100644
--- a/lib/gitlab/middleware/go.rb
+++ b/lib/gitlab/middleware/go.rb
@@ -27,6 +27,8 @@ module Gitlab
           path: request.fullpath
         )
         Rack::Response.new('', 403).finish
+      rescue Gitlab::Auth::MissingPersonalAccessTokenError
+        Rack::Response.new('', 401).finish
       end
 
       private
diff --git a/lib/gitlab/middleware/query_analyzer.rb b/lib/gitlab/middleware/query_analyzer.rb
new file mode 100644
index 00000000000..8d63c644a69
--- /dev/null
+++ b/lib/gitlab/middleware/query_analyzer.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Middleware
+    class QueryAnalyzer
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        ::Gitlab::Database::QueryAnalyzer.instance.within { @app.call(env) }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/middleware/release_env.rb b/lib/gitlab/middleware/release_env.rb
index 0719fb2e8c6..2439e873e0b 100644
--- a/lib/gitlab/middleware/release_env.rb
+++ b/lib/gitlab/middleware/release_env.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Naming/FileName
 # frozen_string_literal: true
 
 module Gitlab
@@ -14,3 +15,5 @@ module Gitlab
     end
   end
 end
+
+# rubocop:enable Naming/FileName
diff --git a/lib/gitlab/pagination/gitaly_keyset_pager.rb b/lib/gitlab/pagination/gitaly_keyset_pager.rb
index a16bf7a379c..99a3145104a 100644
--- a/lib/gitlab/pagination/gitaly_keyset_pager.rb
+++ b/lib/gitlab/pagination/gitaly_keyset_pager.rb
@@ -30,6 +30,8 @@ module Gitlab
 
         if finder.is_a?(BranchesFinder)
           Feature.enabled?(:branch_list_keyset_pagination, project, default_enabled: :yaml)
+        elsif finder.is_a?(TagsFinder)
+          Feature.enabled?(:tag_list_keyset_pagination, project, default_enabled: :yaml)
         elsif finder.is_a?(::Repositories::TreeFinder)
           Feature.enabled?(:repository_tree_gitaly_pagination, project, default_enabled: :yaml)
         else
@@ -42,6 +44,8 @@ module Gitlab
 
         if finder.is_a?(BranchesFinder)
           Feature.enabled?(:branch_list_keyset_pagination, project, default_enabled: :yaml)
+        elsif finder.is_a?(TagsFinder)
+          Feature.enabled?(:tag_list_keyset_pagination, project, default_enabled: :yaml)
         elsif finder.is_a?(::Repositories::TreeFinder)
           Feature.enabled?(:repository_tree_gitaly_pagination, project, default_enabled: :yaml)
         else
diff --git a/lib/gitlab/patch/sidekiq_client.rb b/lib/gitlab/patch/sidekiq_client.rb
new file mode 100644
index 00000000000..2de13560cce
--- /dev/null
+++ b/lib/gitlab/patch/sidekiq_client.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Patch
+    module SidekiqClient
+      private
+
+      # This is a copy of https://github.com/mperham/sidekiq/blob/v6.2.2/lib/sidekiq/client.rb#L187-L194
+      # but using `conn.pipelined` instead of `conn.multi`. The multi call isn't needed here because in
+      # the case of scheduled jobs, only one Redis call is made. For other jobs, we don't really need
+      # the commands to be atomic.
+      def raw_push(payloads)
+        @redis_pool.with do |conn| # rubocop:disable Gitlab/ModuleWithInstanceVariables
+          conn.pipelined do
+            atomic_push(conn, payloads)
+          end
+        end
+        true
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/patch/sidekiq_cron_poller.rb b/lib/gitlab/patch/sidekiq_cron_poller.rb
new file mode 100644
index 00000000000..56ca24c68f5
--- /dev/null
+++ b/lib/gitlab/patch/sidekiq_cron_poller.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Patch
+    module SidekiqCronPoller
+      def enqueue
+        Rails.application.reloader.wrap do
+          ::Gitlab::WithRequestStore.with_request_store do
+            super
+          ensure
+            ::Gitlab::Database::LoadBalancing.release_hosts
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb
index 8875e6320c7..d53b11fe98c 100644
--- a/lib/gitlab/project_template.rb
+++ b/lib/gitlab/project_template.rb
@@ -66,6 +66,7 @@ module Gitlab
           ProjectTemplate.new('nfhexo', 'Netlify/Hexo', _('A Hexo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features'), 'https://gitlab.com/pages/nfhexo', 'illustrations/logos/netlify.svg'),
           ProjectTemplate.new('salesforcedx', 'SalesforceDX', _('A project boilerplate for Salesforce App development with Salesforce Developer tools'), 'https://gitlab.com/gitlab-org/project-templates/salesforcedx'),
           ProjectTemplate.new('serverless_framework', 'Serverless Framework/JS', _('A basic page and serverless function that uses AWS Lambda, AWS API Gateway, and GitLab Pages'), 'https://gitlab.com/gitlab-org/project-templates/serverless-framework', 'illustrations/logos/serverless_framework.svg'),
+          ProjectTemplate.new('tencent_serverless_framework', 'Tencent Serverless Framework/NextjsSSR', _('A project boilerplate for Tencent Serverless Framework that uses Next.js SSR'), 'https://gitlab.com/gitlab-org/project-templates/nextjsssr_demo', 'illustrations/logos/tencent_serverless_framework.svg'),
           ProjectTemplate.new('jsonnet', 'Jsonnet for Dynamic Child Pipelines', _('An example showing how to use Jsonnet with GitLab dynamic child pipelines'), 'https://gitlab.com/gitlab-org/project-templates/jsonnet'),
           ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management'),
           ProjectTemplate.new('kotlin_native_linux', 'Kotlin Native Linux', _('A basic template for developing Linux programs using Kotlin Native'), 'https://gitlab.com/gitlab-org/project-templates/kotlin-native-linux')
diff --git a/lib/gitlab/prometheus/queries/validate_query.rb b/lib/gitlab/prometheus/queries/validate_query.rb
index 1f55f3e9768..160db7d44bc 100644
--- a/lib/gitlab/prometheus/queries/validate_query.rb
+++ b/lib/gitlab/prometheus/queries/validate_query.rb
@@ -7,7 +7,7 @@ module Gitlab
         def query(query)
           client_query(query)
           { valid: true }
-        rescue Gitlab::PrometheusClient::QueryError, Gitlab::HTTP::BlockedUrlError => ex
+        rescue Gitlab::PrometheusClient::QueryError, Gitlab::PrometheusClient::ConnectionError => ex
           { valid: false, error: ex.message }
         end
 
diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb
index 8182dbad4f8..dda28ffdf90 100644
--- a/lib/gitlab/prometheus_client.rb
+++ b/lib/gitlab/prometheus_client.rb
@@ -151,12 +151,8 @@ module Gitlab
 
     def get(path, args)
       Gitlab::HTTP.get(path, { query: args }.merge(http_options) )
-    rescue SocketError
-      raise PrometheusClient::ConnectionError, "Can't connect to #{api_url}"
-    rescue OpenSSL::SSL::SSLError
-      raise PrometheusClient::ConnectionError, "#{api_url} contains invalid SSL data"
-    rescue Errno::ECONNREFUSED
-      raise PrometheusClient::ConnectionError, 'Connection refused'
+    rescue *Gitlab::HTTP::HTTP_ERRORS => e
+      raise PrometheusClient::ConnectionError, e.message
     end
 
     def handle_management_api_response(response)
diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb
index cf5c9296d8c..4bac0643a91 100644
--- a/lib/gitlab/quick_actions/issuable_actions.rb
+++ b/lib/gitlab/quick_actions/issuable_actions.rb
@@ -84,8 +84,7 @@ module Gitlab
         params '~label1 ~"label 2"'
         types Issuable
         condition do
-          parent &&
-            current_user.can?(:"admin_#{quick_action_target.to_ability_name}", parent) &&
+          current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target) &&
             find_labels.any?
         end
         command :label do |labels_param|
@@ -107,7 +106,7 @@ module Gitlab
         condition do
           quick_action_target.persisted? &&
             quick_action_target.labels.any? &&
-            current_user.can?(:"admin_#{quick_action_target.to_ability_name}", parent)
+            current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target)
         end
         command :unlabel, :remove_label do |labels_param = nil|
           if labels_param.present?
@@ -139,7 +138,7 @@ module Gitlab
         condition do
           quick_action_target.persisted? &&
             quick_action_target.labels.any? &&
-            current_user.can?(:"admin_#{quick_action_target.to_ability_name}", parent)
+            current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target)
         end
         command :relabel do |labels_param|
           run_label_command(labels: find_labels(labels_param), command: :relabel, updates_key: :label_ids)
diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb
index c5cf3262039..a55ead519e2 100644
--- a/lib/gitlab/quick_actions/issue_actions.rb
+++ b/lib/gitlab/quick_actions/issue_actions.rb
@@ -19,7 +19,7 @@ module Gitlab
         types Issue
         condition do
           quick_action_target.respond_to?(:due_date) &&
-            current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project)
+            current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target)
         end
         parse_params do |due_date_param|
           Chronic.parse(due_date_param).try(:to_date)
@@ -40,7 +40,7 @@ module Gitlab
           quick_action_target.persisted? &&
             quick_action_target.respond_to?(:due_date) &&
             quick_action_target.due_date? &&
-            current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project)
+            current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target)
         end
         command :remove_due_date do
           @updates[:due_date] = nil
@@ -54,7 +54,7 @@ module Gitlab
         params '~"Target column"'
         types Issue
         condition do
-          current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target) &&
+          current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target) &&
             quick_action_target.project.boards.count == 1
         end
         command :board_move do |target_list_name|
@@ -86,7 +86,7 @@ module Gitlab
         types Issue
         condition do
           quick_action_target.persisted? &&
-            current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target)
+            current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target)
         end
         command :duplicate do |duplicate_param|
           canonical_issue = extract_references(duplicate_param, :issue).first
@@ -172,7 +172,7 @@ module Gitlab
         condition do
           quick_action_target.issue_type_supports?(:confidentiality) &&
             !quick_action_target.confidential? &&
-            current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target)
+            current_user.can?(:set_confidentiality, quick_action_target)
         end
         command :confidential do
           @updates[:confidential] = true
@@ -264,6 +264,27 @@ module Gitlab
           end
         end
 
+        desc _('Promote issue to incident')
+        explanation _('Promotes issue to incident')
+        types Issue
+        condition do
+          quick_action_target.persisted? &&
+            !quick_action_target.incident? &&
+            current_user.can?(:update_issue, quick_action_target)
+        end
+        command :promote_to_incident do
+          issue = ::Issues::UpdateService
+            .new(project: quick_action_target.project, current_user: current_user, params: { issue_type: 'incident' })
+            .execute(quick_action_target)
+
+          @execution_message[:promote_to_incident] =
+            if issue.incident?
+              _('Issue has been promoted to incident')
+            else
+              _('Failed to promote issue to incident')
+            end
+        end
+
         private
 
         def zoom_link_service
diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb
index b53fdd60606..4a75fa0a571 100644
--- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb
+++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb
@@ -26,7 +26,7 @@ module Gitlab
         end
         types Issue, MergeRequest
         condition do
-          quick_action_target.supports_assignee? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project)
+          quick_action_target.supports_assignee? && current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target)
         end
         parse_params do |assignee_param|
           extract_users(assignee_param)
@@ -66,7 +66,7 @@ module Gitlab
         condition do
           quick_action_target.persisted? &&
             quick_action_target.assignees.any? &&
-            current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project)
+            current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target)
         end
         parse_params do |unassign_param|
           # When multiple users are assigned, all will be unassigned if multiple assignees are no longer allowed
@@ -92,7 +92,7 @@ module Gitlab
         types Issue, MergeRequest
         condition do
           quick_action_target.supports_milestone? &&
-          current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) &&
+          current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target) &&
             find_milestones(project, state: 'active').any?
         end
         parse_params do |milestone_param|
@@ -115,7 +115,7 @@ module Gitlab
           quick_action_target.persisted? &&
             quick_action_target.milestone_id? &&
             quick_action_target.supports_milestone? &&
-            current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project)
+            current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target)
         end
         command :remove_milestone do
           @updates[:milestone_id] = nil
@@ -128,7 +128,7 @@ module Gitlab
         params '#issue | !merge_request'
         types Issue, MergeRequest
         condition do
-          current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target)
+          current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target)
         end
         parse_params do |issuable_param|
           extract_references(issuable_param, :issue).first ||
@@ -225,7 +225,7 @@ module Gitlab
         condition do
           quick_action_target.persisted? &&
             !quick_action_target.discussion_locked? &&
-            current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target)
+            current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target)
         end
         command :lock do
           @updates[:discussion_locked] = true
@@ -238,7 +238,7 @@ module Gitlab
         condition do
           quick_action_target.persisted? &&
             quick_action_target.discussion_locked? &&
-            current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target)
+            current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target)
         end
         command :unlock do
           @updates[:discussion_locked] = false
diff --git a/lib/gitlab/redis/hll.rb b/lib/gitlab/redis/hll.rb
index 0d04545688b..4d1855e4637 100644
--- a/lib/gitlab/redis/hll.rb
+++ b/lib/gitlab/redis/hll.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Naming/FileName
 # frozen_string_literal: true
 
 module Gitlab
@@ -51,3 +52,5 @@ module Gitlab
     end
   end
 end
+
+# rubocop:enable Naming/FileName
diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb
new file mode 100644
index 00000000000..f930a0040bc
--- /dev/null
+++ b/lib/gitlab/redis/multi_store.rb
@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Redis
+    class MultiStore
+      include Gitlab::Utils::StrongMemoize
+
+      class ReadFromPrimaryError < StandardError
+        def message
+          'Value not found on the redis primary store. Read from the redis secondary store successful.'
+        end
+      end
+      class MethodMissingError < StandardError
+        def message
+          'Method missing. Falling back to execute method on the redis secondary store.'
+        end
+      end
+
+      attr_reader :primary_store, :secondary_store, :instance_name
+
+      FAILED_TO_READ_ERROR_MESSAGE = 'Failed to read from the redis primary_store.'
+      FAILED_TO_WRITE_ERROR_MESSAGE = 'Failed to write to the redis primary_store.'
+
+      READ_COMMANDS = %i(
+        get
+        mget
+        smembers
+        scard
+      ).freeze
+
+      WRITE_COMMANDS = %i(
+        set
+        setnx
+        setex
+        sadd
+        srem
+        del
+        pipelined
+        flushdb
+      ).freeze
+
+      def initialize(primary_store, secondary_store, instance_name = nil)
+        @primary_store = primary_store
+        @secondary_store = secondary_store
+        @instance_name = instance_name
+
+        validate_stores!
+      end
+
+      READ_COMMANDS.each do |name|
+        define_method(name) do |*args, &block|
+          if multi_store_enabled?
+            read_command(name, *args, &block)
+          else
+            secondary_store.send(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend
+          end
+        end
+      end
+
+      WRITE_COMMANDS.each do |name|
+        define_method(name) do |*args, &block|
+          if multi_store_enabled?
+            write_command(name, *args, &block)
+          else
+            secondary_store.send(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend
+          end
+        end
+      end
+
+      def method_missing(...)
+        return @instance.send(...) if @instance # rubocop:disable GitlabSecurity/PublicSend
+
+        log_method_missing(...)
+
+        secondary_store.send(...) # rubocop:disable GitlabSecurity/PublicSend
+      end
+
+      def respond_to_missing?(command_name, include_private = false)
+        true
+      end
+
+      # This is needed because of Redis::Rack::Connection is requiring Redis::Store
+      # https://github.com/redis-store/redis-rack/blob/a833086ba494083b6a384a1a4e58b36573a9165d/lib/redis/rack/connection.rb#L15
+      # Done similarly in https://github.com/lsegal/yard/blob/main/lib/yard/templates/template.rb#L122
+      def is_a?(klass)
+        return true if klass == secondary_store.class
+
+        super(klass)
+      end
+      alias_method :kind_of?, :is_a?
+
+      def to_s
+        if multi_store_enabled?
+          primary_store.to_s
+        else
+          secondary_store.to_s
+        end
+      end
+
+      private
+
+      def log_method_missing(command_name, *_args)
+        log_error(MethodMissingError.new, command_name)
+        increment_method_missing_count(command_name)
+      end
+
+      def read_command(command_name, *args, &block)
+        if @instance
+          send_command(@instance, command_name, *args, &block)
+        else
+          read_one_with_fallback(command_name, *args, &block)
+        end
+      end
+
+      def write_command(command_name, *args, &block)
+        if @instance
+          send_command(@instance, command_name, *args, &block)
+        else
+          write_both(command_name, *args, &block)
+        end
+      end
+
+      def read_one_with_fallback(command_name, *args, &block)
+        begin
+          value = send_command(primary_store, command_name, *args, &block)
+        rescue StandardError => e
+          log_error(e, command_name,
+            multi_store_error_message: FAILED_TO_READ_ERROR_MESSAGE)
+        end
+
+        value ||= fallback_read(command_name, *args, &block)
+
+        value
+      end
+
+      def fallback_read(command_name, *args, &block)
+        value = send_command(secondary_store, command_name, *args, &block)
+
+        if value
+          log_error(ReadFromPrimaryError.new, command_name)
+          increment_read_fallback_count(command_name)
+        end
+
+        value
+      end
+
+      def write_both(command_name, *args, &block)
+        begin
+          send_command(primary_store, command_name, *args, &block)
+        rescue StandardError => e
+          log_error(e, command_name,
+            multi_store_error_message: FAILED_TO_WRITE_ERROR_MESSAGE)
+        end
+
+        send_command(secondary_store, command_name, *args, &block)
+      end
+
+      def multi_store_enabled?
+        Feature.enabled?(:use_multi_store, default_enabled: :yaml) && !same_redis_store?
+      end
+
+      def same_redis_store?
+        strong_memoize(:same_redis_store) do
+          # "
+          primary_store.inspect == secondary_store.inspect
+        end
+      end
+
+      # rubocop:disable GitlabSecurity/PublicSend
+      def send_command(redis_instance, command_name, *args, &block)
+        if block_given?
+          # Make sure that block is wrapped and executed only on the redis instance that is executing the block
+          redis_instance.send(command_name, *args) do |*params|
+            with_instance(redis_instance, *params, &block)
+          end
+        else
+          redis_instance.send(command_name, *args)
+        end
+      end
+      # rubocop:enable GitlabSecurity/PublicSend
+
+      def with_instance(instance, *params)
+        @instance = instance
+
+        yield(*params)
+      ensure
+        @instance = nil
+      end
+
+      def increment_read_fallback_count(command_name)
+        @read_fallback_counter ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_read_fallback_total, 'Client side Redis MultiStore reading fallback')
+        @read_fallback_counter.increment(command: command_name, instance_name: instance_name)
+      end
+
+      def increment_method_missing_count(command_name)
+        @method_missing_counter ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_method_missing_total, 'Client side Redis MultiStore method missing')
+        @method_missing_counter.increment(command: command_name, innamece_name: instance_name)
+      end
+
+      def validate_stores!
+        raise ArgumentError, 'primary_store is required' unless primary_store
+        raise ArgumentError, 'secondary_store is required' unless secondary_store
+        raise ArgumentError, 'invalid primary_store' unless primary_store.is_a?(::Redis)
+        raise ArgumentError, 'invalid secondary_store' unless secondary_store.is_a?(::Redis)
+      end
+
+      def log_error(exception, command_name, extra = {})
+        Gitlab::ErrorTracking.log_exception(
+          exception,
+          command_name: command_name,
+          extra: extra.merge(instance_name: instance_name))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb
index 7b804038146..985c8dc619c 100644
--- a/lib/gitlab/redis/wrapper.rb
+++ b/lib/gitlab/redis/wrapper.rb
@@ -17,7 +17,7 @@ module Gitlab
   module Redis
     class Wrapper
       class << self
-        delegate :params, :url, to: :new
+        delegate :params, :url, :store, to: :new
 
         def with
           pool.with { |redis| yield redis }
@@ -126,6 +126,10 @@ module Gitlab
         sentinels && !sentinels.empty?
       end
 
+      def store(extras = {})
+        ::Redis::Store::Factory.create(redis_store_options.merge(extras))
+      end
+
       private
 
       def redis_store_options
diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb
index f60cac0aff0..5fbbfd90be1 100644
--- a/lib/gitlab/runtime.rb
+++ b/lib/gitlab/runtime.rb
@@ -63,12 +63,8 @@ module Gitlab
         puma?
       end
 
-      def action_cable?
-        web_server? && (!!defined?(ACTION_CABLE_SERVER) || Gitlab::ActionCable::Config.in_app?)
-      end
-
       def multi_threaded?
-        puma? || sidekiq? || action_cable?
+        puma? || sidekiq?
       end
 
       def puma_in_clustered_mode?
@@ -84,12 +80,15 @@ module Gitlab
         if puma? && Puma.respond_to?(:cli_config)
           threads += Puma.cli_config.options[:max_threads]
         elsif sidekiq?
-          # An extra thread for the poller in Sidekiq Cron:
+          # 2 extra threads for the pollers in Sidekiq and Sidekiq Cron:
           # https://github.com/ondrejbartas/sidekiq-cron#under-the-hood
-          threads += Sidekiq.options[:concurrency] + 1
+          #
+          # These threads execute Sidekiq client middleware when jobs
+          # are enqueued and those can access DB / Redis.
+          threads += Sidekiq.options[:concurrency] + 2
         end
 
-        if action_cable?
+        if web_server?
           threads += Gitlab::ActionCable::Config.worker_pool_size
         end
 
diff --git a/lib/gitlab/saas.rb b/lib/gitlab/saas.rb
index 9220ad1be6c..1e00bd4cbfc 100644
--- a/lib/gitlab/saas.rb
+++ b/lib/gitlab/saas.rb
@@ -38,11 +38,11 @@ module Gitlab
     end
 
     def self.about_pricing_url
-      "https://about.gitlab.com/pricing"
+      "https://about.gitlab.com/pricing/"
     end
 
     def self.about_pricing_faq_url
-      "https://about.gitlab.com/gitlab-com/#faq"
+      "https://about.gitlab.com/pricing#faq"
     end
   end
 end
diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb
index 217a48e740d..37414f9e2b1 100644
--- a/lib/gitlab/search_results.rb
+++ b/lib/gitlab/search_results.rb
@@ -115,6 +115,11 @@ module Gitlab
       {}
     end
 
+    # aggregations are only performed by Elasticsearch backed results
+    def aggregations(scope)
+      []
+    end
+
     private
 
     def collection_for(scope)
diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb
index 751405f1045..3a31f651714 100644
--- a/lib/gitlab/setup_helper.rb
+++ b/lib/gitlab/setup_helper.rb
@@ -104,9 +104,6 @@ module Gitlab
             socket_filename = options[:gitaly_socket] || "gitaly.socket"
             prometheus_listen_addr = options[:prometheus_listen_addr]
 
-            git_bin_path = File.expand_path('../gitaly/_build/deps/git/install/bin/git')
-            git_bin_path = nil unless File.exist?(git_bin_path)
-
             config = {
               # Override the set gitaly_address since Praefect is in the loop
               socket_path: File.join(gitaly_dir, socket_filename),
@@ -116,8 +113,8 @@ module Gitlab
               # sidekiq jobs, and concurrency will be low anyway in test.
               git: {
                 catfile_cache_size: 5,
-                bin_path: git_bin_path
-              }.compact,
+                bin_path: File.expand_path(File.join(gitaly_dir, '_build', 'deps', 'git', 'install', 'bin', 'git'))
+              },
               prometheus_listen_addr: prometheus_listen_addr
             }.compact
 
diff --git a/lib/gitlab/sidekiq_cluster.rb b/lib/gitlab/sidekiq_cluster.rb
deleted file mode 100644
index cc1bd282da8..00000000000
--- a/lib/gitlab/sidekiq_cluster.rb
+++ /dev/null
@@ -1,171 +0,0 @@
-# frozen_string_literal: true
-
-require 'shellwords'
-
-module Gitlab
-  module SidekiqCluster
-    # The signals that should terminate both the master and workers.
-    TERMINATE_SIGNALS = %i(INT TERM).freeze
-
-    # The signals that should simply be forwarded to the workers.
-    FORWARD_SIGNALS = %i(TTIN USR1 USR2 HUP).freeze
-
-    # Traps the given signals and yields the block whenever these signals are
-    # received.
-    #
-    # The block is passed the name of the signal.
-    #
-    # Example:
-    #
-    #     trap_signals(%i(HUP TERM)) do |signal|
-    #       ...
-    #     end
-    def self.trap_signals(signals)
-      signals.each do |signal|
-        trap(signal) do
-          yield signal
-        end
-      end
-    end
-
-    def self.trap_terminate(&block)
-      trap_signals(TERMINATE_SIGNALS, &block)
-    end
-
-    def self.trap_forward(&block)
-      trap_signals(FORWARD_SIGNALS, &block)
-    end
-
-    def self.signal(pid, signal)
-      Process.kill(signal, pid)
-      true
-    rescue Errno::ESRCH
-      false
-    end
-
-    def self.signal_processes(pids, signal)
-      pids.each { |pid| signal(pid, signal) }
-    end
-
-    # Starts Sidekiq workers for the pairs of processes.
-    #
-    # Example:
-    #
-    #     start([ ['foo'], ['bar', 'baz'] ], :production)
-    #
-    # This would start two Sidekiq processes: one processing "foo", and one
-    # processing "bar" and "baz". Each one is placed in its own process group.
-    #
-    # queues - An Array containing Arrays. Each sub Array should specify the
-    #          queues to use for a single process.
-    #
-    # directory - The directory of the Rails application.
-    #
-    # Returns an Array containing the PIDs of the started processes.
-    def self.start(queues, env: :development, directory: Dir.pwd, max_concurrency: 50, min_concurrency: 0, timeout: CLI::DEFAULT_SOFT_TIMEOUT_SECONDS, dryrun: false)
-      queues.map.with_index do |pair, index|
-        start_sidekiq(pair, env: env,
-                            directory: directory,
-                            max_concurrency: max_concurrency,
-                            min_concurrency: min_concurrency,
-                            worker_id: index,
-                            timeout: timeout,
-                            dryrun: dryrun)
-      end
-    end
-
-    # Starts a Sidekiq process that processes _only_ the given queues.
-    #
-    # Returns the PID of the started process.
-    def self.start_sidekiq(queues, env:, directory:, max_concurrency:, min_concurrency:, worker_id:, timeout:, dryrun:)
-      counts = count_by_queue(queues)
-
-      cmd = %w[bundle exec sidekiq]
-      cmd << "-c#{self.concurrency(queues, min_concurrency, max_concurrency)}"
-      cmd << "-e#{env}"
-      cmd << "-t#{timeout}"
-      cmd << "-gqueues:#{proc_details(counts)}"
-      cmd << "-r#{directory}"
-
-      counts.each do |queue, count|
-        cmd << "-q#{queue},#{count}"
-      end
-
-      if dryrun
-        puts Shellwords.join(cmd) # rubocop:disable Rails/Output
-        return
-      end
-
-      pid = Process.spawn(
-        { 'ENABLE_SIDEKIQ_CLUSTER' => '1',
-          'SIDEKIQ_WORKER_ID' => worker_id.to_s },
-        *cmd,
-        pgroup: true,
-        err: $stderr,
-        out: $stdout
-      )
-
-      wait_async(pid)
-
-      pid
-    end
-
-    def self.count_by_queue(queues)
-      queues.tally
-    end
-
-    def self.proc_details(counts)
-      counts.map do |queue, count|
-        if count == 1
-          queue
-        else
-          "#{queue} (#{count})"
-        end
-      end.join(',')
-    end
-
-    def self.concurrency(queues, min_concurrency, max_concurrency)
-      concurrency_from_queues = queues.length + 1
-      max = max_concurrency > 0 ? max_concurrency : concurrency_from_queues
-      min = [min_concurrency, max].min
-
-      concurrency_from_queues.clamp(min, max)
-    end
-
-    # Waits for the given process to complete using a separate thread.
-    def self.wait_async(pid)
-      Thread.new do
-        Process.wait(pid) rescue Errno::ECHILD
-      end
-    end
-
-    # Returns true if all the processes are alive.
-    def self.all_alive?(pids)
-      pids.each do |pid|
-        return false unless process_alive?(pid)
-      end
-
-      true
-    end
-
-    def self.any_alive?(pids)
-      pids_alive(pids).any?
-    end
-
-    def self.pids_alive(pids)
-      pids.select { |pid| process_alive?(pid) }
-    end
-
-    def self.process_alive?(pid)
-      # Signal 0 tests whether the process exists and we have access to send signals
-      # but is otherwise a noop (doesn't actually send a signal to the process)
-      signal(pid, 0)
-    end
-
-    def self.write_pid(path)
-      File.open(path, 'w') do |handle|
-        handle.write(Process.pid.to_s)
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/sidekiq_cluster/cli.rb b/lib/gitlab/sidekiq_cluster/cli.rb
deleted file mode 100644
index 3dee257229d..00000000000
--- a/lib/gitlab/sidekiq_cluster/cli.rb
+++ /dev/null
@@ -1,230 +0,0 @@
-# frozen_string_literal: true
-
-require 'optparse'
-require 'logger'
-require 'time'
-
-module Gitlab
-  module SidekiqCluster
-    class CLI
-      CHECK_TERMINATE_INTERVAL_SECONDS = 1
-
-      # How long to wait when asking for a clean termination.
-      # It maps the Sidekiq default timeout:
-      # https://github.com/mperham/sidekiq/wiki/Signals#term
-      #
-      # This value is passed to Sidekiq's `-t` if none
-      # is given through arguments.
-      DEFAULT_SOFT_TIMEOUT_SECONDS = 25
-
-      # After surpassing the soft timeout.
-      DEFAULT_HARD_TIMEOUT_SECONDS = 5
-
-      CommandError = Class.new(StandardError)
-
-      def initialize(log_output = $stderr)
-        require_relative '../../../lib/gitlab/sidekiq_logging/json_formatter'
-
-        # As recommended by https://github.com/mperham/sidekiq/wiki/Advanced-Options#concurrency
-        @max_concurrency = 50
-        @min_concurrency = 0
-        @environment = ENV['RAILS_ENV'] || 'development'
-        @pid = nil
-        @interval = 5
-        @alive = true
-        @processes = []
-        @logger = Logger.new(log_output)
-        @logger.formatter = ::Gitlab::SidekiqLogging::JSONFormatter.new
-        @rails_path = Dir.pwd
-        @dryrun = false
-        @list_queues = false
-      end
-
-      def run(argv = ARGV)
-        if argv.empty?
-          raise CommandError,
-            'You must specify at least one queue to start a worker for'
-        end
-
-        option_parser.parse!(argv)
-
-        if @dryrun && @list_queues
-          raise CommandError,
-            'The --dryrun and --list-queues options are mutually exclusive'
-        end
-
-        worker_metadatas = SidekiqConfig::CliMethods.worker_metadatas(@rails_path)
-        worker_queues = SidekiqConfig::CliMethods.worker_queues(@rails_path)
-
-        queue_groups = argv.map do |queues_or_query_string|
-          if queues_or_query_string =~ /[\r\n]/
-            raise CommandError,
-              'The queue arguments cannot contain newlines'
-          end
-
-          next worker_queues if queues_or_query_string == SidekiqConfig::WorkerMatcher::WILDCARD_MATCH
-
-          # When using the queue query syntax, we treat each queue group
-          # as a worker attribute query, and resolve the queues for the
-          # queue group using this query.
-
-          if @queue_selector
-            SidekiqConfig::CliMethods.query_queues(queues_or_query_string, worker_metadatas)
-          else
-            SidekiqConfig::CliMethods.expand_queues(queues_or_query_string.split(','), worker_queues)
-          end
-        end
-
-        if @negate_queues
-          queue_groups.map! { |queues| worker_queues - queues }
-        end
-
-        if queue_groups.all?(&:empty?)
-          raise CommandError,
-            'No queues found, you must select at least one queue'
-        end
-
-        if @list_queues
-          puts queue_groups.map(&:sort) # rubocop:disable Rails/Output
-
-          return
-        end
-
-        unless @dryrun
-          @logger.info("Starting cluster with #{queue_groups.length} processes")
-        end
-
-        @processes = SidekiqCluster.start(
-          queue_groups,
-          env: @environment,
-          directory: @rails_path,
-          max_concurrency: @max_concurrency,
-          min_concurrency: @min_concurrency,
-          dryrun: @dryrun,
-          timeout: soft_timeout_seconds
-        )
-
-        return if @dryrun
-
-        write_pid
-        trap_signals
-        start_loop
-      end
-
-      def write_pid
-        SidekiqCluster.write_pid(@pid) if @pid
-      end
-
-      def soft_timeout_seconds
-        @soft_timeout_seconds || DEFAULT_SOFT_TIMEOUT_SECONDS
-      end
-
-      # The amount of time it'll wait for killing the alive Sidekiq processes.
-      def hard_timeout_seconds
-        soft_timeout_seconds + DEFAULT_HARD_TIMEOUT_SECONDS
-      end
-
-      def monotonic_time
-        Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second)
-      end
-
-      def continue_waiting?(deadline)
-        SidekiqCluster.any_alive?(@processes) && monotonic_time < deadline
-      end
-
-      def hard_stop_stuck_pids
-        SidekiqCluster.signal_processes(SidekiqCluster.pids_alive(@processes), "-KILL")
-      end
-
-      def wait_for_termination
-        deadline = monotonic_time + hard_timeout_seconds
-        sleep(CHECK_TERMINATE_INTERVAL_SECONDS) while continue_waiting?(deadline)
-
-        hard_stop_stuck_pids
-      end
-
-      def trap_signals
-        SidekiqCluster.trap_terminate do |signal|
-          @alive = false
-          SidekiqCluster.signal_processes(@processes, signal)
-          wait_for_termination
-        end
-
-        SidekiqCluster.trap_forward do |signal|
-          SidekiqCluster.signal_processes(@processes, signal)
-        end
-      end
-
-      def start_loop
-        while @alive
-          sleep(@interval)
-
-          unless SidekiqCluster.all_alive?(@processes)
-            # If a child process died we'll just terminate the whole cluster. It's up to
-            # runit and such to then restart the cluster.
-            @logger.info('A worker terminated, shutting down the cluster')
-
-            SidekiqCluster.signal_processes(@processes, :TERM)
-            break
-          end
-        end
-      end
-
-      def option_parser
-        OptionParser.new do |opt|
-          opt.banner = "#{File.basename(__FILE__)} [QUEUE,QUEUE] [QUEUE] ... [OPTIONS]"
-
-          opt.separator "\nOptions:\n"
-
-          opt.on('-h', '--help', 'Shows this help message') do
-            abort opt.to_s
-          end
-
-          opt.on('-m', '--max-concurrency INT', 'Maximum threads to use with Sidekiq (default: 50, 0 to disable)') do |int|
-            @max_concurrency = int.to_i
-          end
-
-          opt.on('--min-concurrency INT', 'Minimum threads to use with Sidekiq (default: 0)') do |int|
-            @min_concurrency = int.to_i
-          end
-
-          opt.on('-e', '--environment ENV', 'The application environment') do |env|
-            @environment = env
-          end
-
-          opt.on('-P', '--pidfile PATH', 'Path to the PID file') do |pid|
-            @pid = pid
-          end
-
-          opt.on('-r', '--require PATH', 'Location of the Rails application') do |path|
-            @rails_path = path
-          end
-
-          opt.on('--queue-selector', 'Run workers based on the provided selector') do |queue_selector|
-            @queue_selector = queue_selector
-          end
-
-          opt.on('-n', '--negate', 'Run workers for all queues in sidekiq_queues.yml except the given ones') do
-            @negate_queues = true
-          end
-
-          opt.on('-i', '--interval INT', 'The number of seconds to wait between worker checks') do |int|
-            @interval = int.to_i
-          end
-
-          opt.on('-t', '--timeout INT', 'Graceful timeout for all running processes') do |timeout|
-            @soft_timeout_seconds = timeout.to_i
-          end
-
-          opt.on('-d', '--dryrun', 'Print commands that would be run without this flag, and quit') do |int|
-            @dryrun = true
-          end
-
-          opt.on('--list-queues', 'List matching queues, and quit') do |int|
-            @list_queues = true
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb
index 5663c51bb7a..07ddac209f8 100644
--- a/lib/gitlab/sidekiq_config.rb
+++ b/lib/gitlab/sidekiq_config.rb
@@ -6,11 +6,13 @@ module Gitlab
   module SidekiqConfig
     FOSS_QUEUE_CONFIG_PATH = 'app/workers/all_queues.yml'
     EE_QUEUE_CONFIG_PATH = 'ee/app/workers/all_queues.yml'
+    JH_QUEUE_CONFIG_PATH = 'jh/app/workers/all_queues.yml'
     SIDEKIQ_QUEUES_PATH = 'config/sidekiq_queues.yml'
 
     QUEUE_CONFIG_PATHS = [
       FOSS_QUEUE_CONFIG_PATH,
-      (EE_QUEUE_CONFIG_PATH if Gitlab.ee?)
+      (EE_QUEUE_CONFIG_PATH if Gitlab.ee?),
+      (JH_QUEUE_CONFIG_PATH if Gitlab.jh?)
     ].compact.freeze
 
     # This maps workers not in our application code to queues. We need
@@ -33,7 +35,7 @@ module Gitlab
         weight: 2,
         tags: []
       )
-    }.transform_values { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false) }.freeze
+    }.transform_values { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false, jh: false) }.freeze
 
     class << self
       include Gitlab::SidekiqConfig::CliMethods
@@ -58,10 +60,14 @@ module Gitlab
         @workers ||= begin
           result = []
           result.concat(DEFAULT_WORKERS.values)
-          result.concat(find_workers(Rails.root.join('app', 'workers'), ee: false))
+          result.concat(find_workers(Rails.root.join('app', 'workers'), ee: false, jh: false))
 
           if Gitlab.ee?
-            result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'), ee: true))
+            result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'), ee: true, jh: false))
+          end
+
+          if Gitlab.jh?
+            result.concat(find_workers(Rails.root.join('jh', 'app', 'workers'), ee: false, jh: true))
           end
 
           result
@@ -69,16 +75,26 @@ module Gitlab
       end
 
       def workers_for_all_queues_yml
-        workers.partition(&:ee?).reverse.map(&:sort)
+        workers.each_with_object([[], [], []]) do |worker, array|
+          if worker.jh?
+            array[2].push(worker)
+          elsif worker.ee?
+            array[1].push(worker)
+          else
+            array[0].push(worker)
+          end
+        end.map(&:sort)
       end
 
       # YAML.load_file is OK here as we control the file contents
       def all_queues_yml_outdated?
-        foss_workers, ee_workers = workers_for_all_queues_yml
+        foss_workers, ee_workers, jh_workers = workers_for_all_queues_yml
 
         return true if foss_workers != YAML.load_file(FOSS_QUEUE_CONFIG_PATH)
 
-        Gitlab.ee? && ee_workers != YAML.load_file(EE_QUEUE_CONFIG_PATH)
+        return true if Gitlab.ee? && ee_workers != YAML.load_file(EE_QUEUE_CONFIG_PATH)
+
+        Gitlab.jh? && File.exist?(JH_QUEUE_CONFIG_PATH) && jh_workers != YAML.load_file(JH_QUEUE_CONFIG_PATH)
       end
 
       def queues_for_sidekiq_queues_yml
@@ -120,14 +136,14 @@ module Gitlab
 
       private
 
-      def find_workers(root, ee:)
+      def find_workers(root, ee:, jh:)
         concerns = root.join('concerns').to_s
 
         Dir[root.join('**', '*.rb')]
           .reject { |path| path.start_with?(concerns) }
           .map { |path| worker_from_path(path, root) }
           .select { |worker| worker < Sidekiq::Worker }
-          .map { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: ee) }
+          .map { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: ee, jh: jh) }
       end
 
       def worker_from_path(path, root)
diff --git a/lib/gitlab/sidekiq_config/cli_methods.rb b/lib/gitlab/sidekiq_config/cli_methods.rb
index 8eef15f9ccb..70798f8c3e8 100644
--- a/lib/gitlab/sidekiq_config/cli_methods.rb
+++ b/lib/gitlab/sidekiq_config/cli_methods.rb
@@ -18,6 +18,7 @@ module Gitlab
       QUEUE_CONFIG_PATHS = begin
         result = %w[app/workers/all_queues.yml]
         result << 'ee/app/workers/all_queues.yml' if Gitlab.ee?
+        result << 'jh/app/workers/all_queues.yml' if Gitlab.jh?
         result
       end.freeze
 
diff --git a/lib/gitlab/sidekiq_config/worker.rb b/lib/gitlab/sidekiq_config/worker.rb
index a343573440f..1e3fb675ca7 100644
--- a/lib/gitlab/sidekiq_config/worker.rb
+++ b/lib/gitlab/sidekiq_config/worker.rb
@@ -13,15 +13,20 @@ module Gitlab
                :worker_has_external_dependencies?,
                to: :klass
 
-      def initialize(klass, ee:)
+      def initialize(klass, ee:, jh: false)
         @klass = klass
         @ee = ee
+        @jh = jh
       end
 
       def ee?
         @ee
       end
 
+      def jh?
+        @jh
+      end
+
       def ==(other)
         to_yaml == case other
                    when self.class
diff --git a/lib/gitlab/sidekiq_enq.rb b/lib/gitlab/sidekiq_enq.rb
index d8a01ac8ef4..de0c00fe561 100644
--- a/lib/gitlab/sidekiq_enq.rb
+++ b/lib/gitlab/sidekiq_enq.rb
@@ -1,16 +1,44 @@
 # frozen_string_literal: true
 
-# This is a copy of https://github.com/mperham/sidekiq/blob/32c55e31659a1e6bd42f98334cca5eef2863de8d/lib/sidekiq/scheduled.rb#L11-L34
-#
-# It effectively reverts
-# https://github.com/mperham/sidekiq/commit/9b75467b33759888753191413eddbc15c37a219e
-# because we observe that the extra ZREMs caused by this change can lead to high
-# CPU usage on Redis at peak times:
-# https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1179
-#
 module Gitlab
   class SidekiqEnq
+    LUA_ZPOPBYSCORE = <<~EOS
+      local key, now = KEYS[1], ARGV[1]
+      local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
+      if jobs[1] then
+        redis.call("zrem", key, jobs[1])
+        return jobs[1]
+      end
+    EOS
+
+    LUA_ZPOPBYSCORE_SHA = Digest::SHA1.hexdigest(LUA_ZPOPBYSCORE)
+
     def enqueue_jobs(now = Time.now.to_f.to_s, sorted_sets = Sidekiq::Scheduled::SETS)
+      Rails.application.reloader.wrap do
+        ::Gitlab::WithRequestStore.with_request_store do
+          if Feature.enabled?(:atomic_sidekiq_scheduler, default_enabled: :yaml)
+            atomic_find_jobs_and_enqueue(now, sorted_sets)
+          else
+            find_jobs_and_enqueue(now, sorted_sets)
+          end
+
+        ensure
+          ::Gitlab::Database::LoadBalancing.release_hosts
+        end
+      end
+    end
+
+    private
+
+    # This is a copy of https://github.com/mperham/sidekiq/blob/32c55e31659a1e6bd42f98334cca5eef2863de8d/lib/sidekiq/scheduled.rb#L11-L34
+    #
+    # It effectively reverts
+    # https://github.com/mperham/sidekiq/commit/9b75467b33759888753191413eddbc15c37a219e
+    # because we observe that the extra ZREMs caused by this change can lead to high
+    # CPU usage on Redis at peak times:
+    # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1179
+    #
+    def find_jobs_and_enqueue(now, sorted_sets)
       # A job's "score" in Redis is the time at which it should be processed.
       # Just check Redis for the set of jobs with a timestamp before now.
       Sidekiq.redis do |conn|
@@ -24,8 +52,7 @@ module Gitlab
           # We need to go through the list one at a time to reduce the risk of something
           # going wrong between the time jobs are popped from the scheduled queue and when
           # they are pushed onto a work queue and losing the jobs.
-          while (job = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 1]).first)
-
+          while job = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 1]).first
             # Pop item off the queue and add it to the work queue. If the job can't be popped from
             # the queue, it's because another process already popped it so we can move on to the
             # next one.
@@ -47,5 +74,38 @@ module Gitlab
         end
       end
     end
+
+    def atomic_find_jobs_and_enqueue(now, sorted_sets)
+      Sidekiq.redis do |conn|
+        sorted_sets.each do |sorted_set|
+          start_time = ::Gitlab::Metrics::System.monotonic_time
+          jobs = 0
+
+          Sidekiq.logger.info(message: 'Atomically enqueuing scheduled jobs', status: 'start', sorted_set: sorted_set)
+
+          while job = redis_eval_lua(conn, LUA_ZPOPBYSCORE, LUA_ZPOPBYSCORE_SHA, keys: [sorted_set], argv: [now])
+            jobs += 1
+            Sidekiq::Client.push(Sidekiq.load_json(job))
+          end
+
+          end_time = ::Gitlab::Metrics::System.monotonic_time
+          Sidekiq.logger.info(message: 'Atomically enqueuing scheduled jobs',
+                              status: 'done',
+                              sorted_set: sorted_set,
+                              jobs_count: jobs,
+                              duration_s: end_time - start_time)
+        end
+      end
+    end
+
+    def redis_eval_lua(conn, script, sha, keys: nil, argv: nil)
+      conn.evalsha(sha, keys: keys, argv: argv)
+    rescue ::Redis::CommandError => e
+      if e.message.start_with?('NOSCRIPT')
+        conn.eval(script, keys: keys, argv: argv)
+      else
+        raise
+      end
+    end
   end
 end
diff --git a/lib/gitlab/sidekiq_logging/deduplication_logger.rb b/lib/gitlab/sidekiq_logging/deduplication_logger.rb
index c5654819ffb..f4f6cb2a306 100644
--- a/lib/gitlab/sidekiq_logging/deduplication_logger.rb
+++ b/lib/gitlab/sidekiq_logging/deduplication_logger.rb
@@ -6,7 +6,7 @@ module Gitlab
       include Singleton
       include LogsJobs
 
-      def log(job, deduplication_type, deduplication_options = {})
+      def deduplicated_log(job, deduplication_type, deduplication_options = {})
         payload = parse_job(job)
         payload['job_status'] = 'deduplicated'
         payload['message'] = "#{base_message(payload)}: deduplicated: #{deduplication_type}"
@@ -17,6 +17,14 @@ module Gitlab
 
         Sidekiq.logger.info payload
       end
+
+      def rescheduled_log(job)
+        payload = parse_job(job)
+        payload['job_status'] = 'rescheduled'
+        payload['message'] = "#{base_message(payload)}: rescheduled"
+
+        Sidekiq.logger.info payload
+      end
     end
   end
 end
diff --git a/lib/gitlab/sidekiq_logging/json_formatter.rb b/lib/gitlab/sidekiq_logging/json_formatter.rb
index 8894b48417c..a6281bbdf26 100644
--- a/lib/gitlab/sidekiq_logging/json_formatter.rb
+++ b/lib/gitlab/sidekiq_logging/json_formatter.rb
@@ -6,7 +6,7 @@ require 'json'
 module Gitlab
   module SidekiqLogging
     class JSONFormatter
-      TIMESTAMP_FIELDS = %w[created_at enqueued_at started_at retried_at failed_at completed_at].freeze
+      TIMESTAMP_FIELDS = %w[created_at scheduled_at enqueued_at started_at retried_at failed_at completed_at].freeze
 
       def call(severity, timestamp, progname, data)
         output = {
diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb
index c97b1632bf8..69802fd6217 100644
--- a/lib/gitlab/sidekiq_middleware.rb
+++ b/lib/gitlab/sidekiq_middleware.rb
@@ -33,6 +33,7 @@ module Gitlab
         chain.add ::Gitlab::SidekiqMiddleware::BatchLoader
         chain.add ::Gitlab::SidekiqMiddleware::InstrumentationLogger
         chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Server
+        chain.add ::Gitlab::SidekiqMiddleware::QueryAnalyzer if Gitlab.dev_or_test_env? || Gitlab::Utils.to_boolean(ENV['GITLAB_ENABLE_QUERY_ANALYZERS'], default: false)
         chain.add ::Gitlab::SidekiqVersioning::Middleware
         chain.add ::Gitlab::SidekiqStatus::ServerMiddleware
         chain.add ::Gitlab::SidekiqMiddleware::WorkerContext::Server
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
index e63164efc94..f31262bfcc9 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
@@ -19,11 +19,12 @@ module Gitlab
       class DuplicateJob
         include Gitlab::Utils::StrongMemoize
 
-        DUPLICATE_KEY_TTL = 6.hours
+        DEFAULT_DUPLICATE_KEY_TTL = 6.hours
         WAL_LOCATION_TTL = 60.seconds
         MAX_REDIS_RETRIES = 5
         DEFAULT_STRATEGY = :until_executing
         STRATEGY_NONE = :none
+        DEDUPLICATED_FLAG_VALUE = 1
 
         LUA_SET_WAL_SCRIPT = <<~EOS
           local key, wal, offset, ttl = KEYS[1], ARGV[1], tonumber(ARGV[2]), ARGV[3]
@@ -58,7 +59,7 @@ module Gitlab
         end
 
         # This method will return the jid that was set in redis
-        def check!(expiry = DUPLICATE_KEY_TTL)
+        def check!(expiry = duplicate_key_ttl)
           read_jid = nil
           read_wal_locations = {}
 
@@ -83,7 +84,11 @@ module Gitlab
           Sidekiq.redis do |redis|
             redis.multi do |multi|
               job_wal_locations.each do |connection_name, location|
-                multi.eval(LUA_SET_WAL_SCRIPT, keys: [wal_location_key(connection_name)], argv: [location, pg_wal_lsn_diff(connection_name).to_i, WAL_LOCATION_TTL])
+                multi.eval(
+                  LUA_SET_WAL_SCRIPT,
+                  keys: [wal_location_key(connection_name)],
+                  argv: [location, pg_wal_lsn_diff(connection_name).to_i, WAL_LOCATION_TTL]
+                )
               end
             end
           end
@@ -110,12 +115,18 @@ module Gitlab
         def delete!
           Sidekiq.redis do |redis|
             redis.multi do |multi|
-              multi.del(idempotency_key)
+              multi.del(idempotency_key, deduplicated_flag_key)
               delete_wal_locations!(multi)
             end
           end
         end
 
+        def reschedule
+          Gitlab::SidekiqLogging::DeduplicationLogger.instance.rescheduled_log(job)
+
+          worker_klass.perform_async(*arguments)
+        end
+
         def scheduled?
           scheduled_at.present?
         end
@@ -126,6 +137,22 @@ module Gitlab
           jid != existing_jid
         end
 
+        def set_deduplicated_flag!(expiry = duplicate_key_ttl)
+          return unless reschedulable?
+
+          Sidekiq.redis do |redis|
+            redis.set(deduplicated_flag_key, DEDUPLICATED_FLAG_VALUE, ex: expiry, nx: true)
+          end
+        end
+
+        def should_reschedule?
+          return false unless reschedulable?
+
+          Sidekiq.redis do |redis|
+            redis.get(deduplicated_flag_key).present?
+          end
+        end
+
         def scheduled_at
           job['at']
         end
@@ -145,6 +172,10 @@ module Gitlab
           worker_klass.idempotent?
         end
 
+        def duplicate_key_ttl
+          options[:ttl] || DEFAULT_DUPLICATE_KEY_TTL
+        end
+
         private
 
         attr_writer :existing_wal_locations
@@ -181,7 +212,12 @@ module Gitlab
         end
 
         def pg_wal_lsn_diff(connection_name)
-          Gitlab::Database.databases[connection_name].pg_wal_lsn_diff(job_wal_locations[connection_name], existing_wal_locations[connection_name])
+          model = Gitlab::Database.database_base_models[connection_name]
+
+          model.connection.load_balancer.wal_diff(
+            job_wal_locations[connection_name],
+            existing_wal_locations[connection_name]
+          )
         end
 
         def strategy
@@ -216,6 +252,10 @@ module Gitlab
           @idempotency_key ||= job['idempotency_key'] || "#{namespace}:#{idempotency_hash}"
         end
 
+        def deduplicated_flag_key
+          "#{idempotency_key}:deduplicate_flag"
+        end
+
         def idempotency_hash
           Digest::SHA256.hexdigest(idempotency_string)
         end
@@ -235,6 +275,10 @@ module Gitlab
         def preserve_wal_location?
           Feature.enabled?(:preserve_latest_wal_locations_for_idempotent_jobs, default_enabled: :yaml)
         end
+
+        def reschedulable?
+          !scheduled? && options[:if_deduplicated] == :reschedule_once
+        end
       end
     end
   end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb
index a35edc5774e..6d5d41902ea 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb
@@ -5,7 +5,7 @@ module Gitlab
     module DuplicateJobs
       class Server
         def call(worker, job, queue, &block)
-          DuplicateJob.new(job, queue).perform(&block)
+          ::Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob.new(job, queue).perform(&block)
         end
       end
     end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/base.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/base.rb
index df5df590281..9b3066bae6c 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/base.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/base.rb
@@ -26,8 +26,8 @@ module Gitlab
           end
 
           def check!
-            # The default expiry time is the DuplicateJob::DUPLICATE_KEY_TTL already
-            # Only the strategies de-duplicating when scheduling
+            # The default expiry time is the worker class'
+            # configured deduplication TTL or DuplicateJob::DEFAULT_DUPLICATE_KEY_TTL.
             duplicate_job.check!
           end
         end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb
index b0da85b74a6..0fc95534e2a 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb
@@ -6,6 +6,7 @@ module Gitlab
       module Strategies
         class DeduplicatesWhenScheduling < Base
           extend ::Gitlab::Utils::Override
+          include ::Gitlab::Utils::StrongMemoize
 
           override :initialize
           def initialize(duplicate_job)
@@ -19,8 +20,9 @@ module Gitlab
 
               if duplicate_job.idempotent?
                 duplicate_job.update_latest_wal_location!
+                duplicate_job.set_deduplicated_flag!(expiry)
 
-                Gitlab::SidekiqLogging::DeduplicationLogger.instance.log(
+                Gitlab::SidekiqLogging::DeduplicationLogger.instance.deduplicated_log(
                   job, "dropped #{strategy_name}", duplicate_job.options)
                 return false
               end
@@ -49,11 +51,16 @@ module Gitlab
           end
 
           def expiry
-            return DuplicateJob::DUPLICATE_KEY_TTL unless duplicate_job.scheduled?
+            strong_memoize(:expiry) do
+              next duplicate_job.duplicate_key_ttl unless duplicate_job.scheduled?
 
-            time_diff = duplicate_job.scheduled_at.to_i - Time.now.to_i
+              time_diff = [
+                duplicate_job.scheduled_at.to_i - Time.now.to_i,
+                0
+              ].max
 
-            time_diff > 0 ? time_diff : DuplicateJob::DUPLICATE_KEY_TTL
+              time_diff + duplicate_job.duplicate_key_ttl
+            end
           end
         end
       end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb
index 25f1b8b7c51..8c7e15364f8 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb
@@ -14,7 +14,10 @@ module Gitlab
 
             yield
 
+            should_reschedule = duplicate_job.should_reschedule?
+            # Deleting before rescheduling to make sure we don't deduplicate again.
             duplicate_job.delete!
+            duplicate_job.reschedule if should_reschedule
           end
         end
       end
diff --git a/lib/gitlab/sidekiq_middleware/query_analyzer.rb b/lib/gitlab/sidekiq_middleware/query_analyzer.rb
new file mode 100644
index 00000000000..4478fcd3594
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/query_analyzer.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqMiddleware
+    class QueryAnalyzer
+      def call(worker, job, queue)
+        ::Gitlab::Database::QueryAnalyzer.instance.within { yield }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb
index 71316bbd243..6186c9ad1f4 100644
--- a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb
+++ b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb
@@ -55,18 +55,15 @@ module Gitlab
 
         attr_reader :mode, :size_limit, :compression_threshold
 
-        def initialize(
-          worker_class, job,
-          mode: Gitlab::CurrentSettings.sidekiq_job_limiter_mode,
-          compression_threshold: Gitlab::CurrentSettings.sidekiq_job_limiter_compression_threshold_bytes,
-          size_limit: Gitlab::CurrentSettings.sidekiq_job_limiter_limit_bytes
-        )
+        def initialize(worker_class, job)
           @worker_class = worker_class
           @job = job
 
-          set_mode(mode)
-          set_compression_threshold(compression_threshold)
-          set_size_limit(size_limit)
+          current_settings = Gitlab::CurrentSettings.current_application_settings
+
+          @mode = current_settings.sidekiq_job_limiter_mode
+          @compression_threshold = current_settings.sidekiq_job_limiter_compression_threshold_bytes
+          @size_limit = current_settings.sidekiq_job_limiter_limit_bytes
         end
 
         def validate!
@@ -90,30 +87,6 @@ module Gitlab
 
         private
 
-        def set_mode(mode)
-          @mode = (mode || TRACK_MODE).to_s.strip
-          unless MODES.include?(@mode)
-            ::Sidekiq.logger.warn "Invalid Sidekiq size limiter mode: #{@mode}. Fallback to #{TRACK_MODE} mode."
-            @mode = TRACK_MODE
-          end
-        end
-
-        def set_compression_threshold(compression_threshold)
-          @compression_threshold = (compression_threshold || DEFAULT_COMPRESSION_THRESHOLD_BYTES).to_i
-          if @compression_threshold <= 0
-            ::Sidekiq.logger.warn "Invalid Sidekiq size limiter compression threshold: #{@compression_threshold}"
-            @compression_threshold = DEFAULT_COMPRESSION_THRESHOLD_BYTES
-          end
-        end
-
-        def set_size_limit(size_limit)
-          @size_limit = (size_limit || DEFAULT_SIZE_LIMIT).to_i
-          if @size_limit < 0
-            ::Sidekiq.logger.warn "Invalid Sidekiq size limiter limit: #{@size_limit}"
-            @size_limit = DEFAULT_SIZE_LIMIT
-          end
-        end
-
         def exceed_limit_error(job_args)
           ExceedLimitError.new(@worker_class, job_args.bytesize, @size_limit).tap do |exception|
             # This should belong to Gitlab::ErrorTracking. We'll remove this
diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb
index 623fdd89456..fbf2718d718 100644
--- a/lib/gitlab/sidekiq_status.rb
+++ b/lib/gitlab/sidekiq_status.rb
@@ -7,12 +7,16 @@ module Gitlab
   # To check if a job has been completed, simply pass the job ID to the
   # `completed?` method:
   #
-  #     job_id = SomeWorker.perform_async(...)
+  #     job_id = SomeWorker.with_status.perform_async(...)
   #
   #     if Gitlab::SidekiqStatus.completed?(job_id)
   #       ...
   #     end
   #
+  # If you do not use `with_status`, and the worker class does not declare
+  # `status_expiration` in its `sidekiq_options`, then this status will not be
+  # stored.
+  #
   # For each job ID registered a separate key is stored in Redis, making lookups
   # much faster than using Sidekiq's built-in job finding/status API. These keys
   # expire after a certain period of time to prevent storing too many keys in
diff --git a/lib/gitlab/slash_commands/result.rb b/lib/gitlab/slash_commands/result.rb
index a66a2e0726b..d488606120f 100644
--- a/lib/gitlab/slash_commands/result.rb
+++ b/lib/gitlab/slash_commands/result.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Naming/FileName
 # frozen_string_literal: true
 
 module Gitlab
@@ -5,3 +6,5 @@ module Gitlab
     Result = Struct.new(:type, :message)
   end
 end
+
+# rubocop:enable Naming/FileName
diff --git a/lib/gitlab/spamcheck/client.rb b/lib/gitlab/spamcheck/client.rb
index df6d3eb7d0a..925ca44dfc9 100644
--- a/lib/gitlab/spamcheck/client.rb
+++ b/lib/gitlab/spamcheck/client.rb
@@ -5,6 +5,7 @@ module Gitlab
   module Spamcheck
     class Client
       include ::Spam::SpamConstants
+
       DEFAULT_TIMEOUT_SECS = 2
 
       VERDICT_MAPPING = {
@@ -27,12 +28,7 @@ module Gitlab
         # connect with Spamcheck
         @endpoint_url = @endpoint_url.gsub(%r(^grpc:\/\/), '')
 
-        @creds =
-          if Rails.env.development? || Rails.env.test?
-            :this_channel_is_insecure
-          else
-            GRPC::Core::ChannelCredentials.new
-          end
+        @creds = stub_creds
       end
 
       def issue_spam?(spam_issue:, user:, context: {})
@@ -73,6 +69,8 @@ module Gitlab
         user_pb.emails << build_email(user.email, user.confirmed?)
 
         user.emails.each do |email|
+          next if email.user_primary_email?
+
           user_pb.emails << build_email(email.email, email.confirmed?)
         end
 
@@ -98,6 +96,14 @@ module Gitlab
                                         nanos: ar_timestamp.to_time.nsec)
       end
 
+      def stub_creds
+        if Rails.env.development? || Rails.env.test?
+          :this_channel_is_insecure
+        else
+          GRPC::Core::ChannelCredentials.new ::Gitlab::X509::Certificate.ca_certs_bundle
+        end
+      end
+
       def grpc_client
         @grpc_client ||= ::Spamcheck::SpamcheckService::Stub.new(@endpoint_url, @creds,
                                                         interceptors: interceptors,
diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb
index 9b6bae12057..4f6d25097e2 100644
--- a/lib/gitlab/subscription_portal.rb
+++ b/lib/gitlab/subscription_portal.rb
@@ -4,11 +4,7 @@ module Gitlab
   module SubscriptionPortal
     def self.default_subscriptions_url
       if ::Gitlab.dev_or_test_env?
-        if Feature.enabled?(:new_customersdot_staging_url, default_enabled: :yaml)
-          'https://customers.staging.gitlab.com'
-        else
-          'https://customers.stg.gitlab.com'
-        end
+        'https://customers.staging.gitlab.com'
       else
         'https://customers.gitlab.com'
       end
@@ -43,7 +39,7 @@ module Gitlab
     end
 
     def self.subscriptions_plans_url
-      "#{self.subscriptions_url}/plans"
+      Gitlab::Saas.about_pricing_url
     end
 
     def self.subscriptions_gitlab_plans_url
diff --git a/lib/gitlab/template_parser/ast.rb b/lib/gitlab/template_parser/ast.rb
index 89318ee0d68..c6a5f66c377 100644
--- a/lib/gitlab/template_parser/ast.rb
+++ b/lib/gitlab/template_parser/ast.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Naming/FileName
 # frozen_string_literal: true
 
 module Gitlab
@@ -155,3 +156,5 @@ module Gitlab
     end
   end
 end
+
+# rubocop:enable Naming/FileName
diff --git a/lib/gitlab/testing/request_inspector_middleware.rb b/lib/gitlab/testing/request_inspector_middleware.rb
index 36cdfebcc28..3cbe97cd84c 100644
--- a/lib/gitlab/testing/request_inspector_middleware.rb
+++ b/lib/gitlab/testing/request_inspector_middleware.rb
@@ -9,6 +9,8 @@ module Gitlab
       @@logged_requests = Concurrent::Array.new
       @@inject_headers = Concurrent::Hash.new
 
+      Request = Struct.new(:url, :status_code, :request_headers, :response_headers, :body, keyword_init: true)
+
       # Resets the current request log and starts logging requests
       def self.log_requests!(headers = {})
         @@inject_headers.replace(headers)
@@ -40,7 +42,7 @@ module Gitlab
         full_body = +''
         body.each { |b| full_body << b }
 
-        request = OpenStruct.new(
+        request = Request.new(
           url: url,
           status_code: status,
           request_headers: request_headers,
diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb
index f4fbea50515..ec032cf2d3c 100644
--- a/lib/gitlab/tracking.rb
+++ b/lib/gitlab/tracking.rb
@@ -6,38 +6,37 @@ module Gitlab
 
     class << self
       def enabled?
-        Gitlab::CurrentSettings.snowplow_enabled?
+        snowplow_micro_enabled? || Gitlab::CurrentSettings.snowplow_enabled?
       end
 
       def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists
         contexts = [Tracking::StandardContext.new(project: project, user: user, namespace: namespace, **extra).to_context, *context]
 
         snowplow.event(category, action, label: label, property: property, value: value, context: contexts)
-        product_analytics.event(category, action, label: label, property: property, value: value, context: contexts)
       rescue StandardError => error
         Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action)
       end
 
       def options(group)
-        additional_features = Feature.enabled?(:additional_snowplow_tracking, group)
-        {
-          namespace: SNOWPLOW_NAMESPACE,
-          hostname: Gitlab::CurrentSettings.snowplow_collector_hostname,
-          cookie_domain: Gitlab::CurrentSettings.snowplow_cookie_domain,
-          app_id: Gitlab::CurrentSettings.snowplow_app_id,
-          form_tracking: additional_features,
-          link_click_tracking: additional_features
-        }.transform_keys! { |key| key.to_s.camelize(:lower).to_sym }
+        snowplow.options(group)
+      end
+
+      def collector_hostname
+        snowplow.hostname
       end
 
       private
 
       def snowplow
-        @snowplow ||= Gitlab::Tracking::Destinations::Snowplow.new
+        @snowplow ||= if snowplow_micro_enabled?
+                        Gitlab::Tracking::Destinations::SnowplowMicro.new
+                      else
+                        Gitlab::Tracking::Destinations::Snowplow.new
+                      end
       end
 
-      def product_analytics
-        @product_analytics ||= Gitlab::Tracking::Destinations::ProductAnalytics.new
+      def snowplow_micro_enabled?
+        Rails.env.development? && Gitlab::Utils.to_boolean(ENV['SNOWPLOW_MICRO_ENABLE'])
       end
     end
   end
diff --git a/lib/gitlab/tracking/destinations/product_analytics.rb b/lib/gitlab/tracking/destinations/product_analytics.rb
deleted file mode 100644
index cacedbc5b83..00000000000
--- a/lib/gitlab/tracking/destinations/product_analytics.rb
+++ /dev/null
@@ -1,41 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Tracking
-    module Destinations
-      class ProductAnalytics < Base
-        extend ::Gitlab::Utils::Override
-        include ::Gitlab::Utils::StrongMemoize
-
-        override :event
-        def event(category, action, label: nil, property: nil, value: nil, context: nil)
-          return unless event_allowed?(category, action)
-          return unless enabled?
-
-          tracker.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i)
-        end
-
-        private
-
-        def event_allowed?(category, action)
-          category == 'epics' && action == 'promote'
-        end
-
-        def enabled?
-          Feature.enabled?(:product_analytics_tracking, type: :ops) &&
-            Gitlab::CurrentSettings.usage_ping_enabled? &&
-            Gitlab::CurrentSettings.self_monitoring_project_id.present?
-        end
-
-        def tracker
-          @tracker ||= SnowplowTracker::Tracker.new(
-            SnowplowTracker::AsyncEmitter.new(::ProductAnalytics::Tracker::COLLECTOR_URL, protocol: Gitlab.config.gitlab.protocol),
-            SnowplowTracker::Subject.new,
-            Gitlab::Tracking::SNOWPLOW_NAMESPACE,
-            Gitlab::CurrentSettings.self_monitoring_project_id.to_s
-          )
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/tracking/destinations/snowplow.rb b/lib/gitlab/tracking/destinations/snowplow.rb
index 07a53b0892b..5596e9acd30 100644
--- a/lib/gitlab/tracking/destinations/snowplow.rb
+++ b/lib/gitlab/tracking/destinations/snowplow.rb
@@ -16,25 +16,53 @@ module Gitlab
           increment_total_events_counter
         end
 
+        def options(group)
+          additional_features = Feature.enabled?(:additional_snowplow_tracking, group, type: :ops)
+          {
+            namespace: Gitlab::Tracking::SNOWPLOW_NAMESPACE,
+            hostname: hostname,
+            cookie_domain: cookie_domain,
+            app_id: app_id,
+            form_tracking: additional_features,
+            link_click_tracking: additional_features
+          }.transform_keys! { |key| key.to_s.camelize(:lower).to_sym }
+        end
+
+        def hostname
+          Gitlab::CurrentSettings.snowplow_collector_hostname
+        end
+
         private
 
         def enabled?
           Gitlab::Tracking.enabled?
         end
 
+        def app_id
+          Gitlab::CurrentSettings.snowplow_app_id
+        end
+
+        def protocol
+          'https'
+        end
+
+        def cookie_domain
+          Gitlab::CurrentSettings.snowplow_cookie_domain
+        end
+
         def tracker
           @tracker ||= SnowplowTracker::Tracker.new(
             emitter,
             SnowplowTracker::Subject.new,
             Gitlab::Tracking::SNOWPLOW_NAMESPACE,
-            Gitlab::CurrentSettings.snowplow_app_id
+            app_id
           )
         end
 
         def emitter
           SnowplowTracker::AsyncEmitter.new(
-            Gitlab::CurrentSettings.snowplow_collector_hostname,
-            protocol: 'https',
+            hostname,
+            protocol: protocol,
             on_success: method(:increment_successful_events_emissions),
             on_failure: method(:failure_callback)
           )
@@ -68,8 +96,6 @@ module Gitlab
         end
 
         def log_failures(failures)
-          hostname = Gitlab::CurrentSettings.snowplow_collector_hostname
-
           failures.each do |failure|
             Gitlab::AppLogger.error("#{failure["se_ca"]} #{failure["se_ac"]} failed to be reported to collector at #{hostname}")
           end
diff --git a/lib/gitlab/tracking/destinations/snowplow_micro.rb b/lib/gitlab/tracking/destinations/snowplow_micro.rb
new file mode 100644
index 00000000000..b818d349a6d
--- /dev/null
+++ b/lib/gitlab/tracking/destinations/snowplow_micro.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+#
+module Gitlab
+  module Tracking
+    module Destinations
+      class SnowplowMicro < Snowplow
+        include ::Gitlab::Utils::StrongMemoize
+        extend ::Gitlab::Utils::Override
+
+        DEFAULT_URI = 'http://localhost:9090'
+
+        override :options
+        def options(group)
+          super.update(
+            protocol: uri.scheme,
+            port: uri.port,
+            force_secure_tracker: false
+          )
+        end
+
+        override :hostname
+        def hostname
+          "#{uri.host}:#{uri.port}"
+        end
+
+        private
+
+        def uri
+          strong_memoize(:snowplow_uri) do
+            uri = URI(ENV['SNOWPLOW_MICRO_URI'] || DEFAULT_URI)
+            uri = URI("http://#{ENV['SNOWPLOW_MICRO_URI']}") unless %w[http https].include?(uri.scheme)
+            uri
+          end
+        end
+
+        override :cookie_domain
+        def cookie_domain
+          '.gitlab.com'
+        end
+
+        override :protocol
+        def protocol
+          uri.scheme
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb
index df62e8bbbe6..837390b91fb 100644
--- a/lib/gitlab/tracking/standard_context.rb
+++ b/lib/gitlab/tracking/standard_context.rb
@@ -43,15 +43,8 @@ module Gitlab
           environment: environment,
           source: source,
           plan: plan,
-          extra: extra
-        }.merge(project_and_namespace)
-         .merge(user_data)
-      end
-
-      def project_and_namespace
-        return {} unless ::Feature.enabled?(:add_namespace_and_project_to_snowplow_tracking, default_enabled: :yaml)
-
-        {
+          extra: extra,
+          user_id: user&.id,
           namespace_id: namespace&.id,
           project_id: project_id
         }
@@ -60,10 +53,6 @@ module Gitlab
       def project_id
         project.is_a?(Integer) ? project : project&.id
       end
-
-      def user_data
-        ::Feature.enabled?(:add_actor_based_user_to_snowplow_tracking, user) ? { user_id: user&.id } : {}
-      end
     end
   end
 end
diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb
index 10822f943b6..2c5d76ba41d 100644
--- a/lib/gitlab/url_blocker.rb
+++ b/lib/gitlab/url_blocker.rb
@@ -164,15 +164,21 @@ module Gitlab
       end
 
       def parse_url(url)
-        raise Addressable::URI::InvalidURIError if multiline?(url)
-
-        Addressable::URI.parse(url)
+        Addressable::URI.parse(url).tap do |parsed_url|
+          raise Addressable::URI::InvalidURIError if multiline_blocked?(parsed_url)
+        end
       rescue Addressable::URI::InvalidURIError, URI::InvalidURIError
         raise BlockedUrlError, 'URI is invalid'
       end
 
-      def multiline?(url)
-        CGI.unescape(url.to_s) =~ /\n|\r/
+      def multiline_blocked?(parsed_url)
+        url = parsed_url.to_s
+
+        return true if url =~ /\n|\r/
+        # Google Cloud Storage uses a multi-line, encoded Signature query string
+        return false if %w(http https).include?(parsed_url.scheme&.downcase)
+
+        CGI.unescape(url) =~ /\n|\r/
       end
 
       def validate_port(port, ports)
diff --git a/lib/gitlab/usage/metric.rb b/lib/gitlab/usage/metric.rb
index 5b1ac189c13..24e044c5740 100644
--- a/lib/gitlab/usage/metric.rb
+++ b/lib/gitlab/usage/metric.rb
@@ -25,6 +25,10 @@ module Gitlab
         unflatten_key_path(intrumentation_object.instrumentation)
       end
 
+      def with_suggested_name
+        unflatten_key_path(intrumentation_object.suggested_name)
+      end
+
       private
 
       def unflatten_key_path(value)
diff --git a/lib/gitlab/usage/metrics/names_suggestions/generator.rb b/lib/gitlab/usage/metrics/names_suggestions/generator.rb
index b47dc5689d4..6dcbe5f5fe5 100644
--- a/lib/gitlab/usage/metrics/names_suggestions/generator.rb
+++ b/lib/gitlab/usage/metrics/names_suggestions/generator.rb
@@ -10,14 +10,18 @@ module Gitlab
               uncached_data.deep_stringify_keys.dig(*key_path.split('.'))
             end
 
-            def add_metric(metric, time_frame: 'none')
+            def add_metric(metric, time_frame: 'none', options: {})
               metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize
 
-              metric_class.new(time_frame: time_frame).suggested_name
+              metric_class.new(time_frame: time_frame, options: options).suggested_name
             end
 
             private
 
+            def instrumentation_metrics
+              ::Gitlab::UsageDataMetrics.suggested_names
+            end
+
             def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
               Gitlab::Usage::Metrics::NameSuggestion.for(:count, column: column, relation: relation)
             end
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index dd66f9133bb..20e526aeefa 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -45,23 +45,10 @@ module Gitlab
         clear_memoized
 
         with_finished_at(:recording_ce_finished_at) do
-          license_usage_data
-            .merge(system_usage_data_license)
-            .merge(system_usage_data_settings)
-            .merge(system_usage_data)
-            .merge(system_usage_data_monthly)
-            .merge(system_usage_data_weekly)
-            .merge(features_usage_data)
-            .merge(components_usage_data)
-            .merge(object_store_usage_data)
-            .merge(topology_usage_data)
-            .merge(usage_activity_by_stage)
-            .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, monthly_time_range_db_params))
-            .merge(analytics_unique_visits_data)
-            .merge(compliance_unique_visits_data)
-            .merge(search_unique_visits_data)
-            .merge(redis_hll_counters)
-            .deep_merge(aggregated_metrics_data)
+          usage_data = usage_data_metrics
+          usage_data = usage_data.with_indifferent_access.deep_merge(instrumentation_metrics.with_indifferent_access) if Feature.enabled?(:usage_data_instrumentation)
+
+          usage_data
         end
       end
 
@@ -309,9 +296,11 @@ module Gitlab
             version: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.container_registry_version }
           },
           database: {
-            adapter: alt_usage_data { Gitlab::Database.main.adapter_name },
-            version: alt_usage_data { Gitlab::Database.main.version },
-            pg_system_id: alt_usage_data { Gitlab::Database.main.system_id }
+            # rubocop: disable UsageData/LargeTable
+            adapter: alt_usage_data { ApplicationRecord.database.adapter_name },
+            version: alt_usage_data { ApplicationRecord.database.version },
+            pg_system_id: alt_usage_data { ApplicationRecord.database.system_id }
+            # rubocop: enable UsageData/LargeTable
           },
           mail: {
             smtp_server: alt_usage_data { ActionMailer::Base.smtp_settings[:address] }
@@ -549,7 +538,8 @@ module Gitlab
       # rubocop: disable CodeReuse/ActiveRecord
       def usage_activity_by_stage_manage(time_period)
         {
-          events: distinct_count(::Event.where(time_period), :author_id),
+          # rubocop: disable UsageData/LargeTable
+          events: stage_manage_events(time_period),
           groups: distinct_count(::GroupMember.where(time_period), :user_id),
           users_created: count(::User.where(time_period), start: minimum_id(User), finish: maximum_id(User)),
           omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' },
@@ -628,9 +618,9 @@ module Gitlab
           todos: distinct_count(::Todo.where(time_period), :author_id),
           service_desk_enabled_projects: distinct_count_service_desk_enabled_projects(time_period),
           service_desk_issues: count(::Issue.service_desk.where(time_period)),
-          projects_jira_active: distinct_count(::Project.with_active_integration(::Integrations::Jira) .where(time_period), :creator_id),
-          projects_jira_dvcs_cloud_active: distinct_count(::Project.with_active_integration(::Integrations::Jira) .with_jira_dvcs_cloud.where(time_period), :creator_id),
-          projects_jira_dvcs_server_active: distinct_count(::Project.with_active_integration(::Integrations::Jira) .with_jira_dvcs_server.where(time_period), :creator_id)
+          projects_jira_active: distinct_count(::Project.with_active_integration(::Integrations::Jira).where(time_period), :creator_id),
+          projects_jira_dvcs_cloud_active: distinct_count(::Project.with_active_integration(::Integrations::Jira).with_jira_dvcs_cloud.where(time_period), :creator_id),
+          projects_jira_dvcs_server_active: distinct_count(::Project.with_active_integration(::Integrations::Jira).with_jira_dvcs_server.where(time_period), :creator_id)
         }
       end
       # rubocop: enable CodeReuse/ActiveRecord
@@ -729,6 +719,44 @@ module Gitlab
 
       private
 
+      def stage_manage_events(time_period)
+        if time_period.empty?
+          Gitlab::Utils::UsageData::FALLBACK
+        else
+          # rubocop: disable CodeReuse/ActiveRecord
+          # rubocop: disable UsageData/LargeTable
+          start = ::Event.where(time_period).select(:id).order(created_at: :asc).first&.id
+          finish = ::Event.where(time_period).select(:id).order(created_at: :asc).first&.id
+          estimate_batch_distinct_count(::Event.where(time_period), :author_id, start: start, finish: finish)
+          # rubocop: enable UsageData/LargeTable
+          # rubocop: enable CodeReuse/ActiveRecord
+        end
+      end
+
+      def usage_data_metrics
+        license_usage_data
+          .merge(system_usage_data_license)
+          .merge(system_usage_data_settings)
+          .merge(system_usage_data)
+          .merge(system_usage_data_monthly)
+          .merge(system_usage_data_weekly)
+          .merge(features_usage_data)
+          .merge(components_usage_data)
+          .merge(object_store_usage_data)
+          .merge(topology_usage_data)
+          .merge(usage_activity_by_stage)
+          .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, monthly_time_range_db_params))
+          .merge(analytics_unique_visits_data)
+          .merge(compliance_unique_visits_data)
+          .merge(search_unique_visits_data)
+          .merge(redis_hll_counters)
+          .deep_merge(aggregated_metrics_data)
+      end
+
+      def instrumentation_metrics
+        Gitlab::UsageDataMetrics.uncached_data # rubocop:disable UsageData/LargeTable
+      end
+
       def metric_time_period(time_period)
         time_period.present? ? '28d' : 'none'
       end
@@ -805,7 +833,13 @@ module Gitlab
 
         Users::InProductMarketingEmail.tracks.keys.each_with_object({}) do |track, result|
           # rubocop: enable UsageData/LargeTable:
-          series_amount = Namespaces::InProductMarketingEmailsService::TRACKS[track.to_sym][:interval_days].count
+          series_amount =
+            if track.to_sym == Namespaces::InviteTeamEmailService::TRACK
+              0
+            else
+              Namespaces::InProductMarketingEmailsService::TRACKS[track.to_sym][:interval_days].count
+            end
+
           0.upto(series_amount - 1).map do |series|
             # When there is an error with the query and it's not the Hash we expect, we return what we got from `count`.
             sent_count = sent_emails.is_a?(Hash) ? sent_emails.fetch([track, series], 0) : sent_emails
@@ -881,7 +915,30 @@ module Gitlab
       end
 
       def projects_imported_count(from, time_period)
-        count(::Project.imported_from(from).where(time_period).where.not(import_type: nil)) # rubocop: disable CodeReuse/ActiveRecord
+        # rubocop:disable CodeReuse/ActiveRecord
+        relation = ::Project.imported_from(from).where.not(import_type: nil) # rubocop:disable UsageData/LargeTable
+        if time_period.empty?
+          count(relation)
+        else
+          @project_import_id ||= {}
+          start = time_period[:created_at].first
+          finish = time_period[:created_at].last
+
+          # can be nil values here if no records are in our range and it is possible the same instance
+          # is called with different time periods since it is passed in as a variable
+          unless @project_import_id.key?(start)
+            @project_import_id[start] = ::Project.select(:id).where(Project.arel_table[:created_at].gteq(start)) # rubocop:disable UsageData/LargeTable
+                                                 .order(created_at: :asc).limit(1).first&.id
+          end
+
+          unless @project_import_id.key?(finish)
+            @project_import_id[finish] = ::Project.select(:id).where(Project.arel_table[:created_at].lteq(finish)) # rubocop:disable UsageData/LargeTable
+                                                  .order(created_at: :desc).limit(1).first&.id
+          end
+
+          count(relation, start: @project_import_id[start], finish: @project_import_id[finish])
+        end
+        # rubocop:enable CodeReuse/ActiveRecord
       end
 
       def issue_imports(time_period)
diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml
index 99bdd3ca9e9..40922433635 100644
--- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml
+++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml
@@ -83,6 +83,10 @@
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
+- name: p_ci_templates_security_sast_iac_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
 - name: p_ci_templates_security_dast_runner_validation
   category: ci_templates
   redis_slot: ci_templates
@@ -267,6 +271,10 @@
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
+- name: p_ci_templates_jobs_sast_iac_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
 - name: p_ci_templates_jobs_secret_detection
   category: ci_templates
   redis_slot: ci_templates
@@ -447,6 +455,10 @@
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
+- name: p_ci_templates_implicit_jobs_sast_iac_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
 - name: p_ci_templates_implicit_jobs_secret_detection
   category: ci_templates
   redis_slot: ci_templates
@@ -503,6 +515,10 @@
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
+- name: p_ci_templates_implicit_security_sast_iac_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
 - name: p_ci_templates_implicit_security_dast_runner_validation
   category: ci_templates
   redis_slot: ci_templates
@@ -559,3 +575,7 @@
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
+- name: p_ci_templates_kaniko
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml
index d4a818f8fe0..d4bc060abf9 100644
--- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml
+++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml
@@ -249,3 +249,27 @@
   redis_slot: code_review
   category: code_review
   aggregation: weekly
+- name: i_code_review_widget_nothing_merge_click_new_file
+  redis_slot: code_review
+  category: code_review
+  aggregation: weekly
+- name: i_code_review_post_merge_delete_branch
+  redis_slot: code_review
+  category: code_review
+  aggregation: weekly
+- name: i_code_review_post_merge_click_revert
+  redis_slot: code_review
+  category: code_review
+  aggregation: weekly
+- name: i_code_review_post_merge_click_cherry_pick
+  redis_slot: code_review
+  category: code_review
+  aggregation: weekly
+- name: i_code_review_post_merge_submit_revert_modal
+  redis_slot: code_review
+  category: code_review
+  aggregation: weekly
+- name: i_code_review_post_merge_submit_cherry_pick_modal
+  redis_slot: code_review
+  category: code_review
+  aggregation: weekly
diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml
index 7f77fa8ee02..dff2c4f8d03 100644
--- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml
+++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml
@@ -119,6 +119,10 @@
   category: quickactions
   redis_slot: quickactions
   aggregation: weekly
+- name: i_quickactions_promote_to_incident
+  category: quickactions
+  redis_slot: quickactions
+  aggregation: weekly
 - name: i_quickactions_publish
   category: quickactions
   redis_slot: quickactions
diff --git a/lib/gitlab/usage_data_counters/vs_code_extension_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/vs_code_extension_activity_unique_counter.rb
deleted file mode 100644
index 703c4885b04..00000000000
--- a/lib/gitlab/usage_data_counters/vs_code_extension_activity_unique_counter.rb
+++ /dev/null
@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module UsageDataCounters
-    module VSCodeExtensionActivityUniqueCounter
-      VS_CODE_API_REQUEST_ACTION = 'i_code_review_user_vs_code_api_request'
-      VS_CODE_USER_AGENT_REGEX = /\Avs-code-gitlab-workflow/.freeze
-
-      class << self
-        def track_api_request_when_trackable(user_agent:, user:)
-          user_agent&.match?(VS_CODE_USER_AGENT_REGEX) && track_unique_action_by_user(VS_CODE_API_REQUEST_ACTION, user)
-        end
-
-        private
-
-        def track_unique_action_by_user(action, user)
-          return unless user
-
-          track_unique_action(action, user.id)
-        end
-
-        def track_unique_action(action, value)
-          Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(action, value)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/usage_data_counters/vscode_extension_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/vscode_extension_activity_unique_counter.rb
new file mode 100644
index 00000000000..703c4885b04
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/vscode_extension_activity_unique_counter.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module UsageDataCounters
+    module VSCodeExtensionActivityUniqueCounter
+      VS_CODE_API_REQUEST_ACTION = 'i_code_review_user_vs_code_api_request'
+      VS_CODE_USER_AGENT_REGEX = /\Avs-code-gitlab-workflow/.freeze
+
+      class << self
+        def track_api_request_when_trackable(user_agent:, user:)
+          user_agent&.match?(VS_CODE_USER_AGENT_REGEX) && track_unique_action_by_user(VS_CODE_API_REQUEST_ACTION, user)
+        end
+
+        private
+
+        def track_unique_action_by_user(action, user)
+          return unless user
+
+          track_unique_action(action, user.id)
+        end
+
+        def track_unique_action(action, value)
+          Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(action, value)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage_data_metrics.rb b/lib/gitlab/usage_data_metrics.rb
index 1ef201121d9..48f695d5db1 100644
--- a/lib/gitlab/usage_data_metrics.rb
+++ b/lib/gitlab/usage_data_metrics.rb
@@ -5,7 +5,17 @@ module Gitlab
     class << self
       # Build the Usage Ping JSON payload from metrics YAML definitions which have instrumentation class set
       def uncached_data
-        ::Gitlab::Usage::Metric.all.map(&:with_value).reduce({}, :deep_merge)
+        build_payload(:with_value)
+      end
+
+      def suggested_names
+        build_payload(:with_suggested_name)
+      end
+
+      private
+
+      def build_payload(method_symbol)
+        ::Gitlab::Usage::Metric.all.map(&method_symbol).reduce({}, :deep_merge)
       end
     end
   end
diff --git a/lib/gitlab/usage_data_non_sql_metrics.rb b/lib/gitlab/usage_data_non_sql_metrics.rb
index 1ff4588d091..be5a571fb82 100644
--- a/lib/gitlab/usage_data_non_sql_metrics.rb
+++ b/lib/gitlab/usage_data_non_sql_metrics.rb
@@ -6,13 +6,16 @@ module Gitlab
 
     class << self
       def uncached_data
-        super.with_indifferent_access.deep_merge(instrumentation_metrics_queries.with_indifferent_access)
+        # instrumentation_metrics is already included with feature flag enabled
+        return super if Feature.enabled?(:usage_data_instrumentation)
+
+        super.with_indifferent_access.deep_merge(instrumentation_metrics.with_indifferent_access)
       end
 
-      def add_metric(metric, time_frame: 'none')
+      def add_metric(metric, time_frame: 'none', options: {})
         metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize
 
-        metric_class.new(time_frame: time_frame).instrumentation
+        metric_class.new(time_frame: time_frame, options: options).instrumentation
       end
 
       def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
@@ -50,7 +53,7 @@ module Gitlab
 
       private
 
-      def instrumentation_metrics_queries
+      def instrumentation_metrics
         ::Gitlab::Usage::Metric.all.map(&:with_instrumentation).reduce({}, :deep_merge)
       end
     end
diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb
index f64da2fbf13..f543b29e43f 100644
--- a/lib/gitlab/usage_data_queries.rb
+++ b/lib/gitlab/usage_data_queries.rb
@@ -6,13 +6,16 @@ module Gitlab
   class UsageDataQueries < UsageData
     class << self
       def uncached_data
-        super.with_indifferent_access.deep_merge(instrumentation_metrics_queries.with_indifferent_access)
+        # instrumentation_metrics is already included with feature flag enabled
+        return super if Feature.enabled?(:usage_data_instrumentation)
+
+        super.with_indifferent_access.deep_merge(instrumentation_metrics.with_indifferent_access)
       end
 
-      def add_metric(metric, time_frame: 'none')
+      def add_metric(metric, time_frame: 'none', options: {})
         metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize
 
-        metric_class.new(time_frame: time_frame).instrumentation
+        metric_class.new(time_frame: time_frame, options: options).instrumentation
       end
 
       def count(relation, column = nil, *args, **kwargs)
@@ -71,7 +74,7 @@ module Gitlab
 
       private
 
-      def instrumentation_metrics_queries
+      def instrumentation_metrics
         ::Gitlab::Usage::Metric.all.map(&:with_instrumentation).reduce({}, :deep_merge)
       end
     end
diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb
index d46210f6efe..77f04929661 100644
--- a/lib/gitlab/utils/usage_data.rb
+++ b/lib/gitlab/utils/usage_data.rb
@@ -43,11 +43,16 @@ module Gitlab
       HISTOGRAM_FALLBACK = { '-1' => -1 }.freeze
       DISTRIBUTED_HLL_FALLBACK = -2
       MAX_BUCKET_SIZE = 100
+      INSTRUMENTATION_CLASS_FALLBACK = -100
+
+      def add_metric(metric, time_frame: 'none', options: {})
+        # Results of this method should be overwritten by instrumentation class values
+        # -100 indicates the metric was not properly merged.
+        return INSTRUMENTATION_CLASS_FALLBACK if Feature.enabled?(:usage_data_instrumentation)
 
-      def add_metric(metric, time_frame: 'none')
         metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize
 
-        metric_class.new(time_frame: time_frame).value
+        metric_class.new(time_frame: time_frame, options: options).value
       end
 
       def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
diff --git a/lib/gitlab/webpack/file_loader.rb b/lib/gitlab/webpack/file_loader.rb
new file mode 100644
index 00000000000..35ecb1eb4ed
--- /dev/null
+++ b/lib/gitlab/webpack/file_loader.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+
+module Gitlab
+  module Webpack
+    class FileLoader
+      class BaseError < StandardError
+        attr_reader :original_error, :uri
+
+        def initialize(uri, orig)
+          super orig.message
+          @uri = uri.to_s
+          @original_error = orig
+        end
+      end
+
+      StaticLoadError = Class.new(BaseError)
+      DevServerLoadError = Class.new(BaseError)
+      DevServerSSLError = Class.new(BaseError)
+
+      def self.load(path)
+        if Gitlab.config.webpack.dev_server.enabled
+          self.load_from_dev_server(path)
+        else
+          self.load_from_static(path)
+        end
+      end
+
+      def self.load_from_dev_server(path)
+        host = Gitlab.config.webpack.dev_server.host
+        port = Gitlab.config.webpack.dev_server.port
+        scheme = Gitlab.config.webpack.dev_server.https ? 'https' : 'http'
+        uri = Addressable::URI.new(scheme: scheme, host: host, port: port, path: self.dev_server_path(path))
+
+        # localhost could be blocked via Gitlab::HTTP
+        response = HTTParty.get(uri.to_s, verify: false) # rubocop:disable Gitlab/HTTParty
+
+        return response.body if response.code == 200
+
+        raise "HTTP error #{response.code}"
+      rescue OpenSSL::SSL::SSLError, EOFError => e
+        raise DevServerSSLError.new(uri, e)
+      rescue StandardError => e
+        raise DevServerLoadError.new(uri, e)
+      end
+
+      def self.load_from_static(path)
+        file_uri = ::Rails.root.join(
+          Gitlab.config.webpack.output_dir,
+          path
+        )
+
+        File.read(file_uri)
+      rescue StandardError => e
+        raise StaticLoadError.new(file_uri, e)
+      end
+
+      def self.dev_server_path(path)
+        "/#{Gitlab.config.webpack.public_path}/#{path}"
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/webpack/graphql_known_operations.rb b/lib/gitlab/webpack/graphql_known_operations.rb
new file mode 100644
index 00000000000..7945513667c
--- /dev/null
+++ b/lib/gitlab/webpack/graphql_known_operations.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Webpack
+    class GraphqlKnownOperations
+      class << self
+        include Gitlab::Utils::StrongMemoize
+
+        def clear_memoization!
+          clear_memoization(:graphql_known_operations)
+        end
+
+        def load
+          strong_memoize(:graphql_known_operations) do
+            data = ::Gitlab::Webpack::FileLoader.load("graphql_known_operations.yml")
+
+            YAML.safe_load(data)
+          rescue StandardError
+            []
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/webpack/manifest.rb b/lib/gitlab/webpack/manifest.rb
index b73c2ebb578..06cddc23134 100644
--- a/lib/gitlab/webpack/manifest.rb
+++ b/lib/gitlab/webpack/manifest.rb
@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require 'net/http'
-require 'uri'
-
 module Gitlab
   module Webpack
     class Manifest
@@ -78,49 +75,16 @@ module Gitlab
         end
 
         def load_manifest
-          data = if Gitlab.config.webpack.dev_server.enabled
-                   load_dev_server_manifest
-                 else
-                   load_static_manifest
-                 end
+          data = Gitlab::Webpack::FileLoader.load(Gitlab.config.webpack.manifest_filename)
 
           Gitlab::Json.parse(data)
-        end
-
-        def load_dev_server_manifest
-          host = Gitlab.config.webpack.dev_server.host
-          port = Gitlab.config.webpack.dev_server.port
-          scheme = Gitlab.config.webpack.dev_server.https ? 'https' : 'http'
-          uri = Addressable::URI.new(scheme: scheme, host: host, port: port, path: dev_server_path)
-
-          # localhost could be blocked via Gitlab::HTTP
-          response = HTTParty.get(uri.to_s, verify: false) # rubocop:disable Gitlab/HTTParty
-
-          return response.body if response.code == 200
-
-          raise "HTTP error #{response.code}"
-        rescue OpenSSL::SSL::SSLError, EOFError => e
+        rescue Gitlab::Webpack::FileLoader::StaticLoadError => e
+          raise ManifestLoadError.new("Could not load compiled manifest from #{e.uri}.\n\nHave you run `rake gitlab:assets:compile`?", e.original_error)
+        rescue Gitlab::Webpack::FileLoader::DevServerSSLError => e
           ssl_status = Gitlab.config.webpack.dev_server.https ? ' over SSL' : ''
-          raise ManifestLoadError.new("Could not connect to webpack-dev-server at #{uri}#{ssl_status}.\n\nIs SSL enabled? Check that settings in `gitlab.yml` and webpack-dev-server match.", e)
-        rescue StandardError => e
-          raise ManifestLoadError.new("Could not load manifest from webpack-dev-server at #{uri}.\n\nIs webpack-dev-server running? Try running `gdk status webpack` or `gdk tail webpack`.", e)
-        end
-
-        def load_static_manifest
-          File.read(static_manifest_path)
-        rescue StandardError => e
-          raise ManifestLoadError.new("Could not load compiled manifest from #{static_manifest_path}.\n\nHave you run `rake gitlab:assets:compile`?", e)
-        end
-
-        def static_manifest_path
-          ::Rails.root.join(
-            Gitlab.config.webpack.output_dir,
-            Gitlab.config.webpack.manifest_filename
-          )
-        end
-
-        def dev_server_path
-          "/#{Gitlab.config.webpack.public_path}/#{Gitlab.config.webpack.manifest_filename}"
+          raise ManifestLoadError.new("Could not connect to webpack-dev-server at #{e.uri}#{ssl_status}.\n\nIs SSL enabled? Check that settings in `gitlab.yml` and webpack-dev-server match.", e.original_error)
+        rescue Gitlab::Webpack::FileLoader::DevServerLoadError => e
+          raise ManifestLoadError.new("Could not load manifest from webpack-dev-server at #{e.uri}.\n\nIs webpack-dev-server running? Try running `gdk status webpack` or `gdk tail webpack`.", e.original_error)
         end
       end
     end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index c40aa2273aa..3a905a2e1c5 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -8,6 +8,7 @@ require 'uri'
 module Gitlab
   class Workhorse
     SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data'
+    SEND_DEPENDENCY_CONTENT_TYPE_HEADER = 'Workhorse-Proxy-Content-Type'
     VERSION_FILE = 'GITLAB_WORKHORSE_VERSION'
     INTERNAL_API_CONTENT_TYPE = 'application/vnd.gitlab-workhorse+json'
     INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request'
@@ -170,9 +171,9 @@ module Gitlab
         ]
       end
 
-      def send_dependency(token, url)
+      def send_dependency(headers, url)
         params = {
-          'Header' => { Authorization: ["Bearer #{token}"] },
+          'Header' => headers,
           'Url' => url
         }
 
diff --git a/lib/gitlab/x509/certificate.rb b/lib/gitlab/x509/certificate.rb
index c7289a51b49..752f3c6b004 100644
--- a/lib/gitlab/x509/certificate.rb
+++ b/lib/gitlab/x509/certificate.rb
@@ -19,6 +19,10 @@ module Gitlab
         ca_certs.map(&:to_pem).join('\n') unless ca_certs.blank?
       end
 
+      class << self
+        include ::Gitlab::Utils::StrongMemoize
+      end
+
       def self.from_strings(key_string, cert_string, ca_certs_string = nil)
         key = OpenSSL::PKey::RSA.new(key_string)
         cert = OpenSSL::X509::Certificate.new(cert_string)
@@ -33,6 +37,30 @@ module Gitlab
         from_strings(File.read(key_path), File.read(cert_path), ca_certs_string)
       end
 
+      # Returns all top-level, readable files in the default CA cert directory
+      def self.ca_certs_paths
+        cert_paths = Dir["#{OpenSSL::X509::DEFAULT_CERT_DIR}/*"].select do |path|
+          !File.directory?(path) && File.readable?(path)
+        end
+        cert_paths << OpenSSL::X509::DEFAULT_CERT_FILE if File.exist? OpenSSL::X509::DEFAULT_CERT_FILE
+        cert_paths
+      end
+
+      # Returns a concatenated array of Strings, each being a PEM-coded CA certificate.
+      def self.ca_certs_bundle
+        strong_memoize(:ca_certs_bundle) do
+          ca_certs_paths.flat_map do |cert_file|
+            load_ca_certs_bundle(File.read(cert_file))
+          rescue OpenSSL::OpenSSLError => e
+            Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, cert_file: cert_file)
+          end.uniq.join("\n")
+        end
+      end
+
+      def self.reset_ca_certs_bundle
+        clear_memoization(:ca_certs_bundle)
+      end
+
       # Returns an array of OpenSSL::X509::Certificate objects, empty array if none found
       #
       # Ruby OpenSSL::X509::Certificate.new will only load the first
diff --git a/lib/gitlab/zentao/client.rb b/lib/gitlab/zentao/client.rb
index bdfa4b3a308..8acfb4913f3 100644
--- a/lib/gitlab/zentao/client.rb
+++ b/lib/gitlab/zentao/client.rb
@@ -15,10 +15,8 @@ module Gitlab
       end
 
       def ping
-        response = fetch_product(zentao_product_xid)
-
-        active = response.fetch('deleted') == '0' rescue false
-
+        response = fetch_product(zentao_product_xid) rescue {}
+        active = response['deleted'] == '0'
         if active
           { success: true }
         else
@@ -31,25 +29,30 @@ module Gitlab
       end
 
       def fetch_issues(params = {})
-        get("products/#{zentao_product_xid}/issues",
-            params.reverse_merge(page: 1, limit: 20))
+        get("products/#{zentao_product_xid}/issues", params)
       end
 
       def fetch_issue(issue_id)
+        raise Gitlab::Zentao::Client::Error, 'invalid issue id' unless issue_id_pattern.match(issue_id)
+
         get("issues/#{issue_id}")
       end
 
       private
 
+      def issue_id_pattern
+        /\A\S+-\d+\z/
+      end
+
       def get(path, params = {})
         options = { headers: headers, query: params }
         response = Gitlab::HTTP.get(url(path), options)
 
-        return {} unless response.success?
+        raise Gitlab::Zentao::Client::Error, 'request error' unless response.success?
 
         Gitlab::Json.parse(response.body)
       rescue JSON::ParserError
-        {}
+        raise Gitlab::Zentao::Client::Error, 'invalid response format'
       end
 
       def url(path)
diff --git a/lib/gitlab/zentao/query.rb b/lib/gitlab/zentao/query.rb
new file mode 100644
index 00000000000..d40ee80939a
--- /dev/null
+++ b/lib/gitlab/zentao/query.rb
@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Zentao
+    class Query
+      STATUSES = %w[all opened closed].freeze
+      ISSUES_DEFAULT_LIMIT = 20
+      ISSUES_MAX_LIMIT = 50
+
+      def initialize(integration, params)
+        @client = Client.new(integration)
+        @params = params
+      end
+
+      def issues
+        issues_response = client.fetch_issues(query_options)
+        return [] if issues_response.blank?
+
+        Kaminari.paginate_array(
+          issues_response['issues'],
+          limit: issues_response['limit'],
+          total_count: issues_response['total']
+        )
+      end
+
+      def issue
+        issue_response = client.fetch_issue(params[:id])
+        issue_response['issue']
+      end
+
+      private
+
+      attr_reader :client, :params
+
+      def query_options
+        {
+          order: query_order,
+          status: query_status,
+          labels: query_labels,
+          page: query_page,
+          limit: query_limit,
+          search: query_search
+        }
+      end
+
+      def query_page
+        params[:page].presence || 1
+      end
+
+      def query_limit
+        limit = params[:limit].presence || ISSUES_DEFAULT_LIMIT
+        [limit.to_i, ISSUES_MAX_LIMIT].min
+      end
+
+      def query_search
+        params[:search] || ''
+      end
+
+      def query_order
+        key, order = params['sort'].to_s.split('_', 2)
+        zentao_key = (key == 'created' ? 'openedDate' : 'lastEditedDate')
+        zentao_order = (order == 'asc' ? 'asc' : 'desc')
+
+        "#{zentao_key}_#{zentao_order}"
+      end
+
+      def query_status
+        return params[:state] if params[:state].present? && params[:state].in?(STATUSES)
+
+        'opened'
+      end
+
+      def query_labels
+        (params[:labels].presence || []).join(',')
+      end
+    end
+  end
+end
diff --git a/lib/object_storage/config.rb b/lib/object_storage/config.rb
index 82d9fc07043..056e22278dd 100644
--- a/lib/object_storage/config.rb
+++ b/lib/object_storage/config.rb
@@ -12,16 +12,6 @@ module ObjectStorage
       @options = options.to_hash.deep_symbolize_keys
     end
 
-    def load_provider
-      if aws?
-        require 'fog/aws'
-      elsif google?
-        require 'fog/google'
-      elsif azure?
-        require 'fog/azurerm'
-      end
-    end
-
     def credentials
       @credentials ||= options[:connection] || {}
     end
diff --git a/lib/security/ci_configuration/sast_iac_build_action.rb b/lib/security/ci_configuration/sast_iac_build_action.rb
new file mode 100644
index 00000000000..ddc7db282ef
--- /dev/null
+++ b/lib/security/ci_configuration/sast_iac_build_action.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Security
+  module CiConfiguration
+    class SastIacBuildAction < BaseBuildAction
+      private
+
+      def update_existing_content!
+        @existing_gitlab_ci_content['include'] = generate_includes
+      end
+
+      def template
+        return 'Auto-DevOps.gitlab-ci.yml' if @auto_devops_enabled
+
+        'Security/SAST-IaC.latest.gitlab-ci.yml'
+      end
+    end
+  end
+end
diff --git a/lib/sidebars/groups/menus/customer_relations_menu.rb b/lib/sidebars/groups/menus/customer_relations_menu.rb
new file mode 100644
index 00000000000..fdbbd662ad6
--- /dev/null
+++ b/lib/sidebars/groups/menus/customer_relations_menu.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Sidebars
+  module Groups
+    module Menus
+      class CustomerRelationsMenu < ::Sidebars::Menu
+        override :configure_menu_items
+        def configure_menu_items
+          add_item(contacts_menu_item) if can_read_contact?
+          add_item(organizations_menu_item) if can_read_organization?
+
+          true
+        end
+
+        override :title
+        def title
+          _('Customer relations')
+        end
+
+        override :sprite_icon
+        def sprite_icon
+          'users'
+        end
+
+        override :render?
+        def render?
+          can_read_contact? || can_read_organization?
+        end
+
+        private
+
+        def contacts_menu_item
+          ::Sidebars::MenuItem.new(
+            title: _('Contacts'),
+            link: contacts_group_crm_index_path(context.group),
+            active_routes: { path: 'groups/crm#contacts' },
+            item_id: :crm_contacts
+          )
+        end
+
+        def organizations_menu_item
+          ::Sidebars::MenuItem.new(
+            title: _('Organizations'),
+            link: organizations_group_crm_index_path(context.group),
+            active_routes: { path: 'groups/crm#organizations' },
+            item_id: :crm_organizations
+          )
+        end
+
+        def can_read_contact?
+          can?(context.current_user, :read_crm_contact, context.group)
+        end
+
+        def can_read_organization?
+          can?(context.current_user, :read_crm_organization, context.group)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/sidebars/groups/menus/invite_team_members_menu.rb b/lib/sidebars/groups/menus/invite_team_members_menu.rb
new file mode 100644
index 00000000000..0779b696061
--- /dev/null
+++ b/lib/sidebars/groups/menus/invite_team_members_menu.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Sidebars
+  module Groups
+    module Menus
+      class InviteTeamMembersMenu < ::Sidebars::Menu
+        override :title
+        def title
+          s_('InviteMember|Invite members')
+        end
+
+        override :render?
+        def render?
+          can?(context.current_user, :admin_group_member, context.group) && all_valid_members.size <= 1
+        end
+
+        override :menu_partial
+        def menu_partial
+          'groups/invite_members_side_nav_link'
+        end
+
+        override :menu_partial_options
+        def menu_partial_options
+          {
+            group: context.group,
+            title: title,
+            sidebar_menu: self
+          }
+        end
+
+        override :extra_nav_link_html_options
+        def extra_nav_link_html_options
+          {
+            'data-test-id': 'side-nav-invite-members'
+          }
+        end
+
+        private
+
+        def all_valid_members
+          GroupMembersFinder.new(context.group, context.current_user).execute
+        end
+      end
+    end
+  end
+end
diff --git a/lib/sidebars/groups/menus/packages_registries_menu.rb b/lib/sidebars/groups/menus/packages_registries_menu.rb
index e81e9355e7e..46fcec9f7b8 100644
--- a/lib/sidebars/groups/menus/packages_registries_menu.rb
+++ b/lib/sidebars/groups/menus/packages_registries_menu.rb
@@ -52,7 +52,7 @@ module Sidebars
         end
 
         def dependency_proxy_menu_item
-          unless context.group.dependency_proxy_feature_available?
+          unless can?(context.current_user, :read_dependency_proxy, context.group)
             return ::Sidebars::NilMenuItem.new(item_id: :dependency_proxy)
           end
 
diff --git a/lib/sidebars/groups/panel.rb b/lib/sidebars/groups/panel.rb
index 6efe89d496a..463c2571b14 100644
--- a/lib/sidebars/groups/panel.rb
+++ b/lib/sidebars/groups/panel.rb
@@ -13,13 +13,24 @@ module Sidebars
         add_menu(Sidebars::Groups::Menus::CiCdMenu.new(context))
         add_menu(Sidebars::Groups::Menus::KubernetesMenu.new(context))
         add_menu(Sidebars::Groups::Menus::PackagesRegistriesMenu.new(context))
+        add_menu(Sidebars::Groups::Menus::CustomerRelationsMenu.new(context))
         add_menu(Sidebars::Groups::Menus::SettingsMenu.new(context))
+        add_invite_members_menu
       end
 
       override :aria_label
       def aria_label
         context.group.subgroup? ? _('Subgroup navigation') : _('Group navigation')
       end
+
+      private
+
+      def add_invite_members_menu
+        experiment(:invite_members_in_side_nav, group: context.group) do |e|
+          e.control {}
+          e.candidate { add_menu(Sidebars::Groups::Menus::InviteTeamMembersMenu.new(context)) }
+        end
+      end
     end
   end
 end
diff --git a/lib/sidebars/panel.rb b/lib/sidebars/panel.rb
index 75b3ba65729..e8c02a2d707 100644
--- a/lib/sidebars/panel.rb
+++ b/lib/sidebars/panel.rb
@@ -4,6 +4,7 @@ module Sidebars
   class Panel
     extend ::Gitlab::Utils::Override
     include ::Sidebars::Concerns::PositionableList
+    include Gitlab::Experiment::Dsl
 
     attr_reader :context, :scope_menu, :hidden_menu
 
diff --git a/lib/sidebars/projects/menus/confluence_menu.rb b/lib/sidebars/projects/menus/confluence_menu.rb
index 0d83238fa82..0fd42a57da3 100644
--- a/lib/sidebars/projects/menus/confluence_menu.rb
+++ b/lib/sidebars/projects/menus/confluence_menu.rb
@@ -37,6 +37,11 @@ module Sidebars
         def render?
           context.project.has_confluence?
         end
+
+        override :active_routes
+        def active_routes
+          { controller: :confluences }
+        end
       end
     end
   end
diff --git a/lib/sidebars/projects/menus/infrastructure_menu.rb b/lib/sidebars/projects/menus/infrastructure_menu.rb
index 92e9cbb7040..ccc4787601a 100644
--- a/lib/sidebars/projects/menus/infrastructure_menu.rb
+++ b/lib/sidebars/projects/menus/infrastructure_menu.rb
@@ -91,7 +91,7 @@ module Sidebars
 
         def google_cloud_menu_item
           feature_is_enabled = Feature.enabled?(:incubation_5mp_google_cloud)
-          user_has_permissions = can?(context.current_user, :manage_project_google_cloud, context.project)
+          user_has_permissions = can?(context.current_user, :admin_project_google_cloud, context.project)
 
           unless feature_is_enabled && user_has_permissions
             return ::Sidebars::NilMenuItem.new(item_id: :incubation_5mp_google_cloud)
@@ -100,7 +100,7 @@ module Sidebars
           ::Sidebars::MenuItem.new(
             title: _('Google Cloud'),
             link: project_google_cloud_index_path(context.project),
-            active_routes: {},
+            active_routes: { controller: :google_cloud },
             item_id: :google_cloud
           )
         end
diff --git a/lib/sidebars/projects/menus/invite_team_members_menu.rb b/lib/sidebars/projects/menus/invite_team_members_menu.rb
new file mode 100644
index 00000000000..0db49f1e12a
--- /dev/null
+++ b/lib/sidebars/projects/menus/invite_team_members_menu.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Sidebars
+  module Projects
+    module Menus
+      class InviteTeamMembersMenu < ::Sidebars::Menu
+        override :title
+        def title
+          s_('InviteMember|Invite members')
+        end
+
+        override :render?
+        def render?
+          can?(context.current_user, :admin_project_member, context.project) && all_valid_members.size <= 1
+        end
+
+        override :menu_partial
+        def menu_partial
+          'projects/invite_members_side_nav_link'
+        end
+
+        override :menu_partial_options
+        def menu_partial_options
+          {
+            project: context.project,
+            title: title,
+            sidebar_menu: self
+          }
+        end
+
+        override :extra_nav_link_html_options
+        def extra_nav_link_html_options
+          {
+            'data-test-id': 'side-nav-invite-members'
+          }
+        end
+
+        private
+
+        def all_valid_members
+          MembersFinder.new(context.project, context.current_user)
+                       .execute(include_relations: [:inherited, :direct, :invited_groups])
+        end
+      end
+    end
+  end
+end
diff --git a/lib/sidebars/projects/menus/settings_menu.rb b/lib/sidebars/projects/menus/settings_menu.rb
index 6439c97d0bc..2411ca8263a 100644
--- a/lib/sidebars/projects/menus/settings_menu.rb
+++ b/lib/sidebars/projects/menus/settings_menu.rb
@@ -144,10 +144,6 @@ module Sidebars
         end
 
         def usage_quotas_menu_item
-          unless Feature.enabled?(:project_storage_ui, context.project&.group, default_enabled: :yaml)
-            return ::Sidebars::NilMenuItem.new(item_id: :usage_quotas)
-          end
-
           ::Sidebars::MenuItem.new(
             title: s_('UsageQuota|Usage Quotas'),
             link: project_usage_quotas_path(context.project),
diff --git a/lib/sidebars/projects/menus/zentao_menu.rb b/lib/sidebars/projects/menus/zentao_menu.rb
new file mode 100644
index 00000000000..db9e60326a4
--- /dev/null
+++ b/lib/sidebars/projects/menus/zentao_menu.rb
@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Sidebars
+  module Projects
+    module Menus
+      class ZentaoMenu < ::Sidebars::Menu
+        override :configure_menu_items
+        def configure_menu_items
+          render?.tap { |render| add_items if render }
+        end
+
+        override :link
+        def link
+          zentao_integration.url
+        end
+
+        override :title
+        def title
+          s_('ZentaoIntegration|ZenTao issues')
+        end
+
+        override :title_html_options
+        def title_html_options
+          {
+            id: 'js-onboarding-settings-link'
+          }
+        end
+
+        override :image_path
+        def image_path
+          'logos/zentao.svg'
+        end
+
+        # Hardcode sizes so image doesn't flash before CSS loads https://gitlab.com/gitlab-org/gitlab/-/issues/321022
+        override :image_html_options
+        def image_html_options
+          {
+            size: 16
+          }
+        end
+
+        override :render?
+        def render?
+          return false if zentao_integration.blank?
+
+          zentao_integration.active?
+        end
+
+        def add_items
+          add_item(open_zentao_menu_item)
+        end
+
+        private
+
+        def zentao_integration
+          @zentao_integration ||= context.project.zentao_integration
+        end
+
+        def open_zentao_menu_item
+          ::Sidebars::MenuItem.new(
+            title: s_('ZentaoIntegration|Open ZenTao'),
+            link: zentao_integration.url,
+            active_routes: {},
+            item_id: :open_zentao,
+            sprite_icon: 'external-link',
+            container_html_options: {
+              target: '_blank',
+              rel: 'noopener noreferrer'
+            }
+          )
+        end
+      end
+    end
+  end
+end
+
+::Sidebars::Projects::Menus::ZentaoMenu.prepend_mod
diff --git a/lib/sidebars/projects/panel.rb b/lib/sidebars/projects/panel.rb
index d5311c0a0c1..8fbd71c1543 100644
--- a/lib/sidebars/projects/panel.rb
+++ b/lib/sidebars/projects/panel.rb
@@ -23,6 +23,7 @@ module Sidebars
         add_menu(Sidebars::Projects::Menus::RepositoryMenu.new(context))
         add_menu(Sidebars::Projects::Menus::IssuesMenu.new(context))
         add_menu(Sidebars::Projects::Menus::ExternalIssueTrackerMenu.new(context))
+        add_menu(Sidebars::Projects::Menus::ZentaoMenu.new(context))
         add_menu(Sidebars::Projects::Menus::MergeRequestsMenu.new(context))
         add_menu(Sidebars::Projects::Menus::CiCdMenu.new(context))
         add_menu(Sidebars::Projects::Menus::SecurityComplianceMenu.new(context))
@@ -35,6 +36,14 @@ module Sidebars
         add_menu(Sidebars::Projects::Menus::ExternalWikiMenu.new(context))
         add_menu(Sidebars::Projects::Menus::SnippetsMenu.new(context))
         add_menu(Sidebars::Projects::Menus::SettingsMenu.new(context))
+        add_invite_members_menu
+      end
+
+      def add_invite_members_menu
+        experiment(:invite_members_in_side_nav, group: context.project.group) do |e|
+          e.control {}
+          e.candidate { add_menu(Sidebars::Projects::Menus::InviteTeamMembersMenu.new(context)) }
+        end
       end
 
       def confluence_or_wiki_menu
diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab
index 96e3a015115..1ad89fdc364 100755
--- a/lib/support/init.d/gitlab
+++ b/lib/support/init.d/gitlab
@@ -291,7 +291,9 @@ start_gitlab() {
   if [ "$sidekiq_status" = "0" ]; then
     echo "The Sidekiq job dispatcher is already running with pid $spid, not restarting"
   else
-    RAILS_ENV=$RAILS_ENV SIDEKIQ_WORKERS=$SIDEKIQ_WORKERS bin/background_jobs start &
+    # bin/background_jobs writes to log/sidekiq.log and stdout by
+    # default, so we just need to suppress the latter here.
+    RAILS_ENV=$RAILS_ENV SIDEKIQ_WORKERS=$SIDEKIQ_WORKERS bin/background_jobs start > /dev/null &
   fi
 
   if [ "$gitlab_workhorse_status" = "0" ]; then
@@ -454,7 +456,9 @@ reload_gitlab(){
   echo "Done."
 
   echo "Restarting GitLab Sidekiq since it isn't capable of reloading its config..."
-  RAILS_ENV=$RAILS_ENV SIDEKIQ_WORKERS=$SIDEKIQ_WORKERS bin/background_jobs restart
+  # bin/background_jobs writes to log/sidekiq.log and stdout by default,
+  # so we just need to suppress the latter here.
+  RAILS_ENV=$RAILS_ENV SIDEKIQ_WORKERS=$SIDEKIQ_WORKERS bin/background_jobs restart > /dev/null &
 
   if [ "$mail_room_enabled" != true ]; then
     echo "Restarting GitLab MailRoom since it isn't capable of reloading its config..."
diff --git a/lib/support/systemd/gitlab-gitaly.service b/lib/support/systemd/gitlab-gitaly.service
new file mode 100644
index 00000000000..49f04563292
--- /dev/null
+++ b/lib/support/systemd/gitlab-gitaly.service
@@ -0,0 +1,17 @@
+[Unit]
+Description=GitLab Gitaly
+ReloadPropagatedFrom=gitlab.target
+PartOf=gitlab.target
+After=network.target
+
+[Service]
+Type=simple
+User=git
+WorkingDirectory=/home/git/gitlab
+ExecStart=/home/git/gitaly/_build/bin/gitaly /home/git/gitaly/config.toml
+Restart=on-failure
+SyslogIdentifier=gitlab-gitaly
+Slice=gitlab.slice
+
+[Install]
+WantedBy=gitlab.target
diff --git a/lib/support/systemd/gitlab-mailroom.service b/lib/support/systemd/gitlab-mailroom.service
new file mode 100644
index 00000000000..4186126c128
--- /dev/null
+++ b/lib/support/systemd/gitlab-mailroom.service
@@ -0,0 +1,19 @@
+[Unit]
+Description=GitLab Mailroom
+PartOf=gitlab.target
+After=network.target
+StartLimitIntervalSec=100s
+
+[Service]
+Type=simple
+User=git
+WorkingDirectory=/home/git/gitlab
+Environment=RAILS_ENV=production
+ExecStart=/usr/local/bin/bundle exec mail_room --log-exit-as json --quiet --config /home/git/gitlab/config/mail_room.yml
+Restart=on-failure
+RestartSec=1
+SyslogIdentifier=gitlab-mailroom
+Slice=gitlab.slice
+
+[Install]
+WantedBy=gitlab.target
diff --git a/lib/support/systemd/gitlab-pages.service b/lib/support/systemd/gitlab-pages.service
new file mode 100644
index 00000000000..2060f01be01
--- /dev/null
+++ b/lib/support/systemd/gitlab-pages.service
@@ -0,0 +1,19 @@
+[Unit]
+Description=GitLab Pages
+ReloadPropagatedFrom=gitlab.target
+PartOf=gitlab.target
+After=network.target gitlab-puma.service
+Wants=gitlab-puma.service
+
+[Service]
+Type=simple
+User=git
+WorkingDirectory=/home/git/gitlab
+ExecStart=/home/git/gitlab-pages/gitlab-pages -config /home/git/gitlab-pages/gitlab-pages.conf
+Restart=on-failure
+RestartSec=1
+SyslogIdentifier=gitlab-pages
+Slice=gitlab.slice
+
+[Install]
+WantedBy=gitlab.target
diff --git a/lib/support/systemd/gitlab-puma.service b/lib/support/systemd/gitlab-puma.service
new file mode 100644
index 00000000000..c0affa92ddf
--- /dev/null
+++ b/lib/support/systemd/gitlab-puma.service
@@ -0,0 +1,26 @@
+[Unit]
+Description=GitLab
+Conflicts=gitlab.service
+ReloadPropagatedFrom=gitlab.target
+PartOf=gitlab.target
+After=network.target
+StartLimitIntervalSec=100s
+
+[Service]
+Type=notify
+User=git
+WorkingDirectory=/home/git/gitlab
+Environment=RAILS_ENV=production
+ExecStart=/usr/local/bin/bundle exec puma --config /home/git/gitlab/config/puma.rb --environment production --pidfile /home/git/gitlab/tmp/pids/puma.pid
+ExecReload=/usr/bin/kill -USR2 $MAINPID
+PIDFile=/home/git/gitlab/tmp/pids/puma.pid
+# puma can be slow to start
+TimeoutStartSec=120
+WatchdogSec=10
+Restart=on-failure
+RestartSec=1
+SyslogIdentifier=gitlab-puma
+Slice=gitlab.slice
+
+[Install]
+WantedBy=gitlab.target
diff --git a/lib/support/systemd/gitlab-sidekiq.service b/lib/support/systemd/gitlab-sidekiq.service
new file mode 100644
index 00000000000..81046f5348a
--- /dev/null
+++ b/lib/support/systemd/gitlab-sidekiq.service
@@ -0,0 +1,22 @@
+[Unit]
+Description=GitLab Sidekiq
+ReloadPropagatedFrom=gitlab.target
+PartOf=gitlab.target
+After=network.target
+JoinsNamespaceOf=gitlab-puma.service
+
+[Service]
+Type=simple
+User=git
+WorkingDirectory=/home/git/gitlab
+Environment=RAILS_ENV=production
+ExecStart=/usr/local/bin/bundle exec sidekiq --config /home/git/gitlab/config/sidekiq_queues.yml --environment production
+ExecStop=/usr/local/bin/bundle exec sidekiqctl stop /run/gitlab/sidekiq.pid
+PIDFile=/home/git/gitlab/tmp/pids/sidekiq.pid
+Restart=on-failure
+RestartSec=1
+SyslogIdentifier=gitlab-sidekiq
+Slice=gitlab.slice
+
+[Install]
+WantedBy=gitlab.target
diff --git a/lib/support/systemd/gitlab-workhorse.service b/lib/support/systemd/gitlab-workhorse.service
new file mode 100644
index 00000000000..3e9a72d3cb2
--- /dev/null
+++ b/lib/support/systemd/gitlab-workhorse.service
@@ -0,0 +1,21 @@
+[Unit]
+Description=GitLab Workhorse
+ReloadPropagatedFrom=gitlab.target
+PartOf=gitlab.target
+After=network.target gitlab-puma.service
+Wants=gitlab-puma.service
+
+[Service]
+Type=simple
+User=git
+WorkingDirectory=/home/git/gitlab
+Environment=PATH=/home/git/gitlab-workhorse:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ExecStart=/home/git/gitlab-workhorse/gitlab-workhorse  -listenUmask 0 -listenNetwork unix -listenAddr /home/git/gitlab/tmp/sockets/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket /home/git/gitlab/tmp/sockets/gitlab.socket -documentRoot /home/git/gitlab/public -secretPath /home/git/gitlab/.gitlab_workhorse_secret
+ExecReload=/usr/bin/kill -USR2 $MAINPID
+Restart=on-failure
+RestartSec=1
+SyslogIdentifier=gitlab-workhorse
+Slice=gitlab.slice
+
+[Install]
+WantedBy=gitlab.target
diff --git a/lib/support/systemd/gitlab.slice b/lib/support/systemd/gitlab.slice
new file mode 100644
index 00000000000..2c447f6224a
--- /dev/null
+++ b/lib/support/systemd/gitlab.slice
@@ -0,0 +1,8 @@
+[Unit]
+Description=Slice to bundle all GitLab services
+Before=slices.target
+
+[Slice]
+MemoryAccounting=true
+IOAccounting=true
+CPUAccounting=true
diff --git a/lib/support/systemd/gitlab.target b/lib/support/systemd/gitlab.target
new file mode 100644
index 00000000000..e0538c3ff81
--- /dev/null
+++ b/lib/support/systemd/gitlab.target
@@ -0,0 +1,6 @@
+[Unit]
+Description=GitLab
+Wants=gitlab-gitaly.service gitlab-puma.service gitlab-sidekiq.service gitlab-workhorse.service
+
+[Install]
+WantedBy=multi-user.target
diff --git a/lib/system_check/app/init_script_exists_check.rb b/lib/system_check/app/init_script_exists_check.rb
deleted file mode 100644
index 7be92acdc37..00000000000
--- a/lib/system_check/app/init_script_exists_check.rb
+++ /dev/null
@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module SystemCheck
-  module App
-    class InitScriptExistsCheck < SystemCheck::BaseCheck
-      set_name 'Init script exists?'
-      set_skip_reason 'skipped (omnibus-gitlab has no init script)'
-
-      def skip?
-        omnibus_gitlab?
-      end
-
-      def check?
-        script_path = '/etc/init.d/gitlab'
-        File.exist?(script_path)
-      end
-
-      def show_error
-        try_fixing_it(
-          'Install the init script'
-        )
-        for_more_information(
-          see_installation_guide_section('Install Init Script')
-        )
-        fix_and_rerun
-      end
-    end
-  end
-end
diff --git a/lib/system_check/app/init_script_up_to_date_check.rb b/lib/system_check/app/init_script_up_to_date_check.rb
deleted file mode 100644
index cf841d5e659..00000000000
--- a/lib/system_check/app/init_script_up_to_date_check.rb
+++ /dev/null
@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-module SystemCheck
-  module App
-    class InitScriptUpToDateCheck < SystemCheck::BaseCheck
-      SCRIPT_PATH = '/etc/init.d/gitlab'
-
-      set_name 'Init script up-to-date?'
-      set_skip_reason 'skipped (omnibus-gitlab has no init script)'
-
-      def skip?
-        return true if omnibus_gitlab?
-
-        unless init_file_exists?
-          self.skip_reason = "can't check because of previous errors"
-
-          true
-        end
-      end
-
-      def check?
-        recipe_path = Rails.root.join('lib/support/init.d/', 'gitlab')
-
-        recipe_content = File.read(recipe_path)
-        script_content = File.read(SCRIPT_PATH)
-
-        recipe_content == script_content
-      end
-
-      def show_error
-        try_fixing_it(
-          'Re-download the init script'
-        )
-        for_more_information(
-          see_installation_guide_section('Install Init Script')
-        )
-        fix_and_rerun
-      end
-
-      private
-
-      def init_file_exists?
-        File.exist?(SCRIPT_PATH)
-      end
-    end
-  end
-end
diff --git a/lib/system_check/app/systemd_unit_files_or_init_script_exist_check.rb b/lib/system_check/app/systemd_unit_files_or_init_script_exist_check.rb
new file mode 100644
index 00000000000..b2f059d212b
--- /dev/null
+++ b/lib/system_check/app/systemd_unit_files_or_init_script_exist_check.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module SystemCheck
+  module App
+    class SystemdUnitFilesOrInitScriptExistCheck < SystemCheck::BaseCheck
+      set_name 'Systemd unit files or init script exist?'
+      set_skip_reason 'skipped (omnibus-gitlab has neither init script nor systemd units)'
+
+      def skip?
+        omnibus_gitlab?
+      end
+
+      def check?
+        unit_paths = [
+          '/usr/local/lib/systemd/system/gitlab-gitaly.service',
+          '/usr/local/lib/systemd/system/gitlab-mailroom.service',
+          '/usr/local/lib/systemd/system/gitlab-puma.service',
+          '/usr/local/lib/systemd/system/gitlab-sidekiq.service',
+          '/usr/local/lib/systemd/system/gitlab.slice',
+          '/usr/local/lib/systemd/system/gitlab.target',
+          '/usr/local/lib/systemd/system/gitlab-workhorse.service'
+        ]
+        script_path = '/etc/init.d/gitlab'
+
+        unit_paths.all? { |s| File.exist?(s) } || File.exist?(script_path)
+      end
+
+      def show_error
+        try_fixing_it(
+          'Install the Service'
+        )
+        for_more_information(
+          see_installation_guide_section('Install the Service')
+        )
+        fix_and_rerun
+      end
+    end
+  end
+end
diff --git a/lib/system_check/app/systemd_unit_files_or_init_script_up_to_date_check.rb b/lib/system_check/app/systemd_unit_files_or_init_script_up_to_date_check.rb
new file mode 100644
index 00000000000..10bc772a83c
--- /dev/null
+++ b/lib/system_check/app/systemd_unit_files_or_init_script_up_to_date_check.rb
@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module SystemCheck
+  module App
+    class SystemdUnitFilesOrInitScriptUpToDateCheck < SystemCheck::BaseCheck
+      SCRIPT_PATH = '/etc/init.d/gitlab'
+      UNIT_PATHS = [
+        '/usr/local/lib/systemd/system/gitlab-gitaly.service',
+        '/usr/local/lib/systemd/system/gitlab-mailroom.service',
+        '/usr/local/lib/systemd/system/gitlab-puma.service',
+        '/usr/local/lib/systemd/system/gitlab-sidekiq.service',
+        '/usr/local/lib/systemd/system/gitlab.slice',
+        '/usr/local/lib/systemd/system/gitlab.target',
+        '/usr/local/lib/systemd/system/gitlab-workhorse.service'
+      ].freeze
+
+      set_name 'Systemd unit files or init script up-to-date?'
+      set_skip_reason 'skipped (omnibus-gitlab has neither init script nor systemd units)'
+
+      def skip?
+        return true if omnibus_gitlab?
+
+        unless unit_files_exist? || init_file_exists?
+          self.skip_reason = "can't check because of previous errors"
+
+          true
+        end
+      end
+
+      def check?
+        if unit_files_exist?
+          return unit_files_up_to_date?
+        end
+
+        init_file_up_to_date?
+      end
+
+      def show_error
+        try_fixing_it(
+          'Install the Service'
+        )
+        for_more_information(
+          see_installation_guide_section('Install the Service')
+        )
+        fix_and_rerun
+      end
+
+      private
+
+      def init_file_exists?
+        File.exist?(SCRIPT_PATH)
+      end
+
+      def unit_files_exist?
+        UNIT_PATHS.all? { |s| File.exist?(s) }
+      end
+
+      def init_file_up_to_date?
+        recipe_path = Rails.root.join('lib/support/init.d/', 'gitlab')
+
+        recipe_content = File.read(recipe_path)
+        script_content = File.read(SCRIPT_PATH)
+
+        recipe_content == script_content
+      end
+
+      def unit_files_up_to_date?
+        UNIT_PATHS.all? do |unit|
+          unit_name = File.basename(unit)
+          recipe_path = Rails.root.join('lib/support/systemd/', unit_name)
+
+          recipe_content = File.read(recipe_path)
+          unit_content = File.read(unit)
+
+          recipe_content == unit_content
+        end
+      end
+    end
+  end
+end
diff --git a/lib/system_check/incoming_email/initd_configured_check.rb b/lib/system_check/incoming_email/initd_configured_check.rb
deleted file mode 100644
index acb4b5a9e74..00000000000
--- a/lib/system_check/incoming_email/initd_configured_check.rb
+++ /dev/null
@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module SystemCheck
-  module IncomingEmail
-    class InitdConfiguredCheck < SystemCheck::BaseCheck
-      set_name 'Init.d configured correctly?'
-
-      def skip?
-        omnibus_gitlab?
-      end
-
-      def check?
-        mail_room_configured?
-      end
-
-      def show_error
-        try_fixing_it(
-          'Enable mail_room in the init.d configuration.'
-        )
-        for_more_information(
-          'doc/administration/reply_by_email.md'
-        )
-        fix_and_rerun
-      end
-
-      private
-
-      def mail_room_configured?
-        path = '/etc/default/gitlab'
-        File.exist?(path) && File.read(path).include?('mail_room_enabled=true')
-      end
-    end
-  end
-end
diff --git a/lib/system_check/incoming_email/mail_room_enabled_check.rb b/lib/system_check/incoming_email/mail_room_enabled_check.rb
new file mode 100644
index 00000000000..8e725aabd03
--- /dev/null
+++ b/lib/system_check/incoming_email/mail_room_enabled_check.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module SystemCheck
+  module IncomingEmail
+    class MailRoomEnabledCheck < SystemCheck::BaseCheck
+      include ::SystemCheck::InitHelpers
+      set_name 'Mailroom enabled?'
+
+      def skip?
+        omnibus_gitlab?
+      end
+
+      def check?
+        mail_room_enabled? || mail_room_configured?
+      end
+
+      def show_error
+        try_fixing_it(
+          'Enable mail_room'
+        )
+        for_more_information(
+          'doc/administration/reply_by_email.md'
+        )
+        fix_and_rerun
+      end
+
+      private
+
+      def mail_room_enabled?
+        target = '/usr/local/lib/systemd/system/gitlab.target'
+        service = '/usr/local/lib/systemd/system/gitlab-mailroom.service'
+
+        File.exist?(target) && File.exist?(service) && systemd_get_wants('gitlab.target').include?("gitlab-mailroom.service")
+      end
+
+      def mail_room_configured?
+        path = '/etc/default/gitlab'
+        File.exist?(path) && File.read(path).include?('mail_room_enabled=true')
+      end
+    end
+  end
+end
diff --git a/lib/system_check/incoming_email/mail_room_running_check.rb b/lib/system_check/incoming_email/mail_room_running_check.rb
index b7aead4624e..38bb1e46364 100644
--- a/lib/system_check/incoming_email/mail_room_running_check.rb
+++ b/lib/system_check/incoming_email/mail_room_running_check.rb
@@ -3,12 +3,13 @@
 module SystemCheck
   module IncomingEmail
     class MailRoomRunningCheck < SystemCheck::BaseCheck
+      include ::SystemCheck::InitHelpers
       set_name 'MailRoom running?'
 
       def skip?
         return true if omnibus_gitlab?
 
-        unless mail_room_configured?
+        unless mail_room_enabled? || mail_room_configured?
           self.skip_reason = "can't check because of previous errors"
           true
         end
@@ -20,10 +21,10 @@ module SystemCheck
 
       def show_error
         try_fixing_it(
-          sudo_gitlab('RAILS_ENV=production bin/mail_room start')
+          'Start mail_room'
         )
         for_more_information(
-          see_installation_guide_section('Install Init Script'),
+          'doc/administration/incoming_email.md',
           'see log/mail_room.log for possible errors'
         )
         fix_and_rerun
@@ -31,6 +32,13 @@ module SystemCheck
 
       private
 
+      def mail_room_enabled?
+        target = '/usr/local/lib/systemd/system/gitlab.target'
+        service = '/usr/local/lib/systemd/system/gitlab-mailroom.service'
+
+        File.exist?(target) && File.exist?(service) && systemd_get_wants('gitlab.target').include?("gitlab-mailroom.service")
+      end
+
       def mail_room_configured?
         path = '/etc/default/gitlab'
         File.exist?(path) && File.read(path).include?('mail_room_enabled=true')
diff --git a/lib/system_check/incoming_email_check.rb b/lib/system_check/incoming_email_check.rb
index 84033ada710..3cae9450b94 100644
--- a/lib/system_check/incoming_email_check.rb
+++ b/lib/system_check/incoming_email_check.rb
@@ -14,7 +14,7 @@ module SystemCheck
         end
 
         if Rails.env.production?
-          checks << SystemCheck::IncomingEmail::InitdConfiguredCheck
+          checks << SystemCheck::IncomingEmail::MailRoomEnabledCheck
           checks << SystemCheck::IncomingEmail::MailRoomRunningCheck
         end
 
diff --git a/lib/system_check/init_helpers.rb b/lib/system_check/init_helpers.rb
new file mode 100644
index 00000000000..2573f06b716
--- /dev/null
+++ b/lib/system_check/init_helpers.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+module SystemCheck
+  module InitHelpers
+    # Return the Wants= of a unit, empty if the unit doesn't exist
+    def systemd_get_wants(unitname)
+      stdout, _stderr, status = Open3.capture3("systemctl", "--no-pager", "show", unitname)
+
+      unless status
+        return []
+      end
+
+      wantsline = stdout.lines.find { |line| line.start_with?("Wants=") }
+
+      unless wantsline
+        return []
+      end
+
+      wantsline.delete_prefix("Wants=").strip.split
+    end
+  end
+end
diff --git a/lib/system_check/rake_task/app_task.rb b/lib/system_check/rake_task/app_task.rb
index f7d2bf86c78..892417d67ec 100644
--- a/lib/system_check/rake_task/app_task.rb
+++ b/lib/system_check/rake_task/app_task.rb
@@ -23,8 +23,8 @@ module SystemCheck
           SystemCheck::App::UploadsDirectoryExistsCheck,
           SystemCheck::App::UploadsPathPermissionCheck,
           SystemCheck::App::UploadsPathTmpPermissionCheck,
-          SystemCheck::App::InitScriptExistsCheck,
-          SystemCheck::App::InitScriptUpToDateCheck,
+          SystemCheck::App::SystemdUnitFilesOrInitScriptExistCheck,
+          SystemCheck::App::SystemdUnitFilesOrInitScriptUpToDateCheck,
           SystemCheck::App::ProjectsHaveNamespaceCheck,
           SystemCheck::App::RedisVersionCheck,
           SystemCheck::App::RubyVersionCheck,
diff --git a/lib/system_check/sidekiq_check.rb b/lib/system_check/sidekiq_check.rb
index 7ac1bd58ede..ab048433b37 100644
--- a/lib/system_check/sidekiq_check.rb
+++ b/lib/system_check/sidekiq_check.rb
@@ -39,6 +39,12 @@ module SystemCheck
 
       if (cluster_count == 1 && worker_count > 0) || (cluster_count == 0 && worker_count == 1)
         $stdout.puts "#{cluster_count}/#{worker_count}".color(:green)
+      elsif File.symlink?('/run/systemd/units/invocation:gitlab-sidekiq.service')
+        $stdout.puts "#{cluster_count}/#{worker_count}".color(:red)
+        try_fixing_it(
+          'sudo systemctl restart gitlab-sidekiq.service'
+        )
+        fix_and_rerun
       else
         $stdout.puts "#{cluster_count}/#{worker_count}".color(:red)
         try_fixing_it(
diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake
deleted file mode 100644
index a4600a0ed16..00000000000
--- a/lib/tasks/gemojione.rake
+++ /dev/null
@@ -1,245 +0,0 @@
-# frozen_string_literal: true
-
-namespace :gemojione do
-  desc 'Generates Emoji SHA256 digests'
-
-  task aliases: ['yarn:check', 'environment'] do
-    require 'json'
-
-    aliases = {}
-
-    index_file = File.join(Rails.root, 'fixtures', 'emojis', 'index.json')
-    index = Gitlab::Json.parse(File.read(index_file))
-
-    index.each_pair do |key, data|
-      data['aliases'].each do |a|
-        a.tr!(':', '')
-
-        aliases[a] = key
-      end
-    end
-
-    out = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json')
-    File.open(out, 'w') do |handle|
-      handle.write(Gitlab::Json.pretty_generate(aliases, indent: '   ', space: '', space_before: ''))
-    end
-  end
-
-  task digests: ['yarn:check', 'environment'] do
-    require 'digest/sha2'
-    require 'json'
-
-    # We don't have `node_modules` available in built versions of GitLab
-    FileUtils.cp_r(Rails.root.join('node_modules', 'emoji-unicode-version', 'emoji-unicode-version-map.json'), File.join(Rails.root, 'fixtures', 'emojis'))
-
-    dir = Gemojione.images_path
-    resultant_emoji_map = {}
-    resultant_emoji_map_new = {}
-
-    Gitlab::Emoji.emojis.each do |name, emoji_hash|
-      # Ignore aliases
-      unless Gitlab::Emoji.emojis_aliases.key?(name)
-        fpath = File.join(dir, "#{emoji_hash['unicode']}.png")
-        hash_digest = Digest::SHA256.file(fpath).hexdigest
-
-        category = emoji_hash['category']
-        if name == 'gay_pride_flag'
-          category = 'flags'
-        end
-
-        entry = {
-          category: category,
-          moji: emoji_hash['moji'],
-          description: emoji_hash['description'],
-          unicodeVersion: Gitlab::Emoji.emoji_unicode_version(name),
-          digest: hash_digest
-        }
-
-        resultant_emoji_map[name] = entry
-
-        # Our new map is only characters to make the json substantially smaller
-        new_entry = {
-          c: category,
-          e: emoji_hash['moji'],
-          d: emoji_hash['description'],
-          u: Gitlab::Emoji.emoji_unicode_version(name)
-        }
-
-        resultant_emoji_map_new[name] = new_entry
-      end
-    end
-
-    out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json')
-    File.open(out, 'w') do |handle|
-      handle.write(Gitlab::Json.pretty_generate(resultant_emoji_map))
-    end
-
-    out_new = File.join(Rails.root, 'public', '-', 'emojis', '1', 'emojis.json')
-    File.open(out_new, 'w') do |handle|
-      handle.write(Gitlab::Json.pretty_generate(resultant_emoji_map_new))
-    end
-  end
-
-  # This task will generate a standard and Retina sprite of all of the current
-  # Gemojione Emojis, with the accompanying SCSS map.
-  #
-  # It will not appear in `rake -T` output, and the dependent gems are not
-  # included in the Gemfile by default, because this task will only be needed
-  # occasionally, such as when new Emojis are added to Gemojione.
-  task sprite: :environment do
-    begin
-      require 'sprite_factory'
-      require 'rmagick'
-    rescue LoadError
-      # noop
-    end
-
-    check_requirements!
-
-    SIZE   = 20
-    RETINA = SIZE * 2
-
-    # Update these values to the width and height of the spritesheet when
-    # new emoji are added.
-    SPRITESHEET_WIDTH = 860
-    SPRITESHEET_HEIGHT = 840
-
-    # Set up a map to rename image files
-    emoji_unicode_string_to_name_map = {}
-    Gitlab::Emoji.emojis.each do |name, emoji_hash|
-      # Ignore aliases
-      unless Gitlab::Emoji.emojis_aliases.key?(name)
-        emoji_unicode_string_to_name_map[emoji_hash['unicode']] = name
-      end
-    end
-
-    # Copy the Gemojione assets to the temporary folder for renaming
-    emoji_dir = "app/assets/images/emoji"
-    FileUtils.rm_rf(emoji_dir)
-    FileUtils.mkdir_p(emoji_dir, mode: 0700)
-    FileUtils.cp_r(File.join(Gemojione.images_path, '.'), emoji_dir)
-    Dir[File.join(emoji_dir, "**/*.png")].each do |png|
-      image_path = png
-      rename_to_named_emoji_image!(emoji_unicode_string_to_name_map, image_path)
-    end
-
-    Dir.mktmpdir do |tmpdir|
-      FileUtils.cp_r(File.join(emoji_dir, '.'), tmpdir)
-
-      Dir.chdir(tmpdir) do
-        Dir["**/*.png"].each do |png|
-          tmp_image_path = File.join(tmpdir, png)
-          resize!(tmp_image_path, SIZE)
-        end
-      end
-
-      style_path = Rails.root.join(*%w(app assets stylesheets framework emoji_sprites.scss))
-
-      # Combine the resized assets into a packed sprite and re-generate the SCSS
-      SpriteFactory.cssurl = "image-url('$IMAGE')"
-      SpriteFactory.run!(tmpdir, {
-        output_style: style_path,
-        output_image: "app/assets/images/emoji.png",
-        selector:     '.emoji-',
-        style:        :scss,
-        nocomments:   true,
-        pngcrush:     true,
-        layout:       :packed
-      })
-
-      # SpriteFactory's SCSS is a bit too verbose for our purposes here, so
-      # let's simplify it
-      system(%Q(sed -i '' "s/width: #{SIZE}px; height: #{SIZE}px; background: image-url('emoji.png')/background-position:/" #{style_path}))
-      system(%Q(sed -i '' "s/ no-repeat//" #{style_path}))
-      system(%Q(sed -i '' "s/ 0px/ 0/g" #{style_path}))
-
-      # Append a generic rule that applies to all Emojis
-      File.open(style_path, 'a') do |f|
-        f.puts
-        f.puts <<-CSS.strip_heredoc
-        .emoji-icon {
-          background-image: image-url('emoji.png');
-          background-repeat: no-repeat;
-          color: transparent;
-          text-indent: -99em;
-          height: #{SIZE}px;
-          width: #{SIZE}px;
-
-          @media only screen and (-webkit-min-device-pixel-ratio: 2),
-                 only screen and (min--moz-device-pixel-ratio: 2),
-                 only screen and (-o-min-device-pixel-ratio: 2/1),
-                 only screen and (min-device-pixel-ratio: 2),
-                 only screen and (min-resolution: 192dpi),
-                 only screen and (min-resolution: 2dppx) {
-            background-image: image-url('emoji@2x.png');
-            background-size: #{SPRITESHEET_WIDTH}px #{SPRITESHEET_HEIGHT}px;
-          }
-        }
-        CSS
-      end
-    end
-
-    # Now do it again but for Retina
-    Dir.mktmpdir do |tmpdir|
-      # Copy the Gemojione assets to the temporary folder for resizing
-      FileUtils.cp_r(File.join(emoji_dir, '.'), tmpdir)
-
-      Dir.chdir(tmpdir) do
-        Dir["**/*.png"].each do |png|
-          tmp_image_path = File.join(tmpdir, png)
-          resize!(tmp_image_path, RETINA)
-        end
-      end
-
-      # Combine the resized assets into a packed sprite and re-generate the SCSS
-      SpriteFactory.run!(tmpdir, {
-        output_image: "app/assets/images/emoji@2x.png",
-        style:        false,
-        nocomments:   true,
-        pngcrush:     true,
-        layout:       :packed
-      })
-    end
-  end
-
-  def check_requirements!
-    return if defined?(SpriteFactory) && defined?(Magick)
-
-    puts <<-MSG.strip_heredoc
-      This task is disabled by default and should only be run when the Gemojione
-      gem is updated with new Emojis.
-
-      To enable this task, *temporarily* add the following lines to Gemfile and
-      re-bundle:
-
-      gem 'sprite-factory'
-      gem 'rmagick'
-    MSG
-
-    exit 1
-  end
-
-  def resize!(image_path, size)
-    # Resize the image in-place, save it, and free the object
-    image = Magick::Image.read(image_path).first
-    image.resize!(size, size)
-    image.write(image_path) { self.quality = 100 }
-    image.destroy!
-  end
-
-  EMOJI_IMAGE_PATH_RE = /(.*?)(([0-9a-f]-?)+)\.png$/i.freeze
-  def rename_to_named_emoji_image!(emoji_unicode_string_to_name_map, image_path)
-    # Rename file from unicode to emoji name
-    matches = EMOJI_IMAGE_PATH_RE.match(image_path)
-    preceding_path = matches[1]
-    unicode_string = matches[2]
-    name = emoji_unicode_string_to_name_map[unicode_string]
-    if name
-      new_png_path = File.join(preceding_path, "#{name}.png")
-      FileUtils.mv(image_path, new_png_path)
-      new_png_path
-    else
-      puts "Warning: emoji_unicode_string_to_name_map missing entry for #{unicode_string}. Full path: #{image_path}"
-    end
-  end
-end
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index e2647021914..e83c4cbdb39 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -160,35 +160,44 @@ namespace :gitlab do
       Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
     end
 
-    desc 'reindex a regular index without downtime to eliminate bloat'
-    task :reindex, [:index_name] => :environment do |_, args|
-      unless Feature.enabled?(:database_reindexing, type: :ops)
+    desc 'execute reindexing without downtime to eliminate bloat'
+    task reindex: :environment do
+      unless Feature.enabled?(:database_reindexing, type: :ops, default_enabled: :yaml)
         puts "This feature (database_reindexing) is currently disabled.".color(:yellow)
         exit
       end
 
-      indexes = Gitlab::Database::PostgresIndex.reindexing_support
+      Gitlab::Database::EachDatabase.each_database_connection do |connection, connection_name|
+        Gitlab::Database::SharedModel.logger = Logger.new($stdout) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false)
 
-      if identifier = args[:index_name]
-        raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/
+        # Hack: Before we do actual reindexing work, create async indexes
+        Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops)
 
-        indexes = indexes.where(identifier: identifier)
-
-        raise "Index not found or not supported: #{args[:index_name]}" if indexes.empty?
+        Gitlab::Database::Reindexing.automatic_reindexing
       end
+    rescue StandardError => e
+      Gitlab::AppLogger.error(e)
+      raise
+    end
 
-      ActiveRecord::Base.logger = Logger.new($stdout) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false)
+    desc 'Enqueue an index for reindexing'
+    task :enqueue_reindexing_action, [:index_name, :database] => :environment do |_, args|
+      model = Gitlab::Database.database_base_models[args.fetch(:database, Gitlab::Database::PRIMARY_DATABASE_NAME)]
 
-      # Cleanup leftover temporary indexes from previous, possibly aborted runs (if any)
-      Gitlab::Database::Reindexing.cleanup_leftovers!
+      Gitlab::Database::SharedModel.using_connection(model.connection) do
+        queued_action = Gitlab::Database::PostgresIndex.find(args[:index_name]).queued_reindexing_actions.create!
 
-      # Hack: Before we do actual reindexing work, create async indexes
-      Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops)
+        puts "Queued reindexing action: #{queued_action}"
+        puts "There are #{Gitlab::Database::Reindexing::QueuedAction.queued.size} queued actions in total."
+      end
 
-      Gitlab::Database::Reindexing.perform(indexes)
-    rescue StandardError => e
-      Gitlab::AppLogger.error(e)
-      raise
+      unless Feature.enabled?(:database_reindexing, type: :ops, default_enabled: :yaml)
+        puts <<~NOTE.color(:yellow)
+          Note: database_reindexing feature is currently disabled.
+
+          Enable with: Feature.enable(:database_reindexing)
+        NOTE
+      end
     end
 
     desc 'Check if there have been user additions to the database'
diff --git a/lib/tasks/gitlab/docs/compile_deprecations.rake b/lib/tasks/gitlab/docs/compile_deprecations.rake
index 0fd43775015..dc9788cb0b2 100644
--- a/lib/tasks/gitlab/docs/compile_deprecations.rake
+++ b/lib/tasks/gitlab/docs/compile_deprecations.rake
@@ -21,7 +21,7 @@ namespace :gitlab do
       if doc == contents
         puts "Deprecations doc is up to date."
       else
-        format_output('Deprecations doc is outdated! Please update it by running `bundle exec rake gitlab:docs:compile_deprecations`.')
+        format_output('Deprecations doc is outdated! You (or your technical writer) can update it by running `bin/rake gitlab:docs:compile_deprecations`.')
         abort
       end
     end
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index ef58c9339f1..eabbb8652f1 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -67,7 +67,8 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
             env["BUNDLE_DEPLOYMENT"] = 'false'
           end
 
-          Gitlab::Popen.popen([make_cmd], nil, env)
+          output, status = Gitlab::Popen.popen([make_cmd, 'all', 'git'], nil, env)
+          raise "Gitaly failed to compile: #{output}" unless status&.zero?
         end
       end
     end
diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake
index 68395d10d24..02764b5d46f 100644
--- a/lib/tasks/gitlab/info.rake
+++ b/lib/tasks/gitlab/info.rake
@@ -68,8 +68,8 @@ namespace :gitlab do
       puts "Version:\t#{Gitlab::VERSION}"
       puts "Revision:\t#{Gitlab.revision}"
       puts "Directory:\t#{Rails.root}"
-      puts "DB Adapter:\t#{Gitlab::Database.main.human_adapter_name}"
-      puts "DB Version:\t#{Gitlab::Database.main.version}"
+      puts "DB Adapter:\t#{ApplicationRecord.database.human_adapter_name}"
+      puts "DB Version:\t#{ApplicationRecord.database.version}"
       puts "URL:\t\t#{Gitlab.config.gitlab.url}"
       puts "HTTP Clone URL:\t#{http_clone_url}"
       puts "SSH Clone URL:\t#{ssh_clone_url}"
diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake
index 90ed91221ae..2e383065b64 100644
--- a/lib/tasks/gitlab/sidekiq.rake
+++ b/lib/tasks/gitlab/sidekiq.rake
@@ -36,13 +36,17 @@ namespace :gitlab do
           # Do not edit it manually!
         BANNER
 
-        foss_workers, ee_workers = Gitlab::SidekiqConfig.workers_for_all_queues_yml
+        foss_workers, ee_workers, jh_workers = Gitlab::SidekiqConfig.workers_for_all_queues_yml
 
         write_yaml(Gitlab::SidekiqConfig::FOSS_QUEUE_CONFIG_PATH, banner, foss_workers)
 
         if Gitlab.ee?
           write_yaml(Gitlab::SidekiqConfig::EE_QUEUE_CONFIG_PATH, banner, ee_workers)
         end
+
+        if Gitlab.jh?
+          write_yaml(Gitlab::SidekiqConfig::JH_QUEUE_CONFIG_PATH, banner, jh_workers)
+        end
       end
 
       desc 'GitLab | Sidekiq | Validate that all_queues.yml matches worker definitions'
@@ -57,6 +61,7 @@ namespace :gitlab do
 
             - #{Gitlab::SidekiqConfig::FOSS_QUEUE_CONFIG_PATH}
             - #{Gitlab::SidekiqConfig::EE_QUEUE_CONFIG_PATH}
+            #{"- " + Gitlab::SidekiqConfig::JH_QUEUE_CONFIG_PATH if Gitlab.jh?}
 
           MSG
         end
diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake
index fb9f9b9fe67..eb5eeed531f 100644
--- a/lib/tasks/gitlab/storage.rake
+++ b/lib/tasks/gitlab/storage.rake
@@ -170,7 +170,7 @@ namespace :gitlab do
       inverval = (ENV['MAX_DATABASE_CONNECTION_CHECK_INTERVAL'] || 10).to_f
 
       attempts.to_i.times do
-        unless Gitlab::Database.main.exists?
+        unless ApplicationRecord.database.exists?
           puts "Waiting until database is ready before continuing...".color(:yellow)
           sleep inverval
         end
diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake
index 270793359e1..71e84d3795f 100644
--- a/lib/tasks/haml-lint.rake
+++ b/lib/tasks/haml-lint.rake
@@ -4,16 +4,5 @@ unless Rails.env.production?
   require 'haml_lint/rake_task'
   require Rails.root.join('haml_lint/inline_javascript')
 
-  # Workaround for warnings from parser/current
-  # Keep it even if it no longer emits any warnings,
-  # because we'll still see warnings in console/server anyway,
-  # and we don't need to break static-analysis for this.
-  task :haml_lint do
-    require 'parser'
-    def Parser.warn(*args)
-      puts(*args) # static-analysis ignores stdout if status is 0
-    end
-  end
-
   HamlLint::RakeTask.new
 end
diff --git a/lib/tasks/tanuki_emoji.rake b/lib/tasks/tanuki_emoji.rake
new file mode 100644
index 00000000000..98d3920c07f
--- /dev/null
+++ b/lib/tasks/tanuki_emoji.rake
@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+
+namespace :tanuki_emoji do
+  desc 'Generates Emoji aliases fixtures'
+  task aliases: :environment do
+    aliases = {}
+
+    TanukiEmoji.index.all.each do |emoji|
+      emoji.aliases.each do |emoji_alias|
+        aliases[TanukiEmoji::Character.format_name(emoji_alias)] = emoji.name
+      end
+    end
+
+    aliases_json_file = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json')
+    File.open(aliases_json_file, 'w') do |handle|
+      handle.write(Gitlab::Json.pretty_generate(aliases, indent: '   ', space: '', space_before: ''))
+    end
+  end
+
+  desc 'Generates Emoji SHA256 digests'
+  task digests: :environment do
+    require 'digest/sha2'
+
+    digest_emoji_map = {}
+    emojis_map = {}
+
+    TanukiEmoji.index.all.each do |emoji|
+      emoji_path = Gitlab::Emoji.emoji_public_absolute_path.join("#{emoji.name}.png")
+
+      digest_entry = {
+        category: emoji.category,
+        moji: emoji.codepoints,
+        description: emoji.description,
+        unicodeVersion: emoji.unicode_version,
+        digest: Digest::SHA256.file(emoji_path).hexdigest
+      }
+
+      digest_emoji_map[emoji.name] = digest_entry
+
+      # Our new map is only characters to make the json substantially smaller
+      emoji_entry = {
+        c: emoji.category,
+        e: emoji.codepoints,
+        d: emoji.description,
+        u: emoji.unicode_version
+      }
+
+      emojis_map[emoji.name] = emoji_entry
+    end
+
+    digests_json = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json')
+    File.open(digests_json, 'w') do |handle|
+      handle.write(Gitlab::Json.pretty_generate(digest_emoji_map))
+    end
+
+    emojis_json = Gitlab::Emoji.emoji_public_absolute_path.join('emojis.json')
+    File.open(emojis_json, 'w') do |handle|
+      handle.write(Gitlab::Json.pretty_generate(emojis_map))
+    end
+  end
+
+  desc 'Import emoji assets from TanukiEmoji to versioned folder'
+  task import: :environment do
+    require 'mini_magick'
+
+    # Setting to the same size as previous gemojione images
+    EMOJI_SIZE = 64
+
+    emoji_dir = Gitlab::Emoji.emoji_public_absolute_path
+
+    puts "Importing emojis into: #{emoji_dir} ..."
+
+    # Re-create the assets folder and copy emojis renaming them to use name instead of unicode hex
+    FileUtils.rm_rf(emoji_dir) if Dir.exist?(emoji_dir)
+    FileUtils.mkdir_p(emoji_dir, mode: 0700)
+
+    TanukiEmoji.index.all.each do |emoji|
+      source = File.join(TanukiEmoji.images_path, emoji.image_name)
+      destination = File.join(emoji_dir, "#{emoji.name}.png")
+
+      FileUtils.cp(source, destination)
+      resize!(destination, EMOJI_SIZE)
+      print emoji.codepoints
+    end
+
+    puts
+    puts 'Done!'
+  end
+
+  # This task will generate a standard and Retina sprite of all of the current
+  # TanukiEmoji Emojis, with the accompanying SCSS map.
+  #
+  # It will not appear in `rake -T` output, and the dependent gems are not
+  # included in the Gemfile by default, because this task will only be needed
+  # occasionally, such as when new Emojis are added to TanukiEmoji.
+  task sprite: :environment do
+    begin
+      require 'sprite_factory'
+      # Sprite-Factory still requires rmagick, but maybe could be migrated to support minimagick
+      # Upstream issue: https://github.com/jakesgordon/sprite-factory/issues/47#issuecomment-929302890
+      require 'rmagick'
+    rescue LoadError
+      # noop
+    end
+
+    check_requirements!
+
+    SIZE   = 20
+    RETINA = SIZE * 2
+
+    # Update these values to the width and height of the spritesheet when
+    # new emoji are added.
+    SPRITESHEET_WIDTH = 860
+    SPRITESHEET_HEIGHT = 840
+
+    emoji_dir = Gitlab::Emoji.emoji_public_absolute_path
+
+    puts "Preparing sprites for regular size: #{SIZE}px..."
+
+    Dir.mktmpdir do |tmpdir|
+      FileUtils.cp_r(File.join(emoji_dir, '.'), tmpdir)
+
+      Dir.chdir(tmpdir) do
+        Dir["**/*.png"].each do |png|
+          tmp_image_path = File.join(tmpdir, png)
+          resize!(tmp_image_path, SIZE)
+          print '.'
+        end
+      end
+      puts ' Done!'
+
+      puts "\n"
+
+      style_path = Rails.root.join(*%w(app assets stylesheets emoji_sprites.scss))
+
+      print 'Compiling sprites regular sprites... '
+
+      # Combine the resized assets into a packed sprite and re-generate the SCSS
+      SpriteFactory.cssurl = "image-url('$IMAGE')"
+      SpriteFactory.run!(tmpdir, {
+        output_style: style_path,
+        output_image: "app/assets/images/emoji.png",
+        selector:     '.emoji-',
+        style:        :scss,
+        nocomments:   true,
+        pngcrush:     true,
+        layout:       :packed
+      })
+
+      # SpriteFactory's SCSS is a bit too verbose for our purposes here, so
+      # let's simplify it
+      system(%Q(sed -i '' "s/width: #{SIZE}px; height: #{SIZE}px; background: image-url('emoji.png')/background-position:/" #{style_path}))
+      system(%Q(sed -i '' "s/ no-repeat//" #{style_path}))
+      system(%Q(sed -i '' "s/ 0px/ 0/g" #{style_path}))
+
+      # Append a generic rule that applies to all Emojis
+      File.open(style_path, 'a') do |f|
+        f.puts
+        f.puts <<-CSS.strip_heredoc
+        .emoji-icon {
+          background-image: image-url('emoji.png');
+          background-repeat: no-repeat;
+          color: transparent;
+          text-indent: -99em;
+          height: #{SIZE}px;
+          width: #{SIZE}px;
+
+          /* stylelint-disable media-feature-name-no-vendor-prefix */
+          @media only screen and (-webkit-min-device-pixel-ratio: 2),
+            only screen and (min--moz-device-pixel-ratio: 2),
+            only screen and (-o-min-device-pixel-ratio: 2/1),
+            only screen and (min-device-pixel-ratio: 2),
+            only screen and (min-resolution: 192dpi),
+            only screen and (min-resolution: 2dppx) {
+            background-image: image-url('emoji@2x.png');
+            background-size: #{SPRITESHEET_WIDTH}px #{SPRITESHEET_HEIGHT}px;
+          }
+          /* stylelint-enable media-feature-name-no-vendor-prefix */
+        }
+        CSS
+      end
+    end
+    puts 'Done!'
+
+    puts "\n"
+
+    puts "Preparing sprites for HiDPI size: #{RETINA}px..."
+
+    # Now do it again but for Retina
+    Dir.mktmpdir do |tmpdir|
+      # Copy the TanukiEmoji assets to the temporary folder for resizing
+      FileUtils.cp_r(File.join(emoji_dir, '.'), tmpdir)
+
+      Dir.chdir(tmpdir) do
+        Dir["**/*.png"].each do |png|
+          tmp_image_path = File.join(tmpdir, png)
+          resize!(tmp_image_path, RETINA)
+          print '.'
+        end
+      end
+      puts ' Done!'
+
+      puts "\n"
+
+      print 'Compiling HiDPI sprites...'
+
+      # Combine the resized assets into a packed sprite and re-generate the SCSS
+      SpriteFactory.run!(tmpdir, {
+        output_image: "app/assets/images/emoji@2x.png",
+        style:        false,
+        nocomments:   true,
+        pngcrush:     true,
+        layout:       :packed
+      })
+    end
+
+    puts ' Done!'
+  end
+
+  def check_requirements!
+    unless defined?(Magick)
+      puts <<~MSG
+      This task is disabled by default and should only be run when the TanukiEmoji
+      gem is updated with new Emojis.
+
+      To enable this task, *temporarily* add the following lines to Gemfile and
+      re-bundle:
+
+      gem 'rmagick', '~> 3.2'
+
+      It depends on ImageMagick 6, which can be installed via HomeBrew with:
+
+      brew unlink imagemagick
+      brew install imagemagick@6 && brew link imagemagick@6 --force
+      MSG
+
+      exit 1
+    end
+
+    return if Dir.exist? Gitlab::Emoji.emoji_public_absolute_path
+
+    puts <<~MSG
+    You first need to import the assets for Emoji version: #{Gitlab::Emoji::EMOJI_VERSION}
+
+    Run the following task:
+
+    rake tanuki_emoji:import
+    MSG
+
+    exit 1
+  end
+
+  def resize!(image_path, size)
+    # Resize the image in-place, save it, and free the object
+    image = MiniMagick::Image.open(image_path)
+    image.quality(100)
+    image.resize("#{size}x#{size}")
+    image.write(image_path)
+  end
+end
diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb
index 79920968603..36baf4a3cf8 100644
--- a/lib/uploaded_file.rb
+++ b/lib/uploaded_file.rb
@@ -20,8 +20,9 @@ class UploadedFile
   attr_reader :remote_id
   attr_reader :sha256
   attr_reader :size
+  attr_reader :upload_duration
 
-  def initialize(path, filename: nil, content_type: "application/octet-stream", sha256: nil, remote_id: nil, size: nil)
+  def initialize(path, filename: nil, content_type: "application/octet-stream", sha256: nil, remote_id: nil, size: nil, upload_duration: nil)
     if path.present?
       raise InvalidPathError, "#{path} file does not exist" unless ::File.exist?(path)
 
@@ -35,6 +36,12 @@ class UploadedFile
       end
     end
 
+    begin
+      @upload_duration = Float(upload_duration)
+    rescue ArgumentError, TypeError
+      @upload_duration = 0
+    end
+
     @content_type = content_type
     @original_filename = sanitize_filename(filename || path || '')
     @content_type = content_type
@@ -64,8 +71,11 @@ class UploadedFile
       content_type: params['type'] || 'application/octet-stream',
       sha256: params['sha256'],
       remote_id: remote_id,
-      size: params['size']
-    )
+      size: params['size'],
+      upload_duration: params['upload_duration']
+    ).tap do |uploaded_file|
+      ::Gitlab::Instrumentation::Uploads.track(uploaded_file)
+    end
   end
 
   def self.allowed_path?(file_path, paths)
-- 
cgit v1.2.3