Add latest changes from gitlab-org/gitlab@14-9-stable-eev14.9.0-rc42

17 files changed, 303 insertions, 69 deletions

diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index fb1fcb7937c..48bdee4062b 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -309,6 +309,15 @@
   :weight: 1
   :idempotent: true
   :tags: []
+- :name: cronjob:database_batched_background_migration_ci_database
+  :worker_name: Database::BatchedBackgroundMigration::CiDatabaseWorker
+  :feature_category: :database
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: cronjob:database_drop_detached_partitions
   :worker_name: Database::DropDetachedPartitionsWorker
   :feature_category: :database
@@ -552,6 +561,15 @@
   :weight: 1
   :idempotent:
   :tags: []
+- :name: cronjob:projects_schedule_refresh_build_artifacts_size_statistics
+  :worker_name: Projects::ScheduleRefreshBuildArtifactsSizeStatisticsWorker
+  :feature_category: :build_artifacts
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: cronjob:prune_old_events
   :worker_name: PruneOldEventsWorker
   :feature_category: :users
@@ -561,6 +579,15 @@
   :weight: 1
   :idempotent:
   :tags: []
+- :name: cronjob:quality_test_data_cleanup
+  :worker_name: Quality::TestDataCleanupWorker
+  :feature_category: :quality_management
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: cronjob:releases_manage_evidence
   :worker_name: Releases::ManageEvidenceWorker
   :feature_category: :release_evidence
@@ -1654,7 +1681,7 @@
   :worker_name: Ci::DropPipelineWorker
   :feature_category: :continuous_integration
   :has_external_dependencies:
-  :urgency: :low
+  :urgency: :high
   :resource_boundary: :unknown
   :weight: 3
   :idempotent: true
@@ -2785,6 +2812,15 @@
   :weight: 1
   :idempotent: true
   :tags: []
+- :name: projects_refresh_build_artifacts_size_statistics
+  :worker_name: Projects::RefreshBuildArtifactsSizeStatisticsWorker
+  :feature_category: :build_artifacts
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: projects_schedule_bulk_repository_shard_moves
   :worker_name: Projects::ScheduleBulkRepositoryShardMovesWorker
   :feature_category: :gitaly
diff --git a/app/workers/bulk_imports/export_request_worker.rb b/app/workers/bulk_imports/export_request_worker.rb
index 8bc0acc9b22..21040178cee 100644
--- a/app/workers/bulk_imports/export_request_worker.rb
+++ b/app/workers/bulk_imports/export_request_worker.rb
@@ -14,6 +14,10 @@ module BulkImports
       entity = BulkImports::Entity.find(entity_id)
 
       request_export(entity)
+    rescue BulkImports::NetworkError => e
+      log_export_failure(e, entity)
+
+      entity.fail_op!
     end
 
     private
@@ -28,5 +32,24 @@ module BulkImports
         token: configuration.access_token
       )
     end
+
+    def log_export_failure(exception, entity)
+      attributes = {
+        bulk_import_entity_id: entity.id,
+        pipeline_class: 'ExportRequestWorker',
+        exception_class: exception.class.to_s,
+        exception_message: exception.message.truncate(255),
+        correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id
+      }
+
+      Gitlab::Import::Logger.warn(
+        attributes.merge(
+          bulk_import_id: entity.bulk_import.id,
+          bulk_import_entity_type: entity.source_type
+        )
+      )
+
+      BulkImports::Failure.create(attributes)
+    end
   end
 end
diff --git a/app/workers/bulk_imports/pipeline_worker.rb b/app/workers/bulk_imports/pipeline_worker.rb
index 8e5d7013c2c..03ec2f058ca 100644
--- a/app/workers/bulk_imports/pipeline_worker.rb
+++ b/app/workers/bulk_imports/pipeline_worker.rb
@@ -43,6 +43,10 @@ module BulkImports
     private
 
     def run(pipeline_tracker)
+      if pipeline_tracker.entity.failed?
+        raise(Entity::FailedError, 'Failed entity status')
+      end
+
       if ndjson_pipeline?(pipeline_tracker)
         status = ExportStatus.new(pipeline_tracker, pipeline_tracker.pipeline_class.relation)
 
diff --git a/app/workers/ci/build_finished_worker.rb b/app/workers/ci/build_finished_worker.rb
index 56cfaa7e674..70c234bd4c7 100644
--- a/app/workers/ci/build_finished_worker.rb
+++ b/app/workers/ci/build_finished_worker.rb
@@ -39,6 +39,7 @@ module Ci
       # We execute these async as these are independent operations.
       BuildHooksWorker.perform_async(build.id)
       ChatNotificationWorker.perform_async(build.id) if build.pipeline.chat?
+      build.track_deployment_usage
 
       if build.failed? && !build.auto_retry_expected?
         ::Ci::MergeRequests::AddTodoWhenBuildFailsWorker.perform_async(build.id)
diff --git a/app/workers/ci/drop_pipeline_worker.rb b/app/workers/ci/drop_pipeline_worker.rb
index edb97c3cac5..6018290b3a2 100644
--- a/app/workers/ci/drop_pipeline_worker.rb
+++ b/app/workers/ci/drop_pipeline_worker.rb
@@ -9,6 +9,8 @@ module Ci
     sidekiq_options retry: 3
     include PipelineQueue
 
+    urgency :high
+
     idempotent!
 
     def perform(pipeline_id, failure_reason)
diff --git a/app/workers/concerns/git_garbage_collect_methods.rb b/app/workers/concerns/git_garbage_collect_methods.rb
index c46deeb716f..13b7e7b5b1f 100644
--- a/app/workers/concerns/git_garbage_collect_methods.rb
+++ b/app/workers/concerns/git_garbage_collect_methods.rb
@@ -83,17 +83,27 @@ module GitGarbageCollectMethods
   def gitaly_call(task, resource)
     repository = resource.repository.raw_repository
 
-    client = get_gitaly_client(task, repository)
-
-    case task
-    when :prune, :gc
-      client.garbage_collect(bitmaps_enabled?, prune: task == :prune)
-    when :full_repack
-      client.repack_full(bitmaps_enabled?)
-    when :incremental_repack
-      client.repack_incremental
-    when :pack_refs
-      client.pack_refs
+    if Feature.enabled?(:optimized_housekeeping, container(resource), default_enabled: :yaml)
+      client = repository.gitaly_repository_client
+
+      if task == :prune
+        client.prune_unreachable_objects
+      else
+        client.optimize_repository
+      end
+    else
+      client = get_gitaly_client(task, repository)
+
+      case task
+      when :prune, :gc
+        client.garbage_collect(bitmaps_enabled?, prune: task == :prune)
+      when :full_repack
+        client.repack_full(bitmaps_enabled?)
+      when :incremental_repack
+        client.repack_incremental
+      when :pack_refs
+        client.pack_refs
+      end
     end
   rescue GRPC::NotFound => e
     Gitlab::GitLogger.error("#{__method__} failed:\nRepository not found")
diff --git a/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb b/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb
index 7f7a77d0524..cd3ed5d4c9b 100644
--- a/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb
+++ b/app/workers/container_expiration_policies/cleanup_container_repository_worker.rb
@@ -123,7 +123,7 @@ module ContainerExpirationPolicies
     end
 
     def throttling_enabled?
-      Feature.enabled?(:container_registry_expiration_policies_throttling)
+      Feature.enabled?(:container_registry_expiration_policies_throttling, default_enabled: :yaml)
     end
 
     def max_cleanup_execution_time
diff --git a/app/workers/container_expiration_policy_worker.rb b/app/workers/container_expiration_policy_worker.rb
index 16ac61976eb..308ccfe2cb3 100644
--- a/app/workers/container_expiration_policy_worker.rb
+++ b/app/workers/container_expiration_policy_worker.rb
@@ -99,7 +99,7 @@ class ContainerExpirationPolicyWorker # rubocop:disable Scalability/IdempotentWo
   end
 
   def throttling_enabled?
-    Feature.enabled?(:container_registry_expiration_policies_throttling)
+    Feature.enabled?(:container_registry_expiration_policies_throttling, default_enabled: :yaml)
   end
 
   def lease_timeout
diff --git a/app/workers/database/batched_background_migration/ci_database_worker.rb b/app/workers/database/batched_background_migration/ci_database_worker.rb
new file mode 100644
index 00000000000..98ec6f98123
--- /dev/null
+++ b/app/workers/database/batched_background_migration/ci_database_worker.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+module Database
+  module BatchedBackgroundMigration
+    class CiDatabaseWorker # rubocop:disable Scalability/IdempotentWorker
+      include SingleDatabaseWorker
+
+      def self.tracking_database
+        @tracking_database ||= Gitlab::Database::CI_DATABASE_NAME
+      end
+    end
+  end
+end
diff --git a/app/workers/database/batched_background_migration/single_database_worker.rb b/app/workers/database/batched_background_migration/single_database_worker.rb
new file mode 100644
index 00000000000..78c82a6549f
--- /dev/null
+++ b/app/workers/database/batched_background_migration/single_database_worker.rb
@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Database
+  module BatchedBackgroundMigration
+    module SingleDatabaseWorker
+      extend ActiveSupport::Concern
+
+      include ApplicationWorker
+      include CronjobQueue # rubocop:disable Scalability/CronWorkerContext
+
+      LEASE_TIMEOUT_MULTIPLIER = 3
+      MINIMUM_LEASE_TIMEOUT = 10.minutes.freeze
+      INTERVAL_VARIANCE = 5.seconds.freeze
+
+      included do
+        data_consistency :always
+        feature_category :database
+        idempotent!
+      end
+
+      class_methods do
+        # :nocov:
+        def tracking_database
+          raise NotImplementedError, "#{self.name} does not implement #{__method__}"
+        end
+        # :nocov:
+
+        def lease_key
+          name.demodulize.underscore
+        end
+      end
+
+      def perform
+        unless base_model
+          Sidekiq.logger.info(
+            class: self.class.name,
+            database: self.class.tracking_database,
+            message: 'skipping migration execution for unconfigured database')
+
+          return
+        end
+
+        Gitlab::Database::SharedModel.using_connection(base_model.connection) do
+          break unless Feature.enabled?(:execute_batched_migrations_on_schedule, type: :ops, default_enabled: :yaml) && active_migration
+
+          with_exclusive_lease(active_migration.interval) do
+            # Now that we have the exclusive lease, reload migration in case another process has changed it.
+            # This is a temporary solution until we have better concurrency handling around job execution
+            #
+            # We also have to disable this cop, because ApplicationRecord aliases reset to reload, but our database
+            # models don't inherit from ApplicationRecord
+            active_migration.reload # rubocop:disable Cop/ActiveRecordAssociationReload
+
+            run_active_migration if active_migration.active? && active_migration.interval_elapsed?(variance: INTERVAL_VARIANCE)
+          end
+        end
+      end
+
+      private
+
+      def active_migration
+        @active_migration ||= Gitlab::Database::BackgroundMigration::BatchedMigration.active_migration
+      end
+
+      def run_active_migration
+        Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new(connection: base_model.connection).run_migration_job(active_migration)
+      end
+
+      def base_model
+        @base_model ||= Gitlab::Database.database_base_models[self.class.tracking_database]
+      end
+
+      def with_exclusive_lease(interval)
+        timeout = [interval * LEASE_TIMEOUT_MULTIPLIER, MINIMUM_LEASE_TIMEOUT].max
+        lease = Gitlab::ExclusiveLease.new(self.class.lease_key, timeout: timeout)
+
+        yield if lease.try_obtain
+      ensure
+        lease&.cancel
+      end
+    end
+  end
+end
diff --git a/app/workers/database/batched_background_migration_worker.rb b/app/workers/database/batched_background_migration_worker.rb
index fda539b372d..29804be832d 100644
--- a/app/workers/database/batched_background_migration_worker.rb
+++ b/app/workers/database/batched_background_migration_worker.rb
@@ -1,56 +1,11 @@
 # frozen_string_literal: true
 
 module Database
-  class BatchedBackgroundMigrationWorker
-    include ApplicationWorker
+  class BatchedBackgroundMigrationWorker # rubocop:disable Scalability/IdempotentWorker
+    include BatchedBackgroundMigration::SingleDatabaseWorker
 
-    data_consistency :always
-
-    include CronjobQueue # rubocop:disable Scalability/CronWorkerContext
-
-    feature_category :database
-    idempotent!
-
-    LEASE_TIMEOUT_MULTIPLIER = 3
-    MINIMUM_LEASE_TIMEOUT = 10.minutes.freeze
-    INTERVAL_VARIANCE = 5.seconds.freeze
-
-    def perform
-      return unless Feature.enabled?(:execute_batched_migrations_on_schedule, type: :ops, default_enabled: :yaml) && active_migration
-
-      with_exclusive_lease(active_migration.interval) do
-        # Now that we have the exclusive lease, reload migration in case another process has changed it.
-        # This is a temporary solution until we have better concurrency handling around job execution
-        #
-        # We also have to disable this cop, because ApplicationRecord aliases reset to reload, but our database
-        # models don't inherit from ApplicationRecord
-        active_migration.reload # rubocop:disable Cop/ActiveRecordAssociationReload
-
-        run_active_migration if active_migration.active? && active_migration.interval_elapsed?(variance: INTERVAL_VARIANCE)
-      end
-    end
-
-    private
-
-    def active_migration
-      @active_migration ||= Gitlab::Database::BackgroundMigration::BatchedMigration.active_migration
-    end
-
-    def run_active_migration
-      Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new.run_migration_job(active_migration)
-    end
-
-    def with_exclusive_lease(interval)
-      timeout = [interval * LEASE_TIMEOUT_MULTIPLIER, MINIMUM_LEASE_TIMEOUT].max
-      lease = Gitlab::ExclusiveLease.new(lease_key, timeout: timeout)
-
-      yield if lease.try_obtain
-    ensure
-      lease&.cancel
-    end
-
-    def lease_key
-      self.class.name.demodulize.underscore
+    def self.tracking_database
+      @tracking_database ||= Gitlab::Database::MAIN_DATABASE_NAME.to_sym
     end
   end
 end
diff --git a/app/workers/projects/git_garbage_collect_worker.rb b/app/workers/projects/git_garbage_collect_worker.rb
index d16583975fc..a70c52abde2 100644
--- a/app/workers/projects/git_garbage_collect_worker.rb
+++ b/app/workers/projects/git_garbage_collect_worker.rb
@@ -7,6 +7,12 @@ module Projects
 
     private
 
+    # Used for getting a project/group out of the resource in order to scope a feature flag
+    # Can be removed within https://gitlab.com/gitlab-org/gitlab/-/issues/353607
+    def container(resource)
+      resource
+    end
+
     override :find_resource
     def find_resource(id)
       Project.find(id)
diff --git a/app/workers/projects/refresh_build_artifacts_size_statistics_worker.rb b/app/workers/projects/refresh_build_artifacts_size_statistics_worker.rb
new file mode 100644
index 00000000000..a91af72cc2c
--- /dev/null
+++ b/app/workers/projects/refresh_build_artifacts_size_statistics_worker.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Projects
+  class RefreshBuildArtifactsSizeStatisticsWorker
+    include ApplicationWorker
+    include LimitedCapacity::Worker
+
+    MAX_RUNNING_LOW = 2
+    MAX_RUNNING_MEDIUM = 20
+    MAX_RUNNING_HIGH = 50
+
+    data_consistency :always
+
+    feature_category :build_artifacts
+
+    idempotent!
+
+    def perform_work(*args)
+      refresh = Projects::RefreshBuildArtifactsSizeStatisticsService.new.execute
+      return unless refresh
+
+      log_extra_metadata_on_done(:project_id, refresh.project_id)
+      log_extra_metadata_on_done(:last_job_artifact_id, refresh.last_job_artifact_id)
+      log_extra_metadata_on_done(:last_batch, refresh.destroyed?)
+      log_extra_metadata_on_done(:refresh_started_at, refresh.refresh_started_at)
+    end
+
+    def remaining_work_count(*args)
+      # LimitedCapacity::Worker only needs to know if there is work left to do
+      # so we can get by with an EXISTS query rather than a count.
+      # https://gitlab.com/gitlab-org/gitlab/-/issues/356167
+      if Projects::BuildArtifactsSizeRefresh.remaining.any?
+        1
+      else
+        0
+      end
+    end
+
+    def max_running_jobs
+      if ::Feature.enabled?(:projects_build_artifacts_size_refresh_high)
+        MAX_RUNNING_HIGH
+      elsif ::Feature.enabled?(:projects_build_artifacts_size_refresh_medium)
+        MAX_RUNNING_MEDIUM
+      elsif ::Feature.enabled?(:projects_build_artifacts_size_refresh_low)
+        MAX_RUNNING_LOW
+      else
+        0
+      end
+    end
+  end
+end
diff --git a/app/workers/projects/schedule_refresh_build_artifacts_size_statistics_worker.rb b/app/workers/projects/schedule_refresh_build_artifacts_size_statistics_worker.rb
new file mode 100644
index 00000000000..ed2b642d998
--- /dev/null
+++ b/app/workers/projects/schedule_refresh_build_artifacts_size_statistics_worker.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Projects
+  class ScheduleRefreshBuildArtifactsSizeStatisticsWorker
+    include ApplicationWorker
+    include CronjobQueue # rubocop:disable Scalability/CronWorkerContext
+
+    data_consistency :always
+
+    feature_category :build_artifacts
+
+    idempotent!
+
+    def perform
+      Projects::RefreshBuildArtifactsSizeStatisticsWorker.perform_with_capacity
+    end
+  end
+end
diff --git a/app/workers/quality/test_data_cleanup_worker.rb b/app/workers/quality/test_data_cleanup_worker.rb
new file mode 100644
index 00000000000..68b36cacbbf
--- /dev/null
+++ b/app/workers/quality/test_data_cleanup_worker.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Quality
+  class TestDataCleanupWorker
+    include ApplicationWorker
+
+    data_consistency :always
+    feature_category :quality_management
+    urgency :low
+
+    include CronjobQueue
+    idempotent!
+
+    KEEP_RECENT_DATA_DAY = 3
+    GROUP_PATH_PATTERN = 'test-group-fulfillment'
+    GROUP_OWNER_EMAIL_PATTERN = %w(test-user- gitlab-qa-user qa-user-).freeze
+
+    # Remove test groups generated in E2E tests on gstg
+    # rubocop: disable CodeReuse/ActiveRecord
+    def perform
+      return unless Gitlab.staging?
+
+      Group.where('path like ?', "#{GROUP_PATH_PATTERN}%").where('created_at < ?', KEEP_RECENT_DATA_DAY.days.ago).each do |group|
+        next unless GROUP_OWNER_EMAIL_PATTERN.any? { |pattern| group.owners.first.email.include?(pattern) }
+
+        with_context(namespace: group, user: group.owners.first) do
+          Groups::DestroyService.new(group, group.owners.first).execute
+        end
+      end
+    end
+    # rubocop: enable CodeReuse/ActiveRecord
+  end
+end
diff --git a/app/workers/web_hook_worker.rb b/app/workers/web_hook_worker.rb
index fdcd22128a3..301f3720991 100644
--- a/app/workers/web_hook_worker.rb
+++ b/app/workers/web_hook_worker.rb
@@ -13,9 +13,6 @@ class WebHookWorker
 
   worker_has_external_dependencies!
 
-  # Webhook recursion detection properties may be passed through the `data` arg.
-  # This will be migrated to the `params` arg over the next few releases.
-  # See https://gitlab.com/gitlab-org/gitlab/-/issues/347389.
   def perform(hook_id, data, hook_name, params = {})
     hook = WebHook.find_by_id(hook_id)
     return unless hook
@@ -23,9 +20,6 @@ class WebHookWorker
     data = data.with_indifferent_access
     params.symbolize_keys!
 
-    # TODO: Remove in 14.9 https://gitlab.com/gitlab-org/gitlab/-/issues/347389
-    params[:recursion_detection_request_uuid] ||= data.delete(:_gitlab_recursion_detection_request_uuid)
-
     # Before executing the hook, reapply any recursion detection UUID that was initially
     # present in the request header so the hook can pass this same header value in its request.
     Gitlab::WebHooks::RecursionDetection.set_request_uuid(params[:recursion_detection_request_uuid])
diff --git a/app/workers/wikis/git_garbage_collect_worker.rb b/app/workers/wikis/git_garbage_collect_worker.rb
index 1b455c50618..b00190c6b98 100644
--- a/app/workers/wikis/git_garbage_collect_worker.rb
+++ b/app/workers/wikis/git_garbage_collect_worker.rb
@@ -7,6 +7,12 @@ module Wikis
 
     private
 
+    # Used for getting a project/group out of the resource in order to scope a feature flag
+    # Can be removed within https://gitlab.com/gitlab-org/gitlab/-/issues/353607
+    def container(resource)
+      resource.container
+    end
+
     override :find_resource
     def find_resource(id)
       Project.find(id).wiki