Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2022-01-20 12:16:11 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2022-01-20 12:16:11 +0300
commitedaa33dee2ff2f7ea3fac488d41558eb5f86d68c (patch)
tree11f143effbfeba52329fb7afbd05e6e2a3790241 /app/workers
parentd8a5691316400a0f7ec4f83832698f1988eb27c1 (diff)
Add latest changes from gitlab-org/gitlab@14-7-stable-eev14.7.0-rc42
Diffstat (limited to 'app/workers')
-rw-r--r--app/workers/all_queues.yml49
-rw-r--r--app/workers/ci/build_finished_worker.rb2
-rw-r--r--app/workers/ci/external_pull_requests/create_pipeline_worker.rb2
-rw-r--r--app/workers/ci/job_artifacts/expire_project_build_artifacts_worker.rb20
-rw-r--r--app/workers/clusters/agents/delete_expired_events_worker.rb21
-rw-r--r--app/workers/concerns/application_worker.rb16
-rw-r--r--app/workers/concerns/cluster_agent_queue.rb10
-rw-r--r--app/workers/concerns/dependency_proxy/expireable.rb2
-rw-r--r--app/workers/concerns/packages/cleanup_artifact_worker.rb (renamed from app/workers/concerns/dependency_proxy/cleanup_worker.rb)17
-rw-r--r--app/workers/dependency_proxy/cleanup_blob_worker.rb7
-rw-r--r--app/workers/dependency_proxy/cleanup_dependency_proxy_worker.rb4
-rw-r--r--app/workers/dependency_proxy/cleanup_manifest_worker.rb7
-rw-r--r--app/workers/dependency_proxy/image_ttl_group_policy_worker.rb4
-rw-r--r--app/workers/email_receiver_worker.rb33
-rw-r--r--app/workers/expire_build_artifacts_worker.rb4
-rw-r--r--app/workers/loose_foreign_keys/cleanup_worker.rb28
-rw-r--r--app/workers/merge_requests/update_head_pipeline_worker.rb23
-rw-r--r--app/workers/metrics/dashboard/sync_dashboards_worker.rb2
-rw-r--r--app/workers/packages/cleanup_package_file_worker.rb44
-rw-r--r--app/workers/packages/cleanup_package_registry_worker.rb41
-rw-r--r--app/workers/pages_update_configuration_worker.rb16
-rw-r--r--app/workers/pages_worker.rb9
-rw-r--r--app/workers/update_external_pull_requests_worker.rb2
-rw-r--r--app/workers/update_head_pipeline_for_merge_request_worker.rb2
-rw-r--r--app/workers/web_hook_worker.rb12
25 files changed, 280 insertions, 97 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index f2961d825a0..14fa6599073 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -129,6 +129,15 @@
:weight: 2
:idempotent:
:tags: []
+- :name: cluster_agent:clusters_agents_delete_expired_events
+ :worker_name: Clusters::Agents::DeleteExpiredEventsWorker
+ :feature_category: :kubernetes_management
+ :has_external_dependencies:
+ :urgency: :low
+ :resource_boundary: :unknown
+ :weight: 1
+ :idempotent: true
+ :tags: []
- :name: container_repository:cleanup_container_repository
:worker_name: CleanupContainerRepositoryWorker
:feature_category: :container_registry
@@ -435,6 +444,15 @@
:weight: 1
:idempotent:
:tags: []
+- :name: cronjob:packages_cleanup_package_registry
+ :worker_name: Packages::CleanupPackageRegistryWorker
+ :feature_category: :package_registry
+ :has_external_dependencies:
+ :urgency: :low
+ :resource_boundary: :unknown
+ :weight: 1
+ :idempotent: true
+ :tags: []
- :name: cronjob:packages_composer_cache_cleanup
:worker_name: Packages::Composer::CacheCleanupWorker
:feature_category: :package_registry
@@ -1357,6 +1375,15 @@
:weight: 1
:idempotent:
:tags: []
+- :name: package_cleanup:packages_cleanup_package_file
+ :worker_name: Packages::CleanupPackageFileWorker
+ :feature_category: :package_registry
+ :has_external_dependencies:
+ :urgency: :low
+ :resource_boundary: :unknown
+ :weight: 1
+ :idempotent: true
+ :tags: []
- :name: package_repositories:packages_debian_generate_distribution
:worker_name: Packages::Debian::GenerateDistributionWorker
:feature_category: :package_registry
@@ -1539,7 +1566,7 @@
:tags: []
- :name: pipeline_creation:ci_external_pull_requests_create_pipeline
:worker_name: Ci::ExternalPullRequests::CreatePipelineWorker
- :feature_category: :pipeline_authoring
+ :feature_category: :continuous_integration
:has_external_dependencies:
:urgency: :high
:resource_boundary: :cpu
@@ -1987,6 +2014,15 @@
:weight: 1
:idempotent: true
:tags: []
+- :name: ci_job_artifacts_expire_project_build_artifacts
+ :worker_name: Ci::JobArtifacts::ExpireProjectBuildArtifactsWorker
+ :feature_category: :build_artifacts
+ :has_external_dependencies:
+ :urgency: :low
+ :resource_boundary: :unknown
+ :weight: 1
+ :idempotent: true
+ :tags: []
- :name: create_commit_signature
:worker_name: CreateCommitSignatureWorker
:feature_category: :source_code_management
@@ -2420,6 +2456,15 @@
:weight: 1
:idempotent: true
:tags: []
+- :name: merge_requests_update_head_pipeline
+ :worker_name: MergeRequests::UpdateHeadPipelineWorker
+ :feature_category: :code_review
+ :has_external_dependencies:
+ :urgency: :high
+ :resource_boundary: :cpu
+ :weight: 1
+ :idempotent: true
+ :tags: []
- :name: metrics_dashboard_prune_old_annotations
:worker_name: Metrics::Dashboard::PruneOldAnnotationsWorker
:feature_category: :metrics
@@ -2909,7 +2954,7 @@
:tags: []
- :name: update_external_pull_requests
:worker_name: UpdateExternalPullRequestsWorker
- :feature_category: :source_code_management
+ :feature_category: :continuous_integration
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown
diff --git a/app/workers/ci/build_finished_worker.rb b/app/workers/ci/build_finished_worker.rb
index aa12bdb009e..56cfaa7e674 100644
--- a/app/workers/ci/build_finished_worker.rb
+++ b/app/workers/ci/build_finished_worker.rb
@@ -40,7 +40,7 @@ module Ci
BuildHooksWorker.perform_async(build.id)
ChatNotificationWorker.perform_async(build.id) if build.pipeline.chat?
- if build.failed?
+ if build.failed? && !build.auto_retry_expected?
::Ci::MergeRequests::AddTodoWhenBuildFailsWorker.perform_async(build.id)
end
diff --git a/app/workers/ci/external_pull_requests/create_pipeline_worker.rb b/app/workers/ci/external_pull_requests/create_pipeline_worker.rb
index 211ea1f2990..334ff099ea2 100644
--- a/app/workers/ci/external_pull_requests/create_pipeline_worker.rb
+++ b/app/workers/ci/external_pull_requests/create_pipeline_worker.rb
@@ -7,7 +7,7 @@ module Ci
data_consistency :always
queue_namespace :pipeline_creation
- feature_category :pipeline_authoring
+ feature_category :continuous_integration
urgency :high
worker_resource_boundary :cpu
diff --git a/app/workers/ci/job_artifacts/expire_project_build_artifacts_worker.rb b/app/workers/ci/job_artifacts/expire_project_build_artifacts_worker.rb
new file mode 100644
index 00000000000..299b9bbe3d3
--- /dev/null
+++ b/app/workers/ci/job_artifacts/expire_project_build_artifacts_worker.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Ci
+ module JobArtifacts
+ class ExpireProjectBuildArtifactsWorker
+ include ApplicationWorker
+
+ data_consistency :always
+
+ feature_category :build_artifacts
+ idempotent!
+
+ def perform(project_id)
+ return unless Project.id_in(project_id).exists?
+
+ ExpireProjectBuildArtifactsService.new(project_id, Time.current).execute
+ end
+ end
+ end
+end
diff --git a/app/workers/clusters/agents/delete_expired_events_worker.rb b/app/workers/clusters/agents/delete_expired_events_worker.rb
new file mode 100644
index 00000000000..3414365a243
--- /dev/null
+++ b/app/workers/clusters/agents/delete_expired_events_worker.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Clusters
+ module Agents
+ class DeleteExpiredEventsWorker
+ include ApplicationWorker
+ include ClusterAgentQueue
+
+ deduplicate :until_executed, including_scheduled: true
+ idempotent!
+
+ data_consistency :always
+
+ def perform(agent_id)
+ if agent = Clusters::Agent.find_by_id(agent_id)
+ Clusters::Agents::DeleteExpiredEventsService.new(agent).execute
+ end
+ end
+ end
+ end
+end
diff --git a/app/workers/concerns/application_worker.rb b/app/workers/concerns/application_worker.rb
index d0b09c15289..83261d9e42e 100644
--- a/app/workers/concerns/application_worker.rb
+++ b/app/workers/concerns/application_worker.rb
@@ -189,12 +189,8 @@ module ApplicationWorker
schedule_at = bulk_schedule_at
end
- if Feature.enabled?(:sidekiq_push_bulk_in_batches)
- in_safe_limit_batches(args_list, schedule_at) do |args_batch, schedule_at_for_batch|
- Sidekiq::Client.push_bulk('class' => self, 'args' => args_batch, 'at' => schedule_at_for_batch)
- end
- else
- Sidekiq::Client.push_bulk('class' => self, 'args' => args_list, 'at' => schedule_at)
+ in_safe_limit_batches(args_list, schedule_at) do |args_batch, schedule_at_for_batch|
+ Sidekiq::Client.push_bulk('class' => self, 'args' => args_batch, 'at' => schedule_at_for_batch)
end
end
@@ -207,12 +203,8 @@ module ApplicationWorker
private
def do_push_bulk(args_list)
- if Feature.enabled?(:sidekiq_push_bulk_in_batches)
- in_safe_limit_batches(args_list) do |args_batch, _|
- Sidekiq::Client.push_bulk('class' => self, 'args' => args_batch)
- end
- else
- Sidekiq::Client.push_bulk('class' => self, 'args' => args_list)
+ in_safe_limit_batches(args_list) do |args_batch, _|
+ Sidekiq::Client.push_bulk('class' => self, 'args' => args_batch)
end
end
diff --git a/app/workers/concerns/cluster_agent_queue.rb b/app/workers/concerns/cluster_agent_queue.rb
new file mode 100644
index 00000000000..68de7cca135
--- /dev/null
+++ b/app/workers/concerns/cluster_agent_queue.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module ClusterAgentQueue
+ extend ActiveSupport::Concern
+
+ included do
+ queue_namespace :cluster_agent
+ feature_category :kubernetes_management
+ end
+end
diff --git a/app/workers/concerns/dependency_proxy/expireable.rb b/app/workers/concerns/dependency_proxy/expireable.rb
index 9650ac85c6c..7e37db36bef 100644
--- a/app/workers/concerns/dependency_proxy/expireable.rb
+++ b/app/workers/concerns/dependency_proxy/expireable.rb
@@ -10,7 +10,7 @@ module DependencyProxy
def expire_artifacts(collection)
collection.each_batch(of: UPDATE_BATCH_SIZE) do |batch|
- batch.update_all(status: :expired)
+ batch.update_all(status: :pending_destruction)
end
end
end
diff --git a/app/workers/concerns/dependency_proxy/cleanup_worker.rb b/app/workers/concerns/packages/cleanup_artifact_worker.rb
index b668634f233..db6c7330ea3 100644
--- a/app/workers/concerns/dependency_proxy/cleanup_worker.rb
+++ b/app/workers/concerns/packages/cleanup_artifact_worker.rb
@@ -1,8 +1,9 @@
# frozen_string_literal: true
-module DependencyProxy
- module CleanupWorker
+module Packages
+ module CleanupArtifactWorker
extend ActiveSupport::Concern
+ include LimitedCapacity::Worker
include Gitlab::Utils::StrongMemoize
def perform_work
@@ -15,12 +16,8 @@ module DependencyProxy
artifact&.error!
end
- def max_running_jobs
- ::Gitlab::CurrentSettings.dependency_proxy_ttl_group_policy_worker_capacity
- end
-
def remaining_work_count
- expired_artifacts.limit(max_running_jobs + 1).count
+ artifacts_pending_destruction.limit(max_running_jobs + 1).count
end
private
@@ -52,12 +49,12 @@ module DependencyProxy
end
end
- def expired_artifacts
- model.expired
+ def artifacts_pending_destruction
+ model.pending_destruction
end
def next_item
- expired_artifacts.lock_next_by(:updated_at).first
+ model.next_pending_destruction(order_by: :updated_at)
end
end
end
diff --git a/app/workers/dependency_proxy/cleanup_blob_worker.rb b/app/workers/dependency_proxy/cleanup_blob_worker.rb
index 054bc5854a3..eab536cd191 100644
--- a/app/workers/dependency_proxy/cleanup_blob_worker.rb
+++ b/app/workers/dependency_proxy/cleanup_blob_worker.rb
@@ -3,9 +3,8 @@
module DependencyProxy
class CleanupBlobWorker
include ApplicationWorker
- include LimitedCapacity::Worker
+ include ::Packages::CleanupArtifactWorker
include Gitlab::Utils::StrongMemoize
- include DependencyProxy::CleanupWorker
data_consistency :always
@@ -17,6 +16,10 @@ module DependencyProxy
worker_resource_boundary :unknown
idempotent!
+ def max_running_jobs
+ ::Gitlab::CurrentSettings.dependency_proxy_ttl_group_policy_worker_capacity
+ end
+
private
def model
diff --git a/app/workers/dependency_proxy/cleanup_dependency_proxy_worker.rb b/app/workers/dependency_proxy/cleanup_dependency_proxy_worker.rb
index d77c782267a..6a958d6e9d7 100644
--- a/app/workers/dependency_proxy/cleanup_dependency_proxy_worker.rb
+++ b/app/workers/dependency_proxy/cleanup_dependency_proxy_worker.rb
@@ -11,8 +11,8 @@ module DependencyProxy
feature_category :dependency_proxy
def perform
- enqueue_blob_cleanup_job if DependencyProxy::Blob.expired.any?
- enqueue_manifest_cleanup_job if DependencyProxy::Manifest.expired.any?
+ enqueue_blob_cleanup_job if DependencyProxy::Blob.pending_destruction.any?
+ enqueue_manifest_cleanup_job if DependencyProxy::Manifest.pending_destruction.any?
end
private
diff --git a/app/workers/dependency_proxy/cleanup_manifest_worker.rb b/app/workers/dependency_proxy/cleanup_manifest_worker.rb
index 1186efa2034..4445ee32e6d 100644
--- a/app/workers/dependency_proxy/cleanup_manifest_worker.rb
+++ b/app/workers/dependency_proxy/cleanup_manifest_worker.rb
@@ -3,9 +3,8 @@
module DependencyProxy
class CleanupManifestWorker
include ApplicationWorker
- include LimitedCapacity::Worker
+ include ::Packages::CleanupArtifactWorker
include Gitlab::Utils::StrongMemoize
- include DependencyProxy::CleanupWorker
data_consistency :always
@@ -17,6 +16,10 @@ module DependencyProxy
worker_resource_boundary :unknown
idempotent!
+ def max_running_jobs
+ ::Gitlab::CurrentSettings.dependency_proxy_ttl_group_policy_worker_capacity
+ end
+
private
def model
diff --git a/app/workers/dependency_proxy/image_ttl_group_policy_worker.rb b/app/workers/dependency_proxy/image_ttl_group_policy_worker.rb
index 3de2364fc71..89099ab6398 100644
--- a/app/workers/dependency_proxy/image_ttl_group_policy_worker.rb
+++ b/app/workers/dependency_proxy/image_ttl_group_policy_worker.rb
@@ -26,8 +26,8 @@ module DependencyProxy
def log_counts
use_replica_if_available do
- expired_blob_count = DependencyProxy::Blob.expired.count
- expired_manifest_count = DependencyProxy::Manifest.expired.count
+ expired_blob_count = DependencyProxy::Blob.pending_destruction.count
+ expired_manifest_count = DependencyProxy::Manifest.pending_destruction.count
processing_blob_count = DependencyProxy::Blob.processing.count
processing_manifest_count = DependencyProxy::Manifest.processing.count
error_blob_count = DependencyProxy::Blob.error.count
diff --git a/app/workers/email_receiver_worker.rb b/app/workers/email_receiver_worker.rb
index 321d492f0f3..54689df4d7b 100644
--- a/app/workers/email_receiver_worker.rb
+++ b/app/workers/email_receiver_worker.rb
@@ -90,37 +90,6 @@ class EmailReceiverWorker # rubocop:disable Scalability/IdempotentWorker
def handle_failure(error)
return unless raw.present?
- can_retry = false
- reason =
- case error
- when Gitlab::Email::UnknownIncomingEmail
- s_("EmailError|We couldn't figure out what the email is for. Please create your issue or comment through the web interface.")
- when Gitlab::Email::SentNotificationNotFoundError
- s_("EmailError|We couldn't figure out what the email is in reply to. Please create your comment through the web interface.")
- when Gitlab::Email::ProjectNotFound
- s_("EmailError|We couldn't find the project. Please check if there's any typo.")
- when Gitlab::Email::EmptyEmailError
- can_retry = true
- s_("EmailError|It appears that the email is blank. Make sure your reply is at the top of the email, we can't process inline replies.")
- when Gitlab::Email::UserNotFoundError
- s_("EmailError|We couldn't figure out what user corresponds to the email. Please create your comment through the web interface.")
- when Gitlab::Email::UserBlockedError
- s_("EmailError|Your account has been blocked. If you believe this is in error, contact a staff member.")
- when Gitlab::Email::UserNotAuthorizedError
- s_("EmailError|You are not allowed to perform this action. If you believe this is in error, contact a staff member.")
- when Gitlab::Email::NoteableNotFoundError
- s_("EmailError|The thread you are replying to no longer exists, perhaps it was deleted? If you believe this is in error, contact a staff member.")
- when Gitlab::Email::InvalidAttachment
- error.message
- when Gitlab::Email::InvalidRecordError
- can_retry = true
- error.message
- end
-
- if reason
- receiver.mail.body = nil
-
- EmailRejectionMailer.rejection(reason, receiver.mail.encoded, can_retry).deliver_later
- end
+ Gitlab::Email::FailureHandler.handle(receiver, error)
end
end
diff --git a/app/workers/expire_build_artifacts_worker.rb b/app/workers/expire_build_artifacts_worker.rb
index 295703cc1c3..9edff55b838 100644
--- a/app/workers/expire_build_artifacts_worker.rb
+++ b/app/workers/expire_build_artifacts_worker.rb
@@ -13,8 +13,8 @@ class ExpireBuildArtifactsWorker # rubocop:disable Scalability/IdempotentWorker
feature_category :build_artifacts
def perform
- service = Ci::JobArtifacts::DestroyAllExpiredService.new
- artifacts_count = service.execute
+ artifacts_count = Ci::JobArtifacts::DestroyAllExpiredService.new.execute
+
log_extra_metadata_on_done(:destroyed_job_artifacts_count, artifacts_count)
end
end
diff --git a/app/workers/loose_foreign_keys/cleanup_worker.rb b/app/workers/loose_foreign_keys/cleanup_worker.rb
index b4565dbf624..c3492fed77b 100644
--- a/app/workers/loose_foreign_keys/cleanup_worker.rb
+++ b/app/workers/loose_foreign_keys/cleanup_worker.rb
@@ -6,6 +6,7 @@ module LooseForeignKeys
include Gitlab::ExclusiveLeaseHelpers
include CronjobQueue # rubocop: disable Scalability/CronWorkerContext
+ sidekiq_options retry: false
feature_category :sharding
data_consistency :always
idempotent!
@@ -13,13 +14,30 @@ module LooseForeignKeys
def perform
return if Feature.disabled?(:loose_foreign_key_cleanup, default_enabled: :yaml)
- ttl = ModificationTracker::MAX_RUNTIME + 1.minute
- in_lock(self.class.name.underscore, ttl: ttl, retries: 0) do
- # TODO: Iterate over the connections
- # https://gitlab.com/gitlab-org/gitlab/-/issues/341513
- stats = ProcessDeletedRecordsService.new(connection: ApplicationRecord.connection).execute
+ in_lock(self.class.name.underscore, ttl: ModificationTracker::MAX_RUNTIME, retries: 0) do
+ stats = {}
+
+ connection_name, base_model = current_connection_name_and_base_model
+
+ Gitlab::Database::SharedModel.using_connection(base_model.connection) do
+ stats = ProcessDeletedRecordsService.new(connection: base_model.connection).execute
+ stats[:connection] = connection_name
+ end
+
log_extra_metadata_on_done(:stats, stats)
end
end
+
+ private
+
+ # Rotate the databases every minute
+ #
+ # If one DB is configured: every minute use the configured DB
+ # If two DBs are configured (Main, CI): minute 1 -> Main, minute 2 -> CI
+ def current_connection_name_and_base_model
+ minutes_since_epoch = Time.current.to_i / 60
+ connections_with_name = Gitlab::Database.database_base_models.to_a # this will never be empty
+ connections_with_name[minutes_since_epoch % connections_with_name.count]
+ end
end
end
diff --git a/app/workers/merge_requests/update_head_pipeline_worker.rb b/app/workers/merge_requests/update_head_pipeline_worker.rb
new file mode 100644
index 00000000000..c8dc9d1f7c8
--- /dev/null
+++ b/app/workers/merge_requests/update_head_pipeline_worker.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module MergeRequests
+ class UpdateHeadPipelineWorker
+ include ApplicationWorker
+ include Gitlab::EventStore::Subscriber
+
+ feature_category :code_review
+ urgency :high
+ worker_resource_boundary :cpu
+ data_consistency :always
+
+ idempotent!
+
+ def handle_event(event)
+ Ci::Pipeline.find_by_id(event.data[:pipeline_id]).try do |pipeline|
+ pipeline.all_merge_requests.opened.each do |merge_request|
+ UpdateHeadPipelineForMergeRequestWorker.perform_async(merge_request.id)
+ end
+ end
+ end
+ end
+end
diff --git a/app/workers/metrics/dashboard/sync_dashboards_worker.rb b/app/workers/metrics/dashboard/sync_dashboards_worker.rb
index fe8694582c4..63ca27d9c44 100644
--- a/app/workers/metrics/dashboard/sync_dashboards_worker.rb
+++ b/app/workers/metrics/dashboard/sync_dashboards_worker.rb
@@ -18,7 +18,7 @@ module Metrics
dashboard_paths = ::Gitlab::Metrics::Dashboard::RepoDashboardFinder.list_dashboards(project)
dashboard_paths.each do |dashboard_path|
- ::Gitlab::Metrics::Dashboard::Importer.new(dashboard_path, project).execute!
+ ::Gitlab::Metrics::Dashboard::Importer.new(dashboard_path, project).execute
end
end
end
diff --git a/app/workers/packages/cleanup_package_file_worker.rb b/app/workers/packages/cleanup_package_file_worker.rb
new file mode 100644
index 00000000000..cb2b2a12c5e
--- /dev/null
+++ b/app/workers/packages/cleanup_package_file_worker.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Packages
+ class CleanupPackageFileWorker
+ include ApplicationWorker
+ include ::Packages::CleanupArtifactWorker
+ include Gitlab::Utils::StrongMemoize
+
+ data_consistency :always
+ queue_namespace :package_cleanup
+ feature_category :package_registry
+ urgency :low
+ worker_resource_boundary :unknown
+ idempotent!
+
+ def max_running_jobs
+ ::Gitlab::CurrentSettings.packages_cleanup_package_file_worker_capacity
+ end
+
+ private
+
+ def model
+ Packages::PackageFile
+ end
+
+ def next_item
+ model.next_pending_destruction
+ end
+
+ def log_metadata(package_file)
+ log_extra_metadata_on_done(:package_file_id, package_file.id)
+ log_extra_metadata_on_done(:package_id, package_file.package_id)
+ end
+
+ def log_cleanup_item(package_file)
+ logger.info(
+ structured_payload(
+ package_id: package_file.package_id,
+ package_file_id: package_file.id
+ )
+ )
+ end
+ end
+end
diff --git a/app/workers/packages/cleanup_package_registry_worker.rb b/app/workers/packages/cleanup_package_registry_worker.rb
new file mode 100644
index 00000000000..a849e055b64
--- /dev/null
+++ b/app/workers/packages/cleanup_package_registry_worker.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Packages
+ class CleanupPackageRegistryWorker
+ include ApplicationWorker
+ include CronjobQueue # rubocop:disable Scalability/CronWorkerContext
+
+ data_consistency :always
+ idempotent!
+
+ feature_category :package_registry
+
+ def perform
+ enqueue_package_file_cleanup_job if Packages::PackageFile.pending_destruction.exists?
+
+ log_counts
+ end
+
+ private
+
+ def enqueue_package_file_cleanup_job
+ Packages::CleanupPackageFileWorker.perform_with_capacity
+ end
+
+ def log_counts
+ use_replica_if_available do
+ pending_destruction_package_files_count = Packages::PackageFile.pending_destruction.count
+ processing_package_files_count = Packages::PackageFile.processing.count
+ error_package_files_count = Packages::PackageFile.error.count
+
+ log_extra_metadata_on_done(:pending_destruction_package_files_count, pending_destruction_package_files_count)
+ log_extra_metadata_on_done(:processing_package_files_count, processing_package_files_count)
+ log_extra_metadata_on_done(:error_package_files_count, error_package_files_count)
+ end
+ end
+
+ def use_replica_if_available(&block)
+ ::Gitlab::Database::LoadBalancing::Session.current.use_replicas_for_read_queries(&block)
+ end
+ end
+end
diff --git a/app/workers/pages_update_configuration_worker.rb b/app/workers/pages_update_configuration_worker.rb
index 3dfd82ed517..9c58b40e098 100644
--- a/app/workers/pages_update_configuration_worker.rb
+++ b/app/workers/pages_update_configuration_worker.rb
@@ -1,25 +1,17 @@
# frozen_string_literal: true
+# TODO: remove this in 14.7 https://gitlab.com/gitlab-org/gitlab/-/issues/348582
class PagesUpdateConfigurationWorker
include ApplicationWorker
data_consistency :always
- sidekiq_options retry: 3
+ sidekiq_options retry: 1
idempotent!
feature_category :pages
- def self.perform_async(*args)
- return unless ::Settings.pages.local_store.enabled
-
- super(*args)
- end
-
- def perform(project_id)
- project = Project.find_by_id(project_id)
- return unless project
-
- Projects::UpdatePagesConfigurationService.new(project).execute
+ def perform(_project_id)
+ # Do nothing
end
end
diff --git a/app/workers/pages_worker.rb b/app/workers/pages_worker.rb
index cceafbc7d2d..3aff4b42629 100644
--- a/app/workers/pages_worker.rb
+++ b/app/workers/pages_worker.rb
@@ -16,14 +16,7 @@ class PagesWorker # rubocop:disable Scalability/IdempotentWorker
def deploy(build_id)
build = Ci::Build.find_by_id(build_id)
- update_contents = Projects::UpdatePagesService.new(build.project, build).execute
- if update_contents[:status] == :success
- Projects::UpdatePagesConfigurationService.new(build.project).execute
- end
- end
- def remove(namespace_path, project_path)
- full_path = File.join(Settings.pages.path, namespace_path, project_path)
- FileUtils.rm_r(full_path, force: true)
+ Projects::UpdatePagesService.new(build.project, build).execute
end
end
diff --git a/app/workers/update_external_pull_requests_worker.rb b/app/workers/update_external_pull_requests_worker.rb
index 2a319a7f6f9..4b3cda6b8f3 100644
--- a/app/workers/update_external_pull_requests_worker.rb
+++ b/app/workers/update_external_pull_requests_worker.rb
@@ -7,7 +7,7 @@ class UpdateExternalPullRequestsWorker # rubocop:disable Scalability/IdempotentW
sidekiq_options retry: 3
- feature_category :source_code_management
+ feature_category :continuous_integration
weight 3
loggable_arguments 2
diff --git a/app/workers/update_head_pipeline_for_merge_request_worker.rb b/app/workers/update_head_pipeline_for_merge_request_worker.rb
index 61fe278e016..3a2447b2108 100644
--- a/app/workers/update_head_pipeline_for_merge_request_worker.rb
+++ b/app/workers/update_head_pipeline_for_merge_request_worker.rb
@@ -8,8 +8,10 @@ class UpdateHeadPipelineForMergeRequestWorker
sidekiq_options retry: 3
include PipelineQueue
+ # NOTE: this worker belongs to :code_review since there is no CI logic.
queue_namespace :pipeline_processing
feature_category :continuous_integration
+
urgency :high
worker_resource_boundary :cpu
diff --git a/app/workers/web_hook_worker.rb b/app/workers/web_hook_worker.rb
index 5b4567dde29..952ac94d5e6 100644
--- a/app/workers/web_hook_worker.rb
+++ b/app/workers/web_hook_worker.rb
@@ -13,11 +13,21 @@ class WebHookWorker
worker_has_external_dependencies!
- def perform(hook_id, data, hook_name)
+ # Webhook recursion detection properties are passed through the `data` arg.
+ # This will be migrated to the `params` arg over the next few releases.
+ # See https://gitlab.com/gitlab-org/gitlab/-/issues/347389.
+ def perform(hook_id, data, hook_name, params = {})
hook = WebHook.find_by_id(hook_id)
return unless hook
data = data.with_indifferent_access
+
+ # Before executing the hook, reapply any recursion detection UUID that was
+ # initially present in the request header so the hook can pass this same header
+ # value in its request.
+ recursion_detection_uuid = data.delete(:_gitlab_recursion_detection_request_uuid)
+ Gitlab::WebHooks::RecursionDetection.set_request_uuid(recursion_detection_uuid)
+
WebHookService.new(hook, data, hook_name, jid).execute
end
end