diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-01-10 23:59:00 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-01-10 23:59:00 +0300 |
commit | 7d90f0e86b89b921d29a7a9fc3e64d0f58f6a993 (patch) | |
tree | 71435bdc355e6aeced7cf5354e09865c8d4ffbff /app/workers | |
parent | b55e13ec164336d1e5d5bbdbca939edcc31d557f (diff) |
Add latest changes from gitlab-org/security/gitlab@14-6-stable-ee
Diffstat (limited to 'app/workers')
-rw-r--r-- | app/workers/all_queues.yml | 9 | ||||
-rw-r--r-- | app/workers/concerns/dependency_proxy/expireable.rb | 17 | ||||
-rw-r--r-- | app/workers/dependency_proxy/cleanup_dependency_proxy_worker.rb | 28 | ||||
-rw-r--r-- | app/workers/dependency_proxy/image_ttl_group_policy_worker.rb | 26 | ||||
-rw-r--r-- | app/workers/purge_dependency_proxy_cache_worker.rb | 12 |
5 files changed, 60 insertions, 32 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index e5ac9da37c6..f2961d825a0 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -291,6 +291,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: cronjob:dependency_proxy_cleanup_dependency_proxy + :worker_name: DependencyProxy::CleanupDependencyProxyWorker + :feature_category: :dependency_proxy + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: cronjob:dependency_proxy_image_ttl_group_policy :worker_name: DependencyProxy::ImageTtlGroupPolicyWorker :feature_category: :dependency_proxy diff --git a/app/workers/concerns/dependency_proxy/expireable.rb b/app/workers/concerns/dependency_proxy/expireable.rb new file mode 100644 index 00000000000..9650ac85c6c --- /dev/null +++ b/app/workers/concerns/dependency_proxy/expireable.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module DependencyProxy + module Expireable + extend ActiveSupport::Concern + + UPDATE_BATCH_SIZE = 100 + + private + + def expire_artifacts(collection) + collection.each_batch(of: UPDATE_BATCH_SIZE) do |batch| + batch.update_all(status: :expired) + end + end + end +end diff --git a/app/workers/dependency_proxy/cleanup_dependency_proxy_worker.rb b/app/workers/dependency_proxy/cleanup_dependency_proxy_worker.rb new file mode 100644 index 00000000000..d77c782267a --- /dev/null +++ b/app/workers/dependency_proxy/cleanup_dependency_proxy_worker.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module DependencyProxy + class CleanupDependencyProxyWorker + include ApplicationWorker + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext + + data_consistency :always + idempotent! + + feature_category :dependency_proxy + + def perform + enqueue_blob_cleanup_job if DependencyProxy::Blob.expired.any? + enqueue_manifest_cleanup_job if DependencyProxy::Manifest.expired.any? + end + + private + + def enqueue_blob_cleanup_job + DependencyProxy::CleanupBlobWorker.perform_with_capacity + end + + def enqueue_manifest_cleanup_job + DependencyProxy::CleanupManifestWorker.perform_with_capacity + end + end +end diff --git a/app/workers/dependency_proxy/image_ttl_group_policy_worker.rb b/app/workers/dependency_proxy/image_ttl_group_policy_worker.rb index 6a1de00ce80..3de2364fc71 100644 --- a/app/workers/dependency_proxy/image_ttl_group_policy_worker.rb +++ b/app/workers/dependency_proxy/image_ttl_group_policy_worker.rb @@ -4,20 +4,19 @@ module DependencyProxy class ImageTtlGroupPolicyWorker # rubocop:disable Scalability/IdempotentWorker include ApplicationWorker include CronjobQueue # rubocop:disable Scalability/CronWorkerContext + include DependencyProxy::Expireable data_consistency :always feature_category :dependency_proxy - UPDATE_BATCH_SIZE = 100 - def perform DependencyProxy::ImageTtlGroupPolicy.enabled.each do |policy| qualified_blobs = policy.group.dependency_proxy_blobs.active.read_before(policy.ttl) qualified_manifests = policy.group.dependency_proxy_manifests.active.read_before(policy.ttl) - enqueue_blob_cleanup_job if expire_artifacts(qualified_blobs, DependencyProxy::Blob) - enqueue_manifest_cleanup_job if expire_artifacts(qualified_manifests, DependencyProxy::Manifest) + expire_artifacts(qualified_blobs) + expire_artifacts(qualified_manifests) end log_counts @@ -25,25 +24,6 @@ module DependencyProxy private - def expire_artifacts(artifacts, model) - rows_updated = false - - artifacts.each_batch(of: UPDATE_BATCH_SIZE) do |batch| - rows = batch.update_all(status: :expired) - rows_updated ||= rows > 0 - end - - rows_updated - end - - def enqueue_blob_cleanup_job - DependencyProxy::CleanupBlobWorker.perform_with_capacity - end - - def enqueue_manifest_cleanup_job - DependencyProxy::CleanupManifestWorker.perform_with_capacity - end - def log_counts use_replica_if_available do expired_blob_count = DependencyProxy::Blob.expired.count diff --git a/app/workers/purge_dependency_proxy_cache_worker.rb b/app/workers/purge_dependency_proxy_cache_worker.rb index 615fa81f28e..c0ddf190210 100644 --- a/app/workers/purge_dependency_proxy_cache_worker.rb +++ b/app/workers/purge_dependency_proxy_cache_worker.rb @@ -2,6 +2,7 @@ class PurgeDependencyProxyCacheWorker include ApplicationWorker + include DependencyProxy::Expireable data_consistency :always @@ -12,21 +13,14 @@ class PurgeDependencyProxyCacheWorker queue_namespace :dependency_proxy feature_category :dependency_proxy - UPDATE_BATCH_SIZE = 100 - def perform(current_user_id, group_id) @current_user = User.find_by_id(current_user_id) @group = Group.find_by_id(group_id) return unless valid? - @group.dependency_proxy_blobs.each_batch(of: UPDATE_BATCH_SIZE) do |batch| - batch.update_all(status: :expired) - end - - @group.dependency_proxy_manifests.each_batch(of: UPDATE_BATCH_SIZE) do |batch| - batch.update_all(status: :expired) - end + expire_artifacts(@group.dependency_proxy_blobs) + expire_artifacts(@group.dependency_proxy_manifests) end private |