diff options
Diffstat (limited to 'app/services/ci/job_artifacts/destroy_all_expired_service.rb')
-rw-r--r-- | app/services/ci/job_artifacts/destroy_all_expired_service.rb | 31 |
1 files changed, 7 insertions, 24 deletions
diff --git a/app/services/ci/job_artifacts/destroy_all_expired_service.rb b/app/services/ci/job_artifacts/destroy_all_expired_service.rb index 7fa56677a0c..c089567ec14 100644 --- a/app/services/ci/job_artifacts/destroy_all_expired_service.rb +++ b/app/services/ci/job_artifacts/destroy_all_expired_service.rb @@ -8,13 +8,15 @@ module Ci BATCH_SIZE = 100 LOOP_TIMEOUT = 5.minutes - LOOP_LIMIT = 1000 + SMALL_LOOP_LIMIT = 100 + LARGE_LOOP_LIMIT = 500 EXCLUSIVE_LOCK_KEY = 'expired_job_artifacts:destroy:lock' LOCK_TIMEOUT = 6.minutes def initialize @removed_artifacts_count = 0 @start_at = Time.current + @loop_limit = ::Feature.enabled?(:ci_artifact_fast_removal_large_loop_limit, default_enabled: :yaml) ? LARGE_LOOP_LIMIT : SMALL_LOOP_LIMIT end ## @@ -24,6 +26,8 @@ module Ci # preventing multiple `ExpireBuildArtifactsWorker` CRON jobs run concurrently, # which is scheduled every 7 minutes. def execute + return 0 unless ::Feature.enabled?(:ci_destroy_all_expired_service, default_enabled: :yaml) + in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do if ::Feature.enabled?(:ci_destroy_unlocked_job_artifacts) destroy_unlocked_job_artifacts @@ -38,34 +42,13 @@ module Ci private def destroy_unlocked_job_artifacts - loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do + loop_until(timeout: LOOP_TIMEOUT, limit: @loop_limit) do artifacts = Ci::JobArtifact.expired_before(@start_at).artifact_unlocked.limit(BATCH_SIZE) service_response = destroy_batch(artifacts) @removed_artifacts_count += service_response[:destroyed_artifacts_count] - - update_locked_status_on_unknown_artifacts if service_response[:destroyed_artifacts_count] == 0 - - # Return a truthy value here to prevent exiting #loop_until - @removed_artifacts_count end end - def update_locked_status_on_unknown_artifacts - build_ids = Ci::JobArtifact.expired_before(@start_at).artifact_unknown.limit(BATCH_SIZE).distinct_job_ids - - return unless build_ids.present? - - locked_pipeline_build_ids = ::Ci::Build.with_pipeline_locked_artifacts.id_in(build_ids).pluck_primary_key - unlocked_pipeline_build_ids = build_ids - locked_pipeline_build_ids - - update_unknown_artifacts(locked_pipeline_build_ids, Ci::JobArtifact.lockeds[:artifacts_locked]) - update_unknown_artifacts(unlocked_pipeline_build_ids, Ci::JobArtifact.lockeds[:unlocked]) - end - - def update_unknown_artifacts(build_ids, locked_value) - Ci::JobArtifact.for_job_ids(build_ids).update_all(locked: locked_value) if build_ids.any? - end - def destroy_job_artifacts_with_slow_iteration Ci::JobArtifact.expired_before(@start_at).each_batch(of: BATCH_SIZE, column: :expire_at, order: :desc) do |relation, index| # For performance reasons, join with ci_pipelines after the batch is queried. @@ -76,7 +59,7 @@ module Ci @removed_artifacts_count += service_response[:destroyed_artifacts_count] break if loop_timeout? - break if index >= LOOP_LIMIT + break if index >= @loop_limit end end |