Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to 'app/services/ci/job_artifacts/destroy_all_expired_service.rb')
-rw-r--r--app/services/ci/job_artifacts/destroy_all_expired_service.rb31
1 files changed, 7 insertions, 24 deletions
diff --git a/app/services/ci/job_artifacts/destroy_all_expired_service.rb b/app/services/ci/job_artifacts/destroy_all_expired_service.rb
index 7fa56677a0c..c089567ec14 100644
--- a/app/services/ci/job_artifacts/destroy_all_expired_service.rb
+++ b/app/services/ci/job_artifacts/destroy_all_expired_service.rb
@@ -8,13 +8,15 @@ module Ci
BATCH_SIZE = 100
LOOP_TIMEOUT = 5.minutes
- LOOP_LIMIT = 1000
+ SMALL_LOOP_LIMIT = 100
+ LARGE_LOOP_LIMIT = 500
EXCLUSIVE_LOCK_KEY = 'expired_job_artifacts:destroy:lock'
LOCK_TIMEOUT = 6.minutes
def initialize
@removed_artifacts_count = 0
@start_at = Time.current
+ @loop_limit = ::Feature.enabled?(:ci_artifact_fast_removal_large_loop_limit, default_enabled: :yaml) ? LARGE_LOOP_LIMIT : SMALL_LOOP_LIMIT
end
##
@@ -24,6 +26,8 @@ module Ci
# preventing multiple `ExpireBuildArtifactsWorker` CRON jobs run concurrently,
# which is scheduled every 7 minutes.
def execute
+ return 0 unless ::Feature.enabled?(:ci_destroy_all_expired_service, default_enabled: :yaml)
+
in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do
if ::Feature.enabled?(:ci_destroy_unlocked_job_artifacts)
destroy_unlocked_job_artifacts
@@ -38,34 +42,13 @@ module Ci
private
def destroy_unlocked_job_artifacts
- loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do
+ loop_until(timeout: LOOP_TIMEOUT, limit: @loop_limit) do
artifacts = Ci::JobArtifact.expired_before(@start_at).artifact_unlocked.limit(BATCH_SIZE)
service_response = destroy_batch(artifacts)
@removed_artifacts_count += service_response[:destroyed_artifacts_count]
-
- update_locked_status_on_unknown_artifacts if service_response[:destroyed_artifacts_count] == 0
-
- # Return a truthy value here to prevent exiting #loop_until
- @removed_artifacts_count
end
end
- def update_locked_status_on_unknown_artifacts
- build_ids = Ci::JobArtifact.expired_before(@start_at).artifact_unknown.limit(BATCH_SIZE).distinct_job_ids
-
- return unless build_ids.present?
-
- locked_pipeline_build_ids = ::Ci::Build.with_pipeline_locked_artifacts.id_in(build_ids).pluck_primary_key
- unlocked_pipeline_build_ids = build_ids - locked_pipeline_build_ids
-
- update_unknown_artifacts(locked_pipeline_build_ids, Ci::JobArtifact.lockeds[:artifacts_locked])
- update_unknown_artifacts(unlocked_pipeline_build_ids, Ci::JobArtifact.lockeds[:unlocked])
- end
-
- def update_unknown_artifacts(build_ids, locked_value)
- Ci::JobArtifact.for_job_ids(build_ids).update_all(locked: locked_value) if build_ids.any?
- end
-
def destroy_job_artifacts_with_slow_iteration
Ci::JobArtifact.expired_before(@start_at).each_batch(of: BATCH_SIZE, column: :expire_at, order: :desc) do |relation, index|
# For performance reasons, join with ci_pipelines after the batch is queried.
@@ -76,7 +59,7 @@ module Ci
@removed_artifacts_count += service_response[:destroyed_artifacts_count]
break if loop_timeout?
- break if index >= LOOP_LIMIT
+ break if index >= @loop_limit
end
end