diff options
Diffstat (limited to 'app/services/ci/pipeline_artifacts')
3 files changed, 35 insertions, 19 deletions
diff --git a/app/services/ci/pipeline_artifacts/coverage_report_service.rb b/app/services/ci/pipeline_artifacts/coverage_report_service.rb index 99877603554..9c6fdb7a405 100644 --- a/app/services/ci/pipeline_artifacts/coverage_report_service.rb +++ b/app/services/ci/pipeline_artifacts/coverage_report_service.rb @@ -27,18 +27,13 @@ module Ci end def pipeline_artifact_params - attributes = { + { pipeline: pipeline, file_type: :code_coverage, file: carrierwave_file, - size: carrierwave_file['tempfile'].size + size: carrierwave_file['tempfile'].size, + locked: pipeline.locked } - - if ::Feature.enabled?(:ci_update_unlocked_pipeline_artifacts, pipeline.project) - attributes[:locked] = pipeline.locked - end - - attributes end def carrierwave_file diff --git a/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb b/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb index aeb68a75f88..a0746ef32b2 100644 --- a/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb +++ b/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb @@ -23,20 +23,15 @@ module Ci def artifact_attributes file = build_carrierwave_file! - attributes = { + { project_id: pipeline.project_id, file_type: :code_quality_mr_diff, file_format: Ci::PipelineArtifact::REPORT_TYPES.fetch(:code_quality_mr_diff), size: file["tempfile"].size, file: file, - expire_at: Ci::PipelineArtifact::EXPIRATION_DATE.from_now + expire_at: Ci::PipelineArtifact::EXPIRATION_DATE.from_now, + locked: pipeline.locked } - - if ::Feature.enabled?(:ci_update_unlocked_pipeline_artifacts, pipeline.project) - attributes[:locked] = pipeline.locked - end - - attributes end def merge_requests diff --git a/app/services/ci/pipeline_artifacts/destroy_all_expired_service.rb b/app/services/ci/pipeline_artifacts/destroy_all_expired_service.rb index 17c039885e5..8dddf3c3f6c 100644 --- a/app/services/ci/pipeline_artifacts/destroy_all_expired_service.rb +++ b/app/services/ci/pipeline_artifacts/destroy_all_expired_service.rb @@ -3,20 +3,26 @@ module Ci module PipelineArtifacts class DestroyAllExpiredService + include ::Gitlab::ExclusiveLeaseHelpers include ::Gitlab::LoopHelpers include ::Gitlab::Utils::StrongMemoize BATCH_SIZE = 100 - LOOP_TIMEOUT = 5.minutes LOOP_LIMIT = 1000 + LOOP_TIMEOUT = 5.minutes + LOCK_TIMEOUT = 10.minutes + EXCLUSIVE_LOCK_KEY = 'expired_pipeline_artifacts:destroy:lock' def initialize @removed_artifacts_count = 0 + @start_at = Time.current end def execute - loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do - destroy_artifacts_batch + in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do + destroy_unlocked_pipeline_artifacts + + legacy_destroy_pipeline_artifacts end @removed_artifacts_count @@ -24,10 +30,30 @@ module Ci private + def destroy_unlocked_pipeline_artifacts + loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do + artifacts = Ci::PipelineArtifact.expired_before(@start_at).artifact_unlocked.limit(BATCH_SIZE) + + break if artifacts.empty? + + destroy_batch(artifacts) + end + end + + def legacy_destroy_pipeline_artifacts + loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do + destroy_artifacts_batch + end + end + def destroy_artifacts_batch artifacts = ::Ci::PipelineArtifact.unlocked.expired.limit(BATCH_SIZE).to_a return false if artifacts.empty? + destroy_batch(artifacts) + end + + def destroy_batch(artifacts) artifacts.each(&:destroy!) increment_stats(artifacts.size) |