diff options
Diffstat (limited to 'app/services/ci')
5 files changed, 38 insertions, 21 deletions
diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index 9c3cc803587..eb25aeaf5a5 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -157,6 +157,13 @@ module Ci duration >= LOG_MAX_CREATION_THRESHOLD end + + l.log_when do |observations| + pipeline_includes_count = observations['pipeline_includes_count'] + next false unless pipeline_includes_count + + pipeline_includes_count.to_i > Gitlab::Ci::Config::External::Context::MAX_INCLUDES + end end end end diff --git a/app/services/ci/job_artifacts/create_service.rb b/app/services/ci/job_artifacts/create_service.rb index ee9982cf3ab..6e2ba76682f 100644 --- a/app/services/ci/job_artifacts/create_service.rb +++ b/app/services/ci/job_artifacts/create_service.rb @@ -92,7 +92,8 @@ module Ci file: artifacts_file, file_type: params[:artifact_type], file_format: params[:artifact_format], - file_sha256: artifacts_file.sha256 + file_sha256: artifacts_file.sha256, + accessibility: accessibility(params) ) ) @@ -102,7 +103,8 @@ module Ci file: metadata_file, file_type: :metadata, file_format: :gzip, - file_sha256: metadata_file.sha256 + file_sha256: metadata_file.sha256, + accessibility: accessibility(params) ) ) end @@ -110,6 +112,10 @@ module Ci [artifact, artifact_metadata] end + def accessibility(params) + params[:accessibility] || 'public' + end + def parse_artifact(artifact) case artifact.file_type when 'dotenv' then parse_dotenv_artifact(artifact) diff --git a/app/services/ci/job_artifacts/delete_service.rb b/app/services/ci/job_artifacts/delete_service.rb index c9d590eccc4..fc5c6b12389 100644 --- a/app/services/ci/job_artifacts/delete_service.rb +++ b/app/services/ci/job_artifacts/delete_service.rb @@ -26,8 +26,7 @@ module Ci if result.fetch(:status) == :success ServiceResponse.success(payload: { - destroyed_artifacts_count: result.fetch(:destroyed_artifacts_count), - statistics_updates: result.fetch(:statistics_updates) + destroyed_artifacts_count: result.fetch(:destroyed_artifacts_count) }) else ServiceResponse.error(message: result.fetch(:message)) diff --git a/app/services/ci/job_artifacts/destroy_associations_service.rb b/app/services/ci/job_artifacts/destroy_associations_service.rb index 794d24eadf2..fd3e69a5913 100644 --- a/app/services/ci/job_artifacts/destroy_associations_service.rb +++ b/app/services/ci/job_artifacts/destroy_associations_service.rb @@ -2,27 +2,32 @@ module Ci module JobArtifacts + # This class is used by Ci::JobArtifact's FastDestroyAll implementation. + # Ci::JobArtifact.begin_fast_destroy instantiates this service and calls #destroy_records. + # This will set @statistics_updates instance variables. + # The same instance is passed to Ci::JobArtifact.finalize_fast_destroy, which then calls + # #update_statistics, using @statistics_updates set by #destroy_records. class DestroyAssociationsService BATCH_SIZE = 100 def initialize(job_artifacts_relation) @job_artifacts_relation = job_artifacts_relation - @statistics = {} + @statistics_updates = {} end def destroy_records @job_artifacts_relation.each_batch(of: BATCH_SIZE) do |relation| service = Ci::JobArtifacts::DestroyBatchService.new(relation, pick_up_at: Time.current) result = service.execute(update_stats: false) - updates = result[:statistics_updates] - - @statistics.merge!(updates) { |_key, oldval, newval| newval + oldval } + @statistics_updates.merge!(result[:statistics_updates]) do |_project, existing_updates, new_updates| + existing_updates.concat(new_updates) + end end end def update_statistics - @statistics.each do |project, delta| - project.increment_statistic_value(Ci::JobArtifact.project_statistics_name, delta) + @statistics_updates.each do |project, increments| + ProjectStatistics.bulk_increment_statistic(project, Ci::JobArtifact.project_statistics_name, increments) end end end diff --git a/app/services/ci/job_artifacts/destroy_batch_service.rb b/app/services/ci/job_artifacts/destroy_batch_service.rb index e0307d9bd53..7cb1be95a3e 100644 --- a/app/services/ci/job_artifacts/destroy_batch_service.rb +++ b/app/services/ci/job_artifacts/destroy_batch_service.rb @@ -46,14 +46,13 @@ module Ci after_batch_destroy_hook(@job_artifacts) - # This is executed outside of the transaction because it depends on Redis update_project_statistics! if update_stats + increment_monitoring_statistics(artifacts_count, artifacts_bytes) Gitlab::Ci::Artifacts::Logger.log_deleted(@job_artifacts, 'Ci::JobArtifacts::DestroyBatchService#execute') - success(destroyed_artifacts_count: artifacts_count, - statistics_updates: affected_project_statistics) + success(destroyed_artifacts_count: artifacts_count, statistics_updates: statistics_updates_per_project) end # rubocop: enable CodeReuse/ActiveRecord @@ -74,17 +73,18 @@ module Ci # using ! here since this can't be called inside a transaction def update_project_statistics! - affected_project_statistics.each do |project, delta| - project.increment_statistic_value(Ci::JobArtifact.project_statistics_name, delta) + statistics_updates_per_project.each do |project, increments| + ProjectStatistics.bulk_increment_statistic(project, Ci::JobArtifact.project_statistics_name, increments) end end - def affected_project_statistics - strong_memoize(:affected_project_statistics) do - artifacts_by_project = @job_artifacts.group_by(&:project) - artifacts_by_project.each.with_object({}) do |(project, artifacts), accumulator| - delta = -artifacts.sum { |artifact| artifact.size.to_i } - accumulator[project] = delta + def statistics_updates_per_project + strong_memoize(:statistics_updates_per_project) do + result = Hash.new { |updates, project| updates[project] = [] } + + @job_artifacts.each_with_object(result) do |job_artifact, result| + increment = Gitlab::Counters::Increment.new(amount: -job_artifact.size.to_i, ref: job_artifact.id) + result[job_artifact.project] << increment end end end |