diff options
Diffstat (limited to 'app/services/ci')
29 files changed, 234 insertions, 78 deletions
diff --git a/app/services/ci/after_requeue_job_service.rb b/app/services/ci/after_requeue_job_service.rb index 3858ee9d550..2b611c857c7 100644 --- a/app/services/ci/after_requeue_job_service.rb +++ b/app/services/ci/after_requeue_job_service.rb @@ -4,7 +4,7 @@ module Ci class AfterRequeueJobService < ::BaseService def execute(processable) process_subsequent_jobs(processable) - reset_ancestor_bridges(processable) + reset_source_bridge(processable) end private @@ -15,8 +15,8 @@ module Ci end end - def reset_ancestor_bridges(processable) - processable.pipeline.reset_ancestor_bridges! + def reset_source_bridge(processable) + processable.pipeline.reset_source_bridge!(current_user) end def process(processable) diff --git a/app/services/ci/archive_trace_service.rb b/app/services/ci/archive_trace_service.rb index 9b2c7788897..bc3219fbd79 100644 --- a/app/services/ci/archive_trace_service.rb +++ b/app/services/ci/archive_trace_service.rb @@ -24,7 +24,7 @@ module Ci end rescue ::Gitlab::Ci::Trace::AlreadyArchivedError # It's already archived, thus we can safely ignore this exception. - rescue => e + rescue StandardError => e # Tracks this error with application logs, Sentry, and Prometheus. # If `archive!` keeps failing for over a week, that could incur data loss. # (See more https://docs.gitlab.com/ee/administration/job_logs.html#new-incremental-logging-architecture) diff --git a/app/services/ci/change_variable_service.rb b/app/services/ci/change_variable_service.rb index f515a335d54..83cd6aae14b 100644 --- a/app/services/ci/change_variable_service.rb +++ b/app/services/ci/change_variable_service.rb @@ -30,4 +30,4 @@ module Ci end end -::Ci::ChangeVariableService.prepend_if_ee('EE::Ci::ChangeVariableService') +::Ci::ChangeVariableService.prepend_mod_with('Ci::ChangeVariableService') diff --git a/app/services/ci/change_variables_service.rb b/app/services/ci/change_variables_service.rb index 3337eb09411..7a68bd2e2b3 100644 --- a/app/services/ci/change_variables_service.rb +++ b/app/services/ci/change_variables_service.rb @@ -8,4 +8,4 @@ module Ci end end -::Ci::ChangeVariablesService.prepend_if_ee('EE::Ci::ChangeVariablesService') +::Ci::ChangeVariablesService.prepend_mod_with('Ci::ChangeVariablesService') diff --git a/app/services/ci/create_downstream_pipeline_service.rb b/app/services/ci/create_downstream_pipeline_service.rb index 93f0338fcba..64a99e404c6 100644 --- a/app/services/ci/create_downstream_pipeline_service.rb +++ b/app/services/ci/create_downstream_pipeline_service.rb @@ -85,6 +85,12 @@ module Ci return false end + if has_cyclic_dependency? + @bridge.drop!(:pipeline_loop_detected) + + return false + end + true end @@ -109,11 +115,24 @@ module Ci end end + def has_cyclic_dependency? + return false if @bridge.triggers_child_pipeline? + + if Feature.enabled?(:ci_drop_cyclical_triggered_pipelines, @bridge.project, default_enabled: :yaml) + checksums = @bridge.pipeline.base_and_ancestors.map { |pipeline| config_checksum(pipeline) } + checksums.uniq.length != checksums.length + end + end + def has_max_descendants_depth? return false unless @bridge.triggers_child_pipeline? ancestors_of_new_child = @bridge.pipeline.base_and_ancestors(same_project: true) ancestors_of_new_child.count > MAX_DESCENDANTS_DEPTH end + + def config_checksum(pipeline) + [pipeline.project_id, pipeline.ref].hash + end end end diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index ca936307acc..fd333e24860 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -10,6 +10,7 @@ module Ci Gitlab::Ci::Pipeline::Chain::Build::Associations, Gitlab::Ci::Pipeline::Chain::Validate::Abilities, Gitlab::Ci::Pipeline::Chain::Validate::Repository, + Gitlab::Ci::Pipeline::Chain::Validate::SecurityOrchestrationPolicy, Gitlab::Ci::Pipeline::Chain::Config::Content, Gitlab::Ci::Pipeline::Chain::Config::Process, Gitlab::Ci::Pipeline::Chain::RemoveUnwantedChatJobs, @@ -84,7 +85,6 @@ module Ci if pipeline.persisted? schedule_head_pipeline_update - record_conversion_event create_namespace_onboarding_action end @@ -122,12 +122,6 @@ module Ci end end - def record_conversion_event - return unless project.namespace.recent? - - Experiments::RecordConversionEventWorker.perform_async(:ci_syntax_templates_b, current_user.id) - end - def create_namespace_onboarding_action Namespaces::OnboardingPipelineCreatedWorker.perform_async(project.namespace_id) end @@ -138,4 +132,4 @@ module Ci end end -Ci::CreatePipelineService.prepend_if_ee('EE::Ci::CreatePipelineService') +Ci::CreatePipelineService.prepend_mod_with('Ci::CreatePipelineService') diff --git a/app/services/ci/create_web_ide_terminal_service.rb b/app/services/ci/create_web_ide_terminal_service.rb index 3b89a599180..db8f61c81fa 100644 --- a/app/services/ci/create_web_ide_terminal_service.rb +++ b/app/services/ci/create_web_ide_terminal_service.rb @@ -28,6 +28,11 @@ module Ci def create_pipeline! build_pipeline.tap do |pipeline| pipeline.stages << terminal_stage_seed(pipeline).to_resource + + # Project iid must be called outside a transaction, so we ensure it is set here + # otherwise it may be set within the save! which it will lock the InternalId row for the whole transaction + pipeline.ensure_project_iid! + pipeline.save! Ci::ProcessPipelineService diff --git a/app/services/ci/delete_unit_tests_service.rb b/app/services/ci/delete_unit_tests_service.rb new file mode 100644 index 00000000000..28f96351175 --- /dev/null +++ b/app/services/ci/delete_unit_tests_service.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Ci + class DeleteUnitTestsService + include EachBatch + + BATCH_SIZE = 100 + + def execute + purge_data!(Ci::UnitTestFailure) + purge_data!(Ci::UnitTest) + end + + private + + def purge_data!(klass) + loop do + break unless delete_batch!(klass) + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def delete_batch!(klass) + deleted = 0 + + ActiveRecord::Base.transaction do + ids = klass.deletable.lock('FOR UPDATE SKIP LOCKED').limit(BATCH_SIZE).pluck(:id) + break if ids.empty? + + deleted = klass.where(id: ids).delete_all + end + + deleted > 0 + end + # rubocop: enable CodeReuse/ActiveRecord + end +end diff --git a/app/services/ci/expire_pipeline_cache_service.rb b/app/services/ci/expire_pipeline_cache_service.rb index 2ae60907dab..80c83818d0b 100644 --- a/app/services/ci/expire_pipeline_cache_service.rb +++ b/app/services/ci/expire_pipeline_cache_service.rb @@ -56,6 +56,10 @@ module Ci url_helpers.graphql_etag_pipeline_path(pipeline) end + def graphql_pipeline_sha_path(sha) + url_helpers.graphql_etag_pipeline_sha_path(sha) + end + # Updates ETag caches of a pipeline. # # This logic resides in a separate method so that EE can more easily extend @@ -76,6 +80,7 @@ module Ci pipeline.self_with_ancestors_and_descendants.each do |relative_pipeline| store.touch(project_pipeline_path(relative_pipeline.project, relative_pipeline)) store.touch(graphql_pipeline_path(relative_pipeline)) + store.touch(graphql_pipeline_sha_path(relative_pipeline.sha)) end end diff --git a/app/services/ci/generate_codequality_mr_diff_report_service.rb b/app/services/ci/generate_codequality_mr_diff_report_service.rb index 3b1bd319a4f..117b0a21eaa 100644 --- a/app/services/ci/generate_codequality_mr_diff_report_service.rb +++ b/app/services/ci/generate_codequality_mr_diff_report_service.rb @@ -12,9 +12,9 @@ module Ci { status: :parsed, key: key(base_pipeline, head_pipeline), - data: head_pipeline.pipeline_artifacts.find_by_file_type(:code_quality_mr_diff).present.for_files(merge_request.new_paths) + data: head_pipeline.pipeline_artifacts.find_by_file_type(:code_quality_mr_diff).present.for_files(merge_request) } - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, project_id: project.id) { status: :error, diff --git a/app/services/ci/generate_coverage_reports_service.rb b/app/services/ci/generate_coverage_reports_service.rb index 4e6fbc5462a..12b1f19f4b5 100644 --- a/app/services/ci/generate_coverage_reports_service.rb +++ b/app/services/ci/generate_coverage_reports_service.rb @@ -14,7 +14,7 @@ module Ci key: key(base_pipeline, head_pipeline), data: head_pipeline.pipeline_artifacts.find_by_file_type(:code_coverage).present.for_files(merge_request.new_paths) } - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_exception( e, project_id: project.id, diff --git a/app/services/ci/generate_exposed_artifacts_report_service.rb b/app/services/ci/generate_exposed_artifacts_report_service.rb index 1dbcd192279..dfa7cbd7d98 100644 --- a/app/services/ci/generate_exposed_artifacts_report_service.rb +++ b/app/services/ci/generate_exposed_artifacts_report_service.rb @@ -14,7 +14,7 @@ module Ci key: key(base_pipeline, head_pipeline), data: data } - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, project_id: project.id) { status: :error, diff --git a/app/services/ci/generate_terraform_reports_service.rb b/app/services/ci/generate_terraform_reports_service.rb index d768ce777d4..0ffb2d7e34a 100644 --- a/app/services/ci/generate_terraform_reports_service.rb +++ b/app/services/ci/generate_terraform_reports_service.rb @@ -13,7 +13,7 @@ module Ci key: key(base_pipeline, head_pipeline), data: head_pipeline.terraform_reports.plans } - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, project_id: project.id) { status: :error, diff --git a/app/services/ci/job_artifacts/create_service.rb b/app/services/ci/job_artifacts/create_service.rb index 65752e56c64..a22ac87f660 100644 --- a/app/services/ci/job_artifacts/create_service.rb +++ b/app/services/ci/job_artifacts/create_service.rb @@ -136,7 +136,7 @@ module Ci rescue *OBJECT_STORAGE_ERRORS => error track_exception(error, params) error(error.message, :service_unavailable) - rescue => error + rescue StandardError => error track_exception(error, params) error(error.message, :bad_request) end diff --git a/app/services/ci/job_artifacts/destroy_associations_service.rb b/app/services/ci/job_artifacts/destroy_associations_service.rb new file mode 100644 index 00000000000..794d24eadf2 --- /dev/null +++ b/app/services/ci/job_artifacts/destroy_associations_service.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Ci + module JobArtifacts + class DestroyAssociationsService + BATCH_SIZE = 100 + + def initialize(job_artifacts_relation) + @job_artifacts_relation = job_artifacts_relation + @statistics = {} + end + + def destroy_records + @job_artifacts_relation.each_batch(of: BATCH_SIZE) do |relation| + service = Ci::JobArtifacts::DestroyBatchService.new(relation, pick_up_at: Time.current) + result = service.execute(update_stats: false) + updates = result[:statistics_updates] + + @statistics.merge!(updates) { |_key, oldval, newval| newval + oldval } + end + end + + def update_statistics + @statistics.each do |project, delta| + project.increment_statistic_value(Ci::JobArtifact.project_statistics_name, delta) + end + end + end + end +end diff --git a/app/services/ci/job_artifacts/destroy_batch_service.rb b/app/services/ci/job_artifacts/destroy_batch_service.rb index 95315dd11ec..8536b88ccc0 100644 --- a/app/services/ci/job_artifacts/destroy_batch_service.rb +++ b/app/services/ci/job_artifacts/destroy_batch_service.rb @@ -23,8 +23,8 @@ module Ci end # rubocop: disable CodeReuse/ActiveRecord - def execute - return success(destroyed_artifacts_count: artifacts_count) if @job_artifacts.empty? + def execute(update_stats: true) + return success(destroyed_artifacts_count: 0, statistics_updates: {}) if @job_artifacts.empty? Ci::DeletedObject.transaction do Ci::DeletedObject.bulk_import(@job_artifacts, @pick_up_at) @@ -33,10 +33,11 @@ module Ci end # This is executed outside of the transaction because it depends on Redis - update_project_statistics + update_project_statistics! if update_stats increment_monitoring_statistics(artifacts_count) - success(destroyed_artifacts_count: artifacts_count) + success(destroyed_artifacts_count: artifacts_count, + statistics_updates: affected_project_statistics) end # rubocop: enable CodeReuse/ActiveRecord @@ -45,12 +46,20 @@ module Ci # This method is implemented in EE and it must do only database work def destroy_related_records(artifacts); end - def update_project_statistics - artifacts_by_project = @job_artifacts.group_by(&:project) - artifacts_by_project.each do |project, artifacts| - delta = -artifacts.sum { |artifact| artifact.size.to_i } - ProjectStatistics.increment_statistic( - project, Ci::JobArtifact.project_statistics_name, delta) + # using ! here since this can't be called inside a transaction + def update_project_statistics! + affected_project_statistics.each do |project, delta| + project.increment_statistic_value(Ci::JobArtifact.project_statistics_name, delta) + end + end + + def affected_project_statistics + strong_memoize(:affected_project_statistics) do + artifacts_by_project = @job_artifacts.group_by(&:project) + artifacts_by_project.each.with_object({}) do |(project, artifacts), accumulator| + delta = -artifacts.sum { |artifact| artifact.size.to_i } + accumulator[project] = delta + end end end @@ -71,4 +80,4 @@ module Ci end end -Ci::JobArtifacts::DestroyBatchService.prepend_if_ee('EE::Ci::JobArtifacts::DestroyBatchService') +Ci::JobArtifacts::DestroyBatchService.prepend_mod_with('Ci::JobArtifacts::DestroyBatchService') diff --git a/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb b/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb index 5c52eef7ba6..d6865efac9f 100644 --- a/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb +++ b/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb @@ -2,11 +2,18 @@ module Ci module PipelineArtifacts class CreateCodeQualityMrDiffReportService - def execute(pipeline) + include Gitlab::Utils::StrongMemoize + + def initialize(pipeline) + @pipeline = pipeline + end + + def execute return unless pipeline.can_generate_codequality_reports? return if pipeline.has_codequality_mr_diff_report? + return unless new_errors_introduced? - file = build_carrierwave_file(pipeline) + file = build_carrierwave_file! pipeline.pipeline_artifacts.create!( project_id: pipeline.project_id, @@ -20,18 +27,54 @@ module Ci private - def build_carrierwave_file(pipeline) + attr_reader :pipeline + + def merge_requests + strong_memoize(:merge_requests) do + pipeline.merge_requests_as_head_pipeline + end + end + + def head_report + strong_memoize(:head_report) do + pipeline.codequality_reports + end + end + + def base_report(merge_request) + strong_memoize(:base_report) do + merge_request&.base_pipeline&.codequality_reports + end + end + + def mr_diff_report_by_merge_requests + strong_memoize(:mr_diff_report_by_merge_requests) do + merge_requests.each_with_object({}) do |merge_request, hash| + key = "merge_request_#{merge_request.id}" + new_errors = Gitlab::Ci::Reports::CodequalityReportsComparer.new(base_report(merge_request), head_report).new_errors + next if new_errors.empty? + + hash[key] = Gitlab::Ci::Reports::CodequalityMrDiff.new(new_errors) + end + end + end + + def new_errors_introduced? + mr_diff_report_by_merge_requests.present? + end + + def build_carrierwave_file! CarrierWaveStringFile.new_file( - file_content: build_quality_mr_diff_report(pipeline), + file_content: build_quality_mr_diff_report(mr_diff_report_by_merge_requests), filename: Ci::PipelineArtifact::DEFAULT_FILE_NAMES.fetch(:code_quality_mr_diff), content_type: 'application/json' ) end - def build_quality_mr_diff_report(pipeline) - mr_diff_report = Gitlab::Ci::Reports::CodequalityMrDiff.new(pipeline.codequality_reports) - - Ci::CodequalityMrDiffReportSerializer.new.represent(mr_diff_report).to_json # rubocop: disable CodeReuse/Serializer + def build_quality_mr_diff_report(mr_diff_report) + mr_diff_report.each_with_object({}) do |diff_report, hash| + hash[diff_report.first] = Ci::CodequalityMrDiffReportSerializer.new.represent(diff_report.second) # rubocop: disable CodeReuse/Serializer + end.to_json end end end diff --git a/app/services/ci/pipeline_artifacts/destroy_all_expired_service.rb b/app/services/ci/pipeline_artifacts/destroy_all_expired_service.rb index fed40aef697..7b6590a117c 100644 --- a/app/services/ci/pipeline_artifacts/destroy_all_expired_service.rb +++ b/app/services/ci/pipeline_artifacts/destroy_all_expired_service.rb @@ -25,7 +25,7 @@ module Ci private def destroy_artifacts_batch - artifacts = ::Ci::PipelineArtifact.expired(BATCH_SIZE).to_a + artifacts = ::Ci::PipelineArtifact.unlocked.expired(BATCH_SIZE).to_a return false if artifacts.empty? artifacts.each(&:destroy!) diff --git a/app/services/ci/pipeline_bridge_status_service.rb b/app/services/ci/pipeline_bridge_status_service.rb index e2e5dd386f2..aeac43588f7 100644 --- a/app/services/ci/pipeline_bridge_status_service.rb +++ b/app/services/ci/pipeline_bridge_status_service.rb @@ -17,4 +17,4 @@ module Ci end end -Ci::PipelineBridgeStatusService.prepend_if_ee('EE::Ci::PipelineBridgeStatusService') +Ci::PipelineBridgeStatusService.prepend_mod_with('Ci::PipelineBridgeStatusService') diff --git a/app/services/ci/pipeline_trigger_service.rb b/app/services/ci/pipeline_trigger_service.rb index a5f70d62e13..62c4d6b4599 100644 --- a/app/services/ci/pipeline_trigger_service.rb +++ b/app/services/ci/pipeline_trigger_service.rb @@ -3,6 +3,7 @@ module Ci class PipelineTriggerService < BaseService include Gitlab::Utils::StrongMemoize + include Services::ReturnServiceResponses def execute if trigger_from_token @@ -20,7 +21,7 @@ module Ci private PAYLOAD_VARIABLE_KEY = 'TRIGGER_PAYLOAD' - PAYLOAD_VARIABLE_HIDDEN_PARAMS = %i(token).freeze + PAYLOAD_VARIABLE_HIDDEN_PARAMS = %i[token].freeze def create_pipeline_from_trigger(trigger) # this check is to not leak the presence of the project if user cannot read it @@ -32,10 +33,17 @@ module Ci pipeline.trigger_requests.build(trigger: trigger) end - if pipeline.persisted? + pipeline_service_response(pipeline) + end + + def pipeline_service_response(pipeline) + if pipeline.created_successfully? success(pipeline: pipeline) + elsif pipeline.persisted? + err = pipeline.errors.messages.presence || pipeline.failure_reason.presence || 'Could not create pipeline' + error(err, :unprocessable_entity) else - error(pipeline.errors.messages, 400) + error(pipeline.errors.messages, :bad_request) end end @@ -61,11 +69,7 @@ module Ci pipeline.source_pipeline = source end - if pipeline.persisted? - success(pipeline: pipeline) - else - error(pipeline.errors.messages, 400) - end + pipeline_service_response(pipeline) end def job_from_token diff --git a/app/services/ci/prepare_build_service.rb b/app/services/ci/prepare_build_service.rb index 3f87c711270..ec61c43cce9 100644 --- a/app/services/ci/prepare_build_service.rb +++ b/app/services/ci/prepare_build_service.rb @@ -12,7 +12,7 @@ module Ci prerequisites.each(&:complete!) build.enqueue_preparing! - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, build_id: build.id) build.drop(:unmet_prerequisites) diff --git a/app/services/ci/process_build_service.rb b/app/services/ci/process_build_service.rb index 73cf3308fe7..5271c0fe93d 100644 --- a/app/services/ci/process_build_service.rb +++ b/app/services/ci/process_build_service.rb @@ -40,4 +40,4 @@ module Ci end end -Ci::ProcessBuildService.prepend_if_ee('EE::Ci::ProcessBuildService') +Ci::ProcessBuildService.prepend_mod_with('Ci::ProcessBuildService') diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb index 6c69df0c616..fb26d5d3356 100644 --- a/app/services/ci/process_pipeline_service.rb +++ b/app/services/ci/process_pipeline_service.rb @@ -48,7 +48,13 @@ module Ci # This counter is temporary. It will be used to check whether if we still use this method or not # after setting correct value of `GenericCommitStatus#retried`. # More info: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50465#note_491657115 - metrics.legacy_update_jobs_counter.increment if updated_count > 0 + if updated_count > 0 + Gitlab::AppJsonLogger.info(event: 'update_retried_is_used', + project_id: pipeline.project.id, + pipeline_id: pipeline.id) + + metrics.legacy_update_jobs_counter.increment + end end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/services/ci/prometheus_metrics/observe_histograms_service.rb b/app/services/ci/prometheus_metrics/observe_histograms_service.rb index 527d87f19c2..6bd3d2121ba 100644 --- a/app/services/ci/prometheus_metrics/observe_histograms_service.rb +++ b/app/services/ci/prometheus_metrics/observe_histograms_service.rb @@ -25,8 +25,6 @@ module Ci end def execute - return ServiceResponse.success(http_status: :accepted) unless enabled? - params .fetch(:histograms, []) .each(&method(:observe)) @@ -48,10 +46,6 @@ module Ci .fetch(name) { raise ActiveRecord::RecordNotFound } .call end - - def enabled? - ::Feature.enabled?(:ci_accept_frontend_prometheus_metrics, project, default_enabled: :yaml) - end end end end diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb index 90341b26fd6..461647ffccc 100644 --- a/app/services/ci/register_job_service.rb +++ b/app/services/ci/register_job_service.rb @@ -169,7 +169,7 @@ module Ci @metrics.increment_queue_operation(:build_conflict_transition) Result.new(nil, nil, false) - rescue => ex + rescue StandardError => ex @metrics.increment_queue_operation(:build_conflict_exception) # If an error (e.g. GRPC::DeadlineExceeded) occurred constructing @@ -233,7 +233,7 @@ module Ci Gitlab::OptimisticLocking.retry_lock(build, 3, name: 'register_job_scheduler_failure') do |subject| subject.drop!(:scheduler_failure) end - rescue => ex + rescue StandardError => ex build.doom! # This requires extra exception, otherwise we would loose information @@ -253,17 +253,23 @@ module Ci # rubocop: disable CodeReuse/ActiveRecord def builds_for_shared_runner - new_builds. + relation = new_builds. # don't run projects which have not enabled shared runners and builds joins(:project).where(projects: { shared_runners_enabled: true, pending_delete: false }) .joins('LEFT JOIN project_features ON ci_builds.project_id = project_features.project_id') - .where('project_features.builds_access_level IS NULL or project_features.builds_access_level > 0'). + .where('project_features.builds_access_level IS NULL or project_features.builds_access_level > 0') - # Implement fair scheduling - # this returns builds that are ordered by number of running builds - # we prefer projects that don't use shared runners at all - joins("LEFT JOIN (#{running_builds_for_shared_runners.to_sql}) AS project_builds ON ci_builds.project_id=project_builds.project_id") - .order(Arel.sql('COALESCE(project_builds.running_builds, 0) ASC'), 'ci_builds.id ASC') + if Feature.enabled?(:ci_queueing_disaster_recovery, runner, type: :ops, default_enabled: :yaml) + # if disaster recovery is enabled, we fallback to FIFO scheduling + relation.order('ci_builds.id ASC') + else + # Implement fair scheduling + # this returns builds that are ordered by number of running builds + # we prefer projects that don't use shared runners at all + relation + .joins("LEFT JOIN (#{running_builds_for_shared_runners.to_sql}) AS project_builds ON ci_builds.project_id=project_builds.project_id") + .order(Arel.sql('COALESCE(project_builds.running_builds, 0) ASC'), 'ci_builds.id ASC') + end end # rubocop: enable CodeReuse/ActiveRecord @@ -310,4 +316,4 @@ module Ci end end -Ci::RegisterJobService.prepend_if_ee('EE::Ci::RegisterJobService') +Ci::RegisterJobService.prepend_mod_with('Ci::RegisterJobService') diff --git a/app/services/ci/retry_build_service.rb b/app/services/ci/retry_build_service.rb index e3de7f43fda..e03f2ae3d52 100644 --- a/app/services/ci/retry_build_service.rb +++ b/app/services/ci/retry_build_service.rb @@ -18,16 +18,14 @@ module Ci AfterRequeueJobService.new(project, current_user).execute(build) ::MergeRequests::AddTodoWhenBuildFailsService - .new(project, current_user) + .new(project: project, current_user: current_user) .close(new_build) end end # rubocop: disable CodeReuse/ActiveRecord def reprocess!(build) - unless can?(current_user, :update_build, build) - raise Gitlab::Access::AccessDeniedError - end + check_access!(build) attributes = self.class.clone_accessors.to_h do |attribute| [attribute, build.public_send(attribute)] # rubocop:disable GitlabSecurity/PublicSend @@ -52,6 +50,12 @@ module Ci private + def check_access!(build) + unless can?(current_user, :update_build, build) + raise Gitlab::Access::AccessDeniedError + end + end + def create_build!(attributes) build = project.builds.new(attributes) build.assign_attributes(::Gitlab::Ci::Pipeline::Seed::Build.environment_attributes_for(build)) @@ -64,4 +68,4 @@ module Ci end end -Ci::RetryBuildService.prepend_if_ee('EE::Ci::RetryBuildService') +Ci::RetryBuildService.prepend_mod_with('Ci::RetryBuildService') diff --git a/app/services/ci/retry_pipeline_service.rb b/app/services/ci/retry_pipeline_service.rb index bb8590a769c..5cc6b89bfef 100644 --- a/app/services/ci/retry_pipeline_service.rb +++ b/app/services/ci/retry_pipeline_service.rb @@ -26,10 +26,10 @@ module Ci retry_optimistic_lock(skipped, name: 'ci_retry_pipeline') { |build| build.process(current_user) } end - pipeline.reset_ancestor_bridges! + pipeline.reset_source_bridge!(current_user) ::MergeRequests::AddTodoWhenBuildFailsService - .new(project, current_user) + .new(project: project, current_user: current_user) .close_all(pipeline) Ci::ProcessPipelineService diff --git a/app/services/ci/stop_environments_service.rb b/app/services/ci/stop_environments_service.rb index 81457130fa0..7c9fc44e7f4 100644 --- a/app/services/ci/stop_environments_service.rb +++ b/app/services/ci/stop_environments_service.rb @@ -27,7 +27,7 @@ module Ci stop_actions.each do |stop_action| stop_action.play(stop_action.user) - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, deployable_id: stop_action.id) end end @@ -35,7 +35,7 @@ module Ci private def environments - @environments ||= EnvironmentsByDeploymentsFinder + @environments ||= Environments::EnvironmentsByDeploymentsFinder .new(project, current_user, ref: @ref, recently_updated: true) .execute end diff --git a/app/services/ci/test_failure_history_service.rb b/app/services/ci/test_failure_history_service.rb index 58bbc716ff0..a3f45c1b9cd 100644 --- a/app/services/ci/test_failure_history_service.rb +++ b/app/services/ci/test_failure_history_service.rb @@ -30,7 +30,7 @@ module Ci end def should_track_failures? - return false unless project.default_branch_or_master == pipeline.ref + return false unless project.default_branch_or_main == pipeline.ref # We fetch for up to MAX_TRACKABLE_FAILURES + 1 builds. So if ever we get # 201 total number of builds with the assumption that each job has at least |