diff options
Diffstat (limited to 'app/services/ci')
| -rw-r--r-- | app/services/ci/create_downstream_pipeline_service.rb | 12 | ||||
| -rw-r--r-- | app/services/ci/create_pipeline_service.rb | 1 | ||||
| -rw-r--r-- | app/services/ci/job_artifacts/create_service.rb | 5 | ||||
| -rw-r--r-- | app/services/ci/pipeline_creation/start_pipeline_service.rb | 19 | ||||
| -rw-r--r-- | app/services/ci/pipeline_schedules/calculate_next_run_service.rb | 59 | ||||
| -rw-r--r-- | app/services/ci/play_bridge_service.rb | 10 | ||||
| -rw-r--r-- | app/services/ci/play_build_service.rb | 18 | ||||
| -rw-r--r-- | app/services/ci/register_job_service.rb | 64 | ||||
| -rw-r--r-- | app/services/ci/retry_build_service.rb | 13 | ||||
| -rw-r--r-- | app/services/ci/update_build_queue_service.rb | 100 | ||||
| -rw-r--r-- | app/services/ci/update_build_state_service.rb | 22 |
11 files changed, 255 insertions, 68 deletions
diff --git a/app/services/ci/create_downstream_pipeline_service.rb b/app/services/ci/create_downstream_pipeline_service.rb index 64a99e404c6..1eff76c2e5d 100644 --- a/app/services/ci/create_downstream_pipeline_service.rb +++ b/app/services/ci/create_downstream_pipeline_service.rb @@ -19,13 +19,14 @@ module Ci DuplicateDownstreamPipelineError.new, bridge_id: @bridge.id, project_id: @bridge.project_id ) - return + + return error('Already has a downstream pipeline') end pipeline_params = @bridge.downstream_pipeline_params target_ref = pipeline_params.dig(:target_revision, :ref) - return unless ensure_preconditions!(target_ref) + return error('Pre-conditions not met') unless ensure_preconditions!(target_ref) service = ::Ci::CreatePipelineService.new( pipeline_params.fetch(:project), @@ -119,8 +120,11 @@ module Ci return false if @bridge.triggers_child_pipeline? if Feature.enabled?(:ci_drop_cyclical_triggered_pipelines, @bridge.project, default_enabled: :yaml) - checksums = @bridge.pipeline.base_and_ancestors.map { |pipeline| config_checksum(pipeline) } - checksums.uniq.length != checksums.length + pipeline_checksums = @bridge.pipeline.base_and_ancestors.filter_map do |pipeline| + config_checksum(pipeline) unless pipeline.child? + end + + pipeline_checksums.uniq.length != pipeline_checksums.length end end diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index fd333e24860..c039f31aafc 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -13,6 +13,7 @@ module Ci Gitlab::Ci::Pipeline::Chain::Validate::SecurityOrchestrationPolicy, Gitlab::Ci::Pipeline::Chain::Config::Content, Gitlab::Ci::Pipeline::Chain::Config::Process, + Gitlab::Ci::Pipeline::Chain::Validate::AfterConfig, Gitlab::Ci::Pipeline::Chain::RemoveUnwantedChatJobs, Gitlab::Ci::Pipeline::Chain::Skip, Gitlab::Ci::Pipeline::Chain::SeedBlock, diff --git a/app/services/ci/job_artifacts/create_service.rb b/app/services/ci/job_artifacts/create_service.rb index a22ac87f660..9fc7c3b4d40 100644 --- a/app/services/ci/job_artifacts/create_service.rb +++ b/app/services/ci/job_artifacts/create_service.rb @@ -115,7 +115,6 @@ module Ci case artifact.file_type when 'dotenv' then parse_dotenv_artifact(artifact) - when 'cluster_applications' then parse_cluster_applications_artifact(artifact) else success end end @@ -165,10 +164,6 @@ module Ci def parse_dotenv_artifact(artifact) Ci::ParseDotenvArtifactService.new(project, current_user).execute(artifact) end - - def parse_cluster_applications_artifact(artifact) - Clusters::ParseClusterApplicationsArtifactService.new(job, job.user).execute(artifact) - end end end end diff --git a/app/services/ci/pipeline_creation/start_pipeline_service.rb b/app/services/ci/pipeline_creation/start_pipeline_service.rb new file mode 100644 index 00000000000..27c12caaa0a --- /dev/null +++ b/app/services/ci/pipeline_creation/start_pipeline_service.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Ci + module PipelineCreation + class StartPipelineService + attr_reader :pipeline + + def initialize(pipeline) + @pipeline = pipeline + end + + def execute + Ci::ProcessPipelineService.new(pipeline).execute + end + end + end +end + +::Ci::PipelineCreation::StartPipelineService.prepend_mod_with('Ci::PipelineCreation::StartPipelineService') diff --git a/app/services/ci/pipeline_schedules/calculate_next_run_service.rb b/app/services/ci/pipeline_schedules/calculate_next_run_service.rb new file mode 100644 index 00000000000..9978b2d4775 --- /dev/null +++ b/app/services/ci/pipeline_schedules/calculate_next_run_service.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Ci + module PipelineSchedules + class CalculateNextRunService < BaseService + include Gitlab::Utils::StrongMemoize + + def execute(schedule, fallback_method:) + @schedule = schedule + + return fallback_method.call unless ::Feature.enabled?(:ci_daily_limit_for_pipeline_schedules, project, default_enabled: :yaml) + return fallback_method.call unless plan_cron&.cron_valid? + + now = Time.zone.now + + schedule_next_run = schedule_cron.next_time_from(now) + return schedule_next_run if worker_cron.match?(schedule_next_run) && plan_cron.match?(schedule_next_run) + + plan_next_run = plan_cron.next_time_from(now) + return plan_next_run if worker_cron.match?(plan_next_run) + + worker_next_run = worker_cron.next_time_from(now) + return worker_next_run if plan_cron.match?(worker_next_run) + + worker_cron.next_time_from(plan_next_run) + end + + private + + def schedule_cron + strong_memoize(:schedule_cron) do + Gitlab::Ci::CronParser.new(@schedule.cron, @schedule.cron_timezone) + end + end + + def worker_cron + strong_memoize(:worker_cron) do + Gitlab::Ci::CronParser.new(worker_cron_expression, Time.zone.name) + end + end + + def plan_cron + strong_memoize(:plan_cron) do + daily_limit = @schedule.daily_limit + + next unless daily_limit + + every_x_minutes = (1.day.in_minutes / daily_limit).to_i + + Gitlab::Ci::CronParser.parse_natural("every #{every_x_minutes} minutes", Time.zone.name) + end + end + + def worker_cron_expression + Settings.cron_jobs['pipeline_schedule_worker']['cron'] + end + end + end +end diff --git a/app/services/ci/play_bridge_service.rb b/app/services/ci/play_bridge_service.rb index c5b19a3963a..2f0bcece9e3 100644 --- a/app/services/ci/play_bridge_service.rb +++ b/app/services/ci/play_bridge_service.rb @@ -3,7 +3,7 @@ module Ci class PlayBridgeService < ::BaseService def execute(bridge) - raise Gitlab::Access::AccessDeniedError unless can?(current_user, :play_job, bridge) + check_access!(bridge) bridge.tap do |bridge| bridge.user = current_user @@ -14,5 +14,13 @@ module Ci AfterRequeueJobService.new(project, current_user).execute(bridge) end end + + private + + def check_access!(bridge) + raise Gitlab::Access::AccessDeniedError unless can?(current_user, :play_job, bridge) + end end end + +Ci::PlayBridgeService.prepend_mod_with('Ci::PlayBridgeService') diff --git a/app/services/ci/play_build_service.rb b/app/services/ci/play_build_service.rb index 4953b1ea5fc..073c1a2d0e0 100644 --- a/app/services/ci/play_build_service.rb +++ b/app/services/ci/play_build_service.rb @@ -3,11 +3,7 @@ module Ci class PlayBuildService < ::BaseService def execute(build, job_variables_attributes = nil) - raise Gitlab::Access::AccessDeniedError unless can?(current_user, :play_job, build) - - if job_variables_attributes.present? && !can?(current_user, :set_pipeline_variables, project) - raise Gitlab::Access::AccessDeniedError - end + check_access!(build, job_variables_attributes) # Try to enqueue the build, otherwise create a duplicate. # @@ -23,5 +19,17 @@ module Ci Ci::Build.retry(build, current_user) end end + + private + + def check_access!(build, job_variables_attributes) + raise Gitlab::Access::AccessDeniedError unless can?(current_user, :play_job, build) + + if job_variables_attributes.present? && !can?(current_user, :set_pipeline_variables, project) + raise Gitlab::Access::AccessDeniedError + end + end end end + +Ci::PlayBuildService.prepend_mod_with('Ci::PlayBuildService') diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb index 461647ffccc..6280bf4c986 100644 --- a/app/services/ci/register_job_service.rb +++ b/app/services/ci/register_job_service.rb @@ -22,11 +22,27 @@ module Ci end def execute(params = {}) + db_all_caught_up = ::Gitlab::Database::LoadBalancing::Sticking.all_caught_up?(:runner, runner.id) + @metrics.increment_queue_operation(:queue_attempt) - @metrics.observe_queue_time(:process, @runner.runner_type) do + result = @metrics.observe_queue_time(:process, @runner.runner_type) do process_queue(params) end + + # Since we execute this query against replica it might lead to false-positive + # We might receive the positive response: "hi, we don't have any more builds for you". + # This might not be true. If our DB replica is not up-to date with when runner event was generated + # we might still have some CI builds to be picked. Instead we should say to runner: + # "Hi, we don't have any more builds now, but not everything is right anyway, so try again". + # Runner will retry, but again, against replica, and again will check if replication lag did catch-up. + if !db_all_caught_up && !result.build + metrics.increment_queue_operation(:queue_replication_lag) + + ::Ci::RegisterJobService::Result.new(nil, false) # rubocop:disable Cop/AvoidReturnFromBlocks + else + result + end end private @@ -109,25 +125,23 @@ module Ci builds = builds.queued_before(params[:job_age].seconds.ago) end - if Feature.enabled?(:ci_register_job_service_one_by_one, runner, default_enabled: true) - build_ids = retrieve_queue(-> { builds.pluck(:id) }) - - @metrics.observe_queue_size(-> { build_ids.size }, @runner.runner_type) + build_ids = retrieve_queue(-> { builds.pluck(:id) }) - build_ids.each do |build_id| - yield Ci::Build.find(build_id) - end - else - builds_array = retrieve_queue(-> { builds.to_a }) - - @metrics.observe_queue_size(-> { builds_array.size }, @runner.runner_type) + @metrics.observe_queue_size(-> { build_ids.size }, @runner.runner_type) - builds_array.each(&blk) + build_ids.each do |build_id| + yield Ci::Build.find(build_id) end end # rubocop: enable CodeReuse/ActiveRecord def retrieve_queue(queue_query_proc) + ## + # We want to reset a load balancing session to discard the side + # effects of writes that could have happened prior to this moment. + # + ::Gitlab::Database::LoadBalancing::Session.clear_session + @metrics.observe_queue_time(:retrieve, @runner.runner_type) do queue_query_proc.call end @@ -182,13 +196,7 @@ module Ci end def max_queue_depth - @max_queue_depth ||= begin - if Feature.enabled?(:gitlab_ci_builds_queue_limit, runner, default_enabled: true) - MAX_QUEUE_DEPTH - else - ::Gitlab::Database::MAX_INT_VALUE - end - end + MAX_QUEUE_DEPTH end # Force variables evaluation to occur now @@ -271,15 +279,11 @@ module Ci .order(Arel.sql('COALESCE(project_builds.running_builds, 0) ASC'), 'ci_builds.id ASC') end end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def builds_for_project_runner new_builds.where(project: runner.projects.without_deleted.with_builds_enabled).order('id ASC') end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def builds_for_group_runner # Workaround for weird Rails bug, that makes `runner.groups.to_sql` to return `runner_id = NULL` groups = ::Group.joins(:runner_namespaces).merge(runner.runner_namespaces) @@ -291,17 +295,23 @@ module Ci .without_deleted new_builds.where(project: projects).order('id ASC') end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def running_builds_for_shared_runners Ci::Build.running.where(runner: Ci::Runner.instance_type) .group(:project_id).select(:project_id, 'count(*) AS running_builds') end + + def all_builds + if Feature.enabled?(:ci_pending_builds_queue_join, runner, default_enabled: :yaml) + Ci::Build.joins(:queuing_entry) + else + Ci::Build.all + end + end # rubocop: enable CodeReuse/ActiveRecord def new_builds - builds = Ci::Build.pending.unstarted + builds = all_builds.pending.unstarted builds = builds.ref_protected if runner.ref_protected? builds end diff --git a/app/services/ci/retry_build_service.rb b/app/services/ci/retry_build_service.rb index e03f2ae3d52..ea76771b80a 100644 --- a/app/services/ci/retry_build_service.rb +++ b/app/services/ci/retry_build_service.rb @@ -34,15 +34,9 @@ module Ci attributes[:user] = current_user Ci::Build.transaction do - # mark all other builds of that name as retried - build.pipeline.builds.latest - .where(name: build.name) - .update_all(retried: true, processed: true) - - create_build!(attributes).tap do - # mark existing object as retried/processed without a reload - build.retried = true - build.processed = true + create_build!(attributes).tap do |new_build| + new_build.update_older_statuses_retried! + build.reset # refresh the data to get new values of `retried` and `processed`. end end end @@ -59,7 +53,6 @@ module Ci def create_build!(attributes) build = project.builds.new(attributes) build.assign_attributes(::Gitlab::Ci::Pipeline::Seed::Build.environment_attributes_for(build)) - build.retried = false BulkInsertableAssociations.with_bulk_insert do build.save! end diff --git a/app/services/ci/update_build_queue_service.rb b/app/services/ci/update_build_queue_service.rb index cf629b879b3..eea09e9ac67 100644 --- a/app/services/ci/update_build_queue_service.rb +++ b/app/services/ci/update_build_queue_service.rb @@ -2,13 +2,103 @@ module Ci class UpdateBuildQueueService - def execute(build, metrics = ::Gitlab::Ci::Queue::Metrics) - tick_for(build, build.project.all_runners, metrics) + InvalidQueueTransition = Class.new(StandardError) + + attr_reader :metrics + + def initialize(metrics = ::Gitlab::Ci::Queue::Metrics) + @metrics = metrics + end + + ## + # Add a build to the pending builds queue + # + def push(build, transition) + return unless maintain_pending_builds_queue?(build) + + raise InvalidQueueTransition unless transition.to == 'pending' + + transition.within_transaction do + result = build.create_queuing_entry! + + unless result.empty? + metrics.increment_queue_operation(:build_queue_push) + + result.rows.dig(0, 0) + end + end + end + + ## + # Remove a build from the pending builds queue + # + def pop(build, transition) + return unless maintain_pending_builds_queue?(build) + + raise InvalidQueueTransition unless transition.from == 'pending' + + transition.within_transaction do + removed = build.all_queuing_entries.delete_all + + if removed > 0 + metrics.increment_queue_operation(:build_queue_pop) + + build.id + end + end + end + + ## + # Add shared runner build tracking entry (used for queuing). + # + def track(build, transition) + return unless Feature.enabled?(:ci_track_shared_runner_builds, build.project, default_enabled: :yaml) + return unless build.shared_runner_build? + + raise InvalidQueueTransition unless transition.to == 'running' + + transition.within_transaction do + result = ::Ci::RunningBuild.upsert_shared_runner_build!(build) + + unless result.empty? + metrics.increment_queue_operation(:shared_runner_build_new) + + result.rows.dig(0, 0) + end + end + end + + ## + # Remove a runtime build tracking entry for a shared runner build (used for + # queuing). + # + def untrack(build, transition) + return unless Feature.enabled?(:ci_untrack_shared_runner_builds, build.project, default_enabled: :yaml) + return unless build.shared_runner_build? + + raise InvalidQueueTransition unless transition.from == 'running' + + transition.within_transaction do + removed = build.all_runtime_metadata.delete_all + + if removed > 0 + metrics.increment_queue_operation(:shared_runner_build_done) + + build.id + end + end + end + + ## + # Unblock runner associated with given project / build + # + def tick(build) + tick_for(build, build.project.all_available_runners) end private - def tick_for(build, runners, metrics) + def tick_for(build, runners) runners = runners.with_recent_runner_queue runners = runners.with_tags if Feature.enabled?(:ci_preload_runner_tags, default_enabled: :yaml) @@ -20,5 +110,9 @@ module Ci runner.pick_build!(build) end end + + def maintain_pending_builds_queue?(build) + Feature.enabled?(:ci_pending_builds_queue_maintain, build.project, default_enabled: :yaml) + end end end diff --git a/app/services/ci/update_build_state_service.rb b/app/services/ci/update_build_state_service.rb index 874f4bf459a..abd50d2f110 100644 --- a/app/services/ci/update_build_state_service.rb +++ b/app/services/ci/update_build_state_service.rb @@ -19,8 +19,6 @@ module Ci end def execute - overwrite_trace! if has_trace? - unless accept_available? return update_build_state! end @@ -34,12 +32,6 @@ module Ci private - def overwrite_trace! - metrics.increment_trace_operation(operation: :overwrite) - - build.trace.set(params[:trace]) if Gitlab::Ci::Features.trace_overwrite? - end - def ensure_pending_state! pending_state.created_at end @@ -151,10 +143,6 @@ module Ci params.dig(:state).to_s end - def has_trace? - params.dig(:trace).present? - end - def has_checksum? trace_checksum.present? end @@ -181,7 +169,7 @@ module Ci state: params.fetch(:state), trace_checksum: trace_checksum, trace_bytesize: trace_bytesize, - failure_reason: params.dig(:failure_reason) + failure_reason: failure_reason ) unless build_state.present? @@ -191,6 +179,14 @@ module Ci build_state || build.pending_state end + def failure_reason + reason = params.dig(:failure_reason) + + return unless reason + + Ci::BuildPendingState.failure_reasons.fetch(reason.to_s, 'unknown_failure') + end + ## # This method is releasing an exclusive lock on a build trace the moment we # conclude that build status has been written and the build state update |