Add latest changes from gitlab-org/gitlab@14-3-stable-eev14.3.0-rc42

14 files changed, 286 insertions, 67 deletions

diff --git a/app/services/ci/after_requeue_job_service.rb b/app/services/ci/after_requeue_job_service.rb
index f717dd0862c..9101ae91967 100644
--- a/app/services/ci/after_requeue_job_service.rb
+++ b/app/services/ci/after_requeue_job_service.rb
@@ -10,16 +10,9 @@ module Ci
     private
 
     def process_subsequent_jobs(processable)
-      if Feature.enabled?(:ci_same_stage_job_needs, processable.project, default_enabled: :yaml)
-        (stage_dependent_jobs(processable) | needs_dependent_jobs(processable))
-        .each do |processable|
-          process(processable)
-        end
-      else
-        skipped_jobs(processable).after_stage(processable.stage_idx)
-          .find_each do |job|
-          process(job)
-        end
+      (stage_dependent_jobs(processable) | needs_dependent_jobs(processable))
+      .each do |processable|
+        process(processable)
       end
     end
 
diff --git a/app/services/ci/archive_trace_service.rb b/app/services/ci/archive_trace_service.rb
index bc3219fbd79..995b58c6882 100644
--- a/app/services/ci/archive_trace_service.rb
+++ b/app/services/ci/archive_trace_service.rb
@@ -3,6 +3,13 @@
 module Ci
   class ArchiveTraceService
     def execute(job, worker_name:)
+      unless job.trace.can_attempt_archival_now?
+        Sidekiq.logger.warn(class: worker_name,
+                            message: job.trace.archival_attempts_message,
+                            job_id: job.id)
+        return
+      end
+
       # TODO: Remove this logging once we confirmed new live trace architecture is functional.
       # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667.
       unless job.has_live_trace?
@@ -25,6 +32,8 @@ module Ci
     rescue ::Gitlab::Ci::Trace::AlreadyArchivedError
       # It's already archived, thus we can safely ignore this exception.
     rescue StandardError => e
+      job.trace.increment_archival_attempts!
+
       # Tracks this error with application logs, Sentry, and Prometheus.
       # If `archive!` keeps failing for over a week, that could incur data loss.
       # (See more https://docs.gitlab.com/ee/administration/job_logs.html#new-incremental-logging-architecture)
diff --git a/app/services/ci/drop_pipeline_service.rb b/app/services/ci/drop_pipeline_service.rb
index 16d3abcbfa0..5772ab8f29c 100644
--- a/app/services/ci/drop_pipeline_service.rb
+++ b/app/services/ci/drop_pipeline_service.rb
@@ -2,8 +2,7 @@
 
 module Ci
   class DropPipelineService
-    PRELOADED_COMMIT_STATUS_RELATIONS = [:project, :pipeline].freeze
-    PRELOADED_CI_BUILD_RELATIONS = [:metadata, :deployment, :taggings].freeze
+    PRELOADED_RELATIONS = [:project, :pipeline, :metadata, :deployment, :taggings].freeze
 
     # execute service asynchronously for each cancelable pipeline
     def execute_async_for_all(pipelines, failure_reason, context_user)
@@ -30,11 +29,8 @@ module Ci
 
     private
 
-    # rubocop: disable CodeReuse/ActiveRecord
     def preload_associations_for_drop(commit_status_batch)
-      ActiveRecord::Associations::Preloader.new.preload(commit_status_batch, PRELOADED_COMMIT_STATUS_RELATIONS)
-      ActiveRecord::Associations::Preloader.new.preload(commit_status_batch.select { |job| job.is_a?(Ci::Build) }, PRELOADED_CI_BUILD_RELATIONS)
+      Preloaders::CommitStatusPreloader.new(commit_status_batch).execute(PRELOADED_RELATIONS)
     end
-    # rubocop: enable CodeReuse/ActiveRecord
   end
 end
diff --git a/app/services/ci/external_pull_requests/create_pipeline_service.rb b/app/services/ci/external_pull_requests/create_pipeline_service.rb
index 83499524a8e..dd93ca4708e 100644
--- a/app/services/ci/external_pull_requests/create_pipeline_service.rb
+++ b/app/services/ci/external_pull_requests/create_pipeline_service.rb
@@ -16,8 +16,14 @@ module Ci
       private
 
       def create_pipeline_for(pull_request)
-        Ci::CreatePipelineService.new(project, current_user, create_params(pull_request))
-          .execute(:external_pull_request_event, external_pull_request: pull_request)
+        if ::Feature.enabled?(:ci_create_external_pr_pipeline_async, project, default_enabled: :yaml)
+          Ci::ExternalPullRequests::CreatePipelineWorker.perform_async(
+            project.id, current_user.id, pull_request.id
+          )
+        else
+          Ci::CreatePipelineService.new(project, current_user, create_params(pull_request))
+            .execute(:external_pull_request_event, external_pull_request: pull_request)
+        end
       end
 
       def create_params(pull_request)
diff --git a/app/services/ci/pipeline_schedules/calculate_next_run_service.rb b/app/services/ci/pipeline_schedules/calculate_next_run_service.rb
index 9c8f6b47288..6c8ccb017e9 100644
--- a/app/services/ci/pipeline_schedules/calculate_next_run_service.rb
+++ b/app/services/ci/pipeline_schedules/calculate_next_run_service.rb
@@ -8,7 +8,6 @@ module Ci
       def execute(schedule, fallback_method:)
         @schedule = schedule
 
-        return fallback_method.call unless ::Feature.enabled?(:ci_daily_limit_for_pipeline_schedules, project, default_enabled: :yaml)
         return fallback_method.call unless plan_cron&.cron_valid?
 
         now = Time.zone.now
diff --git a/app/services/ci/pipelines/add_job_service.rb b/app/services/ci/pipelines/add_job_service.rb
index 41f9903e48c..53536b6fdf9 100644
--- a/app/services/ci/pipelines/add_job_service.rb
+++ b/app/services/ci/pipelines/add_job_service.rb
@@ -21,14 +21,14 @@ module Ci
             Ci::Pipeline.transaction do
               yield(job)
 
-              job.update_older_statuses_retried! if Feature.enabled?(:ci_fix_commit_status_retried, pipeline.project, default_enabled: :yaml)
+              job.update_older_statuses_retried!
             end
           end
         else
           Ci::Pipeline.transaction do
             yield(job)
 
-            job.update_older_statuses_retried! if Feature.enabled?(:ci_fix_commit_status_retried, pipeline.project, default_enabled: :yaml)
+            job.update_older_statuses_retried!
           end
         end
 
diff --git a/app/services/ci/queue/build_queue_service.rb b/app/services/ci/queue/build_queue_service.rb
index 99408d529b2..3276c427923 100644
--- a/app/services/ci/queue/build_queue_service.rb
+++ b/app/services/ci/queue/build_queue_service.rb
@@ -24,26 +24,30 @@ module Ci
 
       # rubocop:disable CodeReuse/ActiveRecord
       def builds_for_group_runner
-        # Workaround for weird Rails bug, that makes `runner.groups.to_sql` to return `runner_id = NULL`
-        groups = ::Group.joins(:runner_namespaces).merge(runner.runner_namespaces)
+        if strategy.use_denormalized_namespace_traversal_ids?
+          strategy.builds_for_group_runner
+        else
+          # Workaround for weird Rails bug, that makes `runner.groups.to_sql` to return `runner_id = NULL`
+          groups = ::Group.joins(:runner_namespaces).merge(runner.runner_namespaces)
 
-        hierarchy_groups = Gitlab::ObjectHierarchy
-          .new(groups)
-          .base_and_descendants
+          hierarchy_groups = Gitlab::ObjectHierarchy
+            .new(groups)
+            .base_and_descendants
 
-        projects = Project.where(namespace_id: hierarchy_groups)
-          .with_group_runners_enabled
-          .with_builds_enabled
-          .without_deleted
+          projects = Project.where(namespace_id: hierarchy_groups)
+            .with_group_runners_enabled
+            .with_builds_enabled
+            .without_deleted
 
-        relation = new_builds.where(project: projects)
+          relation = new_builds.where(project: projects)
 
-        order(relation)
+          order(relation)
+        end
       end
 
       def builds_for_project_runner
         relation = new_builds
-          .where(project: runner.projects.without_deleted.with_builds_enabled)
+          .where(project: runner_projects_relation)
 
         order(relation)
       end
@@ -83,6 +87,14 @@ module Ci
           end
         end
       end
+
+      def runner_projects_relation
+        if ::Feature.enabled?(:ci_pending_builds_project_runners_decoupling, runner, default_enabled: :yaml)
+          runner.runner_projects.select(:project_id)
+        else
+          runner.projects.without_deleted.with_builds_enabled
+        end
+      end
     end
   end
 end
diff --git a/app/services/ci/queue/builds_table_strategy.rb b/app/services/ci/queue/builds_table_strategy.rb
index d0a343cb9d4..ac449a5289e 100644
--- a/app/services/ci/queue/builds_table_strategy.rb
+++ b/app/services/ci/queue/builds_table_strategy.rb
@@ -31,6 +31,10 @@ module Ci
         end
       end
 
+      def builds_for_group_runner
+        raise NotImplementedError
+      end
+
       def builds_matching_tag_ids(relation, ids)
         # pick builds that does not have other tags than runner's one
         relation.matches_tag_ids(ids)
@@ -61,6 +65,10 @@ module Ci
         false
       end
 
+      def use_denormalized_namespace_traversal_ids?
+        false
+      end
+
       private
 
       def running_builds_for_shared_runners
diff --git a/app/services/ci/queue/pending_builds_strategy.rb b/app/services/ci/queue/pending_builds_strategy.rb
index efe3a981d3a..7a913e47df4 100644
--- a/app/services/ci/queue/pending_builds_strategy.rb
+++ b/app/services/ci/queue/pending_builds_strategy.rb
@@ -16,12 +16,26 @@ module Ci
         builds_ordered_for_shared_runners(shared_builds)
       end
 
+      def builds_for_group_runner
+        return new_builds.none if runner.namespace_ids.empty?
+
+        new_builds.where('ci_pending_builds.namespace_traversal_ids && ARRAY[?]::int[]', runner.namespace_ids)
+      end
+
       def builds_matching_tag_ids(relation, ids)
-        relation.merge(CommitStatus.matches_tag_ids(ids, table: 'ci_pending_builds', column: 'build_id'))
+        if ::Feature.enabled?(:ci_queueing_denormalize_tags_information, runner, default_enabled: :yaml)
+          relation.for_tags(runner.tags_ids)
+        else
+          relation.merge(CommitStatus.matches_tag_ids(ids, table: 'ci_pending_builds', column: 'build_id'))
+        end
       end
 
       def builds_with_any_tags(relation)
-        relation.merge(CommitStatus.with_any_tags(table: 'ci_pending_builds', column: 'build_id'))
+        if ::Feature.enabled?(:ci_queueing_denormalize_tags_information, runner, default_enabled: :yaml)
+          relation.where('cardinality(tag_ids) > 0')
+        else
+          relation.merge(CommitStatus.with_any_tags(table: 'ci_pending_builds', column: 'build_id'))
+        end
       end
 
       def order(relation)
@@ -44,6 +58,10 @@ module Ci
         ::Feature.enabled?(:ci_queueing_denormalize_ci_minutes_information, runner, type: :development, default_enabled: :yaml)
       end
 
+      def use_denormalized_namespace_traversal_ids?
+        ::Feature.enabled?(:ci_queueing_denormalize_namespace_traversal_ids, runner, type: :development, default_enabled: :yaml)
+      end
+
       private
 
       def builds_available_for_shared_runners
diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb
index dc046e1d164..c46ddd22558 100644
--- a/app/services/ci/register_job_service.rb
+++ b/app/services/ci/register_job_service.rb
@@ -103,40 +103,42 @@ module Ci
 
     # rubocop: disable CodeReuse/ActiveRecord
     def each_build(params, &blk)
-      queue = ::Ci::Queue::BuildQueueService.new(runner)
-
-      builds = begin
-        if runner.instance_type?
-          queue.builds_for_shared_runner
-        elsif runner.group_type?
-          queue.builds_for_group_runner
-        else
-          queue.builds_for_project_runner
+      ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339429') do
+        queue = ::Ci::Queue::BuildQueueService.new(runner)
+
+        builds = begin
+          if runner.instance_type?
+            queue.builds_for_shared_runner
+          elsif runner.group_type?
+            queue.builds_for_group_runner
+          else
+            queue.builds_for_project_runner
+          end
         end
-      end
 
-      if runner.ref_protected?
-        builds = queue.builds_for_protected_runner(builds)
-      end
+        if runner.ref_protected?
+          builds = queue.builds_for_protected_runner(builds)
+        end
 
-      # pick builds that does not have other tags than runner's one
-      builds = queue.builds_matching_tag_ids(builds, runner.tags.ids)
+        # pick builds that does not have other tags than runner's one
+        builds = queue.builds_matching_tag_ids(builds, runner.tags.ids)
 
-      # pick builds that have at least one tag
-      unless runner.run_untagged?
-        builds = queue.builds_with_any_tags(builds)
-      end
+        # pick builds that have at least one tag
+        unless runner.run_untagged?
+          builds = queue.builds_with_any_tags(builds)
+        end
 
-      # pick builds that older than specified age
-      if params.key?(:job_age)
-        builds = queue.builds_queued_before(builds, params[:job_age].seconds.ago)
-      end
+        # pick builds that older than specified age
+        if params.key?(:job_age)
+          builds = queue.builds_queued_before(builds, params[:job_age].seconds.ago)
+        end
 
-      build_ids = retrieve_queue(-> { queue.execute(builds) })
+        build_ids = retrieve_queue(-> { queue.execute(builds) })
 
-      @metrics.observe_queue_size(-> { build_ids.size }, @runner.runner_type)
+        @metrics.observe_queue_size(-> { build_ids.size }, @runner.runner_type)
 
-      build_ids.each { |build_id| yield Ci::Build.find(build_id) }
+        build_ids.each { |build_id| yield Ci::Build.find(build_id) }
+      end
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
@@ -269,6 +271,15 @@ module Ci
         missing_dependency_failure: -> (build, _) { !build.has_valid_build_dependencies? },
         runner_unsupported: -> (build, params) { !build.supported_runner?(params.dig(:info, :features)) },
         archived_failure: -> (build, _) { build.archived? }
+      }.merge(builds_enabled_checks)
+    end
+
+    def builds_enabled_checks
+      return {} unless ::Feature.enabled?(:ci_queueing_builds_enabled_checks, runner, default_enabled: :yaml)
+
+      {
+        project_deleted: -> (build, _) { build.project.pending_delete? },
+        builds_disabled: -> (build, _) { !build.project.builds_enabled? }
       }
     end
   end
diff --git a/app/services/ci/stuck_builds/drop_helpers.rb b/app/services/ci/stuck_builds/drop_helpers.rb
new file mode 100644
index 00000000000..f79b805c23d
--- /dev/null
+++ b/app/services/ci/stuck_builds/drop_helpers.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Ci
+  module StuckBuilds
+    module DropHelpers
+      def drop(builds, failure_reason:)
+        fetch(builds) do |build|
+          drop_build :outdated, build, failure_reason
+        end
+      end
+
+      def drop_stuck(builds, failure_reason:)
+        fetch(builds) do |build|
+          break unless build.stuck?
+
+          drop_build :stuck, build, failure_reason
+        end
+      end
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def fetch(builds)
+        loop do
+          jobs = builds.includes(:tags, :runner, project: [:namespace, :route])
+            .limit(100)
+            .to_a
+
+          break if jobs.empty?
+
+          jobs.each do |job|
+            Gitlab::ApplicationContext.with_context(project: job.project) { yield(job) }
+          end
+        end
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      def drop_build(type, build, reason)
+        Gitlab::AppLogger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{build.status}, failure_reason: #{reason})"
+        Gitlab::OptimisticLocking.retry_lock(build, 3, name: 'stuck_ci_jobs_worker_drop_build') do |b|
+          b.drop(reason)
+        end
+      rescue StandardError => ex
+        build.doom!
+
+        track_exception_for_build(ex, build)
+      end
+
+      def track_exception_for_build(ex, build)
+        Gitlab::ErrorTracking.track_exception(ex,
+            build_id: build.id,
+            build_name: build.name,
+            build_stage: build.stage,
+            pipeline_id: build.pipeline_id,
+            project_id: build.project_id
+        )
+      end
+    end
+  end
+end
diff --git a/app/services/ci/stuck_builds/drop_service.rb b/app/services/ci/stuck_builds/drop_service.rb
new file mode 100644
index 00000000000..3fee9a94381
--- /dev/null
+++ b/app/services/ci/stuck_builds/drop_service.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Ci
+  module StuckBuilds
+    class DropService
+      include DropHelpers
+
+      BUILD_RUNNING_OUTDATED_TIMEOUT = 1.hour
+      BUILD_PENDING_OUTDATED_TIMEOUT = 1.day
+      BUILD_SCHEDULED_OUTDATED_TIMEOUT = 1.hour
+      BUILD_PENDING_STUCK_TIMEOUT = 1.hour
+      BUILD_LOOKBACK = 5.days
+
+      def execute
+        Gitlab::AppLogger.info "#{self.class}: Cleaning stuck builds"
+
+        drop(running_timed_out_builds, failure_reason: :stuck_or_timeout_failure)
+
+        drop(
+          pending_builds(BUILD_PENDING_OUTDATED_TIMEOUT.ago),
+          failure_reason: :stuck_or_timeout_failure
+        )
+
+        drop(scheduled_timed_out_builds, failure_reason: :stale_schedule)
+
+        drop_stuck(
+          pending_builds(BUILD_PENDING_STUCK_TIMEOUT.ago),
+          failure_reason: :stuck_or_timeout_failure
+        )
+      end
+
+      private
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      # We're adding the ordering clause by `created_at` and `project_id`
+      # because we want to force the query planner to use the
+      # `ci_builds_gitlab_monitor_metrics` index all the time.
+      def pending_builds(timeout)
+        if Feature.enabled?(:ci_new_query_for_pending_stuck_jobs)
+          Ci::Build.pending.created_at_before(timeout).updated_at_before(timeout).order(created_at: :asc, project_id: :asc)
+        else
+          Ci::Build.pending.updated_before(lookback: BUILD_LOOKBACK.ago, timeout: timeout)
+        end
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      def scheduled_timed_out_builds
+        Ci::Build.where(status: :scheduled).where( # rubocop: disable CodeReuse/ActiveRecord
+          'ci_builds.scheduled_at IS NOT NULL AND ci_builds.scheduled_at < ?',
+          BUILD_SCHEDULED_OUTDATED_TIMEOUT.ago
+        )
+      end
+
+      def running_timed_out_builds
+        Ci::Build.running.where( # rubocop: disable CodeReuse/ActiveRecord
+          'ci_builds.updated_at < ?',
+          BUILD_RUNNING_OUTDATED_TIMEOUT.ago
+        )
+      end
+    end
+  end
+end
diff --git a/app/services/ci/update_build_queue_service.rb b/app/services/ci/update_build_queue_service.rb
index eea09e9ac67..c1cbf031ca1 100644
--- a/app/services/ci/update_build_queue_service.rb
+++ b/app/services/ci/update_build_queue_service.rb
@@ -99,15 +99,17 @@ module Ci
     private
 
     def tick_for(build, runners)
-      runners = runners.with_recent_runner_queue
-      runners = runners.with_tags if Feature.enabled?(:ci_preload_runner_tags, default_enabled: :yaml)
+      ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339937') do
+        runners = runners.with_recent_runner_queue
+        runners = runners.with_tags if Feature.enabled?(:ci_preload_runner_tags, default_enabled: :yaml)
 
-      metrics.observe_active_runners(-> { runners.to_a.size })
+        metrics.observe_active_runners(-> { runners.to_a.size })
 
-      runners.each do |runner|
-        metrics.increment_runner_tick(runner)
+        runners.each do |runner|
+          metrics.increment_runner_tick(runner)
 
-        runner.pick_build!(build)
+          runner.pick_build!(build)
+        end
       end
     end
 
diff --git a/app/services/ci/update_pending_build_service.rb b/app/services/ci/update_pending_build_service.rb
new file mode 100644
index 00000000000..dcba06e60bf
--- /dev/null
+++ b/app/services/ci/update_pending_build_service.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Ci
+  class UpdatePendingBuildService
+    VALID_PARAMS = %i[instance_runners_enabled].freeze
+
+    InvalidParamsError = Class.new(StandardError)
+    InvalidModelError = Class.new(StandardError)
+
+    def initialize(model, update_params)
+      @model = model
+      @update_params = update_params
+
+      validations!
+    end
+
+    def execute
+      return unless ::Feature.enabled?(:ci_pending_builds_maintain_shared_runners_data, @model, default_enabled: :yaml)
+
+      @model.pending_builds.each_batch do |relation|
+        relation.update_all(@update_params)
+      end
+    end
+
+    private
+
+    def validations!
+      validate_model! && validate_params!
+    end
+
+    def validate_model!
+      raise InvalidModelError unless @model.is_a?(::Project) || @model.is_a?(::Group)
+
+      true
+    end
+
+    def validate_params!
+      extra_params = @update_params.keys - VALID_PARAMS
+
+      raise InvalidParamsError, "Unvalid params: #{extra_params.join(', ')}" unless extra_params.empty?
+
+      true
+    end
+  end
+end