Add latest changes from gitlab-org/gitlab@14-4-stable-eev14.4.0-rc42

15 files changed, 144 insertions, 112 deletions

diff --git a/app/services/ci/archive_trace_service.rb b/app/services/ci/archive_trace_service.rb
index 995b58c6882..17cac38ace2 100644
--- a/app/services/ci/archive_trace_service.rb
+++ b/app/services/ci/archive_trace_service.rb
@@ -3,10 +3,15 @@
 module Ci
   class ArchiveTraceService
     def execute(job, worker_name:)
+      unless job.trace.archival_attempts_available?
+        Sidekiq.logger.warn(class: worker_name, message: 'The job is out of archival attempts.', job_id: job.id)
+
+        job.trace.attempt_archive_cleanup!
+        return
+      end
+
       unless job.trace.can_attempt_archival_now?
-        Sidekiq.logger.warn(class: worker_name,
-                            message: job.trace.archival_attempts_message,
-                            job_id: job.id)
+        Sidekiq.logger.warn(class: worker_name, message: 'The job can not be archived right now.', job_id: job.id)
         return
       end
 
diff --git a/app/services/ci/destroy_pipeline_service.rb b/app/services/ci/destroy_pipeline_service.rb
index dd5c8e0379f..476c7523d60 100644
--- a/app/services/ci/destroy_pipeline_service.rb
+++ b/app/services/ci/destroy_pipeline_service.rb
@@ -9,6 +9,9 @@ module Ci
 
       pipeline.cancel_running if pipeline.cancelable?
 
+      # Ci::Pipeline#destroy triggers `use_fast_destroy :job_artifacts` and
+      # ci_builds has ON DELETE CASCADE to ci_pipelines. The pipeline, the builds,
+      # job and pipeline artifacts all get destroyed here.
       pipeline.reset.destroy!
 
       ServiceResponse.success(message: 'Pipeline not found')
diff --git a/app/services/ci/pipelines/add_job_service.rb b/app/services/ci/pipelines/add_job_service.rb
index 53536b6fdf9..703bb22fb5d 100644
--- a/app/services/ci/pipelines/add_job_service.rb
+++ b/app/services/ci/pipelines/add_job_service.rb
@@ -16,15 +16,7 @@ module Ci
       def execute!(job, &block)
         assign_pipeline_attributes(job)
 
-        if Feature.enabled?(:ci_pipeline_add_job_with_lock, pipeline.project, default_enabled: :yaml)
-          in_lock("ci:pipelines:#{pipeline.id}:add-job", ttl: LOCK_TIMEOUT, sleep_sec: LOCK_SLEEP, retries: LOCK_RETRIES) do
-            Ci::Pipeline.transaction do
-              yield(job)
-
-              job.update_older_statuses_retried!
-            end
-          end
-        else
+        in_lock("ci:pipelines:#{pipeline.id}:add-job", ttl: LOCK_TIMEOUT, sleep_sec: LOCK_SLEEP, retries: LOCK_RETRIES) do
           Ci::Pipeline.transaction do
             yield(job)
 
diff --git a/app/services/ci/pipelines/hook_service.rb b/app/services/ci/pipelines/hook_service.rb
new file mode 100644
index 00000000000..629ed7e1ebd
--- /dev/null
+++ b/app/services/ci/pipelines/hook_service.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Ci
+  module Pipelines
+    class HookService
+      include Gitlab::Utils::StrongMemoize
+
+      HOOK_NAME = :pipeline_hooks
+
+      def initialize(pipeline)
+        @pipeline = pipeline
+      end
+
+      def execute
+        project.execute_hooks(hook_data, HOOK_NAME) if project.has_active_hooks?(HOOK_NAME)
+        project.execute_integrations(hook_data, HOOK_NAME) if project.has_active_integrations?(HOOK_NAME)
+      end
+
+      private
+
+      attr_reader :pipeline
+
+      def project
+        @project ||= pipeline.project
+      end
+
+      def hook_data
+        strong_memoize(:hook_data) do
+          Gitlab::DataBuilder::Pipeline.build(pipeline)
+        end
+      end
+    end
+  end
+end
diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb
index fb26d5d3356..664915c5e2f 100644
--- a/app/services/ci/process_pipeline_service.rb
+++ b/app/services/ci/process_pipeline_service.rb
@@ -11,8 +11,6 @@ module Ci
     def execute
       increment_processing_counter
 
-      update_retried
-
       Ci::PipelineProcessing::AtomicProcessingService
         .new(pipeline)
         .execute
@@ -24,41 +22,6 @@ module Ci
 
     private
 
-    # This method is for compatibility and data consistency and should be removed with 9.3 version of GitLab
-    # This replicates what is db/post_migrate/20170416103934_upate_retried_for_ci_build.rb
-    # and ensures that functionality will not be broken before migration is run
-    # this updates only when there are data that needs to be updated, there are two groups with no retried flag
-    # rubocop: disable CodeReuse/ActiveRecord
-    def update_retried
-      return if Feature.enabled?(:ci_remove_update_retried_from_process_pipeline, pipeline.project, default_enabled: :yaml)
-
-      # find the latest builds for each name
-      latest_statuses = pipeline.latest_statuses
-        .group(:name)
-        .having('count(*) > 1')
-        .pluck(Arel.sql('MAX(id)'), 'name')
-
-      # mark builds that are retried
-      if latest_statuses.any?
-        updated_count = pipeline.latest_statuses
-                          .where(name: latest_statuses.map(&:second))
-                          .where.not(id: latest_statuses.map(&:first))
-                          .update_all(retried: true)
-
-        # This counter is temporary. It will be used to check whether if we still use this method or not
-        # after setting correct value of `GenericCommitStatus#retried`.
-        # More info: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50465#note_491657115
-        if updated_count > 0
-          Gitlab::AppJsonLogger.info(event: 'update_retried_is_used',
-                                     project_id: pipeline.project.id,
-                                     pipeline_id: pipeline.id)
-
-          metrics.legacy_update_jobs_counter.increment
-        end
-      end
-    end
-    # rubocop: enable CodeReuse/ActiveRecord
-
     def increment_processing_counter
       metrics.pipeline_processing_events_counter.increment
     end
diff --git a/app/services/ci/queue/build_queue_service.rb b/app/services/ci/queue/build_queue_service.rb
index 3276c427923..3c886cb023f 100644
--- a/app/services/ci/queue/build_queue_service.rb
+++ b/app/services/ci/queue/build_queue_service.rb
@@ -90,7 +90,7 @@ module Ci
 
       def runner_projects_relation
         if ::Feature.enabled?(:ci_pending_builds_project_runners_decoupling, runner, default_enabled: :yaml)
-          runner.runner_projects.select(:project_id)
+          runner.runner_projects.select('"ci_runner_projects"."project_id"::bigint')
         else
           runner.projects.without_deleted.with_builds_enabled
         end
diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb
index c46ddd22558..67ef4f10709 100644
--- a/app/services/ci/register_job_service.rb
+++ b/app/services/ci/register_job_service.rb
@@ -22,7 +22,8 @@ module Ci
     end
 
     def execute(params = {})
-      db_all_caught_up = ::Gitlab::Database::LoadBalancing::Sticking.all_caught_up?(:runner, runner.id)
+      db_all_caught_up =
+        ::Ci::Runner.sticking.all_caught_up?(:runner, runner.id)
 
       @metrics.increment_queue_operation(:queue_attempt)
 
@@ -103,42 +104,40 @@ module Ci
 
     # rubocop: disable CodeReuse/ActiveRecord
     def each_build(params, &blk)
-      ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339429') do
-        queue = ::Ci::Queue::BuildQueueService.new(runner)
-
-        builds = begin
-          if runner.instance_type?
-            queue.builds_for_shared_runner
-          elsif runner.group_type?
-            queue.builds_for_group_runner
-          else
-            queue.builds_for_project_runner
-          end
-        end
+      queue = ::Ci::Queue::BuildQueueService.new(runner)
 
-        if runner.ref_protected?
-          builds = queue.builds_for_protected_runner(builds)
+      builds = begin
+        if runner.instance_type?
+          queue.builds_for_shared_runner
+        elsif runner.group_type?
+          queue.builds_for_group_runner
+        else
+          queue.builds_for_project_runner
         end
+      end
 
-        # pick builds that does not have other tags than runner's one
-        builds = queue.builds_matching_tag_ids(builds, runner.tags.ids)
+      if runner.ref_protected?
+        builds = queue.builds_for_protected_runner(builds)
+      end
 
-        # pick builds that have at least one tag
-        unless runner.run_untagged?
-          builds = queue.builds_with_any_tags(builds)
-        end
+      # pick builds that does not have other tags than runner's one
+      builds = queue.builds_matching_tag_ids(builds, runner.tags.ids)
 
-        # pick builds that older than specified age
-        if params.key?(:job_age)
-          builds = queue.builds_queued_before(builds, params[:job_age].seconds.ago)
-        end
+      # pick builds that have at least one tag
+      unless runner.run_untagged?
+        builds = queue.builds_with_any_tags(builds)
+      end
 
-        build_ids = retrieve_queue(-> { queue.execute(builds) })
+      # pick builds that older than specified age
+      if params.key?(:job_age)
+        builds = queue.builds_queued_before(builds, params[:job_age].seconds.ago)
+      end
 
-        @metrics.observe_queue_size(-> { build_ids.size }, @runner.runner_type)
+      build_ids = retrieve_queue(-> { queue.execute(builds) })
 
-        build_ids.each { |build_id| yield Ci::Build.find(build_id) }
-      end
+      @metrics.observe_queue_size(-> { build_ids.size }, @runner.runner_type)
+
+      build_ids.each { |build_id| yield Ci::Build.find(build_id) }
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
diff --git a/app/services/ci/resource_groups/assign_resource_from_resource_group_service.rb b/app/services/ci/resource_groups/assign_resource_from_resource_group_service.rb
index 1d329fe7b53..dfd97498fc8 100644
--- a/app/services/ci/resource_groups/assign_resource_from_resource_group_service.rb
+++ b/app/services/ci/resource_groups/assign_resource_from_resource_group_service.rb
@@ -9,7 +9,7 @@ module Ci
 
         free_resources = resource_group.resources.free.count
 
-        resource_group.processables.waiting_for_resource.take(free_resources).each do |processable|
+        resource_group.upcoming_processables.take(free_resources).each do |processable|
           processable.enqueue_waiting_for_resource
         end
       end
diff --git a/app/services/ci/retry_build_service.rb b/app/services/ci/retry_build_service.rb
index 08520c9514c..07cfbb9ce3c 100644
--- a/app/services/ci/retry_build_service.rb
+++ b/app/services/ci/retry_build_service.rb
@@ -17,7 +17,7 @@ module Ci
     def execute(build)
       build.ensure_scheduling_type!
 
-      reprocess!(build).tap do |new_build|
+      clone!(build).tap do |new_build|
         check_assignable_runners!(new_build)
         next if new_build.failed?
 
@@ -31,7 +31,12 @@ module Ci
     end
 
     # rubocop: disable CodeReuse/ActiveRecord
-    def reprocess!(build)
+    def clone!(build)
+      # Cloning a build requires a strict type check to ensure
+      # the attributes being used for the clone are taken straight
+      # from the model and not overridden by other abstractions.
+      raise TypeError unless build.instance_of?(Ci::Build)
+
       check_access!(build)
 
       new_build = clone_build(build)
diff --git a/app/services/ci/retry_pipeline_service.rb b/app/services/ci/retry_pipeline_service.rb
index 02ee40d2cf6..9ad46ca7585 100644
--- a/app/services/ci/retry_pipeline_service.rb
+++ b/app/services/ci/retry_pipeline_service.rb
@@ -9,20 +9,15 @@ module Ci
         raise Gitlab::Access::AccessDeniedError
       end
 
-      needs = Set.new
-
       pipeline.ensure_scheduling_type!
 
       builds_relation(pipeline).find_each do |build|
         next unless can_be_retried?(build)
 
-        Ci::RetryBuildService.new(project, current_user)
-          .reprocess!(build)
-
-        needs += build.needs.map(&:name)
+        Ci::RetryBuildService.new(project, current_user).clone!(build)
       end
 
-      pipeline.builds.latest.skipped.find_each do |skipped|
+      pipeline.processables.latest.skipped.find_each do |skipped|
         retry_optimistic_lock(skipped, name: 'ci_retry_pipeline') { |build| build.process(current_user) }
       end
 
diff --git a/app/services/ci/stuck_builds/drop_service.rb b/app/services/ci/stuck_builds/drop_pending_service.rb
index 3fee9a94381..4653e701973 100644
--- a/app/services/ci/stuck_builds/drop_service.rb
+++ b/app/services/ci/stuck_builds/drop_pending_service.rb
@@ -2,27 +2,21 @@
 
 module Ci
   module StuckBuilds
-    class DropService
+    class DropPendingService
       include DropHelpers
 
-      BUILD_RUNNING_OUTDATED_TIMEOUT = 1.hour
       BUILD_PENDING_OUTDATED_TIMEOUT = 1.day
-      BUILD_SCHEDULED_OUTDATED_TIMEOUT = 1.hour
       BUILD_PENDING_STUCK_TIMEOUT = 1.hour
       BUILD_LOOKBACK = 5.days
 
       def execute
-        Gitlab::AppLogger.info "#{self.class}: Cleaning stuck builds"
-
-        drop(running_timed_out_builds, failure_reason: :stuck_or_timeout_failure)
+        Gitlab::AppLogger.info "#{self.class}: Cleaning pending timed-out builds"
 
         drop(
           pending_builds(BUILD_PENDING_OUTDATED_TIMEOUT.ago),
           failure_reason: :stuck_or_timeout_failure
         )
 
-        drop(scheduled_timed_out_builds, failure_reason: :stale_schedule)
-
         drop_stuck(
           pending_builds(BUILD_PENDING_STUCK_TIMEOUT.ago),
           failure_reason: :stuck_or_timeout_failure
@@ -43,20 +37,6 @@ module Ci
         end
       end
       # rubocop: enable CodeReuse/ActiveRecord
-
-      def scheduled_timed_out_builds
-        Ci::Build.where(status: :scheduled).where( # rubocop: disable CodeReuse/ActiveRecord
-          'ci_builds.scheduled_at IS NOT NULL AND ci_builds.scheduled_at < ?',
-          BUILD_SCHEDULED_OUTDATED_TIMEOUT.ago
-        )
-      end
-
-      def running_timed_out_builds
-        Ci::Build.running.where( # rubocop: disable CodeReuse/ActiveRecord
-          'ci_builds.updated_at < ?',
-          BUILD_RUNNING_OUTDATED_TIMEOUT.ago
-        )
-      end
     end
   end
 end
diff --git a/app/services/ci/stuck_builds/drop_running_service.rb b/app/services/ci/stuck_builds/drop_running_service.rb
new file mode 100644
index 00000000000..a79224cc231
--- /dev/null
+++ b/app/services/ci/stuck_builds/drop_running_service.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Ci
+  module StuckBuilds
+    class DropRunningService
+      include DropHelpers
+
+      BUILD_RUNNING_OUTDATED_TIMEOUT = 1.hour
+
+      def execute
+        Gitlab::AppLogger.info "#{self.class}: Cleaning running, timed-out builds"
+
+        drop(running_timed_out_builds, failure_reason: :stuck_or_timeout_failure)
+      end
+
+      private
+
+      def running_timed_out_builds
+        if Feature.enabled?(:ci_new_query_for_running_stuck_jobs, default_enabled: :yaml)
+          Ci::Build
+            .running
+            .created_at_before(BUILD_RUNNING_OUTDATED_TIMEOUT.ago)
+            .updated_at_before(BUILD_RUNNING_OUTDATED_TIMEOUT.ago)
+            .order(created_at: :asc, project_id: :asc) # rubocop:disable CodeReuse/ActiveRecord
+        else
+          Ci::Build.running.updated_at_before(BUILD_RUNNING_OUTDATED_TIMEOUT.ago)
+        end
+      end
+    end
+  end
+end
diff --git a/app/services/ci/stuck_builds/drop_scheduled_service.rb b/app/services/ci/stuck_builds/drop_scheduled_service.rb
new file mode 100644
index 00000000000..d4f4252c2c0
--- /dev/null
+++ b/app/services/ci/stuck_builds/drop_scheduled_service.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Ci
+  module StuckBuilds
+    class DropScheduledService
+      include DropHelpers
+
+      BUILD_SCHEDULED_OUTDATED_TIMEOUT = 1.hour
+
+      def execute
+        Gitlab::AppLogger.info "#{self.class}: Cleaning scheduled, timed-out builds"
+
+        drop(scheduled_timed_out_builds, failure_reason: :stale_schedule)
+      end
+
+      private
+
+      def scheduled_timed_out_builds
+        Ci::Build.scheduled.scheduled_at_before(BUILD_SCHEDULED_OUTDATED_TIMEOUT.ago)
+      end
+    end
+  end
+end
diff --git a/app/services/ci/update_build_state_service.rb b/app/services/ci/update_build_state_service.rb
index abd50d2f110..3b403f92486 100644
--- a/app/services/ci/update_build_state_service.rb
+++ b/app/services/ci/update_build_state_service.rb
@@ -73,9 +73,11 @@ module Ci
       ::Gitlab::Ci::Trace::Checksum.new(build).then do |checksum|
         unless checksum.valid?
           metrics.increment_trace_operation(operation: :invalid)
+          metrics.increment_error_counter(type: :chunks_invalid_checksum)
 
           if checksum.corrupted?
             metrics.increment_trace_operation(operation: :corrupted)
+            metrics.increment_error_counter(type: :chunks_invalid_size)
           end
 
           next unless log_invalid_chunks?
diff --git a/app/services/ci/update_pending_build_service.rb b/app/services/ci/update_pending_build_service.rb
index dcba06e60bf..d546dbcfe3d 100644
--- a/app/services/ci/update_pending_build_service.rb
+++ b/app/services/ci/update_pending_build_service.rb
@@ -2,7 +2,7 @@
 
 module Ci
   class UpdatePendingBuildService
-    VALID_PARAMS = %i[instance_runners_enabled].freeze
+    VALID_PARAMS = %i[instance_runners_enabled namespace_id namespace_traversal_ids].freeze
 
     InvalidParamsError = Class.new(StandardError)
     InvalidModelError = Class.new(StandardError)