Add latest changes from gitlab-org/gitlab@14-1-stable-eev14.1.0-rc42

4 files changed, 101 insertions, 2 deletions

diff --git a/app/workers/ci/archive_trace_worker.rb b/app/workers/ci/archive_trace_worker.rb
new file mode 100644
index 00000000000..16288faf370
--- /dev/null
+++ b/app/workers/ci/archive_trace_worker.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Ci
+  class ArchiveTraceWorker # rubocop:disable Scalability/IdempotentWorker
+    include ApplicationWorker
+
+    sidekiq_options retry: 3
+    include PipelineBackgroundQueue
+
+    # rubocop: disable CodeReuse/ActiveRecord
+    def perform(job_id)
+      Ci::Build.without_archived_trace.find_by(id: job_id).try do |job|
+        Ci::ArchiveTraceService.new.execute(job, worker_name: self.class.name)
+      end
+    end
+    # rubocop: enable CodeReuse/ActiveRecord
+  end
+end
diff --git a/app/workers/ci/archive_traces_cron_worker.rb b/app/workers/ci/archive_traces_cron_worker.rb
index c748bc33ada..5fe3adf870f 100644
--- a/app/workers/ci/archive_traces_cron_worker.rb
+++ b/app/workers/ci/archive_traces_cron_worker.rb
@@ -12,7 +12,7 @@ module Ci
     # rubocop: disable CodeReuse/ActiveRecord
     def perform
       # Archive stale live traces which still resides in redis or database
-      # This could happen when ArchiveTraceWorker sidekiq jobs were lost by receiving SIGKILL
+      # This could happen when Ci::ArchiveTraceWorker sidekiq jobs were lost by receiving SIGKILL
       # More details in https://gitlab.com/gitlab-org/gitlab-foss/issues/36791
       Ci::Build.with_stale_live_trace.find_each(batch_size: 100) do |build|
         Ci::ArchiveTraceService.new.execute(build, worker_name: self.class.name)
diff --git a/app/workers/ci/build_finished_worker.rb b/app/workers/ci/build_finished_worker.rb
new file mode 100644
index 00000000000..1d6e3b1fa3c
--- /dev/null
+++ b/app/workers/ci/build_finished_worker.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Ci
+  class BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker
+    include ApplicationWorker
+
+    sidekiq_options retry: 3
+    include PipelineQueue
+
+    queue_namespace :pipeline_processing
+    urgency :high
+    worker_resource_boundary :cpu
+
+    ARCHIVE_TRACES_IN = 2.minutes.freeze
+
+    # rubocop: disable CodeReuse/ActiveRecord
+    def perform(build_id)
+      Ci::Build.find_by(id: build_id).try do |build|
+        process_build(build)
+      end
+    end
+    # rubocop: enable CodeReuse/ActiveRecord
+
+    private
+
+    # Processes a single CI build that has finished.
+    #
+    # This logic resides in a separate method so that EE can extend it more
+    # easily.
+    #
+    # @param [Ci::Build] build The build to process.
+    def process_build(build)
+      # We execute these in sync to reduce IO.
+      build.parse_trace_sections!
+      build.update_coverage
+      Ci::BuildReportResultService.new.execute(build)
+
+      # We execute these async as these are independent operations.
+      BuildHooksWorker.perform_async(build.id)
+      ChatNotificationWorker.perform_async(build.id) if build.pipeline.chat?
+
+      if build.failed?
+        ::Ci::MergeRequests::AddTodoWhenBuildFailsWorker.perform_async(build.id)
+      end
+
+      ##
+      # We want to delay sending a build trace to object storage operation to
+      # validate that this fixes a race condition between this and flushing live
+      # trace chunks and chunks being removed after consolidation and putting
+      # them into object storage archive.
+      #
+      # TODO This is temporary fix we should improve later, after we validate
+      # that this is indeed the culprit.
+      #
+      # See https://gitlab.com/gitlab-org/gitlab/-/issues/267112 for more
+      # details.
+      #
+      archive_trace_worker_class(build).perform_in(ARCHIVE_TRACES_IN, build.id)
+    end
+
+    def archive_trace_worker_class(build)
+      if Feature.enabled?(:ci_build_finished_worker_namespace_changed, build.project, default_enabled: :yaml)
+        Ci::ArchiveTraceWorker
+      else
+        ::ArchiveTraceWorker
+      end
+    end
+  end
+end
+
+Ci::BuildFinishedWorker.prepend_mod_with('Ci::BuildFinishedWorker')
diff --git a/app/workers/ci/resource_groups/assign_resource_from_resource_group_worker.rb b/app/workers/ci/resource_groups/assign_resource_from_resource_group_worker.rb
index 15ed89fd00e..ad0ed3d16f1 100644
--- a/app/workers/ci/resource_groups/assign_resource_from_resource_group_worker.rb
+++ b/app/workers/ci/resource_groups/assign_resource_from_resource_group_worker.rb
@@ -2,7 +2,10 @@
 
 module Ci
   module ResourceGroups
-    class AssignResourceFromResourceGroupWorker # rubocop:disable Scalability/IdempotentWorker
+    # This worker is to assign a resource to a pipeline job from a resource group
+    # and enqueue the job to be executed by a runner.
+    # See https://docs.gitlab.com/ee/ci/yaml/#resource_group for more information.
+    class AssignResourceFromResourceGroupWorker
       include ApplicationWorker
 
       sidekiq_options retry: 3
@@ -11,6 +14,13 @@ module Ci
       queue_namespace :pipeline_processing
       feature_category :continuous_delivery
 
+      # This worker is idempotent that it produces the same result
+      # as long as the same resource group id is passed as an argument.
+      # Therefore, we can deduplicate the sidekiq jobs until the on-going
+      # assignment process has been finished.
+      idempotent!
+      deduplicate :until_executed
+
       def perform(resource_group_id)
         ::Ci::ResourceGroup.find_by_id(resource_group_id).try do |resource_group|
           Ci::ResourceGroups::AssignResourceFromResourceGroupService.new(resource_group.project, nil)