Add latest changes from gitlab-org/gitlab@13-5-stable-eev13.5.0-rc42

18 files changed, 260 insertions, 76 deletions

diff --git a/app/services/ci/archive_trace_service.rb b/app/services/ci/archive_trace_service.rb
index 073ef465e13..9b2c7788897 100644
--- a/app/services/ci/archive_trace_service.rb
+++ b/app/services/ci/archive_trace_service.rb
@@ -27,7 +27,7 @@ module Ci
     rescue => e
       # Tracks this error with application logs, Sentry, and Prometheus.
       # If `archive!` keeps failing for over a week, that could incur data loss.
-      # (See more https://docs.gitlab.com/ee/administration/job_traces.html#new-live-trace-architecture)
+      # (See more https://docs.gitlab.com/ee/administration/job_logs.html#new-incremental-logging-architecture)
       # In order to avoid interrupting the system, we do not raise an exception here.
       archive_error(e, job, worker_name)
     end
diff --git a/app/services/ci/build_report_result_service.rb b/app/services/ci/build_report_result_service.rb
index ca66ad8249d..76ecf428f11 100644
--- a/app/services/ci/build_report_result_service.rb
+++ b/app/services/ci/build_report_result_service.rb
@@ -2,12 +2,20 @@
 
 module Ci
   class BuildReportResultService
+    include Gitlab::Utils::UsageData
+
+    EVENT_NAME = 'i_testing_test_case_parsed'
+
     def execute(build)
       return unless build.has_test_reports?
 
+      test_suite = generate_test_suite_report(build)
+
+      track_test_cases(build, test_suite)
+
       build.report_results.create!(
         project_id: build.project_id,
-        data: tests_params(build)
+        data: tests_params(test_suite)
       )
     end
 
@@ -17,9 +25,7 @@ module Ci
       build.collect_test_reports!(Gitlab::Ci::Reports::TestReports.new)
     end
 
-    def tests_params(build)
-      test_suite = generate_test_suite_report(build)
-
+    def tests_params(test_suite)
       {
         tests: {
           name: test_suite.name,
@@ -31,5 +37,20 @@ module Ci
         }
       }
     end
+
+    def track_test_cases(build, test_suite)
+      return if Feature.disabled?(:track_unique_test_cases_parsed, build.project)
+
+      track_usage_event(EVENT_NAME, test_case_hashes(build, test_suite))
+    end
+
+    def test_case_hashes(build, test_suite)
+      [].tap do |hashes|
+        test_suite.each_test_case do |test_case|
+          key = "#{build.project_id}-#{test_case.key}"
+          hashes << Digest::SHA256.hexdigest(key)
+        end
+      end
+    end
   end
 end
diff --git a/app/services/ci/create_downstream_pipeline_service.rb b/app/services/ci/create_downstream_pipeline_service.rb
index 0394cfb6119..86d0cf079fc 100644
--- a/app/services/ci/create_downstream_pipeline_service.rb
+++ b/app/services/ci/create_downstream_pipeline_service.rb
@@ -33,7 +33,7 @@ module Ci
         pipeline_params.fetch(:target_revision))
 
       downstream_pipeline = service.execute(
-        pipeline_params.fetch(:source), pipeline_params[:execute_params]) do |pipeline|
+        pipeline_params.fetch(:source), **pipeline_params[:execute_params]) do |pipeline|
           pipeline.variables.build(@bridge.downstream_variables)
         end
 
@@ -77,16 +77,9 @@ module Ci
 
       # TODO: Remove this condition if favour of model validation
       # https://gitlab.com/gitlab-org/gitlab/issues/38338
-      if ::Gitlab::Ci::Features.child_of_child_pipeline_enabled?(project)
-        if has_max_descendants_depth?
-          @bridge.drop!(:reached_max_descendant_pipelines_depth)
-          return false
-        end
-      else
-        if @bridge.triggers_child_pipeline? && @bridge.pipeline.parent_pipeline.present?
-          @bridge.drop!(:bridge_pipeline_is_child_pipeline)
-          return false
-        end
+      if has_max_descendants_depth?
+        @bridge.drop!(:reached_max_descendant_pipelines_depth)
+        return false
       end
 
       unless can_create_downstream_pipeline?(target_ref)
diff --git a/app/services/ci/create_job_artifacts_service.rb b/app/services/ci/create_job_artifacts_service.rb
index 1fe65898d55..5efb3805bf7 100644
--- a/app/services/ci/create_job_artifacts_service.rb
+++ b/app/services/ci/create_job_artifacts_service.rb
@@ -52,24 +52,15 @@ module Ci
     attr_reader :job, :project
 
     def validate_requirements(artifact_type:, filesize:)
-      return forbidden_type_error(artifact_type) if forbidden_type?(artifact_type)
       return too_large_error if too_large?(artifact_type, filesize)
 
       success
     end
 
-    def forbidden_type?(type)
-      lsif?(type) && !code_navigation_enabled?
-    end
-
     def too_large?(type, size)
       size > max_size(type) if size
     end
 
-    def code_navigation_enabled?
-      Feature.enabled?(:code_navigation, project, default_enabled: true)
-    end
-
     def lsif?(type)
       type == LSIF_ARTIFACT_TYPE
     end
diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb
index 70ad18e80eb..e7ede98fea4 100644
--- a/app/services/ci/create_pipeline_service.rb
+++ b/app/services/ci/create_pipeline_service.rb
@@ -70,7 +70,7 @@ module Ci
         push_options: params[:push_options] || {},
         chat_data: params[:chat_data],
         bridge: bridge,
-        **extra_options(options))
+        **extra_options(**options))
 
       # Ensure we never persist the pipeline when dry_run: true
       @pipeline.readonly! if command.dry_run?
@@ -82,8 +82,7 @@ module Ci
       schedule_head_pipeline_update if pipeline.persisted?
 
       # If pipeline is not persisted, try to recover IID
-      pipeline.reset_project_iid unless pipeline.persisted? ||
-          Feature.disabled?(:ci_pipeline_rewind_iid, project, default_enabled: true)
+      pipeline.reset_project_iid unless pipeline.persisted?
 
       pipeline
     end
diff --git a/app/services/ci/daily_build_group_report_result_service.rb b/app/services/ci/daily_build_group_report_result_service.rb
index 6cdf3c88f8c..c32fc27c274 100644
--- a/app/services/ci/daily_build_group_report_result_service.rb
+++ b/app/services/ci/daily_build_group_report_result_service.rb
@@ -3,8 +3,6 @@
 module Ci
   class DailyBuildGroupReportResultService
     def execute(pipeline)
-      return unless Feature.enabled?(:ci_daily_code_coverage, pipeline.project, default_enabled: true)
-
       DailyBuildGroupReportResult.upsert_reports(coverage_reports(pipeline))
     end
 
diff --git a/app/services/ci/delete_objects_service.rb b/app/services/ci/delete_objects_service.rb
new file mode 100644
index 00000000000..bac99abadc9
--- /dev/null
+++ b/app/services/ci/delete_objects_service.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Ci
+  class DeleteObjectsService
+    TransactionInProgressError = Class.new(StandardError)
+    TRANSACTION_MESSAGE = "can't perform network calls inside a database transaction"
+    BATCH_SIZE = 100
+    RETRY_IN = 10.minutes
+
+    def execute
+      objects = load_next_batch
+      destroy_everything(objects)
+    end
+
+    def remaining_batches_count(max_batch_count:)
+      Ci::DeletedObject
+        .ready_for_destruction(max_batch_count * BATCH_SIZE)
+        .size
+        .fdiv(BATCH_SIZE)
+        .ceil
+    end
+
+    private
+
+    # rubocop: disable CodeReuse/ActiveRecord
+    def load_next_batch
+      # `find_by_sql` performs a write in this case and we need to wrap it in
+      # a transaction to stick to the primary database.
+      Ci::DeletedObject.transaction do
+        Ci::DeletedObject.find_by_sql([
+          next_batch_sql, new_pick_up_at: RETRY_IN.from_now
+        ])
+      end
+    end
+    # rubocop: enable CodeReuse/ActiveRecord
+
+    def next_batch_sql
+      <<~SQL.squish
+      UPDATE "ci_deleted_objects"
+        SET "pick_up_at" = :new_pick_up_at
+        WHERE "ci_deleted_objects"."id" IN (#{locked_object_ids_sql})
+        RETURNING *
+      SQL
+    end
+
+    def locked_object_ids_sql
+      Ci::DeletedObject.lock_for_destruction(BATCH_SIZE).to_sql
+    end
+
+    def destroy_everything(objects)
+      raise TransactionInProgressError, TRANSACTION_MESSAGE if transaction_open?
+      return unless objects.any?
+
+      deleted = objects.select(&:delete_file_from_storage)
+      Ci::DeletedObject.id_in(deleted.map(&:id)).delete_all
+    end
+
+    def transaction_open?
+      Ci::DeletedObject.connection.transaction_open?
+    end
+  end
+end
diff --git a/app/services/ci/destroy_expired_job_artifacts_service.rb b/app/services/ci/destroy_expired_job_artifacts_service.rb
index ca6e60f819a..438b5c7496d 100644
--- a/app/services/ci/destroy_expired_job_artifacts_service.rb
+++ b/app/services/ci/destroy_expired_job_artifacts_service.rb
@@ -39,6 +39,13 @@ module Ci
       return false if artifacts.empty?
 
       artifacts.each(&:destroy!)
+      run_after_destroy(artifacts)
+
+      true # This is required because of the design of `loop_until` method.
     end
+
+    def run_after_destroy(artifacts); end
   end
 end
+
+Ci::DestroyExpiredJobArtifactsService.prepend_if_ee('EE::Ci::DestroyExpiredJobArtifactsService')
diff --git a/app/services/ci/expire_pipeline_cache_service.rb b/app/services/ci/expire_pipeline_cache_service.rb
index 32abd1a7626..8343e0f8cd0 100644
--- a/app/services/ci/expire_pipeline_cache_service.rb
+++ b/app/services/ci/expire_pipeline_cache_service.rb
@@ -32,11 +32,18 @@ module Ci
       Gitlab::Routing.url_helpers.project_new_merge_request_path(project, format: :json)
     end
 
+    def pipelines_project_merge_request_path(merge_request)
+      Gitlab::Routing.url_helpers.pipelines_project_merge_request_path(merge_request.target_project, merge_request, format: :json)
+    end
+
+    def merge_request_widget_path(merge_request)
+      Gitlab::Routing.url_helpers.cached_widget_project_json_merge_request_path(merge_request.project, merge_request, format: :json)
+    end
+
     def each_pipelines_merge_request_path(pipeline)
       pipeline.all_merge_requests.each do |merge_request|
-        path = Gitlab::Routing.url_helpers.pipelines_project_merge_request_path(merge_request.target_project, merge_request, format: :json)
-
-        yield(path)
+        yield(pipelines_project_merge_request_path(merge_request))
+        yield(merge_request_widget_path(merge_request))
       end
     end
 
diff --git a/app/services/ci/list_config_variables_service.rb b/app/services/ci/list_config_variables_service.rb
new file mode 100644
index 00000000000..b5dc192b512
--- /dev/null
+++ b/app/services/ci/list_config_variables_service.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Ci
+  class ListConfigVariablesService < ::BaseService
+    def execute(sha)
+      config = project.ci_config_for(sha)
+      return {} unless config
+
+      result = Gitlab::Ci::YamlProcessor.new(config).execute
+      result.valid? ? result.variables_with_data : {}
+    end
+  end
+end
diff --git a/app/services/ci/pipelines/create_artifact_service.rb b/app/services/ci/pipelines/create_artifact_service.rb
index b7d334e436d..bfaf317241a 100644
--- a/app/services/ci/pipelines/create_artifact_service.rb
+++ b/app/services/ci/pipelines/create_artifact_service.rb
@@ -3,7 +3,6 @@ module Ci
   module Pipelines
     class CreateArtifactService
       def execute(pipeline)
-        return unless ::Gitlab::Ci::Features.coverage_report_view?(pipeline.project)
         return unless pipeline.can_generate_coverage_reports?
         return if pipeline.has_coverage_reports?
 
diff --git a/app/services/ci/play_bridge_service.rb b/app/services/ci/play_bridge_service.rb
new file mode 100644
index 00000000000..70c4a8e6136
--- /dev/null
+++ b/app/services/ci/play_bridge_service.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Ci
+  class PlayBridgeService < ::BaseService
+    def execute(bridge)
+      raise Gitlab::Access::AccessDeniedError unless can?(current_user, :play_job, bridge)
+
+      bridge.tap do |bridge|
+        bridge.user = current_user
+        bridge.enqueue!
+      end
+    end
+  end
+end
diff --git a/app/services/ci/play_build_service.rb b/app/services/ci/play_build_service.rb
index 9f922ffde81..6adeca624a8 100644
--- a/app/services/ci/play_build_service.rb
+++ b/app/services/ci/play_build_service.rb
@@ -3,9 +3,7 @@
 module Ci
   class PlayBuildService < ::BaseService
     def execute(build, job_variables_attributes = nil)
-      unless can?(current_user, :update_build, build)
-        raise Gitlab::Access::AccessDeniedError
-      end
+      raise Gitlab::Access::AccessDeniedError unless can?(current_user, :play_job, build)
 
       # Try to enqueue the build, otherwise create a duplicate.
       #
diff --git a/app/services/ci/play_manual_stage_service.rb b/app/services/ci/play_manual_stage_service.rb
index 2497fc52e6b..c6fa7803e52 100644
--- a/app/services/ci/play_manual_stage_service.rb
+++ b/app/services/ci/play_manual_stage_service.rb
@@ -9,12 +9,12 @@ module Ci
     end
 
     def execute(stage)
-      stage.builds.manual.each do |build|
-        next unless build.playable?
+      stage.processables.manual.each do |processable|
+        next unless processable.playable?
 
-        build.play(current_user)
+        processable.play(current_user)
       rescue Gitlab::Access::AccessDeniedError
-        logger.error(message: 'Unable to play manual action', build_id: build.id)
+        logger.error(message: 'Unable to play manual action', processable_id: processable.id)
       end
     end
 
diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb
index 18bae26613f..e511e26adfe 100644
--- a/app/services/ci/process_pipeline_service.rb
+++ b/app/services/ci/process_pipeline_service.rb
@@ -31,14 +31,14 @@ module Ci
     # rubocop: disable CodeReuse/ActiveRecord
     def update_retried
       # find the latest builds for each name
-      latest_statuses = pipeline.statuses.latest
+      latest_statuses = pipeline.latest_statuses
         .group(:name)
         .having('count(*) > 1')
         .pluck(Arel.sql('MAX(id)'), 'name')
 
       # mark builds that are retried
       if latest_statuses.any?
-        pipeline.statuses.latest
+        pipeline.latest_statuses
           .where(name: latest_statuses.map(&:second))
           .where.not(id: latest_statuses.map(&:first))
           .update_all(retried: true)
diff --git a/app/services/ci/retry_build_service.rb b/app/services/ci/retry_build_service.rb
index 6b2e6c245f3..f397ada0696 100644
--- a/app/services/ci/retry_build_service.rb
+++ b/app/services/ci/retry_build_service.rb
@@ -58,7 +58,7 @@ module Ci
       build = project.builds.new(attributes)
       build.assign_attributes(::Gitlab::Ci::Pipeline::Seed::Build.environment_attributes_for(build))
       build.retried = false
-      BulkInsertableAssociations.with_bulk_insert(enabled: ::Gitlab::Ci::Features.bulk_insert_on_create?(project)) do
+      BulkInsertableAssociations.with_bulk_insert do
         build.save!
       end
       build
diff --git a/app/services/ci/update_build_queue_service.rb b/app/services/ci/update_build_queue_service.rb
index 31c7178c9e7..241eba733ea 100644
--- a/app/services/ci/update_build_queue_service.rb
+++ b/app/services/ci/update_build_queue_service.rb
@@ -9,9 +9,7 @@ module Ci
     private
 
     def tick_for(build, runners)
-      if Feature.enabled?(:ci_update_queues_for_online_runners, build.project, default_enabled: true)
-        runners = runners.with_recent_runner_queue
-      end
+      runners = runners.with_recent_runner_queue
 
       runners.each do |runner|
         runner.pick_build!(build)
diff --git a/app/services/ci/update_build_state_service.rb b/app/services/ci/update_build_state_service.rb
index 61e4c77c1e5..22a27906700 100644
--- a/app/services/ci/update_build_state_service.rb
+++ b/app/services/ci/update_build_state_service.rb
@@ -2,7 +2,11 @@
 
 module Ci
   class UpdateBuildStateService
-    Result = Struct.new(:status, keyword_init: true)
+    include ::Gitlab::Utils::StrongMemoize
+    include ::Gitlab::ExclusiveLeaseHelpers
+
+    Result = Struct.new(:status, :backoff, keyword_init: true)
+    InvalidTraceError = Class.new(StandardError)
 
     ACCEPT_TIMEOUT = 5.minutes.freeze
 
@@ -17,43 +21,77 @@ module Ci
     def execute
       overwrite_trace! if has_trace?
 
-      if accept_request?
-        accept_build_state!
-      else
-        check_migration_state
-        update_build_state!
+      unless accept_available?
+        return update_build_state!
+      end
+
+      ensure_pending_state!
+
+      in_build_trace_lock do
+        process_build_state!
       end
     end
 
     private
 
-    def accept_build_state!
-      if Time.current - ensure_pending_state.created_at > ACCEPT_TIMEOUT
-        metrics.increment_trace_operation(operation: :discarded)
+    def overwrite_trace!
+      metrics.increment_trace_operation(operation: :overwrite)
 
-        return update_build_state!
+      build.trace.set(params[:trace]) if Gitlab::Ci::Features.trace_overwrite?
+    end
+
+    def ensure_pending_state!
+      pending_state.created_at
+    end
+
+    def process_build_state!
+      if live_chunks_pending?
+        if pending_state_outdated?
+          discard_build_trace!
+          update_build_state!
+        else
+          accept_build_state!
+        end
+      else
+        validate_build_trace!
+        update_build_state!
       end
+    end
 
+    def accept_build_state!
       build.trace_chunks.live.find_each do |chunk|
         chunk.schedule_to_persist!
       end
 
       metrics.increment_trace_operation(operation: :accepted)
 
-      Result.new(status: 202)
+      ::Gitlab::Ci::Runner::Backoff.new(pending_state.created_at).then do |backoff|
+        Result.new(status: 202, backoff: backoff.to_seconds)
+      end
     end
 
-    def overwrite_trace!
-      metrics.increment_trace_operation(operation: :overwrite)
+    def validate_build_trace!
+      return unless has_chunks?
 
-      build.trace.set(params[:trace]) if Gitlab::Ci::Features.trace_overwrite?
-    end
+      unless live_chunks_pending?
+        metrics.increment_trace_operation(operation: :finalized)
+        metrics.observe_migration_duration(pending_state_seconds)
+      end
 
-    def check_migration_state
-      return unless accept_available?
+      ::Gitlab::Ci::Trace::Checksum.new(build).then do |checksum|
+        unless checksum.valid?
+          metrics.increment_trace_operation(operation: :invalid)
 
-      if has_chunks? && !live_chunks_pending?
-        metrics.increment_trace_operation(operation: :finalized)
+          next unless log_invalid_chunks?
+
+          ::Gitlab::ErrorTracking.log_exception(InvalidTraceError.new,
+            project_path: build.project.full_path,
+            build_id: build.id,
+            state_crc32: checksum.state_crc32,
+            chunks_crc32: checksum.chunks_crc32,
+            chunks_count: checksum.chunks_count
+          )
+        end
       end
     end
 
@@ -76,12 +114,32 @@ module Ci
       end
     end
 
+    def discard_build_trace!
+      metrics.increment_trace_operation(operation: :discarded)
+    end
+
     def accept_available?
       !build_running? && has_checksum? && chunks_migration_enabled?
     end
 
-    def accept_request?
-      accept_available? && live_chunks_pending?
+    def live_chunks_pending?
+      build.trace_chunks.live.any?
+    end
+
+    def has_chunks?
+      build.trace_chunks.any?
+    end
+
+    def pending_state_outdated?
+      pending_state_duration > ACCEPT_TIMEOUT
+    end
+
+    def pending_state_duration
+      Time.current - pending_state.created_at
+    end
+
+    def pending_state_seconds
+      pending_state_duration.seconds
     end
 
     def build_state
@@ -96,18 +154,14 @@ module Ci
       params.dig(:checksum).present?
     end
 
-    def has_chunks?
-      build.trace_chunks.any?
-    end
-
-    def live_chunks_pending?
-      build.trace_chunks.live.any?
-    end
-
     def build_running?
       build_state == 'running'
     end
 
+    def pending_state
+      strong_memoize(:pending_state) { ensure_pending_state }
+    end
+
     def ensure_pending_state
       Ci::BuildPendingState.create_or_find_by!(
         build_id: build.id,
@@ -121,8 +175,38 @@ module Ci
       build.pending_state
     end
 
+    ##
+    # This method is releasing an exclusive lock on a build trace the moment we
+    # conclude that build status has been written and the build state update
+    # has been committed to the database.
+    #
+    # Because a build state machine schedules a bunch of workers to run after
+    # build status transition to complete, we do not want to keep the lease
+    # until all the workers are scheduled because it opens a possibility of
+    # race conditions happening.
+    #
+    # Instead of keeping the lease until the transition is fully done and
+    # workers are scheduled, we immediately release the lock after the database
+    # commit happens.
+    #
+    def in_build_trace_lock(&block)
+      build.trace.lock do |_, lease| # rubocop:disable CodeReuse/ActiveRecord
+        build.run_on_status_commit { lease.cancel }
+
+        yield
+      end
+    rescue ::Gitlab::Ci::Trace::LockedError
+      metrics.increment_trace_operation(operation: :locked)
+
+      accept_build_state!
+    end
+
     def chunks_migration_enabled?
       ::Gitlab::Ci::Features.accept_trace?(build.project)
     end
+
+    def log_invalid_chunks?
+      ::Gitlab::Ci::Features.log_invalid_trace_chunks?(build.project)
+    end
   end
 end