Add latest changes from gitlab-org/gitlab@15-6-stable-eev15.6.0-rc42

18 files changed, 130 insertions, 119 deletions

diff --git a/app/services/ci/after_requeue_job_service.rb b/app/services/ci/after_requeue_job_service.rb
index 9d54207d75d..4374ccd52e0 100644
--- a/app/services/ci/after_requeue_job_service.rb
+++ b/app/services/ci/after_requeue_job_service.rb
@@ -23,8 +23,6 @@ module Ci
 
     # rubocop: disable CodeReuse/ActiveRecord
     def dependent_jobs
-      return legacy_dependent_jobs unless ::Feature.enabled?(:ci_requeue_with_dag_object_hierarchy, project)
-
       ordered_by_dag(
         @processable.pipeline.processables
           .from_union(needs_dependent_jobs, stage_dependent_jobs)
@@ -50,24 +48,6 @@ module Ci
       ).descendants
     end
 
-    def legacy_skipped_jobs
-      @legacy_skipped_jobs ||= @processable.pipeline.processables.skipped
-    end
-
-    def legacy_dependent_jobs
-      ordered_by_dag(
-        legacy_stage_dependent_jobs.or(legacy_needs_dependent_jobs).ordered_by_stage.preload(:needs)
-      )
-    end
-
-    def legacy_stage_dependent_jobs
-      legacy_skipped_jobs.after_stage(@processable.stage_idx)
-    end
-
-    def legacy_needs_dependent_jobs
-      legacy_skipped_jobs.scheduling_type_dag.with_needs([@processable.name])
-    end
-
     def ordered_by_dag(jobs)
       sorted_job_names = sort_jobs(jobs).each_with_index.to_h
 
diff --git a/app/services/ci/archive_trace_service.rb b/app/services/ci/archive_trace_service.rb
index 566346a4b09..3d548c824c8 100644
--- a/app/services/ci/archive_trace_service.rb
+++ b/app/services/ci/archive_trace_service.rb
@@ -68,7 +68,7 @@ module Ci
       Gitlab::ErrorTracking
         .track_and_raise_for_dev_exception(error,
                           issue_url: 'https://gitlab.com/gitlab-org/gitlab-foss/issues/51502',
-                          job_id: job.id )
+                          job_id: job.id)
     end
   end
 end
diff --git a/app/services/ci/build_erase_service.rb b/app/services/ci/build_erase_service.rb
index 8a468e094eb..71b4c5481b3 100644
--- a/app/services/ci/build_erase_service.rb
+++ b/app/services/ci/build_erase_service.rb
@@ -33,9 +33,7 @@ module Ci
     attr_reader :build, :current_user
 
     def destroy_artifacts
-      # fix_expire_at is false because in this case we want to explicitly delete the job artifacts
-      # this flag is a workaround that will be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/355833
-      Ci::JobArtifacts::DestroyBatchService.new(build.job_artifacts, fix_expire_at: false).execute
+      Ci::JobArtifacts::DestroyBatchService.new(build.job_artifacts).execute
     end
 
     def erase_trace!
diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb
index 0b49beffcb5..4106dfe0ecc 100644
--- a/app/services/ci/create_pipeline_service.rb
+++ b/app/services/ci/create_pipeline_service.rb
@@ -30,6 +30,7 @@ module Ci
                 Gitlab::Ci::Pipeline::Chain::Limit::Deployments,
                 Gitlab::Ci::Pipeline::Chain::Validate::External,
                 Gitlab::Ci::Pipeline::Chain::Populate,
+                Gitlab::Ci::Pipeline::Chain::PopulateMetadata,
                 Gitlab::Ci::Pipeline::Chain::StopDryRun,
                 Gitlab::Ci::Pipeline::Chain::EnsureEnvironments,
                 Gitlab::Ci::Pipeline::Chain::EnsureResourceGroups,
@@ -118,17 +119,6 @@ module Ci
     end
     # rubocop: enable Metrics/ParameterLists
 
-    def execute!(*args, &block)
-      source = args[0]
-      params = Hash(args[1])
-
-      execute(source, **params, &block).tap do |response|
-        unless response.payload.persisted?
-          raise CreateError, pipeline.full_error_messages
-        end
-      end
-    end
-
     private
 
     def commit
diff --git a/app/services/ci/job_artifacts/destroy_associations_service.rb b/app/services/ci/job_artifacts/destroy_associations_service.rb
index 08d7f7f6f02..794d24eadf2 100644
--- a/app/services/ci/job_artifacts/destroy_associations_service.rb
+++ b/app/services/ci/job_artifacts/destroy_associations_service.rb
@@ -12,7 +12,7 @@ module Ci
 
       def destroy_records
         @job_artifacts_relation.each_batch(of: BATCH_SIZE) do |relation|
-          service = Ci::JobArtifacts::DestroyBatchService.new(relation, pick_up_at: Time.current, fix_expire_at: false)
+          service = Ci::JobArtifacts::DestroyBatchService.new(relation, pick_up_at: Time.current)
           result  = service.execute(update_stats: false)
           updates = result[:statistics_updates]
 
diff --git a/app/services/ci/job_artifacts/destroy_batch_service.rb b/app/services/ci/job_artifacts/destroy_batch_service.rb
index 54ec2c671c6..e0307d9bd53 100644
--- a/app/services/ci/job_artifacts/destroy_batch_service.rb
+++ b/app/services/ci/job_artifacts/destroy_batch_service.rb
@@ -17,10 +17,9 @@ module Ci
       # +pick_up_at+:: When to pick up for deletion of files
       # Returns:
       # +Hash+:: A hash with status and destroyed_artifacts_count keys
-      def initialize(job_artifacts, pick_up_at: nil, fix_expire_at: fix_expire_at?, skip_projects_on_refresh: false)
+      def initialize(job_artifacts, pick_up_at: nil, skip_projects_on_refresh: false)
         @job_artifacts = job_artifacts.with_destroy_preloads.to_a
         @pick_up_at = pick_up_at
-        @fix_expire_at = fix_expire_at
         @skip_projects_on_refresh = skip_projects_on_refresh
       end
 
@@ -32,9 +31,7 @@ module Ci
           track_artifacts_undergoing_stats_refresh
         end
 
-        # Detect and fix artifacts that had `expire_at` wrongly backfilled by migration
-        # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47723
-        detect_and_fix_wrongly_expired_artifacts
+        exclude_trace_artifacts
 
         return success(destroyed_artifacts_count: 0, statistics_updates: {}) if @job_artifacts.empty?
 
@@ -113,55 +110,9 @@ module Ci
         end
       end
 
-      # This detects and fixes job artifacts that have `expire_at` wrongly backfilled by the migration
-      # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47723.
-      # These job artifacts will not be deleted and will have their `expire_at` removed.
-      #
-      # The migration would have backfilled `expire_at`
-      # to midnight on the 22nd of the month of the local timezone,
-      # storing it as UTC time in the database.
-      #
-      # If the timezone setting has changed since the migration,
-      # the `expire_at` stored in the database could have changed to a different local time other than midnight.
-      # For example:
-      # - changing timezone from UTC+02:00 to UTC+02:30 would change the `expire_at` in local time 00:00:00 to 00:30:00.
-      # - changing timezone from UTC+00:00 to UTC-01:00 would change the `expire_at` in local time 00:00:00 to 23:00:00 on the previous day (21st).
-      #
-      # Therefore job artifacts that have `expire_at` exactly on the 00, 30 or 45 minute mark
-      # on the dates 21, 22, 23 of the month will not be deleted.
-      # https://en.wikipedia.org/wiki/List_of_UTC_time_offsets
-      def detect_and_fix_wrongly_expired_artifacts
-        return unless @fix_expire_at
-
-        wrongly_expired_artifacts, @job_artifacts = @job_artifacts.partition { |artifact| wrongly_expired?(artifact) }
-
-        remove_expire_at(wrongly_expired_artifacts) if wrongly_expired_artifacts.any?
-      end
-
-      def fix_expire_at?
-        Feature.enabled?(:ci_detect_wrongly_expired_artifacts)
-      end
-
-      def wrongly_expired?(artifact)
-        return false unless artifact.expire_at.present?
-
-        # Although traces should never have expiration dates that don't match time & date here.
-        # we can explicitly exclude them by type since they should never be destroyed.
-        artifact.trace? || (match_date?(artifact.expire_at) && match_time?(artifact.expire_at))
-      end
-
-      def match_date?(expire_at)
-        [21, 22, 23].include?(expire_at.day)
-      end
-
-      def match_time?(expire_at)
-        %w[00:00.000 30:00.000 45:00.000].include?(expire_at.strftime('%M:%S.%L'))
-      end
-
-      def remove_expire_at(artifacts)
-        Ci::JobArtifact.id_in(artifacts).update_all(expire_at: nil)
-
-        Gitlab::AppLogger.info(message: "Fixed expire_at from artifacts.", fixed_artifacts_expire_at_count: artifacts.count)
+      # Traces should never be destroyed.
+      def exclude_trace_artifacts
+        _trace_artifacts, @job_artifacts = @job_artifacts.partition(&:trace?)
       end
 
       def track_artifacts_undergoing_stats_refresh
diff --git a/app/services/ci/job_artifacts/track_artifact_report_service.rb b/app/services/ci/job_artifacts/track_artifact_report_service.rb
index 1be1d98394f..0230a5e19ce 100644
--- a/app/services/ci/job_artifacts/track_artifact_report_service.rb
+++ b/app/services/ci/job_artifacts/track_artifact_report_service.rb
@@ -5,7 +5,7 @@ module Ci
     class TrackArtifactReportService
       include Gitlab::Utils::UsageData
 
-      REPORT_TRACKED = %i[test].freeze
+      REPORT_TRACKED = %i[test coverage].freeze
 
       def execute(pipeline)
         REPORT_TRACKED.each do |report|
diff --git a/app/services/ci/list_config_variables_service.rb b/app/services/ci/list_config_variables_service.rb
index 3890882b3d4..df4963d1b33 100644
--- a/app/services/ci/list_config_variables_service.rb
+++ b/app/services/ci/list_config_variables_service.rb
@@ -30,7 +30,7 @@ module Ci
                                                              user: current_user,
                                                              sha: sha).execute
 
-      result.valid? ? result.variables_with_data : {}
+      result.valid? ? result.root_variables_with_prefill_data : {}
     end
 
     # Required for ReactiveCaching, it is also used in `reactive_cache_worker_finder`
diff --git a/app/services/ci/pipeline_artifacts/coverage_report_service.rb b/app/services/ci/pipeline_artifacts/coverage_report_service.rb
index 9c6fdb7a405..de66a4cb045 100644
--- a/app/services/ci/pipeline_artifacts/coverage_report_service.rb
+++ b/app/services/ci/pipeline_artifacts/coverage_report_service.rb
@@ -39,7 +39,7 @@ module Ci
       def carrierwave_file
         strong_memoize(:carrier_wave_file) do
           CarrierWaveStringFile.new_file(
-            file_content: report.to_json,
+            file_content: Gitlab::Json.dump(report),
             filename: Ci::PipelineArtifact::DEFAULT_FILE_NAMES.fetch(:code_coverage),
             content_type: 'application/json'
           )
diff --git a/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb b/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb
index a0746ef32b2..57b663dc293 100644
--- a/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb
+++ b/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb
@@ -77,9 +77,9 @@ module Ci
       end
 
       def build_quality_mr_diff_report(mr_diff_report)
-        mr_diff_report.each_with_object({}) do |diff_report, hash|
+        Gitlab::Json.dump(mr_diff_report.each_with_object({}) do |diff_report, hash|
           hash[diff_report.first] = Ci::CodequalityMrDiffReportSerializer.new.represent(diff_report.second) # rubocop: disable CodeReuse/Serializer
-        end.to_json
+        end)
       end
     end
   end
diff --git a/app/services/ci/pipeline_schedules/take_ownership_service.rb b/app/services/ci/pipeline_schedules/take_ownership_service.rb
new file mode 100644
index 00000000000..9b4001c74bd
--- /dev/null
+++ b/app/services/ci/pipeline_schedules/take_ownership_service.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Ci
+  module PipelineSchedules
+    class TakeOwnershipService
+      def initialize(schedule, user)
+        @schedule = schedule
+        @user = user
+      end
+
+      def execute
+        return forbidden unless allowed?
+
+        if schedule.update(owner: user)
+          ServiceResponse.success(payload: schedule)
+        else
+          ServiceResponse.error(message: schedule.errors.full_messages)
+        end
+      end
+
+      private
+
+      attr_reader :schedule, :user
+
+      def allowed?
+        user.can?(:take_ownership_pipeline_schedule, schedule)
+      end
+
+      def forbidden
+        ServiceResponse.error(message: _('Failed to change the owner'), reason: :access_denied)
+      end
+    end
+  end
+end
diff --git a/app/services/ci/pipeline_trigger_service.rb b/app/services/ci/pipeline_trigger_service.rb
index 39ac9bf33e9..d7065680053 100644
--- a/app/services/ci/pipeline_trigger_service.rb
+++ b/app/services/ci/pipeline_trigger_service.rb
@@ -93,7 +93,7 @@ module Ci
 
     def payload_variable
       { key: PAYLOAD_VARIABLE_KEY,
-        value: params.except(*PAYLOAD_VARIABLE_HIDDEN_PARAMS).to_json,
+        value: Gitlab::Json.dump(params.except(*PAYLOAD_VARIABLE_HIDDEN_PARAMS)),
         variable_type: :file }
     end
 
diff --git a/app/services/ci/play_build_service.rb b/app/services/ci/play_build_service.rb
index fbf2aad1991..b7aec57f3e3 100644
--- a/app/services/ci/play_build_service.rb
+++ b/app/services/ci/play_build_service.rb
@@ -5,21 +5,27 @@ module Ci
     def execute(build, job_variables_attributes = nil)
       check_access!(build, job_variables_attributes)
 
-      # Try to enqueue the build, otherwise create a duplicate.
-      #
-      if build.enqueue
-        build.tap do |build|
-          build.update!(user: current_user, job_variables_attributes: job_variables_attributes || [])
-
-          AfterRequeueJobService.new(project, current_user).execute(build)
-        end
+      if build.can_enqueue?
+        build.user = current_user
+        build.job_variables_attributes = job_variables_attributes || []
+        build.enqueue!
+
+        AfterRequeueJobService.new(project, current_user).execute(build)
+
+        build
       else
-        Ci::RetryJobService.new(project, current_user).execute(build)[:job]
+        retry_build(build)
       end
+    rescue StateMachines::InvalidTransition
+      retry_build(build.reset)
     end
 
     private
 
+    def retry_build(build)
+      Ci::RetryJobService.new(project, current_user).execute(build)[:job]
+    end
+
     def check_access!(build, job_variables_attributes)
       raise Gitlab::Access::AccessDeniedError unless can?(current_user, :play_job, build)
 
diff --git a/app/services/ci/process_build_service.rb b/app/services/ci/process_build_service.rb
index 22cd267806d..cb51d918fc2 100644
--- a/app/services/ci/process_build_service.rb
+++ b/app/services/ci/process_build_service.rb
@@ -15,6 +15,8 @@ module Ci
     private
 
     def process(build)
+      return enqueue(build) if Feature.enabled?(:ci_retry_job_fix, project) && build.enqueue_immediately?
+
       if build.schedulable?
         build.schedule
       elsif build.action?
@@ -25,7 +27,7 @@ module Ci
     end
 
     def enqueue(build)
-      return build.drop!(:failed_outdated_deployment_job) if build.prevent_rollback_deployment?
+      return build.drop!(:failed_outdated_deployment_job) if build.outdated_deployment?
 
       build.enqueue
     end
diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb
index 0bd4bf8cc86..f11577feb88 100644
--- a/app/services/ci/register_job_service.rb
+++ b/app/services/ci/register_job_service.rb
@@ -42,7 +42,7 @@ module Ci
       if !db_all_caught_up && !result.build
         metrics.increment_queue_operation(:queue_replication_lag)
 
-        ::Ci::RegisterJobService::Result.new(nil, false) # rubocop:disable Cop/AvoidReturnFromBlocks
+        ::Ci::RegisterJobService::Result.new(nil, nil, false) # rubocop:disable Cop/AvoidReturnFromBlocks
       else
         result
       end
@@ -226,7 +226,7 @@ module Ci
       log_artifacts_context(build)
       log_build_dependencies_size(presented_build)
 
-      build_json = ::API::Entities::Ci::JobRequest::Response.new(presented_build).to_json
+      build_json = Gitlab::Json.dump(::API::Entities::Ci::JobRequest::Response.new(presented_build))
       Result.new(build, build_json, true)
     end
 
diff --git a/app/services/ci/retry_job_service.rb b/app/services/ci/retry_job_service.rb
index 25bda8a6380..74ebaef48b1 100644
--- a/app/services/ci/retry_job_service.rb
+++ b/app/services/ci/retry_job_service.rb
@@ -19,7 +19,7 @@ module Ci
     end
 
     # rubocop: disable CodeReuse/ActiveRecord
-    def clone!(job, variables: [])
+    def clone!(job, variables: [], enqueue_if_actionable: false)
       # Cloning a job requires a strict type check to ensure
       # the attributes being used for the clone are taken straight
       # from the model and not overridden by other abstractions.
@@ -28,6 +28,9 @@ module Ci
       check_access!(job)
 
       new_job = job.clone(current_user: current_user, new_job_variables_attributes: variables)
+      if Feature.enabled?(:ci_retry_job_fix, project) && enqueue_if_actionable && new_job.action?
+        new_job.set_enqueue_immediately!
+      end
 
       new_job.run_after_commit do
         ::Ci::CopyCrossDatabaseAssociationsService.new.execute(job, new_job)
@@ -56,13 +59,20 @@ module Ci
     def check_assignable_runners!(job); end
 
     def retry_job(job, variables: [])
-      clone!(job, variables: variables).tap do |new_job|
+      clone!(job, variables: variables, enqueue_if_actionable: true).tap do |new_job|
         check_assignable_runners!(new_job) if new_job.is_a?(Ci::Build)
 
         next if new_job.failed?
 
-        Gitlab::OptimisticLocking.retry_lock(new_job, name: 'retry_build', &:enqueue)
+        Gitlab::OptimisticLocking.retry_lock(new_job, name: 'retry_build', &:enqueue) if Feature.disabled?(
+          :ci_retry_job_fix, project)
+
         AfterRequeueJobService.new(project, current_user).execute(job)
+
+        if Feature.enabled?(:ci_retry_job_fix, project)
+          Ci::PipelineCreation::StartPipelineService.new(job.pipeline).execute
+          new_job.reset
+        end
       end
     end
 
diff --git a/app/services/ci/runners/bulk_delete_runners_service.rb b/app/services/ci/runners/bulk_delete_runners_service.rb
index ce07aa541c2..b6b07746e61 100644
--- a/app/services/ci/runners/bulk_delete_runners_service.rb
+++ b/app/services/ci/runners/bulk_delete_runners_service.rb
@@ -7,29 +7,69 @@ module Ci
 
       RUNNER_LIMIT = 50
 
-      # @param runners [Array<Ci::Runner, Integer>] the runners to unregister/destroy
-      def initialize(runners:)
+      # @param runners [Array<Ci::Runner>] the runners to unregister/destroy
+      # @param current_user [User] the user performing the operation
+      def initialize(runners:, current_user:)
         @runners = runners
+        @current_user = current_user
       end
 
       def execute
         if @runners
           # Delete a few runners immediately
-          return ServiceResponse.success(payload: delete_runners)
+          return delete_runners
         end
 
-        ServiceResponse.success(payload: { deleted_count: 0, deleted_ids: [] })
+        ServiceResponse.success(payload: { deleted_count: 0, deleted_ids: [], errors: [] })
       end
 
       private
 
       def delete_runners
+        runner_count = @runners.limit(RUNNER_LIMIT + 1).count
+        authorized_runners_ids, unauthorized_runners_ids = compute_authorized_runners
         # rubocop:disable CodeReuse/ActiveRecord
-        runners_to_be_deleted = Ci::Runner.where(id: @runners).limit(RUNNER_LIMIT)
+        runners_to_be_deleted =
+          Ci::Runner
+            .where(id: authorized_runners_ids)
+            .preload([:taggings, :runner_namespaces, :runner_projects])
         # rubocop:enable CodeReuse/ActiveRecord
-        deleted_ids = runners_to_be_deleted.destroy_all.map(&:id) # rubocop: disable Cop/DestroyAll
+        deleted_ids = runners_to_be_deleted.destroy_all.map(&:id) # rubocop:disable Cop/DestroyAll
 
-        { deleted_count: deleted_ids.count, deleted_ids: deleted_ids }
+        ServiceResponse.success(
+          payload: {
+            deleted_count: deleted_ids.count,
+            deleted_ids: deleted_ids,
+            errors: error_messages(runner_count, authorized_runners_ids, unauthorized_runners_ids)
+          })
+      end
+
+      def compute_authorized_runners
+        # rubocop:disable CodeReuse/ActiveRecord
+        @current_user.ci_owned_runners.load # preload the owned runners to avoid an N+1
+        authorized_runners, unauthorized_runners =
+          @runners.limit(RUNNER_LIMIT)
+                  .partition { |runner| Ability.allowed?(@current_user, :delete_runner, runner) }
+        # rubocop:enable CodeReuse/ActiveRecord
+
+        [authorized_runners.map(&:id), unauthorized_runners.map(&:id)]
+      end
+
+      def error_messages(runner_count, authorized_runners_ids, unauthorized_runners_ids)
+        errors = []
+
+        if runner_count > RUNNER_LIMIT
+          errors << "Can only delete up to #{RUNNER_LIMIT} runners per call. Ignored the remaining runner(s)."
+        end
+
+        if authorized_runners_ids.empty?
+          errors << "User does not have permission to delete any of the runners"
+        elsif unauthorized_runners_ids.any?
+          failed_ids = unauthorized_runners_ids.map { |runner_id| "##{runner_id}" }.join(', ')
+          errors << "User does not have permission to delete runner(s) #{failed_ids}"
+        end
+
+        errors
       end
     end
   end
diff --git a/app/services/ci/runners/set_runner_associated_projects_service.rb b/app/services/ci/runners/set_runner_associated_projects_service.rb
index 7930776749d..5e33fdae2f4 100644
--- a/app/services/ci/runners/set_runner_associated_projects_service.rb
+++ b/app/services/ci/runners/set_runner_associated_projects_service.rb
@@ -17,7 +17,7 @@ module Ci
           return ServiceResponse.error(message: 'user not allowed to assign runner', http_status: :forbidden)
         end
 
-        return ServiceResponse.success if project_ids.blank?
+        return ServiceResponse.success if project_ids.nil?
 
         set_associated_projects
       end