Add latest changes from gitlab-org/gitlab@13-8-stable-eev13.8.0-rc42

14 files changed, 134 insertions, 49 deletions

diff --git a/app/services/ci/create_downstream_pipeline_service.rb b/app/services/ci/create_downstream_pipeline_service.rb
index 86d0cf079fc..629d85b041f 100644
--- a/app/services/ci/create_downstream_pipeline_service.rb
+++ b/app/services/ci/create_downstream_pipeline_service.rb
@@ -33,9 +33,7 @@ module Ci
         pipeline_params.fetch(:target_revision))
 
       downstream_pipeline = service.execute(
-        pipeline_params.fetch(:source), **pipeline_params[:execute_params]) do |pipeline|
-          pipeline.variables.build(@bridge.downstream_variables)
-        end
+        pipeline_params.fetch(:source), **pipeline_params[:execute_params])
 
       downstream_pipeline.tap do |pipeline|
         update_bridge_status!(@bridge, pipeline)
diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb
index dbe81521cfc..d3001e54288 100644
--- a/app/services/ci/create_pipeline_service.rb
+++ b/app/services/ci/create_pipeline_service.rb
@@ -27,6 +27,7 @@ module Ci
                 Gitlab::Ci::Pipeline::Chain::Limit::JobActivity,
                 Gitlab::Ci::Pipeline::Chain::CancelPendingPipelines,
                 Gitlab::Ci::Pipeline::Chain::Metrics,
+                Gitlab::Ci::Pipeline::Chain::TemplateUsage,
                 Gitlab::Ci::Pipeline::Chain::Pipeline::Process].freeze
 
     # Create a new pipeline in the specified project.
@@ -81,7 +82,11 @@ module Ci
         .new(pipeline, command, SEQUENCE)
         .build!
 
-      schedule_head_pipeline_update if pipeline.persisted?
+      if pipeline.persisted?
+        schedule_head_pipeline_update
+        record_conversion_event
+        create_namespace_onboarding_action
+      end
 
       # If pipeline is not persisted, try to recover IID
       pipeline.reset_project_iid unless pipeline.persisted?
@@ -116,6 +121,15 @@ module Ci
       end
     end
 
+    def record_conversion_event
+      Experiments::RecordConversionEventWorker.perform_async(:ci_syntax_templates, current_user.id)
+      Experiments::RecordConversionEventWorker.perform_async(:pipelines_empty_state, current_user.id)
+    end
+
+    def create_namespace_onboarding_action
+      Namespaces::OnboardingPipelineCreatedWorker.perform_async(project.namespace_id)
+    end
+
     def extra_options(content: nil, dry_run: false)
       { content: content, dry_run: dry_run }
     end
diff --git a/app/services/ci/create_web_ide_terminal_service.rb b/app/services/ci/create_web_ide_terminal_service.rb
index a78281aed16..785d82094b9 100644
--- a/app/services/ci/create_web_ide_terminal_service.rb
+++ b/app/services/ci/create_web_ide_terminal_service.rb
@@ -6,7 +6,7 @@ module Ci
 
     TerminalCreationError = Class.new(StandardError)
 
-    TERMINAL_NAME = 'terminal'.freeze
+    TERMINAL_NAME = 'terminal'
 
     attr_reader :terminal
 
diff --git a/app/services/ci/destroy_expired_job_artifacts_service.rb b/app/services/ci/destroy_expired_job_artifacts_service.rb
index 6e7caba8545..7d8a3c17abe 100644
--- a/app/services/ci/destroy_expired_job_artifacts_service.rb
+++ b/app/services/ci/destroy_expired_job_artifacts_service.rb
@@ -12,6 +12,10 @@ module Ci
     EXCLUSIVE_LOCK_KEY = 'expired_job_artifacts:destroy:lock'
     LOCK_TIMEOUT = 6.minutes
 
+    def initialize
+      @removed_artifacts_count = 0
+    end
+
     ##
     # Destroy expired job artifacts on GitLab instance
     #
@@ -20,40 +24,22 @@ module Ci
     # which is scheduled every 7 minutes.
     def execute
       in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do
-        loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do
-          destroy_artifacts_batch
-        end
+        destroy_job_artifacts_with_slow_iteration(Time.current)
       end
-    end
-
-    private
 
-    def destroy_artifacts_batch
-      destroy_job_artifacts_batch || destroy_pipeline_artifacts_batch
+      @removed_artifacts_count
     end
 
-    def destroy_job_artifacts_batch
-      artifacts = Ci::JobArtifact
-        .expired(BATCH_SIZE)
-        .unlocked
-        .with_destroy_preloads
-        .to_a
-
-      return false if artifacts.empty?
-
-      parallel_destroy_batch(artifacts)
-      true
-    end
-
-    # TODO: Make sure this can also be parallelized
-    # https://gitlab.com/gitlab-org/gitlab/-/issues/270973
-    def destroy_pipeline_artifacts_batch
-      artifacts = Ci::PipelineArtifact.expired(BATCH_SIZE).to_a
-      return false if artifacts.empty?
+    private
 
-      artifacts.each(&:destroy!)
+    def destroy_job_artifacts_with_slow_iteration(start_at)
+      Ci::JobArtifact.expired_before(start_at).each_batch(of: BATCH_SIZE, column: :expire_at, order: :desc) do |relation, index|
+        artifacts = relation.unlocked.with_destroy_preloads.to_a
 
-      true
+        parallel_destroy_batch(artifacts) if artifacts.any?
+        break if loop_timeout?(start_at)
+        break if index >= LOOP_LIMIT
+      end
     end
 
     def parallel_destroy_batch(job_artifacts)
@@ -64,14 +50,14 @@ module Ci
       end
 
       # This is executed outside of the transaction because it depends on Redis
-      update_statistics_for(job_artifacts)
-      destroyed_artifacts_counter.increment({}, job_artifacts.size)
+      update_project_statistics_for(job_artifacts)
+      increment_monitoring_statistics(job_artifacts.size)
     end
 
     # This method is implemented in EE and it must do only database work
     def destroy_related_records_for(job_artifacts); end
 
-    def update_statistics_for(job_artifacts)
+    def update_project_statistics_for(job_artifacts)
       artifacts_by_project = job_artifacts.group_by(&:project)
       artifacts_by_project.each do |project, artifacts|
         delta = -artifacts.sum { |artifact| artifact.size.to_i }
@@ -80,6 +66,11 @@ module Ci
       end
     end
 
+    def increment_monitoring_statistics(size)
+      destroyed_artifacts_counter.increment({}, size)
+      @removed_artifacts_count += size
+    end
+
     def destroyed_artifacts_counter
       strong_memoize(:destroyed_artifacts_counter) do
         name = :destroyed_job_artifacts_count_total
@@ -88,6 +79,10 @@ module Ci
         ::Gitlab::Metrics.counter(name, comment)
       end
     end
+
+    def loop_timeout?(start_at)
+      Time.current > start_at + LOOP_TIMEOUT
+    end
   end
 end
 
diff --git a/app/services/ci/pipelines/create_artifact_service.rb b/app/services/ci/pipeline_artifacts/coverage_report_service.rb
index bfaf317241a..9f5c445c91a 100644
--- a/app/services/ci/pipelines/create_artifact_service.rb
+++ b/app/services/ci/pipeline_artifacts/coverage_report_service.rb
@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 module Ci
-  module Pipelines
-    class CreateArtifactService
+  module PipelineArtifacts
+    class CoverageReportService
       def execute(pipeline)
         return unless pipeline.can_generate_coverage_reports?
         return if pipeline.has_coverage_reports?
diff --git a/app/services/ci/pipeline_artifacts/destroy_expired_artifacts_service.rb b/app/services/ci/pipeline_artifacts/destroy_expired_artifacts_service.rb
new file mode 100644
index 00000000000..0dbabe178da
--- /dev/null
+++ b/app/services/ci/pipeline_artifacts/destroy_expired_artifacts_service.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Ci
+  module PipelineArtifacts
+    class DestroyExpiredArtifactsService
+      include ::Gitlab::LoopHelpers
+      include ::Gitlab::Utils::StrongMemoize
+
+      BATCH_SIZE = 100
+      LOOP_TIMEOUT = 5.minutes
+      LOOP_LIMIT = 1000
+
+      def initialize
+        @removed_artifacts_count = 0
+      end
+
+      def execute
+        loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do
+          destroy_artifacts_batch
+        end
+
+        @removed_artifacts_count
+      end
+
+      private
+
+      def destroy_artifacts_batch
+        artifacts = ::Ci::PipelineArtifact.expired(BATCH_SIZE).to_a
+        return false if artifacts.empty?
+
+        artifacts.each(&:destroy!)
+        increment_stats(artifacts.size)
+
+        true
+      end
+
+      def increment_stats(size)
+        destroyed_artifacts_counter.increment({}, size)
+        @removed_artifacts_count += size
+      end
+
+      def destroyed_artifacts_counter
+        strong_memoize(:destroyed_artifacts_counter) do
+          name = :destroyed_pipeline_artifacts_count_total
+          comment = 'Counter of destroyed expired pipeline artifacts'
+
+          ::Gitlab::Metrics.counter(name, comment)
+        end
+      end
+    end
+  end
+end
diff --git a/app/services/ci/pipeline_trigger_service.rb b/app/services/ci/pipeline_trigger_service.rb
index d9f41b7040e..a31f5e9056e 100644
--- a/app/services/ci/pipeline_trigger_service.rb
+++ b/app/services/ci/pipeline_trigger_service.rb
@@ -21,10 +21,10 @@ module Ci
       # this check is to not leak the presence of the project if user cannot read it
       return unless trigger.project == project
 
-      pipeline = Ci::CreatePipelineService.new(project, trigger.owner, ref: params[:ref])
+      pipeline = Ci::CreatePipelineService
+        .new(project, trigger.owner, ref: params[:ref], variables_attributes: variables)
         .execute(:trigger, ignore_skip_ci: true) do |pipeline|
           pipeline.trigger_requests.build(trigger: trigger)
-          pipeline.variables.build(variables)
         end
 
       if pipeline.persisted?
@@ -44,7 +44,8 @@ module Ci
       # this check is to not leak the presence of the project if user cannot read it
       return unless can?(job.user, :read_project, project)
 
-      pipeline = Ci::CreatePipelineService.new(project, job.user, ref: params[:ref])
+      pipeline = Ci::CreatePipelineService
+        .new(project, job.user, ref: params[:ref], variables_attributes: variables)
         .execute(:pipeline, ignore_skip_ci: true) do |pipeline|
           source = job.sourced_pipelines.build(
             source_pipeline: job.pipeline,
@@ -53,7 +54,6 @@ module Ci
             project: project)
 
           pipeline.source_pipeline = source
-          pipeline.variables.build(variables)
         end
 
       if pipeline.persisted?
diff --git a/app/services/ci/play_build_service.rb b/app/services/ci/play_build_service.rb
index 6adeca624a8..ebc980a9053 100644
--- a/app/services/ci/play_build_service.rb
+++ b/app/services/ci/play_build_service.rb
@@ -5,6 +5,10 @@ module Ci
     def execute(build, job_variables_attributes = nil)
       raise Gitlab::Access::AccessDeniedError unless can?(current_user, :play_job, build)
 
+      if job_variables_attributes.present? && !can?(current_user, :set_pipeline_variables, project)
+        raise Gitlab::Access::AccessDeniedError
+      end
+
       # Try to enqueue the build, otherwise create a duplicate.
       #
       if build.enqueue
diff --git a/app/services/ci/process_build_service.rb b/app/services/ci/process_build_service.rb
index 12cdca24066..dd7b562cdb7 100644
--- a/app/services/ci/process_build_service.rb
+++ b/app/services/ci/process_build_service.rb
@@ -26,6 +26,27 @@ module Ci
     end
 
     def valid_statuses_for_build(build)
+      if ::Feature.enabled?(:skip_dag_manual_and_delayed_jobs, default_enabled: :yaml)
+        current_valid_statuses_for_build(build)
+      else
+        legacy_valid_statuses_for_build(build)
+      end
+    end
+
+    def current_valid_statuses_for_build(build)
+      case build.when
+      when 'on_success', 'manual', 'delayed'
+        build.scheduling_type_dag? ? %w[success] : %w[success skipped]
+      when 'on_failure'
+        %w[failed]
+      when 'always'
+        %w[success failed skipped]
+      else
+        []
+      end
+    end
+
+    def legacy_valid_statuses_for_build(build)
       case build.when
       when 'on_success'
         build.scheduling_type_dag? ? %w[success] : %w[success skipped]
diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb
index 04d620d1d38..59691fe4ef3 100644
--- a/app/services/ci/register_job_service.rb
+++ b/app/services/ci/register_job_service.rb
@@ -8,8 +8,8 @@ module Ci
 
     JOB_QUEUE_DURATION_SECONDS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze
     JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET = 5.freeze
-    METRICS_SHARD_TAG_PREFIX = 'metrics_shard::'.freeze
-    DEFAULT_METRICS_SHARD = 'default'.freeze
+    METRICS_SHARD_TAG_PREFIX = 'metrics_shard::'
+    DEFAULT_METRICS_SHARD = 'default'
 
     Result = Struct.new(:build, :build_json, :valid?)
 
diff --git a/app/services/ci/retry_build_service.rb b/app/services/ci/retry_build_service.rb
index f397ada0696..e5e79f70616 100644
--- a/app/services/ci/retry_build_service.rb
+++ b/app/services/ci/retry_build_service.rb
@@ -2,6 +2,8 @@
 
 module Ci
   class RetryBuildService < ::BaseService
+    include Gitlab::OptimisticLocking
+
     def self.clone_accessors
       %i[pipeline project ref tag options name
          allow_failure stage stage_id stage_idx trigger_request
@@ -65,8 +67,8 @@ module Ci
     end
 
     def mark_subsequent_stages_as_processable(build)
-      build.pipeline.processables.skipped.after_stage(build.stage_idx).find_each do |processable|
-        Gitlab::OptimisticLocking.retry_lock(processable, &:process)
+      build.pipeline.processables.skipped.after_stage(build.stage_idx).find_each do |skipped|
+        retry_optimistic_lock(skipped) { |build| build.process(current_user) }
       end
     end
   end
diff --git a/app/services/ci/retry_pipeline_service.rb b/app/services/ci/retry_pipeline_service.rb
index 45244d16393..dea4bf73a4c 100644
--- a/app/services/ci/retry_pipeline_service.rb
+++ b/app/services/ci/retry_pipeline_service.rb
@@ -23,7 +23,7 @@ module Ci
       end
 
       pipeline.builds.latest.skipped.find_each do |skipped|
-        retry_optimistic_lock(skipped) { |build| build.process }
+        retry_optimistic_lock(skipped) { |build| build.process(current_user) }
       end
 
       pipeline.reset_ancestor_bridges!
diff --git a/app/services/ci/test_failure_history_service.rb b/app/services/ci/test_failure_history_service.rb
index 99a2592ec06..61fda79a4a2 100644
--- a/app/services/ci/test_failure_history_service.rb
+++ b/app/services/ci/test_failure_history_service.rb
@@ -30,7 +30,6 @@ module Ci
     end
 
     def should_track_failures?
-      return false unless Feature.enabled?(:test_failure_history, project)
       return false unless project.default_branch_or_master == pipeline.ref
 
       # We fetch for up to MAX_TRACKABLE_FAILURES + 1 builds. So if ever we get
diff --git a/app/services/ci/update_build_state_service.rb b/app/services/ci/update_build_state_service.rb
index f01d41d9414..874f4bf459a 100644
--- a/app/services/ci/update_build_state_service.rb
+++ b/app/services/ci/update_build_state_service.rb
@@ -111,7 +111,7 @@ module Ci
 
         Result.new(status: 200)
       when 'failed'
-        build.drop!(params[:failure_reason] || :unknown_failure)
+        build.drop_with_exit_code!(params[:failure_reason] || :unknown_failure, params[:exit_code])
 
         Result.new(status: 200)
       else