Add latest changes from gitlab-org/gitlab@13-9-stable-eev13.9.0-rc42

12 files changed, 167 insertions, 11 deletions

diff --git a/app/services/ci/create_job_artifacts_service.rb b/app/services/ci/create_job_artifacts_service.rb
index 5efb3805bf7..f1fdc8e2490 100644
--- a/app/services/ci/create_job_artifacts_service.rb
+++ b/app/services/ci/create_job_artifacts_service.rb
@@ -7,6 +7,7 @@ module Ci
     ArtifactsExistError = Class.new(StandardError)
 
     LSIF_ARTIFACT_TYPE = 'lsif'
+    METRICS_REPORT_UPLOAD_EVENT_NAME = 'i_testing_metrics_report_artifact_uploaders'
 
     OBJECT_STORAGE_ERRORS = [
       Errno::EIO,
@@ -42,6 +43,8 @@ module Ci
       artifact, artifact_metadata = build_artifact(artifacts_file, params, metadata_file)
       result = parse_artifact(artifact)
 
+      track_artifact_uploader(artifact)
+
       return result unless result[:status] == :success
 
       persist_artifact(artifact, artifact_metadata, params)
@@ -152,6 +155,12 @@ module Ci
       )
     end
 
+    def track_artifact_uploader(artifact)
+      return unless artifact.file_type == 'metrics'
+
+      track_usage_event(METRICS_REPORT_UPLOAD_EVENT_NAME, @job.user_id)
+    end
+
     def parse_dotenv_artifact(artifact)
       Ci::ParseDotenvArtifactService.new(project, current_user).execute(artifact)
     end
diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb
index d3001e54288..dc42411dfa1 100644
--- a/app/services/ci/create_pipeline_service.rb
+++ b/app/services/ci/create_pipeline_service.rb
@@ -123,7 +123,6 @@ module Ci
 
     def record_conversion_event
       Experiments::RecordConversionEventWorker.perform_async(:ci_syntax_templates, current_user.id)
-      Experiments::RecordConversionEventWorker.perform_async(:pipelines_empty_state, current_user.id)
     end
 
     def create_namespace_onboarding_action
diff --git a/app/services/ci/daily_build_group_report_result_service.rb b/app/services/ci/daily_build_group_report_result_service.rb
index bc966fb9634..820e6e08fc5 100644
--- a/app/services/ci/daily_build_group_report_result_service.rb
+++ b/app/services/ci/daily_build_group_report_result_service.rb
@@ -14,7 +14,8 @@ module Ci
         ref_path: pipeline.source_ref_path,
         date: pipeline.created_at.to_date,
         last_pipeline_id: pipeline.id,
-        default_branch: pipeline.default_branch?
+        default_branch: pipeline.default_branch?,
+        group_id: pipeline.project&.group&.id
       }
 
       aggregate(pipeline.builds.with_coverage).map do |group_name, group|
diff --git a/app/services/ci/generate_codequality_mr_diff_report_service.rb b/app/services/ci/generate_codequality_mr_diff_report_service.rb
new file mode 100644
index 00000000000..3b1bd319a4f
--- /dev/null
+++ b/app/services/ci/generate_codequality_mr_diff_report_service.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Ci
+  # TODO: a couple of points with this approach:
+  # + reuses existing architecture and reactive caching
+  # - it's not a report comparison and some comparing features must be turned off.
+  # see CompareReportsBaseService for more notes.
+  # issue: https://gitlab.com/gitlab-org/gitlab/issues/34224
+  class GenerateCodequalityMrDiffReportService < CompareReportsBaseService
+    def execute(base_pipeline, head_pipeline)
+      merge_request = MergeRequest.find_by_id(params[:id])
+      {
+        status: :parsed,
+        key: key(base_pipeline, head_pipeline),
+        data: head_pipeline.pipeline_artifacts.find_by_file_type(:code_quality_mr_diff).present.for_files(merge_request.new_paths)
+      }
+    rescue => e
+      Gitlab::ErrorTracking.track_exception(e, project_id: project.id)
+      {
+        status: :error,
+        key: key(base_pipeline, head_pipeline),
+        status_reason: _('An error occurred while fetching codequality mr diff reports.')
+      }
+    end
+
+    def latest?(base_pipeline, head_pipeline, data)
+      data&.fetch(:key, nil) == key(base_pipeline, head_pipeline)
+    end
+  end
+end
diff --git a/app/services/ci/generate_coverage_reports_service.rb b/app/services/ci/generate_coverage_reports_service.rb
index 063fb966183..b3aa7b3091b 100644
--- a/app/services/ci/generate_coverage_reports_service.rb
+++ b/app/services/ci/generate_coverage_reports_service.rb
@@ -12,7 +12,7 @@ module Ci
       {
         status: :parsed,
         key: key(base_pipeline, head_pipeline),
-        data: head_pipeline.pipeline_artifacts.find_with_code_coverage.present.for_files(merge_request.new_paths)
+        data: head_pipeline.pipeline_artifacts.find_by_file_type(:code_coverage).present.for_files(merge_request.new_paths)
       }
     rescue => e
       Gitlab::ErrorTracking.track_exception(e, project_id: project.id)
diff --git a/app/services/ci/pipeline_artifacts/coverage_report_service.rb b/app/services/ci/pipeline_artifacts/coverage_report_service.rb
index 9f5c445c91a..8209639fa22 100644
--- a/app/services/ci/pipeline_artifacts/coverage_report_service.rb
+++ b/app/services/ci/pipeline_artifacts/coverage_report_service.rb
@@ -11,7 +11,7 @@ module Ci
         pipeline.pipeline_artifacts.create!(
           project_id: pipeline.project_id,
           file_type: :code_coverage,
-          file_format: :raw,
+          file_format: Ci::PipelineArtifact::REPORT_TYPES.fetch(:code_coverage),
           size: file["tempfile"].size,
           file: file,
           expire_at: Ci::PipelineArtifact::EXPIRATION_DATE.from_now
diff --git a/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb b/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb
new file mode 100644
index 00000000000..5c52eef7ba6
--- /dev/null
+++ b/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+module Ci
+  module PipelineArtifacts
+    class CreateCodeQualityMrDiffReportService
+      def execute(pipeline)
+        return unless pipeline.can_generate_codequality_reports?
+        return if pipeline.has_codequality_mr_diff_report?
+
+        file = build_carrierwave_file(pipeline)
+
+        pipeline.pipeline_artifacts.create!(
+          project_id: pipeline.project_id,
+          file_type: :code_quality_mr_diff,
+          file_format: Ci::PipelineArtifact::REPORT_TYPES.fetch(:code_quality_mr_diff),
+          size: file["tempfile"].size,
+          file: file,
+          expire_at: Ci::PipelineArtifact::EXPIRATION_DATE.from_now
+        )
+      end
+
+      private
+
+      def build_carrierwave_file(pipeline)
+        CarrierWaveStringFile.new_file(
+          file_content: build_quality_mr_diff_report(pipeline),
+          filename: Ci::PipelineArtifact::DEFAULT_FILE_NAMES.fetch(:code_quality_mr_diff),
+          content_type: 'application/json'
+        )
+      end
+
+      def build_quality_mr_diff_report(pipeline)
+        mr_diff_report = Gitlab::Ci::Reports::CodequalityMrDiff.new(pipeline.codequality_reports)
+
+        Ci::CodequalityMrDiffReportSerializer.new.represent(mr_diff_report).to_json # rubocop: disable CodeReuse/Serializer
+      end
+    end
+  end
+end
diff --git a/app/services/ci/pipeline_trigger_service.rb b/app/services/ci/pipeline_trigger_service.rb
index a31f5e9056e..dbbaefb2b2f 100644
--- a/app/services/ci/pipeline_trigger_service.rb
+++ b/app/services/ci/pipeline_trigger_service.rb
@@ -17,6 +17,9 @@ module Ci
 
     private
 
+    PAYLOAD_VARIABLE_KEY = 'TRIGGER_PAYLOAD'
+    PAYLOAD_VARIABLE_HIDDEN_PARAMS = %i(token).freeze
+
     def create_pipeline_from_trigger(trigger)
       # this check is to not leak the presence of the project if user cannot read it
       return unless trigger.project == project
@@ -70,9 +73,23 @@ module Ci
     end
 
     def variables
+      if ::Feature.enabled?(:ci_trigger_payload_into_pipeline, project, default_enabled: :yaml)
+        param_variables + [payload_variable]
+      else
+        param_variables
+      end
+    end
+
+    def param_variables
       params[:variables].to_h.map do |key, value|
         { key: key, value: value }
       end
     end
+
+    def payload_variable
+      { key: PAYLOAD_VARIABLE_KEY,
+        value: params.except(*PAYLOAD_VARIABLE_HIDDEN_PARAMS).to_json,
+        variable_type: :file }
+    end
   end
 end
diff --git a/app/services/ci/process_build_service.rb b/app/services/ci/process_build_service.rb
index dd7b562cdb7..733aa75f255 100644
--- a/app/services/ci/process_build_service.rb
+++ b/app/services/ci/process_build_service.rb
@@ -26,7 +26,7 @@ module Ci
     end
 
     def valid_statuses_for_build(build)
-      if ::Feature.enabled?(:skip_dag_manual_and_delayed_jobs, default_enabled: :yaml)
+      if ::Feature.enabled?(:skip_dag_manual_and_delayed_jobs, build.project, default_enabled: :yaml)
         current_valid_statuses_for_build(build)
       else
         legacy_valid_statuses_for_build(build)
diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb
index e511e26adfe..678b386fbbf 100644
--- a/app/services/ci/process_pipeline_service.rb
+++ b/app/services/ci/process_pipeline_service.rb
@@ -38,10 +38,15 @@ module Ci
 
       # mark builds that are retried
       if latest_statuses.any?
-        pipeline.latest_statuses
-          .where(name: latest_statuses.map(&:second))
-          .where.not(id: latest_statuses.map(&:first))
-          .update_all(retried: true)
+        updated_count = pipeline.latest_statuses
+                          .where(name: latest_statuses.map(&:second))
+                          .where.not(id: latest_statuses.map(&:first))
+                          .update_all(retried: true)
+
+        # This counter is temporary. It will be used to check whether if we still use this method or not
+        # after setting correct value of `GenericCommitStatus#retried`.
+        # More info: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50465#note_491657115
+        metrics.legacy_update_jobs_counter.increment if updated_count > 0
       end
     end
     # rubocop: enable CodeReuse/ActiveRecord
diff --git a/app/services/ci/prometheus_metrics/observe_histograms_service.rb b/app/services/ci/prometheus_metrics/observe_histograms_service.rb
new file mode 100644
index 00000000000..527d87f19c2
--- /dev/null
+++ b/app/services/ci/prometheus_metrics/observe_histograms_service.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Ci
+  module PrometheusMetrics
+    class ObserveHistogramsService
+      class << self
+        def available_histograms
+          @available_histograms ||= [
+            histogram(:pipeline_graph_link_calculation_duration_seconds, 'Total time spent calculating links, in seconds', {}, [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.8, 1, 2]),
+            histogram(:pipeline_graph_links_total, 'Number of links per graph', {}, [1, 5, 10, 25, 50, 100, 200]),
+            histogram(:pipeline_graph_links_per_job_ratio, 'Ratio of links to job per graph', {}, [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1])
+          ].to_h
+        end
+
+        private
+
+        def histogram(name, *attrs)
+          [name.to_s, proc { Gitlab::Metrics.histogram(name, *attrs) }]
+        end
+      end
+
+      def initialize(project, params)
+        @project = project
+        @params = params
+      end
+
+      def execute
+        return ServiceResponse.success(http_status: :accepted) unless enabled?
+
+        params
+          .fetch(:histograms, [])
+          .each(&method(:observe))
+
+        ServiceResponse.success(http_status: :created)
+      end
+
+      private
+
+      attr_reader :project, :params
+
+      def observe(data)
+        histogram = find_histogram(data[:name])
+        histogram.observe({}, data[:value].to_f)
+      end
+
+      def find_histogram(name)
+        self.class.available_histograms
+          .fetch(name) { raise ActiveRecord::RecordNotFound }
+          .call
+      end
+
+      def enabled?
+        ::Feature.enabled?(:ci_accept_frontend_prometheus_metrics, project, default_enabled: :yaml)
+      end
+    end
+  end
+end
diff --git a/app/services/ci/resource_groups/assign_resource_from_resource_group_service.rb b/app/services/ci/resource_groups/assign_resource_from_resource_group_service.rb
index a4bcca8e8b3..9e3e6de3928 100644
--- a/app/services/ci/resource_groups/assign_resource_from_resource_group_service.rb
+++ b/app/services/ci/resource_groups/assign_resource_from_resource_group_service.rb
@@ -7,8 +7,8 @@ module Ci
       def execute(resource_group)
         free_resources = resource_group.resources.free.count
 
-        resource_group.builds.waiting_for_resource.take(free_resources).each do |build|
-          build.enqueue_waiting_for_resource
+        resource_group.processables.waiting_for_resource.take(free_resources).each do |processable|
+          processable.enqueue_waiting_for_resource
         end
       end
       # rubocop: enable CodeReuse/ActiveRecord