Add latest changes from gitlab-org/gitlab@13-6-stable-eev13.6.0-rc42

11 files changed, 200 insertions, 27 deletions

diff --git a/app/services/ci/append_build_trace_service.rb b/app/services/ci/append_build_trace_service.rb
new file mode 100644
index 00000000000..602f8c5030d
--- /dev/null
+++ b/app/services/ci/append_build_trace_service.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Ci
+  class AppendBuildTraceService
+    Result = Struct.new(:status, :stream_size, keyword_init: true)
+    TraceRangeError = Class.new(StandardError)
+
+    attr_reader :build, :params
+
+    def initialize(build, params)
+      @build = build
+      @params = params
+    end
+
+    def execute(body_data)
+      # TODO:
+      # it seems that `Content-Range` as formatted by runner is wrong,
+      # the `byte_end` should point to final byte, but it points byte+1
+      # that means that we have to calculate end of body,
+      # as we cannot use `content_length[1]`
+      # Issue: https://gitlab.com/gitlab-org/gitlab-runner/issues/3275
+
+      content_range = stream_range.split('-')
+      body_start = content_range[0].to_i
+      body_end = body_start + body_data.bytesize
+
+      stream_size = build.trace.append(body_data, body_start)
+
+      unless stream_size == body_end
+        log_range_error(stream_size, body_end)
+
+        return Result.new(status: 416, stream_size: stream_size)
+      end
+
+      Result.new(status: 202, stream_size: stream_size)
+    end
+
+    private
+
+    def stream_range
+      params.fetch(:content_range)
+    end
+
+    def log_range_error(stream_size, body_end)
+      extra = {
+        build_id: build.id,
+        body_end: body_end,
+        stream_size: stream_size,
+        stream_class: stream_size.class,
+        stream_range: stream_range
+      }
+
+      build.trace_chunks.last.try do |chunk|
+        extra.merge!(
+          chunk_index: chunk.chunk_index,
+          chunk_store: chunk.data_store,
+          chunks_count: build.trace_chunks.count
+        )
+      end
+
+      ::Gitlab::ErrorTracking
+        .log_exception(TraceRangeError.new, extra)
+    end
+  end
+end
diff --git a/app/services/ci/build_report_result_service.rb b/app/services/ci/build_report_result_service.rb
index 76ecf428f11..f138aa91236 100644
--- a/app/services/ci/build_report_result_service.rb
+++ b/app/services/ci/build_report_result_service.rb
@@ -39,8 +39,6 @@ module Ci
     end
 
     def track_test_cases(build, test_suite)
-      return if Feature.disabled?(:track_unique_test_cases_parsed, build.project)
-
       track_usage_event(EVENT_NAME, test_case_hashes(build, test_suite))
     end
 
diff --git a/app/services/ci/compare_reports_base_service.rb b/app/services/ci/compare_reports_base_service.rb
index 2e84f914db3..9aba3a50ec1 100644
--- a/app/services/ci/compare_reports_base_service.rb
+++ b/app/services/ci/compare_reports_base_service.rb
@@ -8,7 +8,9 @@ module Ci
   # issue: https://gitlab.com/gitlab-org/gitlab/issues/34224
   class CompareReportsBaseService < ::BaseService
     def execute(base_pipeline, head_pipeline)
-      comparer = build_comparer(base_pipeline, head_pipeline)
+      base_report = get_report(base_pipeline)
+      head_report = get_report(head_pipeline)
+      comparer = build_comparer(base_report, head_report)
 
       {
         status: :parsed,
@@ -31,8 +33,8 @@ module Ci
 
     protected
 
-    def build_comparer(base_pipeline, head_pipeline)
-      comparer_class.new(get_report(base_pipeline), get_report(head_pipeline))
+    def build_comparer(base_report, head_report)
+      comparer_class.new(base_report, head_report)
     end
 
     private
diff --git a/app/services/ci/compare_test_reports_service.rb b/app/services/ci/compare_test_reports_service.rb
index 382d5b8995f..ed85ca8274c 100644
--- a/app/services/ci/compare_test_reports_service.rb
+++ b/app/services/ci/compare_test_reports_service.rb
@@ -13,5 +13,19 @@ module Ci
     def get_report(pipeline)
       pipeline&.test_reports
     end
+
+    def build_comparer(base_report, head_report)
+      # We need to load the test failure history on the test comparer because we display
+      # this on the MR widget
+      super.tap do |test_reports_comparer|
+        ::Gitlab::Ci::Reports::TestFailureHistory.new(failed_test_cases(test_reports_comparer), project).load!
+      end
+    end
+
+    def failed_test_cases(test_reports_comparer)
+      test_reports_comparer.suite_comparers.flat_map do |suite_comparer|
+        suite_comparer.limited_tests.new_failures + suite_comparer.limited_tests.existing_failures
+      end
+    end
   end
 end
diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb
index e7ede98fea4..e3bab2de44e 100644
--- a/app/services/ci/create_pipeline_service.rb
+++ b/app/services/ci/create_pipeline_service.rb
@@ -14,6 +14,7 @@ module Ci
                 Gitlab::Ci::Pipeline::Chain::Config::Process,
                 Gitlab::Ci::Pipeline::Chain::RemoveUnwantedChatJobs,
                 Gitlab::Ci::Pipeline::Chain::Skip,
+                Gitlab::Ci::Pipeline::Chain::SeedBlock,
                 Gitlab::Ci::Pipeline::Chain::EvaluateWorkflowRules,
                 Gitlab::Ci::Pipeline::Chain::Seed,
                 Gitlab::Ci::Pipeline::Chain::Limit::Size,
diff --git a/app/services/ci/daily_build_group_report_result_service.rb b/app/services/ci/daily_build_group_report_result_service.rb
index c32fc27c274..bc966fb9634 100644
--- a/app/services/ci/daily_build_group_report_result_service.rb
+++ b/app/services/ci/daily_build_group_report_result_service.rb
@@ -13,7 +13,8 @@ module Ci
         project_id: pipeline.project_id,
         ref_path: pipeline.source_ref_path,
         date: pipeline.created_at.to_date,
-        last_pipeline_id: pipeline.id
+        last_pipeline_id: pipeline.id,
+        default_branch: pipeline.default_branch?
       }
 
       aggregate(pipeline.builds.with_coverage).map do |group_name, group|
diff --git a/app/services/ci/destroy_expired_job_artifacts_service.rb b/app/services/ci/destroy_expired_job_artifacts_service.rb
index 438b5c7496d..6e7caba8545 100644
--- a/app/services/ci/destroy_expired_job_artifacts_service.rb
+++ b/app/services/ci/destroy_expired_job_artifacts_service.rb
@@ -4,47 +4,90 @@ module Ci
   class DestroyExpiredJobArtifactsService
     include ::Gitlab::ExclusiveLeaseHelpers
     include ::Gitlab::LoopHelpers
+    include ::Gitlab::Utils::StrongMemoize
 
     BATCH_SIZE = 100
-    LOOP_TIMEOUT = 45.minutes
+    LOOP_TIMEOUT = 5.minutes
     LOOP_LIMIT = 1000
     EXCLUSIVE_LOCK_KEY = 'expired_job_artifacts:destroy:lock'
-    LOCK_TIMEOUT = 50.minutes
+    LOCK_TIMEOUT = 6.minutes
 
     ##
     # Destroy expired job artifacts on GitLab instance
     #
-    # This destroy process cannot run for more than 45 minutes. This is for
+    # This destroy process cannot run for more than 6 minutes. This is for
     # preventing multiple `ExpireBuildArtifactsWorker` CRON jobs run concurrently,
-    # which is scheduled at every hour.
+    # which is scheduled every 7 minutes.
     def execute
       in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do
         loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do
-          destroy_batch(Ci::JobArtifact) || destroy_batch(Ci::PipelineArtifact)
+          destroy_artifacts_batch
         end
       end
     end
 
     private
 
-    def destroy_batch(klass)
-      artifact_batch = if klass == Ci::JobArtifact
-                         klass.expired(BATCH_SIZE).unlocked
-                       else
-                         klass.expired(BATCH_SIZE)
-                       end
+    def destroy_artifacts_batch
+      destroy_job_artifacts_batch || destroy_pipeline_artifacts_batch
+    end
+
+    def destroy_job_artifacts_batch
+      artifacts = Ci::JobArtifact
+        .expired(BATCH_SIZE)
+        .unlocked
+        .with_destroy_preloads
+        .to_a
+
+      return false if artifacts.empty?
 
-      artifacts = artifact_batch.to_a
+      parallel_destroy_batch(artifacts)
+      true
+    end
 
+    # TODO: Make sure this can also be parallelized
+    # https://gitlab.com/gitlab-org/gitlab/-/issues/270973
+    def destroy_pipeline_artifacts_batch
+      artifacts = Ci::PipelineArtifact.expired(BATCH_SIZE).to_a
       return false if artifacts.empty?
 
       artifacts.each(&:destroy!)
-      run_after_destroy(artifacts)
 
-      true # This is required because of the design of `loop_until` method.
+      true
+    end
+
+    def parallel_destroy_batch(job_artifacts)
+      Ci::DeletedObject.transaction do
+        Ci::DeletedObject.bulk_import(job_artifacts)
+        Ci::JobArtifact.id_in(job_artifacts.map(&:id)).delete_all
+        destroy_related_records_for(job_artifacts)
+      end
+
+      # This is executed outside of the transaction because it depends on Redis
+      update_statistics_for(job_artifacts)
+      destroyed_artifacts_counter.increment({}, job_artifacts.size)
+    end
+
+    # This method is implemented in EE and it must do only database work
+    def destroy_related_records_for(job_artifacts); end
+
+    def update_statistics_for(job_artifacts)
+      artifacts_by_project = job_artifacts.group_by(&:project)
+      artifacts_by_project.each do |project, artifacts|
+        delta = -artifacts.sum { |artifact| artifact.size.to_i }
+        ProjectStatistics.increment_statistic(
+          project, Ci::JobArtifact.project_statistics_name, delta)
+      end
     end
 
-    def run_after_destroy(artifacts); end
+    def destroyed_artifacts_counter
+      strong_memoize(:destroyed_artifacts_counter) do
+        name = :destroyed_job_artifacts_count_total
+        comment = 'Counter of destroyed expired job artifacts'
+
+        ::Gitlab::Metrics.counter(name, comment)
+      end
+    end
   end
 end
 
diff --git a/app/services/ci/list_config_variables_service.rb b/app/services/ci/list_config_variables_service.rb
index b5dc192b512..4a5b3a92a2c 100644
--- a/app/services/ci/list_config_variables_service.rb
+++ b/app/services/ci/list_config_variables_service.rb
@@ -6,7 +6,10 @@ module Ci
       config = project.ci_config_for(sha)
       return {} unless config
 
-      result = Gitlab::Ci::YamlProcessor.new(config).execute
+      result = Gitlab::Ci::YamlProcessor.new(config, project: project,
+                                                     user:    current_user,
+                                                     sha:     sha).execute
+
       result.valid? ? result.variables_with_data : {}
     end
   end
diff --git a/app/services/ci/parse_dotenv_artifact_service.rb b/app/services/ci/parse_dotenv_artifact_service.rb
index 71b306864b2..2ee9be476bb 100644
--- a/app/services/ci/parse_dotenv_artifact_service.rb
+++ b/app/services/ci/parse_dotenv_artifact_service.rb
@@ -3,7 +3,7 @@
 module Ci
   class ParseDotenvArtifactService < ::BaseService
     MAX_ACCEPTABLE_DOTENV_SIZE = 5.kilobytes
-    MAX_ACCEPTABLE_VARIABLES_COUNT = 10
+    MAX_ACCEPTABLE_VARIABLES_COUNT = 20
 
     SizeLimitError = Class.new(StandardError)
     ParserError = Class.new(StandardError)
diff --git a/app/services/ci/test_cases_service.rb b/app/services/ci/test_cases_service.rb
new file mode 100644
index 00000000000..3139b567571
--- /dev/null
+++ b/app/services/ci/test_cases_service.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Ci
+  class TestCasesService
+    MAX_TRACKABLE_FAILURES = 200
+
+    def execute(build)
+      return unless Feature.enabled?(:test_failure_history, build.project)
+      return unless build.has_test_reports?
+      return unless build.project.default_branch_or_master == build.ref
+
+      test_suite = generate_test_suite_report(build)
+
+      track_failures(build, test_suite)
+    end
+
+    private
+
+    def generate_test_suite_report(build)
+      build.collect_test_reports!(Gitlab::Ci::Reports::TestReports.new)
+    end
+
+    def track_failures(build, test_suite)
+      return if test_suite.failed_count > MAX_TRACKABLE_FAILURES
+
+      test_suite.failed.keys.each_slice(100) do |keys|
+        Ci::TestCase.transaction do
+          test_cases = Ci::TestCase.find_or_create_by_batch(build.project, keys)
+          Ci::TestCaseFailure.insert_all(test_case_failures(test_cases, build))
+        end
+      end
+    end
+
+    def test_case_failures(test_cases, build)
+      test_cases.map do |test_case|
+        {
+          test_case_id: test_case.id,
+          build_id: build.id,
+          failed_at: build.finished_at
+        }
+      end
+    end
+  end
+end
diff --git a/app/services/ci/update_build_state_service.rb b/app/services/ci/update_build_state_service.rb
index 22a27906700..fb67b0d2355 100644
--- a/app/services/ci/update_build_state_service.rb
+++ b/app/services/ci/update_build_state_service.rb
@@ -163,16 +163,18 @@ module Ci
     end
 
     def ensure_pending_state
-      Ci::BuildPendingState.create_or_find_by!(
+      build_state = Ci::BuildPendingState.safe_find_or_create_by(
         build_id: build.id,
         state: params.fetch(:state),
         trace_checksum: params.fetch(:checksum),
         failure_reason: params.dig(:failure_reason)
       )
-    rescue ActiveRecord::RecordNotFound
-      metrics.increment_trace_operation(operation: :conflict)
 
-      build.pending_state
+      unless build_state.present?
+        metrics.increment_trace_operation(operation: :conflict)
+      end
+
+      build_state || build.pending_state
     end
 
     ##