Add latest changes from gitlab-org/gitlab@13-11-stable-eev13.11.0-rc43

16 files changed, 192 insertions, 110 deletions

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 824e35a6480..3d8e9f4c126 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -14,8 +14,6 @@ module Ci
 
     BuildArchivedError = Class.new(StandardError)
 
-    ignore_columns :artifacts_file, :artifacts_file_store, :artifacts_metadata, :artifacts_metadata_store, :artifacts_size, :commands, remove_after: '2019-12-15', remove_with: '12.7'
-
     belongs_to :project, inverse_of: :builds
     belongs_to :runner
     belongs_to :trigger_request
@@ -35,6 +33,7 @@ module Ci
     }.freeze
 
     DEGRADATION_THRESHOLD_VARIABLE_NAME = 'DEGRADATION_THRESHOLD'
+    RUNNERS_STATUS_CACHE_EXPIRATION = 1.minute
 
     has_one :deployment, as: :deployable, class_name: 'Deployment'
     has_one :pending_state, class_name: 'Ci::BuildPendingState', inverse_of: :build
@@ -75,7 +74,14 @@ module Ci
       return unless has_environment?
 
       strong_memoize(:persisted_environment) do
-        Environment.find_by(name: expanded_environment_name, project: project)
+        # This code path has caused N+1s in the past, since environments are only indirectly
+        # associated to builds and pipelines; see https://gitlab.com/gitlab-org/gitlab/-/issues/326445
+        # We therefore batch-load them to prevent dormant N+1s until we found a proper solution.
+        BatchLoader.for(expanded_environment_name).batch(key: project_id) do |names, loader, args|
+          Environment.where(name: names, project: args[:key]).find_each do |environment|
+            loader.call(environment.name, environment)
+          end
+        end
       end
     end
 
@@ -88,8 +94,7 @@ module Ci
     validates :ref, presence: true
 
     scope :not_interruptible, -> do
-      joins(:metadata).where('ci_builds_metadata.id NOT IN (?)',
-        Ci::BuildMetadata.scoped_build.with_interruptible.select(:id))
+      joins(:metadata).where.not('ci_builds_metadata.id' => Ci::BuildMetadata.scoped_build.with_interruptible.select(:id))
     end
 
     scope :unstarted, -> { where(runner_id: nil) }
@@ -319,7 +324,7 @@ module Ci
         end
       end
 
-      before_transition any => [:failed] do |build|
+      after_transition any => [:failed] do |build|
         next unless build.project
         next unless build.deployment
 
@@ -372,11 +377,11 @@ module Ci
     end
 
     def other_manual_actions
-      pipeline.manual_actions.where.not(name: name)
+      pipeline.manual_actions.reject { |action| action.name == self.name }
     end
 
     def other_scheduled_actions
-      pipeline.scheduled_actions.where.not(name: name)
+      pipeline.scheduled_actions.reject { |action| action.name == self.name }
     end
 
     def pages_generator?
@@ -698,7 +703,23 @@ module Ci
     end
 
     def any_runners_online?
-      project.any_active_runners? { |runner| runner.match_build_if_online?(self) }
+      if Feature.enabled?(:runners_cached_states, project, default_enabled: :yaml)
+        cache_for_online_runners do
+          project.any_online_runners? { |runner| runner.match_build_if_online?(self) }
+        end
+      else
+        project.any_active_runners? { |runner| runner.match_build_if_online?(self) }
+      end
+    end
+
+    def any_runners_available?
+      if Feature.enabled?(:runners_cached_states, project, default_enabled: :yaml)
+        cache_for_available_runners do
+          project.active_runners.exists?
+        end
+      else
+        project.any_active_runners?
+      end
     end
 
     def stuck?
@@ -1103,6 +1124,20 @@ module Ci
         .to_a
         .include?(exit_code)
     end
+
+    def cache_for_online_runners(&block)
+      Rails.cache.fetch(
+        ['has-online-runners', id],
+        expires_in: RUNNERS_STATUS_CACHE_EXPIRATION
+      ) { yield }
+    end
+
+    def cache_for_available_runners(&block)
+      Rails.cache.fetch(
+        ['has-available-runners', project.id],
+        expires_in: RUNNERS_STATUS_CACHE_EXPIRATION
+      ) { yield }
+    end
   end
 end
 
diff --git a/app/models/ci/build_dependencies.rb b/app/models/ci/build_dependencies.rb
index b50ecf99439..8ae921f1416 100644
--- a/app/models/ci/build_dependencies.rb
+++ b/app/models/ci/build_dependencies.rb
@@ -21,8 +21,7 @@ module Ci
       deps = model_class.where(pipeline_id: processable.pipeline_id).latest
       deps = from_previous_stages(deps)
       deps = from_needs(deps)
-      deps = from_dependencies(deps)
-      deps
+      from_dependencies(deps)
     end
 
     # Dependencies from the same parent-pipeline hierarchy excluding
diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb
index d4f9f78a1ac..7e03d709f24 100644
--- a/app/models/ci/build_trace_chunk.rb
+++ b/app/models/ci/build_trace_chunk.rb
@@ -30,9 +30,9 @@ module Ci
       fog: 3
     }.freeze
 
-    STORE_TYPES = DATA_STORES.keys.map do |store|
+    STORE_TYPES = DATA_STORES.keys.to_h do |store|
       [store, "Ci::BuildTraceChunks::#{store.capitalize}".constantize]
-    end.to_h.freeze
+    end.freeze
 
     enum data_store: DATA_STORES
 
diff --git a/app/models/ci/build_trace_chunks/redis.rb b/app/models/ci/build_trace_chunks/redis.rb
index 58d50b39c11..003ec107895 100644
--- a/app/models/ci/build_trace_chunks/redis.rb
+++ b/app/models/ci/build_trace_chunks/redis.rb
@@ -4,7 +4,7 @@ module Ci
   module BuildTraceChunks
     class Redis
       CHUNK_REDIS_TTL = 1.week
-      LUA_APPEND_CHUNK = <<~EOS.freeze
+      LUA_APPEND_CHUNK = <<~EOS
         local key, new_data, offset = KEYS[1], ARGV[1], ARGV[2]
         local length = new_data:len()
         local expire = #{CHUNK_REDIS_TTL.seconds}
diff --git a/app/models/ci/group.rb b/app/models/ci/group.rb
index 4ba09fd8152..47b91fcf2ce 100644
--- a/app/models/ci/group.rb
+++ b/app/models/ci/group.rb
@@ -22,6 +22,13 @@ module Ci
       @jobs = jobs
     end
 
+    def ==(other)
+      other.present? && other.is_a?(self.class) &&
+        project == other.project &&
+        stage == other.stage &&
+        name == other.name
+    end
+
     def status
       strong_memoize(:status) do
         status_struct.status
diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb
index d5e88f2be5b..50e21a1c323 100644
--- a/app/models/ci/job_artifact.rb
+++ b/app/models/ci/job_artifact.rb
@@ -131,8 +131,6 @@ module Ci
     update_project_statistics project_statistics_name: :build_artifacts_size
 
     scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) }
-    scope :with_files_stored_locally, -> { where(file_store: ::JobArtifactUploader::Store::LOCAL) }
-    scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) }
     scope :for_sha, ->(sha, project_id) { joins(job: :pipeline).where(ci_pipelines: { sha: sha, project_id: project_id }) }
     scope :for_job_name, ->(name) { joins(:job).where(ci_builds: { name: name }) }
 
@@ -292,8 +290,12 @@ module Ci
         end
     end
 
+    def archived_trace_exists?
+      file&.file&.exists?
+    end
+
     def self.archived_trace_exists_for?(job_id)
-      where(job_id: job_id).trace.take&.file&.file&.exists?
+      where(job_id: job_id).trace.take&.archived_trace_exists?
     end
 
     def self.max_artifact_size(type:, project:)
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index b63ec0c8a97..c9ab69317e1 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -286,9 +286,11 @@ module Ci
       end
 
       after_transition any => [:failed] do |pipeline|
-        next unless pipeline.auto_devops_source?
+        pipeline.run_after_commit do
+          ::Gitlab::Ci::Pipeline::Metrics.pipeline_failure_reason_counter.increment(reason: pipeline.failure_reason)
 
-        pipeline.run_after_commit { AutoDevops::DisableWorker.perform_async(pipeline.id) }
+          AutoDevops::DisableWorker.perform_async(pipeline.id) if pipeline.auto_devops_source?
+        end
       end
     end
 
@@ -309,6 +311,7 @@ module Ci
     scope :created_after, -> (time) { where('ci_pipelines.created_at > ?', time) }
     scope :created_before_id, -> (id) { where('ci_pipelines.id < ?', id) }
     scope :before_pipeline, -> (pipeline) { created_before_id(pipeline.id).outside_pipeline_family(pipeline) }
+    scope :eager_load_project, -> { eager_load(project: [:route, { namespace: :route }]) }
 
     scope :outside_pipeline_family, ->(pipeline) do
       where.not(id: pipeline.same_family_pipeline_ids)
@@ -393,26 +396,13 @@ module Ci
     #       given we simply get the latest pipelines for the commits, regardless
     #       of what refs the pipelines belong to.
     def self.latest_pipeline_per_commit(commits, ref = nil)
-      p1 = arel_table
-      p2 = arel_table.alias
-
-      # This LEFT JOIN will filter out all but the newest row for every
-      # combination of (project_id, sha) or (project_id, sha, ref) if a ref is
-      # given.
-      cond = p1[:sha].eq(p2[:sha])
-        .and(p1[:project_id].eq(p2[:project_id]))
-        .and(p1[:id].lt(p2[:id]))
-
-      cond = cond.and(p1[:ref].eq(p2[:ref])) if ref
-      join = p1.join(p2, Arel::Nodes::OuterJoin).on(cond)
+      sql = select('DISTINCT ON (sha) *')
+              .where(sha: commits)
+              .order(:sha, id: :desc)
 
-      relation = where(sha: commits)
-        .where(p2[:id].eq(nil))
-        .joins(join.join_sources)
+      sql = sql.where(ref: ref) if ref
 
-      relation = relation.where(ref: ref) if ref
-
-      relation.each_with_object({}) do |pipeline, hash|
+      sql.each_with_object({}) do |pipeline, hash|
         hash[pipeline.sha] = pipeline
       end
     end
@@ -445,6 +435,10 @@ module Ci
       @auto_devops_pipelines_completed_total ||= Gitlab::Metrics.counter(:auto_devops_pipelines_completed_total, 'Number of completed auto devops pipelines')
     end
 
+    def uses_needs?
+      builds.where(scheduling_type: :dag).any?
+    end
+
     def stages_count
       statuses.select(:stage).distinct.count
     end
@@ -510,6 +504,12 @@ module Ci
       end
     end
 
+    def git_author_full_text
+      strong_memoize(:git_author_full_text) do
+        commit.try(:author_full_text)
+      end
+    end
+
     def git_commit_message
       strong_memoize(:git_commit_message) do
         commit.try(:message)
@@ -573,10 +573,18 @@ module Ci
     end
 
     def cancel_running(retries: nil)
-      retry_optimistic_lock(cancelable_statuses, retries, name: 'ci_pipeline_cancel_running') do |cancelable|
-        cancelable.find_each do |job|
-          yield(job) if block_given?
-          job.cancel
+      commit_status_relations = [:project, :pipeline]
+      ci_build_relations = [:deployment, :taggings]
+
+      retry_optimistic_lock(cancelable_statuses, retries, name: 'ci_pipeline_cancel_running') do |cancelables|
+        cancelables.find_in_batches do |batch|
+          ActiveRecord::Associations::Preloader.new.preload(batch, commit_status_relations)
+          ActiveRecord::Associations::Preloader.new.preload(batch.select { |job| job.is_a?(Ci::Build) }, ci_build_relations)
+
+          batch.each do |job|
+            yield(job) if block_given?
+            job.cancel
+          end
         end
       end
     end
@@ -664,7 +672,9 @@ module Ci
     end
 
     def has_kubernetes_active?
-      project.deployment_platform&.active?
+      strong_memoize(:has_kubernetes_active) do
+        project.deployment_platform&.active?
+      end
     end
 
     def freeze_period?
@@ -822,6 +832,7 @@ module Ci
         variables.append(key: 'CI_COMMIT_DESCRIPTION', value: git_commit_description.to_s)
         variables.append(key: 'CI_COMMIT_REF_PROTECTED', value: (!!protected_ref?).to_s)
         variables.append(key: 'CI_COMMIT_TIMESTAMP', value: git_commit_timestamp.to_s)
+        variables.append(key: 'CI_COMMIT_AUTHOR', value: git_author_full_text.to_s)
 
         # legacy variables
         variables.append(key: 'CI_BUILD_REF', value: sha)
diff --git a/app/models/ci/pipeline_artifact.rb b/app/models/ci/pipeline_artifact.rb
index f538a4cd808..9dfe4252e95 100644
--- a/app/models/ci/pipeline_artifact.rb
+++ b/app/models/ci/pipeline_artifact.rb
@@ -57,3 +57,5 @@ module Ci
     end
   end
 end
+
+Ci::PipelineArtifact.prepend_ee_mod
diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb
index 2fae077dd87..3c17246bc34 100644
--- a/app/models/ci/pipeline_schedule.rb
+++ b/app/models/ci/pipeline_schedule.rb
@@ -7,6 +7,7 @@ module Ci
     include StripAttribute
     include Schedulable
     include Limitable
+    include EachBatch
 
     self.limit_name = 'ci_pipeline_schedules'
     self.limit_scope = :project
@@ -28,6 +29,7 @@ module Ci
     scope :active, -> { where(active: true) }
     scope :inactive, -> { where(active: false) }
     scope :preloaded, -> { preload(:owner, project: [:route]) }
+    scope :owned_by, ->(user) { where(owner: user) }
 
     accepts_nested_attributes_for :variables, allow_destroy: true
 
diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb
index 0ad1ed2fce8..3b61840805a 100644
--- a/app/models/ci/processable.rb
+++ b/app/models/ci/processable.rb
@@ -165,7 +165,13 @@ module Ci
     end
 
     def all_dependencies
-      dependencies.all
+      if Feature.enabled?(:preload_associations_jobs_request_api_endpoint, project, default_enabled: :yaml)
+        strong_memoize(:all_dependencies) do
+          dependencies.all
+        end
+      else
+        dependencies.all
+      end
     end
 
     private
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index d1a20bc93c3..05126853e0f 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -45,8 +45,6 @@ module Ci
     FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level maximum_timeout_human_readable].freeze
     MINUTES_COST_FACTOR_FIELDS = %i[public_projects_minutes_cost_factor private_projects_minutes_cost_factor].freeze
 
-    ignore_column :is_shared, remove_after: '2019-12-15', remove_with: '12.6'
-
     has_many :builds
     has_many :runner_projects, inverse_of: :runner, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
     has_many :projects, through: :runner_projects
diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb
index 03a97355574..9dd75150ac7 100644
--- a/app/models/ci/stage.rb
+++ b/app/models/ci/stage.rb
@@ -14,11 +14,20 @@ module Ci
 
     has_many :statuses, class_name: 'CommitStatus', foreign_key: :stage_id
     has_many :latest_statuses, -> { ordered.latest }, class_name: 'CommitStatus', foreign_key: :stage_id
+    has_many :retried_statuses, -> { ordered.retried }, class_name: 'CommitStatus', foreign_key: :stage_id
     has_many :processables, class_name: 'Ci::Processable', foreign_key: :stage_id
     has_many :builds, foreign_key: :stage_id
     has_many :bridges, foreign_key: :stage_id
 
     scope :ordered, -> { order(position: :asc) }
+    scope :in_pipelines, ->(pipelines) { where(pipeline: pipelines) }
+    scope :by_name, ->(names) { where(name: names) }
+    scope :with_latest_and_retried_statuses, -> do
+      includes(
+        latest_statuses: [:pipeline, project: :namespace],
+        retried_statuses: [:pipeline, project: :namespace]
+      )
+    end
 
     with_options unless: :importing? do
       validates :project, presence: true
@@ -35,7 +44,7 @@ module Ci
       next if position.present?
 
       self.position = statuses.select(:stage_idx)
-        .where('stage_idx IS NOT NULL')
+        .where.not(stage_idx: nil)
         .group(:stage_idx)
         .order('COUNT(*) DESC')
         .first&.stage_idx.to_i
diff --git a/app/models/ci/test_case.rb b/app/models/ci/test_case.rb
deleted file mode 100644
index 19ecc177436..00000000000
--- a/app/models/ci/test_case.rb
+++ /dev/null
@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-module Ci
-  class TestCase < ApplicationRecord
-    extend Gitlab::Ci::Model
-
-    validates :project, :key_hash, presence: true
-
-    has_many :test_case_failures, class_name: 'Ci::TestCaseFailure'
-
-    belongs_to :project
-
-    scope :by_project_and_keys, -> (project, keys) { where(project_id: project.id, key_hash: keys) }
-
-    class << self
-      def find_or_create_by_batch(project, test_case_keys)
-        # Insert records first. Existing ones will be skipped.
-        insert_all(test_case_attrs(project, test_case_keys))
-
-        # Find all matching records now that we are sure they all are persisted.
-        by_project_and_keys(project, test_case_keys)
-      end
-
-      private
-
-      def test_case_attrs(project, test_case_keys)
-        # NOTE: Rails 6.1 will add support for insert_all on relation so that
-        # we will be able to do project.test_cases.insert_all.
-        test_case_keys.map do |hashed_key|
-          { project_id: project.id, key_hash: hashed_key }
-        end
-      end
-    end
-  end
-end
diff --git a/app/models/ci/test_case_failure.rb b/app/models/ci/test_case_failure.rb
deleted file mode 100644
index 8867b954240..00000000000
--- a/app/models/ci/test_case_failure.rb
+++ /dev/null
@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Ci
-  class TestCaseFailure < ApplicationRecord
-    extend Gitlab::Ci::Model
-
-    REPORT_WINDOW = 14.days
-
-    validates :test_case, :build, :failed_at, presence: true
-
-    belongs_to :test_case, class_name: "Ci::TestCase", foreign_key: :test_case_id
-    belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id
-
-    def self.recent_failures_count(project:, test_case_keys:, date_range: REPORT_WINDOW.ago..Time.current)
-      joins(:test_case)
-        .where(
-          ci_test_cases: {
-            project_id: project.id,
-            key_hash: test_case_keys
-          },
-          ci_test_case_failures: {
-            failed_at: date_range
-          }
-        )
-        .group(:key_hash)
-        .count('ci_test_case_failures.id')
-    end
-  end
-end
diff --git a/app/models/ci/unit_test.rb b/app/models/ci/unit_test.rb
new file mode 100644
index 00000000000..81623b4f6ad
--- /dev/null
+++ b/app/models/ci/unit_test.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Ci
+  class UnitTest < ApplicationRecord
+    extend Gitlab::Ci::Model
+
+    MAX_NAME_SIZE = 255
+    MAX_SUITE_NAME_SIZE = 255
+
+    validates :project, :key_hash, :name, :suite_name, presence: true
+
+    has_many :unit_test_failures, class_name: 'Ci::UnitTestFailure'
+
+    belongs_to :project
+
+    scope :by_project_and_keys, -> (project, keys) { where(project_id: project.id, key_hash: keys) }
+
+    class << self
+      def find_or_create_by_batch(project, unit_test_attrs)
+        # Insert records first. Existing ones will be skipped.
+        insert_all(build_insert_attrs(project, unit_test_attrs))
+
+        # Find all matching records now that we are sure they all are persisted.
+        by_project_and_keys(project, gather_keys(unit_test_attrs))
+      end
+
+      private
+
+      def build_insert_attrs(project, unit_test_attrs)
+        # NOTE: Rails 6.1 will add support for insert_all on relation so that
+        # we will be able to do project.test_cases.insert_all.
+        unit_test_attrs.map do |attrs|
+          attrs.merge(
+            project_id: project.id,
+            name: attrs[:name].truncate(MAX_NAME_SIZE),
+            suite_name: attrs[:suite_name].truncate(MAX_SUITE_NAME_SIZE)
+          )
+        end
+      end
+
+      def gather_keys(unit_test_attrs)
+        unit_test_attrs.map { |attrs| attrs[:key_hash] }
+      end
+    end
+  end
+end
diff --git a/app/models/ci/unit_test_failure.rb b/app/models/ci/unit_test_failure.rb
new file mode 100644
index 00000000000..653a56bd2b3
--- /dev/null
+++ b/app/models/ci/unit_test_failure.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Ci
+  class UnitTestFailure < ApplicationRecord
+    extend Gitlab::Ci::Model
+
+    REPORT_WINDOW = 14.days
+
+    validates :unit_test, :build, :failed_at, presence: true
+
+    belongs_to :unit_test, class_name: "Ci::UnitTest", foreign_key: :unit_test_id
+    belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id
+
+    def self.recent_failures_count(project:, unit_test_keys:, date_range: REPORT_WINDOW.ago..Time.current)
+      joins(:unit_test)
+        .where(
+          ci_unit_tests: {
+            project_id: project.id,
+            key_hash: unit_test_keys
+          },
+          ci_unit_test_failures: {
+            failed_at: date_range
+          }
+        )
+        .group(:key_hash)
+        .count('ci_unit_test_failures.id')
+    end
+  end
+end