Add latest changes from gitlab-org/gitlab@14-1-stable-eev14.1.0-rc42

13 files changed, 184 insertions, 89 deletions

diff --git a/app/models/ci/base_model.rb b/app/models/ci/base_model.rb
new file mode 100644
index 00000000000..8fb752ead1d
--- /dev/null
+++ b/app/models/ci/base_model.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Ci
+  # TODO: https://gitlab.com/groups/gitlab-org/-/epics/6168
+  #
+  # Do not use this yet outside of `ci_instance_variables`.
+  # This class is part of a migration to move all CI classes to a new separate database.
+  # Initially we are only going to be moving the `Ci::InstanceVariable` model and it will be duplicated in the main and CI tables
+  # Do not extend this class in any other models.
+  class BaseModel < ::ApplicationRecord
+    self.abstract_class = true
+
+    if Gitlab::Database.has_config?(:ci)
+      connects_to database: { writing: :ci, reading: :ci }
+    end
+  end
+end
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index fdfffd9b0cd..4328f3f7a4b 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -11,7 +11,6 @@ module Ci
     include Importable
     include Ci::HasRef
     include IgnorableColumns
-    include TaggableQueries
 
     BuildArchivedError = Class.new(StandardError)
 
@@ -136,6 +135,7 @@ module Ci
 
     scope :eager_load_job_artifacts, -> { includes(:job_artifacts) }
     scope :eager_load_job_artifacts_archive, -> { includes(:job_artifacts_archive) }
+    scope :eager_load_tags, -> { includes(:tags) }
 
     scope :eager_load_everything, -> do
       includes(
@@ -178,25 +178,6 @@ module Ci
       joins(:metadata).where("ci_builds_metadata.config_options -> 'artifacts' -> 'reports' ?| array[:job_types]", job_types: job_types)
     end
 
-    scope :matches_tag_ids, -> (tag_ids) do
-      matcher = ::ActsAsTaggableOn::Tagging
-        .where(taggable_type: CommitStatus.name)
-        .where(context: 'tags')
-        .where('taggable_id = ci_builds.id')
-        .where.not(tag_id: tag_ids).select('1')
-
-      where("NOT EXISTS (?)", matcher)
-    end
-
-    scope :with_any_tags, -> do
-      matcher = ::ActsAsTaggableOn::Tagging
-        .where(taggable_type: CommitStatus.name)
-        .where(context: 'tags')
-        .where('taggable_id = ci_builds.id').select('1')
-
-      where("EXISTS (?)", matcher)
-    end
-
     scope :queued_before, ->(time) { where(arel_table[:queued_at].lt(time)) }
 
     scope :preload_project_and_pipeline_project, -> do
@@ -212,7 +193,7 @@ module Ci
 
     acts_as_taggable
 
-    add_authentication_token_field :token, encrypted: :optional
+    add_authentication_token_field :token, encrypted: :required
 
     before_save :ensure_token
     before_destroy { unscoped_project }
@@ -344,7 +325,11 @@ module Ci
         build.run_after_commit do
           build.run_status_commit_hooks!
 
-          BuildFinishedWorker.perform_async(id)
+          if Feature.enabled?(:ci_build_finished_worker_namespace_changed, build.project, default_enabled: :yaml)
+            Ci::BuildFinishedWorker.perform_async(id)
+          else
+            ::BuildFinishedWorker.perform_async(id)
+          end
         end
       end
 
@@ -758,6 +743,14 @@ module Ci
       self.token && ActiveSupport::SecurityUtils.secure_compare(token, self.token)
     end
 
+    def tag_list
+      if tags.loaded?
+        tags.map(&:name)
+      else
+        super
+      end
+    end
+
     def has_tags?
       tag_list.any?
     end
@@ -782,7 +775,7 @@ module Ci
       return unless project
 
       project.execute_hooks(build_data.dup, :job_hooks) if project.has_active_hooks?(:job_hooks)
-      project.execute_services(build_data.dup, :job_hooks) if project.has_active_services?(:job_hooks)
+      project.execute_integrations(build_data.dup, :job_hooks) if project.has_active_integrations?(:job_hooks)
     end
 
     def browsable_artifacts?
@@ -939,8 +932,7 @@ module Ci
     end
 
     def supports_artifacts_exclude?
-      options&.dig(:artifacts, :exclude)&.any? &&
-        Gitlab::Ci::Features.artifacts_exclude_enabled?
+      options&.dig(:artifacts, :exclude)&.any?
     end
 
     def multi_build_steps?
diff --git a/app/models/ci/build_dependencies.rb b/app/models/ci/build_dependencies.rb
index d39e0411a79..c4a04d42a1e 100644
--- a/app/models/ci/build_dependencies.rb
+++ b/app/models/ci/build_dependencies.rb
@@ -37,12 +37,20 @@ module Ci
         next [] unless processable.pipeline_id # we don't have any dependency when creating the pipeline
 
         deps = model_class.where(pipeline_id: processable.pipeline_id).latest
-        deps = from_previous_stages(deps)
-        deps = from_needs(deps)
+        deps = find_dependencies(processable, deps)
+
         from_dependencies(deps).to_a
       end
     end
 
+    def find_dependencies(processable, deps)
+      if processable.scheduling_type_dag?
+        from_needs(deps)
+      else
+        from_previous_stages(deps)
+      end
+    end
+
     # Dependencies from the same parent-pipeline hierarchy excluding
     # the current job's pipeline
     def cross_pipeline
@@ -125,8 +133,6 @@ module Ci
     end
 
     def from_needs(scope)
-      return scope unless processable.scheduling_type_dag?
-
       needs_names = processable.needs.artifacts.select(:name)
       scope.where(name: needs_names)
     end
diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb
index f009e4c6aa1..50775f578f0 100644
--- a/app/models/ci/build_metadata.rb
+++ b/app/models/ci/build_metadata.rb
@@ -22,8 +22,8 @@ module Ci
     validates :build, presence: true
     validates :secrets, json_schema: { filename: 'build_metadata_secrets' }
 
-    serialize :config_options, Serializers::Json # rubocop:disable Cop/ActiveRecordSerialize
-    serialize :config_variables, Serializers::Json # rubocop:disable Cop/ActiveRecordSerialize
+    serialize :config_options, Serializers::SymbolizedJson # rubocop:disable Cop/ActiveRecordSerialize
+    serialize :config_variables, Serializers::SymbolizedJson # rubocop:disable Cop/ActiveRecordSerialize
 
     chronic_duration_attr_reader :timeout_human_readable, :timeout
 
diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb
index 25f4a06088d..3fa9a484b0c 100644
--- a/app/models/ci/build_trace_chunk.rb
+++ b/app/models/ci/build_trace_chunk.rb
@@ -14,13 +14,7 @@ module Ci
 
     belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id
 
-    default_value_for :data_store do
-      if Feature.enabled?(:dedicated_redis_trace_chunks, type: :ops)
-        :redis_trace_chunks
-      else
-        :redis
-      end
-    end
+    default_value_for :data_store, :redis_trace_chunks
 
     after_create { metrics.increment_trace_operation(operation: :chunked) }
 
@@ -115,7 +109,7 @@ module Ci
       raise ArgumentError, 'Offset is out of range' if offset > size || offset < 0
       return if offset == size # Skip the following process as it doesn't affect anything
 
-      self.append("", offset)
+      self.append(+"", offset)
     end
 
     def append(new_data, offset)
diff --git a/app/models/ci/build_trace_chunks/fog.rb b/app/models/ci/build_trace_chunks/fog.rb
index fab85fae33d..3bfac2b33c0 100644
--- a/app/models/ci/build_trace_chunks/fog.rb
+++ b/app/models/ci/build_trace_chunks/fog.rb
@@ -25,14 +25,36 @@ module Ci
         files.create(create_attributes(model, new_data))
       end
 
+      # This is the sequence that causes append_data to be called:
+      #
+      # 1. Runner sends a PUT /api/v4/jobs/:id to indicate the job is canceled or finished.
+      # 2. UpdateBuildStateService#accept_build_state! persists all live job logs to object storage (or filesystem).
+      # 3. UpdateBuildStateService#accept_build_state! returns a 202 to the runner.
+      # 4. The runner continues to send PATCH requests with job logs until all logs have been sent and received.
+      # 5. If the last PATCH request arrives after the job log has been persisted, we
+      #    retrieve the data from object storage to append the remaining lines.
       def append_data(model, new_data, offset)
         if offset > 0
           truncated_data = data(model).to_s.byteslice(0, offset)
-          new_data = truncated_data + new_data
+          new_data = append_strings(truncated_data, new_data)
         end
 
         set_data(model, new_data)
         new_data.bytesize
+      rescue Encoding::CompatibilityError => e
+        Gitlab::ErrorTracking.track_and_raise_exception(
+          e,
+          build_id: model.build_id,
+          chunk_index: model.chunk_index,
+          chunk_start_offset: model.start_offset,
+          chunk_end_offset: model.end_offset,
+          chunk_size: model.size,
+          chunk_data_store: model.data_store,
+          offset: offset,
+          old_data_encoding: truncated_data.encoding.to_s,
+          new_data: new_data,
+          new_data_size: new_data.bytesize,
+          new_data_encoding: new_data.encoding.to_s)
       end
 
       def size(model)
@@ -57,6 +79,17 @@ module Ci
 
       private
 
+      def append_strings(old_data, new_data)
+        if Feature.enabled?(:ci_job_trace_force_encode, default_enabled: :yaml)
+          # When object storage is in use, old_data may be retrieved in UTF-8.
+          old_data = old_data.force_encoding(Encoding::ASCII_8BIT)
+          # new_data should already be in ASCII-8BIT, but just in case it isn't, do this.
+          new_data = new_data.force_encoding(Encoding::ASCII_8BIT)
+        end
+
+        old_data + new_data
+      end
+
       def key(model)
         key_raw(model.build_id, model.chunk_index)
       end
diff --git a/app/models/ci/group.rb b/app/models/ci/group.rb
index 47b91fcf2ce..e5cb2026503 100644
--- a/app/models/ci/group.rb
+++ b/app/models/ci/group.rb
@@ -10,6 +10,7 @@ module Ci
   class Group
     include StaticModel
     include Gitlab::Utils::StrongMemoize
+    include GlobalID::Identification
 
     attr_reader :project, :stage, :name, :jobs
 
@@ -22,6 +23,10 @@ module Ci
       @jobs = jobs
     end
 
+    def id
+      "#{stage.id}-#{name}"
+    end
+
     def ==(other)
       other.present? && other.is_a?(self.class) &&
         project == other.project &&
diff --git a/app/models/ci/instance_variable.rb b/app/models/ci/instance_variable.rb
index e083caa8751..5aee4c924af 100644
--- a/app/models/ci/instance_variable.rb
+++ b/app/models/ci/instance_variable.rb
@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Ci
-  class InstanceVariable < ApplicationRecord
+  class InstanceVariable < ::Ci::BaseModel
     extend Gitlab::Ci::Model
     extend Gitlab::ProcessMemoryCache::Helper
     include Ci::NewHasVariable
diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb
index 6a7a2b3f6bd..46c976d5616 100644
--- a/app/models/ci/job_artifact.rb
+++ b/app/models/ci/job_artifact.rb
@@ -33,6 +33,7 @@ module Ci
       secret_detection: 'gl-secret-detection-report.json',
       dependency_scanning: 'gl-dependency-scanning-report.json',
       container_scanning: 'gl-container-scanning-report.json',
+      cluster_image_scanning: 'gl-cluster-image-scanning-report.json',
       dast: 'gl-dast-report.json',
       license_scanning: 'gl-license-scanning-report.json',
       performance: 'performance.json',
@@ -71,6 +72,7 @@ module Ci
       secret_detection: :raw,
       dependency_scanning: :raw,
       container_scanning: :raw,
+      cluster_image_scanning: :raw,
       dast: :raw,
       license_scanning: :raw,
 
@@ -108,6 +110,7 @@ module Ci
       sast
       secret_detection
       requirements
+      cluster_image_scanning
     ].freeze
 
     TYPE_AND_FORMAT_PAIRS = INTERNAL_TYPES.merge(REPORT_TYPES).freeze
@@ -181,6 +184,8 @@ module Ci
     scope :with_destroy_preloads, -> { includes(project: [:route, :statistics]) }
 
     scope :scoped_project, -> { where('ci_job_artifacts.project_id = projects.id') }
+    scope :for_project, ->(project) { where(project_id: project) }
+    scope :created_in_time_range, ->(from: nil, to: nil) { where(created_at: from..to) }
 
     delegate :filename, :exists?, :open, to: :file
 
@@ -210,7 +215,8 @@ module Ci
       coverage_fuzzing: 23, ## EE-specific
       browser_performance: 24, ## EE-specific
       load_performance: 25, ## EE-specific
-      api_fuzzing: 26 ## EE-specific
+      api_fuzzing: 26, ## EE-specific
+      cluster_image_scanning: 27 ## EE-specific
     }
 
     # `file_location` indicates where actual files are stored.
diff --git a/app/models/ci/job_token/project_scope_link.rb b/app/models/ci/job_token/project_scope_link.rb
index 283ad4a190d..99118f8090b 100644
--- a/app/models/ci/job_token/project_scope_link.rb
+++ b/app/models/ci/job_token/project_scope_link.rb
@@ -19,6 +19,10 @@ module Ci
       validates :target_project, presence: true
       validate :not_self_referential_link
 
+      def self.for_source_and_target(source_project, target_project)
+        self.find_by(source_project: source_project, target_project: target_project)
+      end
+
       private
 
       def not_self_referential_link
diff --git a/app/models/ci/pending_build.rb b/app/models/ci/pending_build.rb
index b9a8a44bd6b..0663052f51d 100644
--- a/app/models/ci/pending_build.rb
+++ b/app/models/ci/pending_build.rb
@@ -7,12 +7,52 @@ module Ci
     belongs_to :project
     belongs_to :build, class_name: 'Ci::Build'
 
+    scope :ref_protected, -> { where(protected: true) }
+    scope :queued_before, ->(time) { where(arel_table[:created_at].lt(time)) }
+
     def self.upsert_from_build!(build)
-      entry = self.new(build: build, project: build.project, protected: build.protected?)
+      entry = self.new(args_from_build(build))
 
       entry.validate!
 
       self.upsert(entry.attributes.compact, returning: %w[build_id], unique_by: :build_id)
     end
+
+    def self.args_from_build(build)
+      args = {
+        build: build,
+        project: build.project,
+        protected: build.protected?
+      }
+
+      if Feature.enabled?(:ci_pending_builds_maintain_shared_runners_data, type: :development, default_enabled: :yaml)
+        args.merge(instance_runners_enabled: shareable?(build))
+      else
+        args
+      end
+    end
+    private_class_method :args_from_build
+
+    def self.shareable?(build)
+      shared_runner_enabled?(build) &&
+        builds_access_level?(build) &&
+        project_not_removed?(build)
+    end
+    private_class_method :shareable?
+
+    def self.shared_runner_enabled?(build)
+      build.project.shared_runners.exists?
+    end
+    private_class_method :shared_runner_enabled?
+
+    def self.project_not_removed?(build)
+      !build.project.pending_delete?
+    end
+    private_class_method :project_not_removed?
+
+    def self.builds_access_level?(build)
+      build.project.project_feature.builds_access_level.nil? || build.project.project_feature.builds_access_level > 0
+    end
+    private_class_method :builds_access_level?
   end
 end
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index 159d9d10878..5d079f57267 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -29,6 +29,8 @@ module Ci
 
     BridgeStatusError = Class.new(StandardError)
 
+    paginates_per 15
+
     sha_attribute :source_sha
     sha_attribute :target_sha
 
@@ -222,7 +224,7 @@ module Ci
       end
 
       after_transition [:created, :waiting_for_resource, :preparing, :pending, :running] => :success do |pipeline|
-        # We wait a little bit to ensure that all BuildFinishedWorkers finish first
+        # We wait a little bit to ensure that all Ci::BuildFinishedWorkers finish first
         # because this is where some metrics like code coverage is parsed and stored
         # in CI build records which the daily build metrics worker relies on.
         pipeline.run_after_commit { Ci::DailyBuildGroupReportResultsWorker.perform_in(10.minutes, pipeline.id) }
@@ -577,11 +579,11 @@ module Ci
       canceled? && auto_canceled_by_id?
     end
 
-    def cancel_running(retries: nil)
+    def cancel_running(retries: 1)
       commit_status_relations = [:project, :pipeline]
       ci_build_relations = [:deployment, :taggings]
 
-      retry_optimistic_lock(cancelable_statuses, retries, name: 'ci_pipeline_cancel_running') do |cancelables|
+      retry_lock(cancelable_statuses, retries, name: 'ci_pipeline_cancel_running') do |cancelables|
         cancelables.find_in_batches do |batch|
           ActiveRecord::Associations::Preloader.new.preload(batch, commit_status_relations)
           ActiveRecord::Associations::Preloader.new.preload(batch.select { |job| job.is_a?(Ci::Build) }, ci_build_relations)
@@ -594,7 +596,7 @@ module Ci
       end
     end
 
-    def auto_cancel_running(pipeline, retries: nil)
+    def auto_cancel_running(pipeline, retries: 1)
       update(auto_canceled_by: pipeline)
 
       cancel_running(retries: retries) do |job|
@@ -610,8 +612,6 @@ module Ci
     # rubocop: enable CodeReuse/ServiceClass
 
     def lazy_ref_commit
-      return unless ::Gitlab::Ci::Features.pipeline_latest?
-
       BatchLoader.for(ref).batch do |refs, loader|
         next unless project.repository_exists?
 
@@ -623,11 +623,6 @@ module Ci
 
     def latest?
       return false unless git_ref && commit.present?
-
-      unless ::Gitlab::Ci::Features.pipeline_latest?
-        return project.commit(git_ref) == commit
-      end
-
       return false if lazy_ref_commit.nil?
 
       lazy_ref_commit.id == commit.id
@@ -861,7 +856,7 @@ module Ci
 
     def execute_hooks
       project.execute_hooks(pipeline_data, :pipeline_hooks) if project.has_active_hooks?(:pipeline_hooks)
-      project.execute_services(pipeline_data, :pipeline_hooks) if project.has_active_services?(:pipeline_hooks)
+      project.execute_integrations(pipeline_data, :pipeline_hooks) if project.has_active_integrations?(:pipeline_hooks)
     end
 
     # All the merge requests for which the current pipeline runs/ran against
@@ -911,7 +906,7 @@ module Ci
 
     def same_family_pipeline_ids
       ::Gitlab::Ci::PipelineObjectHierarchy.new(
-        self.class.default_scoped.where(id: root_ancestor), options: { same_project: true }
+        self.class.default_scoped.where(id: root_ancestor), options: { project_condition: :same }
       ).base_and_descendants.select(:id)
     end
 
@@ -932,29 +927,34 @@ module Ci
       Environment.where(id: environment_ids)
     end
 
-    # Without using `unscoped`, caller scope is also included into the query.
-    # Using `unscoped` here will be redundant after Rails 6.1
+    # With multi-project and parent-child pipelines
+    def self_and_upstreams
+      object_hierarchy.base_and_ancestors
+    end
+
+    # With multi-project and parent-child pipelines
+    def self_with_upstreams_and_downstreams
+      object_hierarchy.all_objects
+    end
+
+    # With only parent-child pipelines
+    def self_and_ancestors
+      object_hierarchy(project_condition: :same).base_and_ancestors
+    end
+
+    # With only parent-child pipelines
     def self_and_descendants
-      ::Gitlab::Ci::PipelineObjectHierarchy
-        .new(self.class.unscoped.where(id: id), options: { same_project: true })
-        .base_and_descendants
+      object_hierarchy(project_condition: :same).base_and_descendants
     end
 
     def root_ancestor
       return self unless child?
 
-      Gitlab::Ci::PipelineObjectHierarchy
-        .new(self.class.unscoped.where(id: id), options: { same_project: true })
+      object_hierarchy(project_condition: :same)
         .base_and_ancestors(hierarchy_order: :desc)
         .first
     end
 
-    def self_with_ancestors_and_descendants(same_project: false)
-      ::Gitlab::Ci::PipelineObjectHierarchy
-        .new(self.class.unscoped.where(id: id), options: { same_project: same_project })
-        .all_objects
-    end
-
     def bridge_triggered?
       source_bridge.present?
     end
@@ -1026,8 +1026,6 @@ module Ci
     end
 
     def can_generate_codequality_reports?
-      return false unless ::Gitlab::Ci::Features.display_quality_on_mr_diff?(project)
-
       has_reports?(Ci::JobArtifact.codequality_reports)
     end
 
@@ -1214,14 +1212,6 @@ module Ci
       self.ci_ref = Ci::Ref.ensure_for(self)
     end
 
-    def base_and_ancestors(same_project: false)
-      # Without using `unscoped`, caller scope is also included into the query.
-      # Using `unscoped` here will be redundant after Rails 6.1
-      ::Gitlab::Ci::PipelineObjectHierarchy
-        .new(self.class.unscoped.where(id: id), options: { same_project: same_project })
-        .base_and_ancestors
-    end
-
     # We need `base_and_ancestors` in a specific order to "break" when needed.
     # If we use `find_each`, then the order is broken.
     # rubocop:disable Rails/FindEach
@@ -1232,7 +1222,7 @@ module Ci
         source_bridge.pending!
         Ci::AfterRequeueJobService.new(project, current_user).execute(source_bridge) # rubocop:disable CodeReuse/ServiceClass
       else
-        base_and_ancestors.includes(:source_bridge).each do |pipeline|
+        self_and_upstreams.includes(:source_bridge).each do |pipeline|
           break unless pipeline.bridge_waiting?
 
           pipeline.source_bridge.pending!
@@ -1315,6 +1305,13 @@ module Ci
 
       project.repository.keep_around(self.sha, self.before_sha)
     end
+
+    # Without using `unscoped`, caller scope is also included into the query.
+    # Using `unscoped` here will be redundant after Rails 6.1
+    def object_hierarchy(options = {})
+      ::Gitlab::Ci::PipelineObjectHierarchy
+        .new(self.class.unscoped.where(id: id), options: options)
+    end
   end
 end
 
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index 71110ef0696..a541dca47de 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -13,7 +13,7 @@ module Ci
     include Gitlab::Utils::StrongMemoize
     include TaggableQueries
 
-    add_authentication_token_field :token, encrypted: -> { Feature.enabled?(:ci_runners_tokens_optional_encryption, default_enabled: true) ? :optional : :required }
+    add_authentication_token_field :token, encrypted: :optional
 
     enum access_level: {
       not_protected: 0,
@@ -214,15 +214,15 @@ module Ci
         Arel.sql("(#{arel_tag_names_array.to_sql})")
       ]
 
-      # we use distinct to de-duplicate data
-      distinct.pluck(*unique_params).map do |values|
+      group(*unique_params).pluck('array_agg(ci_runners.id)', *unique_params).map do |values|
         Gitlab::Ci::Matching::RunnerMatcher.new({
-          runner_type: values[0],
-          public_projects_minutes_cost_factor: values[1],
-          private_projects_minutes_cost_factor: values[2],
-          run_untagged: values[3],
-          access_level: values[4],
-          tag_list: values[5]
+          runner_ids: values[0],
+          runner_type: values[1],
+          public_projects_minutes_cost_factor: values[2],
+          private_projects_minutes_cost_factor: values[3],
+          run_untagged: values[4],
+          access_level: values[5],
+          tag_list: values[6]
         })
       end
     end
@@ -230,6 +230,7 @@ module Ci
     def runner_matcher
       strong_memoize(:runner_matcher) do
         Gitlab::Ci::Matching::RunnerMatcher.new({
+          runner_ids: [id],
           runner_type: runner_type,
           public_projects_minutes_cost_factor: public_projects_minutes_cost_factor,
           private_projects_minutes_cost_factor: private_projects_minutes_cost_factor,