Add latest changes from gitlab-org/gitlab@15-0-stable-eev15.0.0-rc42

10 files changed, 106 insertions, 39 deletions

diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb
index ff444ddefa3..a06b920342c 100644
--- a/app/models/ci/bridge.rb
+++ b/app/models/ci/bridge.rb
@@ -57,6 +57,14 @@ module Ci
       end
     end
 
+    def retryable?
+      return false unless Feature.enabled?(:ci_recreate_downstream_pipeline, project)
+
+      return false if failed? && (pipeline_loop_detected? || reached_max_descendant_pipelines_depth?)
+
+      super
+    end
+
     def self.with_preloads
       preload(
         :metadata,
@@ -65,8 +73,11 @@ module Ci
       )
     end
 
-    def retryable?
-      false
+    def self.clone_accessors
+      %i[pipeline project ref tag options name
+         allow_failure stage stage_id stage_idx
+         yaml_variables when description needs_attributes
+         scheduling_type].freeze
     end
 
     def inherit_status_from_downstream!(pipeline)
@@ -204,7 +215,7 @@ module Ci
     end
 
     def downstream_variables
-      if ::Feature.enabled?(:ci_trigger_forward_variables, project, default_enabled: :yaml)
+      if ::Feature.enabled?(:ci_trigger_forward_variables, project)
         calculate_downstream_variables
           .reverse # variables priority
           .uniq { |var| var[:key] } # only one variable key to pass
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index a8ad55fd5a4..eea8086d71d 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -45,6 +45,7 @@ module Ci
     has_one :runtime_metadata, class_name: 'Ci::RunningBuild', foreign_key: :build_id
     has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id, inverse_of: :build
     has_many :report_results, class_name: 'Ci::BuildReportResult', inverse_of: :build
+    has_one :namespace, through: :project
 
     # Projects::DestroyService destroys Ci::Pipelines, which use_fast_destroy on :job_artifacts
     # before we delete builds. By doing this, the relation should be empty and not fire any
@@ -74,6 +75,7 @@ module Ci
     delegate :gitlab_deploy_token, to: :project
     delegate :harbor_integration, to: :project
     delegate :trigger_short_token, to: :trigger_request, allow_nil: true
+    delegate :ensure_persistent_ref, to: :pipeline
 
     ##
     # Since Gitlab 11.5, deployments records started being created right after
@@ -325,7 +327,7 @@ module Ci
 
       after_transition pending: :running do |build|
         build.run_after_commit do
-          build.pipeline.persistent_ref.create
+          build.ensure_persistent_ref
 
           BuildHooksWorker.perform_async(id)
         end
@@ -335,7 +337,7 @@ module Ci
         build.run_after_commit do
           build.run_status_commit_hooks!
 
-          if Feature.enabled?(:ci_build_finished_worker_namespace_changed, build.project, default_enabled: :yaml)
+          if Feature.enabled?(:ci_build_finished_worker_namespace_changed, build.project)
             Ci::BuildFinishedWorker.perform_async(id)
           else
             ::BuildFinishedWorker.perform_async(id)
@@ -504,7 +506,7 @@ module Ci
         if metadata&.expanded_environment_name.present?
           metadata.expanded_environment_name
         else
-          if ::Feature.enabled?(:ci_expand_environment_name_and_url, project, default_enabled: :yaml)
+          if ::Feature.enabled?(:ci_expand_environment_name_and_url, project)
             ExpandVariables.expand(environment, -> { simple_variables.sort_and_expand_all })
           else
             ExpandVariables.expand(environment, -> { simple_variables })
@@ -675,7 +677,7 @@ module Ci
     end
 
     def has_archived_trace?
-      trace.archived_trace_exist?
+      trace.archived?
     end
 
     def artifacts_file
@@ -752,7 +754,7 @@ module Ci
     end
 
     def valid_token?(token)
-      self.token && ActiveSupport::SecurityUtils.secure_compare(token, self.token)
+      self.token && token.present? && ActiveSupport::SecurityUtils.secure_compare(token, self.token)
     end
 
     # acts_as_taggable uses this method create/remove tags with contexts
@@ -823,7 +825,6 @@ module Ci
       end
     end
 
-    # and use that for `ExpireBuildInstanceArtifactsWorker`?
     def erase_erasable_artifacts!
       job_artifacts.erasable.destroy_all # rubocop: disable Cop/DestroyAll
     end
@@ -884,10 +885,6 @@ module Ci
       job_artifacts.find_by(file_type: file_types_ids)&.file
     end
 
-    def coverage_regex
-      super || project.try(:build_coverage_regex)
-    end
-
     def steps
       [Gitlab::Ci::Build::Step.from_commands(self),
        Gitlab::Ci::Build::Step.from_release(self),
@@ -911,6 +908,8 @@ module Ci
         end
       end
 
+      return cache unless project.ci_separated_caches
+
       type_suffix = pipeline.protected_ref? ? 'protected' : 'non_protected'
       cache.map do |entry|
         entry.merge(key: "#{entry[:key]}-#{type_suffix}")
@@ -1224,7 +1223,7 @@ module Ci
 
     def job_jwt_variables
       Gitlab::Ci::Variables::Collection.new.tap do |variables|
-        break variables unless Feature.enabled?(:ci_job_jwt, project, default_enabled: true)
+        break variables unless Feature.enabled?(:ci_job_jwt, project)
 
         jwt = Gitlab::Ci::Jwt.for_build(self)
         jwt_v2 = Gitlab::Ci::JwtV2.for_build(self)
diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb
index ca68989002c..4ee661d89f4 100644
--- a/app/models/ci/build_metadata.rb
+++ b/app/models/ci/build_metadata.rb
@@ -38,7 +38,7 @@ module Ci
         job_timeout_source: 4
     }
 
-    ignore_columns :runner_features, remove_with: '14.7', remove_after: '2021-11-22'
+    ignore_columns :runner_features, remove_with: '15.1', remove_after: '2022-05-22'
 
     def update_timeout_state
       timeout = timeout_with_highest_precedence
diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb
index dff8bb89021..c831ef12501 100644
--- a/app/models/ci/job_artifact.rb
+++ b/app/models/ci/job_artifact.rb
@@ -45,7 +45,7 @@ module Ci
       dotenv: '.env',
       cobertura: 'cobertura-coverage.xml',
       terraform: 'tfplan.json',
-      cluster_applications: 'gl-cluster-applications.json', # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/333441
+      cluster_applications: 'gl-cluster-applications.json', # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/361094
       requirements: 'requirements.json',
       coverage_fuzzing: 'gl-coverage-fuzzing.json',
       api_fuzzing: 'gl-api-fuzzing-report.json'
@@ -64,7 +64,7 @@ module Ci
       network_referee: :gzip,
       dotenv: :gzip,
       cobertura: :gzip,
-      cluster_applications: :gzip,
+      cluster_applications: :gzip, # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/361094
       lsif: :zip,
 
       # Security reports and license scanning reports are raw artifacts
@@ -187,7 +187,6 @@ module Ci
     scope :downloadable, -> { where(file_type: DOWNLOADABLE_TYPES) }
     scope :unlocked, -> { joins(job: :pipeline).merge(::Ci::Pipeline.unlocked) }
     scope :order_expired_asc, -> { order(expire_at: :asc) }
-    scope :order_expired_desc, -> { order(expire_at: :desc) }
     scope :with_destroy_preloads, -> { includes(project: [:route, :statistics]) }
 
     scope :for_project, ->(project) { where(project_id: project) }
@@ -323,12 +322,12 @@ module Ci
         end
     end
 
-    def archived_trace_exists?
+    def stored?
       file&.file&.exists?
     end
 
     def self.archived_trace_exists_for?(job_id)
-      where(job_id: job_id).trace.take&.archived_trace_exists?
+      where(job_id: job_id).trace.take&.stored?
     end
 
     def self.max_artifact_size(type:, project:)
diff --git a/app/models/ci/namespace_settings.rb b/app/models/ci/namespace_settings.rb
new file mode 100644
index 00000000000..d519a48311f
--- /dev/null
+++ b/app/models/ci/namespace_settings.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# CI::NamespaceSettings mixin
+#
+# This module is intended to encapsulate CI/CD settings-specific logic
+# and be prepended in the `Namespace` model
+module Ci
+  module NamespaceSettings
+    # Overridden in EE::Namespace
+    def allow_stale_runner_pruning?
+      false
+    end
+
+    # Overridden in EE::Namespace
+    def allow_stale_runner_pruning=(_value)
+      raise NotImplementedError
+    end
+  end
+end
diff --git a/app/models/ci/pending_build.rb b/app/models/ci/pending_build.rb
index 41dc74ef050..d900a056242 100644
--- a/app/models/ci/pending_build.rb
+++ b/app/models/ci/pending_build.rb
@@ -31,7 +31,7 @@ module Ci
       end
 
       def maintain_denormalized_data?
-        ::Feature.enabled?(:ci_pending_builds_maintain_denormalized_data, default_enabled: :yaml)
+        ::Feature.enabled?(:ci_pending_builds_maintain_denormalized_data)
       end
 
       private
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index 2d0479e02a3..c10069382f2 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -36,10 +36,10 @@ module Ci
 
     # Ci::CreatePipelineService returns Ci::Pipeline so this is the only place
     # where we can pass additional information from the service. This accessor
-    # is used for storing the processed CI YAML contents for linting purposes.
+    # is used for storing the processed metadata for linting purposes.
     # There is an open issue to address this:
     # https://gitlab.com/gitlab-org/gitlab/-/issues/259010
-    attr_accessor :merged_yaml
+    attr_accessor :config_metadata
 
     # This is used to retain access to the method defined by `Ci::HasRef`
     # before being overridden in this class.
@@ -198,7 +198,7 @@ module Ci
       # Create a separate worker for each new operation
 
       before_transition [:created, :waiting_for_resource, :preparing, :pending] => :running do |pipeline|
-        pipeline.started_at = Time.current
+        pipeline.started_at ||= Time.current
       end
 
       before_transition any => [:success, :failed, :canceled] do |pipeline|
@@ -253,8 +253,6 @@ module Ci
 
       after_transition any => ::Ci::Pipeline.completed_statuses do |pipeline|
         pipeline.run_after_commit do
-          pipeline.persistent_ref.delete
-
           pipeline.all_merge_requests.each do |merge_request|
             next unless merge_request.auto_merge_enabled?
 
@@ -288,6 +286,12 @@ module Ci
         end
       end
 
+      after_transition any => ::Ci::Pipeline.stopped_statuses do |pipeline|
+        pipeline.run_after_commit do
+          pipeline.persistent_ref.delete
+        end
+      end
+
       after_transition any => [:success, :failed] do |pipeline|
         ref_status = pipeline.ci_ref&.update_status_by!(pipeline)
 
@@ -336,7 +340,7 @@ module Ci
     scope :with_only_interruptible_builds, -> do
       where('NOT EXISTS (?)',
         Ci::Build.where('ci_builds.commit_id = ci_pipelines.id')
-                 .with_status(:running, :success, :failed)
+                 .with_status(STARTED_STATUSES)
                  .not_interruptible
       )
     end
@@ -978,7 +982,7 @@ module Ci
     end
 
     # With multi-project and parent-child pipelines
-    def self_with_upstreams_and_downstreams
+    def all_pipelines_in_hierarchy
       object_hierarchy.all_objects
     end
 
@@ -992,6 +996,7 @@ module Ci
       object_hierarchy(project_condition: :same).base_and_descendants
     end
 
+    # Follow the parent-child relationships and return the top-level parent
     def root_ancestor
       return self unless child?
 
@@ -1000,6 +1005,12 @@ module Ci
         .first
     end
 
+    # Follow the upstream pipeline relationships, regardless of multi-project or
+    # parent-child, and return the top-level ancestor.
+    def upstream_root
+      object_hierarchy.base_and_ancestors(hierarchy_order: :desc).first
+    end
+
     def bridge_triggered?
       source_bridge.present?
     end
@@ -1257,6 +1268,12 @@ module Ci
       self.ci_ref = Ci::Ref.ensure_for(self)
     end
 
+    def ensure_persistent_ref
+      return if persistent_ref.exist?
+
+      persistent_ref.create
+    end
+
     def reset_source_bridge!(current_user)
       return unless bridge_waiting?
 
@@ -1271,10 +1288,11 @@ module Ci
 
     def security_reports(report_types: [])
       reports_scope = report_types.empty? ? ::Ci::JobArtifact.security_reports : ::Ci::JobArtifact.security_reports(file_types: report_types)
+      types_to_collect = report_types.empty? ? ::Ci::JobArtifact::SECURITY_REPORT_FILE_TYPES : report_types
 
       ::Gitlab::Ci::Reports::Security::Reports.new(self).tap do |security_reports|
         latest_report_builds(reports_scope).each do |build|
-          build.collect_security_reports!(security_reports)
+          build.collect_security_reports!(security_reports, report_types: types_to_collect)
         end
       end
     end
diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb
index d79ff74753a..f666629c8fd 100644
--- a/app/models/ci/processable.rb
+++ b/app/models/ci/processable.rb
@@ -101,6 +101,21 @@ module Ci
       :merge_train_pipeline?,
       to: :pipeline
 
+    def clone(current_user:)
+      new_attributes = self.class.clone_accessors.to_h do |attribute|
+        [attribute, public_send(attribute)] # rubocop:disable GitlabSecurity/PublicSend
+      end
+
+      if persisted_environment.present?
+        new_attributes[:metadata_attributes] ||= {}
+        new_attributes[:metadata_attributes][:expanded_environment_name] = expanded_environment_name
+      end
+
+      new_attributes[:user] = current_user
+
+      self.class.new(new_attributes)
+    end
+
     def retryable?
       return false if retried? || archived? || deployment_rejected?
 
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index b9ba9d8e1b0..7a1d52f5aea 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -12,6 +12,7 @@ module Ci
     include Gitlab::Utils::StrongMemoize
     include TaggableQueries
     include Presentable
+    include EachBatch
 
     add_authentication_token_field :token, encrypted: :optional, expires_at: :compute_token_expiration, expiration_enforced?: :token_expiration_enforced?
 
@@ -59,7 +60,7 @@ module Ci
 
     AVAILABLE_TYPES_LEGACY = %w[specific shared].freeze
     AVAILABLE_TYPES = runner_types.keys.freeze
-    AVAILABLE_STATUSES = %w[active paused online offline not_connected never_contacted stale].freeze # TODO: Remove in %15.0: not_connected. In %16.0: active, paused. Relevant issues: https://gitlab.com/gitlab-org/gitlab/-/issues/347303, https://gitlab.com/gitlab-org/gitlab/-/issues/347305, https://gitlab.com/gitlab-org/gitlab/-/issues/344648
+    AVAILABLE_STATUSES = %w[active paused online offline never_contacted stale].freeze # TODO: Remove in %16.0: active, paused. Relevant issue: https://gitlab.com/gitlab-org/gitlab/-/issues/344648
     AVAILABLE_SCOPES = (AVAILABLE_TYPES_LEGACY + AVAILABLE_TYPES + AVAILABLE_STATUSES).freeze
 
     FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level maximum_timeout_human_readable].freeze
@@ -83,7 +84,6 @@ module Ci
     scope :recent, -> { where('ci_runners.created_at >= :date OR ci_runners.contacted_at >= :date', date: stale_deadline) }
     scope :stale, -> { where('ci_runners.created_at < :date AND (ci_runners.contacted_at IS NULL OR ci_runners.contacted_at < :date)', date: stale_deadline) }
     scope :offline, -> { where(arel_table[:contacted_at].lteq(online_contact_time_deadline)) }
-    scope :not_connected, -> { where(contacted_at: nil) } # TODO: Remove in 15.0
     scope :never_contacted, -> { where(contacted_at: nil) }
     scope :ordered, -> { order(id: :desc) }
 
@@ -289,7 +289,7 @@ module Ci
 
     def assign_to(project, current_user = nil)
       if instance_type?
-        self.runner_type = :project_type
+        raise ArgumentError, 'Transitioning an instance runner to a project runner is not supported'
       elsif group_type?
         raise ArgumentError, 'Transitioning a group runner to a project runner is not supported'
       end
@@ -322,6 +322,9 @@ module Ci
     end
 
     def status(legacy_mode = nil)
+      # TODO Deprecate legacy_mode in %16.0 and make it a no-op
+      #   (see https://gitlab.com/gitlab-org/gitlab/-/issues/360545)
+      # TODO Remove legacy_mode in %17.0
       return deprecated_rest_status if legacy_mode == '14.5'
 
       return :stale if stale?
@@ -331,10 +334,12 @@ module Ci
     end
 
     # DEPRECATED
-    # TODO Remove in %16.0 in favor of `status` for REST calls
+    # TODO Remove in %16.0 in favor of `status` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/344648
     def deprecated_rest_status
-      if contacted_at.nil? # TODO Remove in %15.0, see https://gitlab.com/gitlab-org/gitlab/-/issues/344648
-        :not_connected
+      return :stale if stale?
+
+      if contacted_at.nil?
+        :never_contacted
       elsif active?
         online? ? :online : :offline
       else
@@ -462,7 +467,7 @@ module Ci
     end
 
     def self.token_expiration_enforced?
-      Feature.enabled?(:enforce_runner_token_expires_at, default_enabled: :yaml)
+      Feature.enabled?(:enforce_runner_token_expires_at)
     end
 
     private
diff --git a/app/models/ci/secure_file.rb b/app/models/ci/secure_file.rb
index 6a26a5341aa..9c82e106d6e 100644
--- a/app/models/ci/secure_file.rb
+++ b/app/models/ci/secure_file.rb
@@ -3,8 +3,11 @@
 module Ci
   class SecureFile < Ci::ApplicationRecord
     include FileStoreMounter
+    include IgnorableColumns
     include Limitable
 
+    ignore_column :permissions, remove_with: '15.2', remove_after: '2022-06-22'
+
     FILE_SIZE_LIMIT = 5.megabytes.freeze
     CHECKSUM_ALGORITHM = 'sha256'
 
@@ -14,14 +17,12 @@ module Ci
     belongs_to :project, optional: false
 
     validates :file, presence: true, file_size: { maximum: FILE_SIZE_LIMIT }
-    validates :checksum, :file_store, :name, :permissions, :project_id, presence: true
+    validates :checksum, :file_store, :name, :project_id, presence: true
     validates :name, uniqueness: { scope: :project }
 
     after_initialize :generate_key_data
     before_validation :assign_checksum
 
-    enum permissions: { read_only: 0, read_write: 1, execute: 2 }
-
     default_value_for(:file_store) { Ci::SecureFileUploader.default_store }
 
     mount_file_store_uploader Ci::SecureFileUploader