Add latest changes from gitlab-org/gitlab@14-3-stable-eev14.3.0-rc42

11 files changed, 189 insertions, 99 deletions

diff --git a/app/models/ci/application_record.rb b/app/models/ci/application_record.rb
index 9d4a8f0648e..913e7a62c66 100644
--- a/app/models/ci/application_record.rb
+++ b/app/models/ci/application_record.rb
@@ -2,6 +2,7 @@
 
 module Ci
   class ApplicationRecord < ::ApplicationRecord
+    self.gitlab_schema = :gitlab_ci
     self.abstract_class = true
 
     def self.table_name_prefix
diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb
index 577bca282ef..97fb8233d34 100644
--- a/app/models/ci/bridge.rb
+++ b/app/models/ci/bridge.rb
@@ -28,10 +28,10 @@ module Ci
 
     state_machine :status do
       after_transition [:created, :manual, :waiting_for_resource] => :pending do |bridge|
-        next unless bridge.downstream_project
+        next unless bridge.triggers_downstream_pipeline?
 
         bridge.run_after_commit do
-          bridge.schedule_downstream_pipeline!
+          ::Ci::CreateCrossProjectPipelineWorker.perform_async(bridge.id)
         end
       end
 
@@ -64,12 +64,6 @@ module Ci
       )
     end
 
-    def schedule_downstream_pipeline!
-      raise InvalidBridgeTypeError unless downstream_project
-
-      ::Ci::CreateCrossProjectPipelineWorker.perform_async(self.id)
-    end
-
     def inherit_status_from_downstream!(pipeline)
       case pipeline.status
       when 'success'
@@ -112,10 +106,18 @@ module Ci
       pipeline if triggers_child_pipeline?
     end
 
+    def triggers_downstream_pipeline?
+      triggers_child_pipeline? || triggers_cross_project_pipeline?
+    end
+
     def triggers_child_pipeline?
       yaml_for_downstream.present?
     end
 
+    def triggers_cross_project_pipeline?
+      downstream_project_path.present?
+    end
+
     def tags
       [:bridge]
     end
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 1ca291a659b..e2e24247679 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -90,6 +90,10 @@ module Ci
       end
     end
 
+    def persisted_environment=(environment)
+      strong_memoize(:persisted_environment) { environment }
+    end
+
     serialize :options # rubocop:disable Cop/ActiveRecordSerialize
     serialize :yaml_variables, Gitlab::Serializer::Ci::Variables # rubocop:disable Cop/ActiveRecordSerialize
 
@@ -166,8 +170,6 @@ module Ci
     scope :with_stale_live_trace, -> { with_live_trace.finished_before(12.hours.ago) }
     scope :finished_before, -> (date) { finished.where('finished_at < ?', date) }
 
-    scope :with_secure_reports_from_options, -> (job_type) { where('options like :job_type', job_type: "%:artifacts:%:reports:%:#{job_type}:%") }
-
     scope :with_secure_reports_from_config_options, -> (job_types) do
       joins(:metadata).where("ci_builds_metadata.config_options -> 'artifacts' -> 'reports' ?| array[:job_types]", job_types: job_types)
     end
@@ -306,7 +308,9 @@ module Ci
       end
 
       after_transition pending: :running do |build|
-        build.deployment&.run
+        Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do
+          build.deployment&.run
+        end
 
         build.run_after_commit do
           build.pipeline.persistent_ref.create
@@ -328,7 +332,9 @@ module Ci
       end
 
       after_transition any => [:success] do |build|
-        build.deployment&.succeed
+        Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do
+          build.deployment&.succeed
+        end
 
         build.run_after_commit do
           BuildSuccessWorker.perform_async(id)
@@ -341,7 +347,9 @@ module Ci
         next unless build.deployment
 
         begin
-          build.deployment.drop!
+          Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do
+            build.deployment.drop!
+          end
         rescue StandardError => e
           Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, build_id: build.id)
         end
@@ -362,10 +370,12 @@ module Ci
       end
 
       after_transition any => [:skipped, :canceled] do |build, transition|
-        if transition.to_name == :skipped
-          build.deployment&.skip
-        else
-          build.deployment&.cancel
+        Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do
+          if transition.to_name == :skipped
+            build.deployment&.skip
+          else
+            build.deployment&.cancel
+          end
         end
       end
     end
@@ -712,6 +722,10 @@ module Ci
       update_column(:trace, nil)
     end
 
+    def ensure_trace_metadata!
+      Ci::BuildTraceMetadata.find_or_upsert_for!(id)
+    end
+
     def artifacts_expose_as
       options.dig(:artifacts, :expose_as)
     end
@@ -748,7 +762,9 @@ module Ci
 
     def any_runners_available?
       cache_for_available_runners do
-        project.active_runners.exists?
+        ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339937') do
+          project.active_runners.exists?
+        end
       end
     end
 
@@ -1013,9 +1029,10 @@ module Ci
 
     # Consider this object to have a structural integrity problems
     def doom!
-      update_columns(
-        status: :failed,
-        failure_reason: :data_integrity_failure)
+      transaction do
+        update_columns(status: :failed, failure_reason: :data_integrity_failure)
+        all_queuing_entries.delete_all
+      end
     end
 
     def degradation_threshold
diff --git a/app/models/ci/build_trace_chunks/fog.rb b/app/models/ci/build_trace_chunks/fog.rb
index 3bfac2b33c0..1cae2279434 100644
--- a/app/models/ci/build_trace_chunks/fog.rb
+++ b/app/models/ci/build_trace_chunks/fog.rb
@@ -80,12 +80,10 @@ module Ci
       private
 
       def append_strings(old_data, new_data)
-        if Feature.enabled?(:ci_job_trace_force_encode, default_enabled: :yaml)
-          # When object storage is in use, old_data may be retrieved in UTF-8.
-          old_data = old_data.force_encoding(Encoding::ASCII_8BIT)
-          # new_data should already be in ASCII-8BIT, but just in case it isn't, do this.
-          new_data = new_data.force_encoding(Encoding::ASCII_8BIT)
-        end
+        # When object storage is in use, old_data may be retrieved in UTF-8.
+        old_data = old_data.force_encoding(Encoding::ASCII_8BIT)
+        # new_data should already be in ASCII-8BIT, but just in case it isn't, do this.
+        new_data = new_data.force_encoding(Encoding::ASCII_8BIT)
 
         old_data + new_data
       end
diff --git a/app/models/ci/build_trace_metadata.rb b/app/models/ci/build_trace_metadata.rb
index 05bdb3d8b7b..901b84ceec6 100644
--- a/app/models/ci/build_trace_metadata.rb
+++ b/app/models/ci/build_trace_metadata.rb
@@ -2,6 +2,7 @@
 
 module Ci
   class BuildTraceMetadata < Ci::ApplicationRecord
+    MAX_ATTEMPTS = 5
     self.table_name = 'ci_build_trace_metadata'
     self.primary_key = :build_id
 
@@ -9,5 +10,49 @@ module Ci
     belongs_to :trace_artifact, class_name: 'Ci::JobArtifact'
 
     validates :build, presence: true
+    validates :archival_attempts, presence: true
+
+    def self.find_or_upsert_for!(build_id)
+      record = find_by(build_id: build_id)
+      return record if record
+
+      upsert({ build_id: build_id }, unique_by: :build_id)
+      find_by!(build_id: build_id)
+    end
+
+    # The job is retried around 5 times during the 7 days retention period for
+    # trace chunks as defined in `Ci::BuildTraceChunks::RedisBase::CHUNK_REDIS_TTL`
+    def can_attempt_archival_now?
+      return false unless archival_attempts_available?
+      return true unless last_archival_attempt_at
+
+      last_archival_attempt_at + backoff < Time.current
+    end
+
+    def archival_attempts_available?
+      archival_attempts <= MAX_ATTEMPTS
+    end
+
+    def increment_archival_attempts!
+      increment!(:archival_attempts, touch: :last_archival_attempt_at)
+    end
+
+    def track_archival!(trace_artifact_id)
+      update!(trace_artifact_id: trace_artifact_id, archived_at: Time.current)
+    end
+
+    def archival_attempts_message
+      if archival_attempts_available?
+        'The job can not be archived right now.'
+      else
+        'The job is out of archival attempts.'
+      end
+    end
+
+    private
+
+    def backoff
+      ::Gitlab::Ci::Trace::Backoff.new(archival_attempts).value_with_jitter
+    end
   end
 end
diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb
index 1f0da4345f2..ad3e867f9d5 100644
--- a/app/models/ci/job_artifact.rb
+++ b/app/models/ci/job_artifact.rb
@@ -10,6 +10,9 @@ module Ci
     include Artifactable
     include FileStoreMounter
     include EachBatch
+    include IgnorableColumns
+
+    ignore_columns %i[id_convert_to_bigint job_id_convert_to_bigint], remove_with: '14.5', remove_after: '2021-11-22'
 
     TEST_REPORT_FILE_TYPES = %w[junit].freeze
     COVERAGE_REPORT_FILE_TYPES = %w[cobertura].freeze
@@ -182,7 +185,6 @@ module Ci
     scope :order_expired_desc, -> { order(expire_at: :desc) }
     scope :with_destroy_preloads, -> { includes(project: [:route, :statistics]) }
 
-    scope :scoped_project, -> { where('ci_job_artifacts.project_id = projects.id') }
     scope :for_project, ->(project) { where(project_id: project) }
     scope :created_in_time_range, ->(from: nil, to: nil) { where(created_at: from..to) }
 
@@ -232,6 +234,17 @@ module Ci
       hashed_path: 2
     }
 
+    # `locked` will be populated from the source of truth on Ci::Pipeline
+    # in order to clean up expired job artifacts in a performant way.
+    # The values should be the same as `Ci::Pipeline.lockeds` with the
+    # additional value of `unknown` to indicate rows that have not
+    # yet been populated from the parent Ci::Pipeline
+    enum locked: {
+      unlocked: 0,
+      artifacts_locked: 1,
+      unknown: 2
+    }, _prefix: :artifact
+
     def validate_file_format!
       unless TYPE_AND_FORMAT_PAIRS[self.file_type&.to_sym] == self.file_format&.to_sym
         errors.add(:base, _('Invalid file format with specified file type'))
diff --git a/app/models/ci/pending_build.rb b/app/models/ci/pending_build.rb
index 7cf3a387516..ccad6290fac 100644
--- a/app/models/ci/pending_build.rb
+++ b/app/models/ci/pending_build.rb
@@ -2,6 +2,8 @@
 
 module Ci
   class PendingBuild < Ci::ApplicationRecord
+    include EachBatch
+
     belongs_to :project
     belongs_to :build, class_name: 'Ci::Build'
     belongs_to :namespace, inverse_of: :pending_builds, class_name: 'Namespace'
@@ -11,52 +13,62 @@ module Ci
     scope :ref_protected, -> { where(protected: true) }
     scope :queued_before, ->(time) { where(arel_table[:created_at].lt(time)) }
     scope :with_instance_runners, -> { where(instance_runners_enabled: true) }
+    scope :for_tags, ->(tag_ids) do
+      if tag_ids.present?
+        where('ci_pending_builds.tag_ids <@ ARRAY[?]::int[]', Array.wrap(tag_ids))
+      else
+        where("ci_pending_builds.tag_ids = '{}'")
+      end
+    end
 
-    def self.upsert_from_build!(build)
-      entry = self.new(args_from_build(build))
+    class << self
+      def upsert_from_build!(build)
+        entry = self.new(args_from_build(build))
 
-      entry.validate!
+        entry.validate!
 
-      self.upsert(entry.attributes.compact, returning: %w[build_id], unique_by: :build_id)
-    end
+        self.upsert(entry.attributes.compact, returning: %w[build_id], unique_by: :build_id)
+      end
 
-    def self.args_from_build(build)
-      args = {
-        build: build,
-        project: build.project,
-        protected: build.protected?,
-        namespace: build.project.namespace
-      }
+      private
+
+      def args_from_build(build)
+        project = build.project
+
+        args = {
+          build: build,
+          project: project,
+          protected: build.protected?,
+          namespace: project.namespace
+        }
+
+        if Feature.enabled?(:ci_pending_builds_maintain_tags_data, type: :development, default_enabled: :yaml)
+          args.store(:tag_ids, build.tags_ids)
+        end
+
+        if Feature.enabled?(:ci_pending_builds_maintain_shared_runners_data, type: :development, default_enabled: :yaml)
+          args.store(:instance_runners_enabled, shared_runners_enabled?(project))
+        end
+
+        if Feature.enabled?(:ci_pending_builds_maintain_namespace_traversal_ids, type: :development, default_enabled: :yaml)
+          args.store(:namespace_traversal_ids, project.namespace.traversal_ids) if group_runners_enabled?(project)
+        end
 
-      if Feature.enabled?(:ci_pending_builds_maintain_shared_runners_data, type: :development, default_enabled: :yaml)
-        args.merge(instance_runners_enabled: shareable?(build))
-      else
         args
       end
-    end
-    private_class_method :args_from_build
-
-    def self.shareable?(build)
-      shared_runner_enabled?(build) &&
-        builds_access_level?(build) &&
-        project_not_removed?(build)
-    end
-    private_class_method :shareable?
 
-    def self.shared_runner_enabled?(build)
-      build.project.shared_runners.exists?
-    end
-    private_class_method :shared_runner_enabled?
+      def shared_runners_enabled?(project)
+        builds_enabled?(project) && project.shared_runners_enabled?
+      end
 
-    def self.project_not_removed?(build)
-      !build.project.pending_delete?
-    end
-    private_class_method :project_not_removed?
+      def group_runners_enabled?(project)
+        builds_enabled?(project) && project.group_runners_enabled?
+      end
 
-    def self.builds_access_level?(build)
-      build.project.project_feature.builds_access_level.nil? || build.project.project_feature.builds_access_level > 0
+      def builds_enabled?(project)
+        project.builds_enabled? && !project.pending_delete?
+      end
     end
-    private_class_method :builds_access_level?
   end
 end
 
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index 70e67953e31..1a0cec3c935 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -66,6 +66,7 @@ module Ci
     has_many :processables, class_name: 'Ci::Processable', foreign_key: :commit_id, inverse_of: :pipeline
     has_many :bridges, class_name: 'Ci::Bridge', foreign_key: :commit_id, inverse_of: :pipeline
     has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline
+    has_many :generic_commit_statuses, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'GenericCommitStatus'
     has_many :job_artifacts, through: :builds
     has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent
     has_many :variables, class_name: 'Ci::PipelineVariable'
@@ -307,6 +308,7 @@ module Ci
     scope :ci_and_parent_sources, -> { where(source: Enums::Ci::Pipeline.ci_and_parent_sources.values) }
     scope :for_user, -> (user) { where(user: user) }
     scope :for_sha, -> (sha) { where(sha: sha) }
+    scope :where_not_sha, -> (sha) { where.not(sha: sha) }
     scope :for_source_sha, -> (source_sha) { where(source_sha: source_sha) }
     scope :for_sha_or_source_sha, -> (sha) { for_sha(sha).or(for_source_sha(sha)) }
     scope :for_ref, -> (ref) { where(ref: ref) }
@@ -317,7 +319,6 @@ module Ci
     scope :created_after, -> (time) { where('ci_pipelines.created_at > ?', time) }
     scope :created_before_id, -> (id) { where('ci_pipelines.id < ?', id) }
     scope :before_pipeline, -> (pipeline) { created_before_id(pipeline.id).outside_pipeline_family(pipeline) }
-    scope :eager_load_project, -> { eager_load(project: [:route, { namespace: :route }]) }
     scope :with_pipeline_source, -> (source) { where(source: source)}
 
     scope :outside_pipeline_family, ->(pipeline) do
@@ -588,13 +589,11 @@ module Ci
     end
 
     def cancel_running(retries: 1)
-      commit_status_relations = [:project, :pipeline]
-      ci_build_relations = [:deployment, :taggings]
+      preloaded_relations = [:project, :pipeline, :deployment, :taggings]
 
       retry_lock(cancelable_statuses, retries, name: 'ci_pipeline_cancel_running') do |cancelables|
         cancelables.find_in_batches do |batch|
-          ActiveRecord::Associations::Preloader.new.preload(batch, commit_status_relations)
-          ActiveRecord::Associations::Preloader.new.preload(batch.select { |job| job.is_a?(Ci::Build) }, ci_build_relations)
+          Preloaders::CommitStatusPreloader.new(batch).execute(preloaded_relations)
 
           batch.each do |job|
             yield(job) if block_given?
@@ -1108,7 +1107,7 @@ module Ci
           merge_request.modified_paths
         elsif branch_updated?
           push_details.modified_paths
-        elsif external_pull_request? && ::Feature.enabled?(:ci_modified_paths_of_external_prs, project, default_enabled: :yaml)
+        elsif external_pull_request?
           external_pull_request.modified_paths
         end
       end
@@ -1220,24 +1219,12 @@ module Ci
       self.ci_ref = Ci::Ref.ensure_for(self)
     end
 
-    # We need `base_and_ancestors` in a specific order to "break" when needed.
-    # If we use `find_each`, then the order is broken.
-    # rubocop:disable Rails/FindEach
     def reset_source_bridge!(current_user)
-      if ::Feature.enabled?(:ci_reset_bridge_with_subsequent_jobs, project, default_enabled: :yaml)
-        return unless bridge_waiting?
+      return unless bridge_waiting?
 
-        source_bridge.pending!
-        Ci::AfterRequeueJobService.new(project, current_user).execute(source_bridge) # rubocop:disable CodeReuse/ServiceClass
-      else
-        self_and_upstreams.includes(:source_bridge).each do |pipeline|
-          break unless pipeline.bridge_waiting?
-
-          pipeline.source_bridge.pending!
-        end
-      end
+      source_bridge.pending!
+      Ci::AfterRequeueJobService.new(project, current_user).execute(source_bridge) # rubocop:disable CodeReuse/ServiceClass
     end
-    # rubocop:enable Rails/FindEach
 
     # EE-only
     def merge_train_pipeline?
diff --git a/app/models/ci/pipeline_variable.rb b/app/models/ci/pipeline_variable.rb
index a0e8886414b..3dca77af051 100644
--- a/app/models/ci/pipeline_variable.rb
+++ b/app/models/ci/pipeline_variable.rb
@@ -8,7 +8,7 @@ module Ci
 
     alias_attribute :secret_value, :value
 
-    validates :key, uniqueness: { scope: :pipeline_id }
+    validates :key, presence: true
 
     def hook_attrs
       { key: key, value: value }
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index 432c3a408a9..4aa232ad26b 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -208,16 +208,18 @@ module Ci
         Arel.sql("(#{arel_tag_names_array.to_sql})")
       ]
 
-      group(*unique_params).pluck('array_agg(ci_runners.id)', *unique_params).map do |values|
-        Gitlab::Ci::Matching::RunnerMatcher.new({
-          runner_ids: values[0],
-          runner_type: values[1],
-          public_projects_minutes_cost_factor: values[2],
-          private_projects_minutes_cost_factor: values[3],
-          run_untagged: values[4],
-          access_level: values[5],
-          tag_list: values[6]
-        })
+      ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339621') do
+        group(*unique_params).pluck('array_agg(ci_runners.id)', *unique_params).map do |values|
+          Gitlab::Ci::Matching::RunnerMatcher.new({
+            runner_ids: values[0],
+            runner_type: values[1],
+            public_projects_minutes_cost_factor: values[2],
+            private_projects_minutes_cost_factor: values[3],
+            run_untagged: values[4],
+            access_level: values[5],
+            tag_list: values[6]
+          })
+        end
       end
     end
 
@@ -385,6 +387,12 @@ module Ci
       read_attribute(:contacted_at)
     end
 
+    def namespace_ids
+      strong_memoize(:namespace_ids) do
+        runner_namespaces.pluck(:namespace_id).compact
+      end
+    end
+
     private
 
     def cleanup_runner_queue
@@ -420,14 +428,18 @@ module Ci
     end
 
     def no_projects
-      if projects.any?
-        errors.add(:runner, 'cannot have projects assigned')
+      ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338659') do
+        if projects.any?
+          errors.add(:runner, 'cannot have projects assigned')
+        end
       end
     end
 
     def no_groups
-      if groups.any?
-        errors.add(:runner, 'cannot have groups assigned')
+      ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338659') do
+        if groups.any?
+          errors.add(:runner, 'cannot have groups assigned')
+        end
       end
     end
 
diff --git a/app/models/ci/sources/pipeline.rb b/app/models/ci/sources/pipeline.rb
index f78caf710a6..95842d944f9 100644
--- a/app/models/ci/sources/pipeline.rb
+++ b/app/models/ci/sources/pipeline.rb
@@ -4,6 +4,9 @@ module Ci
   module Sources
     class Pipeline < Ci::ApplicationRecord
       include Ci::NamespacedModelName
+      include IgnorableColumns
+
+      ignore_columns 'source_job_id_convert_to_bigint', remove_with: '14.5', remove_after: '2021-11-22'
 
       self.table_name = "ci_sources_pipelines"