Add latest changes from gitlab-org/gitlab@14-7-stable-eev14.7.0-rc42

7 files changed, 154 insertions, 56 deletions

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 428e440afba..c4d1a2c740b 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -268,6 +268,10 @@ module Ci
         !build.any_unmet_prerequisites? # If false is returned, it stops the transition
       end
 
+      before_transition on: :enqueue do |build|
+        !build.waiting_for_deployment_approval? # If false is returned, it stops the transition
+      end
+
       after_transition created: :scheduled do |build|
         build.run_after_commit do
           Ci::BuildScheduleWorker.perform_at(build.scheduled_at, build.id)
@@ -393,6 +397,10 @@ module Ci
       auto_retry.allowed?
     end
 
+    def auto_retry_expected?
+      failed? && auto_retry_allowed?
+    end
+
     def detailed_status(current_user)
       Gitlab::Ci::Status::Build::Factory
         .new(self.present, current_user)
@@ -416,6 +424,10 @@ module Ci
       true
     end
 
+    def save_tags
+      super unless Thread.current['ci_bulk_insert_tags']
+    end
+
     def archived?
       return true if degenerated?
 
@@ -424,7 +436,11 @@ module Ci
     end
 
     def playable?
-      action? && !archived? && (manual? || scheduled? || retryable?)
+      action? && !archived? && (manual? || scheduled? || retryable?) && !waiting_for_deployment_approval?
+    end
+
+    def waiting_for_deployment_approval?
+      manual? && starts_environment? && deployment&.blocked?
     end
 
     def schedulable?
@@ -751,9 +767,7 @@ module Ci
 
     def any_runners_available?
       cache_for_available_runners do
-        ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339937') do
-          project.active_runners.exists?
-        end
+        project.active_runners.exists?
       end
     end
 
@@ -1021,7 +1035,15 @@ module Ci
       transaction do
         update_columns(status: :failed, failure_reason: :data_integrity_failure)
         all_queuing_entries.delete_all
+        all_runtime_metadata.delete_all
       end
+
+      Gitlab::AppLogger.info(
+        message: 'Build doomed',
+        class: self.class.name,
+        build_id: id,
+        pipeline_id: pipeline_id,
+        project_id: project_id)
     end
 
     def degradation_threshold
@@ -1051,10 +1073,7 @@ module Ci
     end
 
     def drop_with_exit_code!(failure_reason, exit_code)
-      transaction do
-        conditionally_allow_failure!(exit_code)
-        drop!(failure_reason)
-      end
+      drop!(::Gitlab::Ci::Build::Status::Reason.new(self, failure_reason, exit_code))
     end
 
     def exit_codes_defined?
@@ -1065,6 +1084,10 @@ module Ci
       ::Ci::PendingBuild.upsert_from_build!(self)
     end
 
+    def create_runtime_metadata!
+      ::Ci::RunningBuild.upsert_shared_runner_build!(self)
+    end
+
     ##
     # We can have only one queuing entry or running build tracking entry,
     # because there is a unique index on `build_id` in each table, but we need
@@ -1093,6 +1116,13 @@ module Ci
       end
     end
 
+    def allowed_to_fail_with_code?(exit_code)
+      options
+        .dig(:allow_failure_criteria, :exit_codes)
+        .to_a
+        .include?(exit_code)
+    end
+
     protected
 
     def run_status_commit_hooks!
@@ -1174,27 +1204,15 @@ module Ci
         break variables unless Feature.enabled?(:ci_job_jwt, project, default_enabled: true)
 
         jwt = Gitlab::Ci::Jwt.for_build(self)
+        jwt_v2 = Gitlab::Ci::JwtV2.for_build(self)
         variables.append(key: 'CI_JOB_JWT', value: jwt, public: false, masked: true)
+        variables.append(key: 'CI_JOB_JWT_V1', value: jwt, public: false, masked: true)
+        variables.append(key: 'CI_JOB_JWT_V2', value: jwt_v2, public: false, masked: true)
       rescue OpenSSL::PKey::RSAError, Gitlab::Ci::Jwt::NoSigningKeyError => e
         Gitlab::ErrorTracking.track_exception(e)
       end
     end
 
-    def conditionally_allow_failure!(exit_code)
-      return unless exit_code
-
-      if allowed_to_fail_with_code?(exit_code)
-        update_columns(allow_failure: true)
-      end
-    end
-
-    def allowed_to_fail_with_code?(exit_code)
-      options
-        .dig(:allow_failure_criteria, :exit_codes)
-        .to_a
-        .include?(exit_code)
-    end
-
     def cache_for_online_runners(&block)
       Rails.cache.fetch(
         ['has-online-runners', id],
diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb
index e6dd62fab34..3426c4d5248 100644
--- a/app/models/ci/job_artifact.rb
+++ b/app/models/ci/job_artifact.rb
@@ -181,9 +181,7 @@ module Ci
     end
 
     scope :erasable, -> do
-      types = self.file_types.reject { |file_type| NON_ERASABLE_FILE_TYPES.include?(file_type) }.values
-
-      where(file_type: types)
+      where(file_type: self.erasable_file_types)
     end
 
     scope :downloadable, -> { where(file_type: DOWNLOADABLE_TYPES) }
@@ -263,6 +261,10 @@ module Ci
       [file_type]
     end
 
+    def self.erasable_file_types
+      self.file_types.keys - NON_ERASABLE_FILE_TYPES
+    end
+
     def self.total_size
       self.sum(:size)
     end
@@ -271,10 +273,6 @@ module Ci
       self.where(project: project).sum(:size)
     end
 
-    def self.distinct_job_ids
-      distinct.pluck(:job_id)
-    end
-
     ##
     # FastDestroyAll concerns
     # rubocop: disable CodeReuse/ServiceClass
@@ -350,9 +348,7 @@ module Ci
 
     def store_after_commit?
       strong_memoize(:store_after_commit) do
-        trace? &&
-          JobArtifactUploader.direct_upload_enabled? &&
-          Feature.enabled?(:ci_store_trace_outside_transaction, project, default_enabled: :yaml)
+        trace? && JobArtifactUploader.direct_upload_enabled?
       end
     end
 
diff --git a/app/models/ci/namespace_mirror.rb b/app/models/ci/namespace_mirror.rb
index 8a4be3139e8..ce3faf3546b 100644
--- a/app/models/ci/namespace_mirror.rb
+++ b/app/models/ci/namespace_mirror.rb
@@ -10,6 +10,12 @@ module Ci
       where('traversal_ids @> ARRAY[?]::int[]', id)
     end
 
+    scope :contains_any_of_namespaces, -> (ids) do
+      where('traversal_ids && ARRAY[?]::int[]', ids)
+    end
+
+    scope :by_namespace_id, -> (namespace_id) { where(namespace_id: namespace_id) }
+
     class << self
       def sync!(event)
         namespace = event.namespace
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index a90bd739741..00d331df4c3 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -69,6 +69,7 @@ module Ci
     has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline
     has_many :generic_commit_statuses, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'GenericCommitStatus'
     has_many :job_artifacts, through: :builds
+    has_many :build_trace_chunks, class_name: 'Ci::BuildTraceChunk', through: :builds, source: :trace_chunks
     has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent
     has_many :variables, class_name: 'Ci::PipelineVariable'
     has_many :deployments, through: :builds
@@ -130,6 +131,7 @@ module Ci
     after_create :keep_around_commits, unless: :importing?
 
     use_fast_destroy :job_artifacts
+    use_fast_destroy :build_trace_chunks
 
     # We use `Enums::Ci::Pipeline.sources` here so that EE can more easily extend
     # this `Hash` with new values.
@@ -242,11 +244,7 @@ module Ci
             ::JiraConnect::SyncBuildsWorker.perform_async(pipeline.id, seq_id)
           end
 
-          if Feature.enabled?(:expire_job_and_pipeline_cache_synchronously, pipeline.project, default_enabled: :yaml)
-            Ci::ExpirePipelineCacheService.new.execute(pipeline) # rubocop: disable CodeReuse/ServiceClass
-          else
-            ExpirePipelineCacheWorker.perform_async(pipeline.id)
-          end
+          Ci::ExpirePipelineCacheService.new.execute(pipeline) # rubocop: disable CodeReuse/ServiceClass
         end
       end
 
@@ -466,6 +464,14 @@ module Ci
       statuses.count(:id)
     end
 
+    def tags_count
+      ActsAsTaggableOn::Tagging.where(taggable: builds).count
+    end
+
+    def distinct_tags_count
+      ActsAsTaggableOn::Tagging.where(taggable: builds).count('distinct(tag_id)')
+    end
+
     def stages_names
       statuses.order(:stage_idx).distinct
         .pluck(:stage, :stage_idx).map(&:first)
@@ -1284,6 +1290,12 @@ module Ci
       end
     end
 
+    def use_variables_builder_definitions?
+      strong_memoize(:use_variables_builder_definitions) do
+        ::Feature.enabled?(:ci_use_variables_builder_definitions, project, default_enabled: :yaml)
+      end
+    end
+
     private
 
     def add_message(severity, content)
diff --git a/app/models/ci/project_mirror.rb b/app/models/ci/project_mirror.rb
index d6aaa3f50c1..9000d1791a6 100644
--- a/app/models/ci/project_mirror.rb
+++ b/app/models/ci/project_mirror.rb
@@ -6,6 +6,9 @@ module Ci
   class ProjectMirror < ApplicationRecord
     belongs_to :project
 
+    scope :by_namespace_id, -> (namespace_id) { where(namespace_id: namespace_id) }
+    scope :by_project_id, -> (project_id) { where(project_id: project_id) }
+
     class << self
       def sync!(event)
         upsert({ project_id: event.project_id, namespace_id: event.project.namespace_id },
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index a80fd02080f..809c245d2b9 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -95,7 +95,33 @@ module Ci
       joins(:runner_projects).where(ci_runner_projects: { project_id: project_id })
     }
 
-    scope :belonging_to_group, -> (group_id, include_ancestors: false) {
+    scope :belonging_to_group, -> (group_id) {
+      joins(:runner_namespaces)
+        .where(ci_runner_namespaces: { namespace_id: group_id })
+    }
+
+    scope :belonging_to_group_or_project_descendants, -> (group_id) {
+      group_ids = Ci::NamespaceMirror.contains_namespace(group_id).select(:namespace_id)
+      project_ids = Ci::ProjectMirror.by_namespace_id(group_ids).select(:project_id)
+
+      group_runners = joins(:runner_namespaces).where(ci_runner_namespaces: { namespace_id: group_ids })
+      project_runners = joins(:runner_projects).where(ci_runner_projects: { project_id: project_ids })
+
+      union_sql = ::Gitlab::SQL::Union.new([group_runners, project_runners]).to_sql
+
+      from("(#{union_sql}) #{table_name}")
+    }
+
+    scope :belonging_to_group_and_ancestors, -> (group_id) {
+      group_self_and_ancestors_ids = ::Group.find_by(id: group_id)&.self_and_ancestor_ids
+
+      joins(:runner_namespaces)
+        .where(ci_runner_namespaces: { namespace_id: group_self_and_ancestors_ids })
+    }
+
+    # deprecated
+    # split this into: belonging_to_group & belonging_to_group_and_ancestors
+    scope :legacy_belonging_to_group, -> (group_id, include_ancestors: false) {
       groups = ::Group.where(id: group_id)
       groups = groups.self_and_ancestors if include_ancestors
 
@@ -104,7 +130,8 @@ module Ci
         .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433')
     }
 
-    scope :belonging_to_group_or_project, -> (group_id, project_id) {
+    # deprecated
+    scope :legacy_belonging_to_group_or_project, -> (group_id, project_id) {
       groups = ::Group.where(id: group_id)
 
       group_runners = joins(:runner_namespaces).where(ci_runner_namespaces: { namespace_id: groups })
@@ -117,11 +144,11 @@ module Ci
     }
 
     scope :belonging_to_parent_group_of_project, -> (project_id) {
+      raise ArgumentError, "only 1 project_id allowed for performance reasons" unless project_id.is_a?(Integer)
+
       project_groups = ::Group.joins(:projects).where(projects: { id: project_id })
 
-      joins(:groups)
-        .where(namespaces: { id: project_groups.self_and_ancestors.as_ids })
-        .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433')
+      belonging_to_group(project_groups.self_and_ancestors.pluck(:id))
     }
 
     scope :owned_or_instance_wide, -> (project_id) do
@@ -132,7 +159,7 @@ module Ci
           instance_type
         ],
         remove_duplicates: false
-      ).allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433')
+      )
     end
 
     scope :assignable_for, ->(project) do
@@ -183,6 +210,8 @@ module Ci
 
     validates :config, json_schema: { filename: 'ci_runner_config' }
 
+    validates :maintainer_note, length: { maximum: 255 }
+
     # Searches for runners matching the given query.
     #
     # This method uses ILIKE on PostgreSQL for the description field and performs a full match on tokens.
@@ -233,18 +262,16 @@ module Ci
         Arel.sql("(#{arel_tag_names_array.to_sql})")
       ]
 
-      ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339621') do
-        group(*unique_params).pluck('array_agg(ci_runners.id)', *unique_params).map do |values|
-          Gitlab::Ci::Matching::RunnerMatcher.new({
-            runner_ids: values[0],
-            runner_type: values[1],
-            public_projects_minutes_cost_factor: values[2],
-            private_projects_minutes_cost_factor: values[3],
-            run_untagged: values[4],
-            access_level: values[5],
-            tag_list: values[6]
-          })
-        end
+      group(*unique_params).pluck('array_agg(ci_runners.id)', *unique_params).map do |values|
+        Gitlab::Ci::Matching::RunnerMatcher.new({
+          runner_ids: values[0],
+          runner_type: values[1],
+          public_projects_minutes_cost_factor: values[2],
+          private_projects_minutes_cost_factor: values[3],
+          run_untagged: values[4],
+          access_level: values[5],
+          tag_list: values[6]
+        })
       end
     end
 
@@ -441,6 +468,7 @@ module Ci
     private
 
     EXECUTOR_NAME_TO_TYPES = {
+      'unknown' => :unknown,
       'custom' => :custom,
       'shell' => :shell,
       'docker' => :docker,
@@ -454,6 +482,8 @@ module Ci
       'kubernetes' => :kubernetes
     }.freeze
 
+    EXECUTOR_TYPE_TO_NAMES = EXECUTOR_NAME_TO_TYPES.invert.freeze
+
     def cleanup_runner_queue
       Gitlab::Redis::SharedState.with do |redis|
         redis.del(runner_queue_key)
diff --git a/app/models/ci/secure_file.rb b/app/models/ci/secure_file.rb
new file mode 100644
index 00000000000..56f632b6232
--- /dev/null
+++ b/app/models/ci/secure_file.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Ci
+  class SecureFile < Ci::ApplicationRecord
+    include FileStoreMounter
+
+    FILE_SIZE_LIMIT = 5.megabytes.freeze
+    CHECKSUM_ALGORITHM = 'sha256'
+
+    belongs_to :project, optional: false
+
+    validates :file, presence: true, file_size: { maximum: FILE_SIZE_LIMIT }
+    validates :checksum, :file_store, :name, :permissions, :project_id, presence: true
+
+    before_validation :assign_checksum
+
+    enum permissions: { read_only: 0, read_write: 1, execute: 2 }
+
+    default_value_for(:file_store) { Ci::SecureFileUploader.default_store }
+
+    mount_file_store_uploader Ci::SecureFileUploader
+
+    def checksum_algorithm
+      CHECKSUM_ALGORITHM
+    end
+
+    private
+
+    def assign_checksum
+      self.checksum = file.checksum if file.present? && file_changed?
+    end
+  end
+end