Add latest changes from gitlab-org/gitlab@14-4-stable-eev14.4.0-rc42

16 files changed, 135 insertions, 69 deletions

diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb
index 97fb8233d34..50bda64d537 100644
--- a/app/models/ci/bridge.rb
+++ b/app/models/ci/bridge.rb
@@ -31,7 +31,7 @@ module Ci
         next unless bridge.triggers_downstream_pipeline?
 
         bridge.run_after_commit do
-          ::Ci::CreateCrossProjectPipelineWorker.perform_async(bridge.id)
+          ::Ci::CreateDownstreamPipelineWorker.perform_async(bridge.id)
         end
       end
 
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index e2e24247679..990ef71a457 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -42,6 +42,10 @@ module Ci
     has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id, inverse_of: :build
     has_many :report_results, class_name: 'Ci::BuildReportResult', inverse_of: :build
 
+    # Projects::DestroyService destroys Ci::Pipelines, which use_fast_destroy on :job_artifacts
+    # before we delete builds. By doing this, the relation should be empty and not fire any
+    # DELETE queries when the Ci::Build is destroyed. The next step is to remove `dependent: :destroy`.
+    # Details: https://gitlab.com/gitlab-org/gitlab/-/issues/24644#note_689472685
     has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent
     has_many :job_variables, class_name: 'Ci::JobVariable', foreign_key: :job_id
     has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_job_id
@@ -55,6 +59,8 @@ module Ci
     has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build
     has_one :trace_metadata, class_name: 'Ci::BuildTraceMetadata', inverse_of: :build
 
+    has_many :terraform_state_versions, class_name: 'Terraform::StateVersion', dependent: :nullify, inverse_of: :build, foreign_key: :ci_build_id # rubocop:disable Cop/ActiveRecordDependent
+
     accepts_nested_attributes_for :runner_session, update_only: true
     accepts_nested_attributes_for :job_variables
 
@@ -64,8 +70,8 @@ module Ci
     delegate :gitlab_deploy_token, to: :project
     delegate :trigger_short_token, to: :trigger_request, allow_nil: true
 
-    ignore_columns :id_convert_to_bigint, remove_with: '14.1', remove_after: '2021-07-22'
-    ignore_columns :stage_id_convert_to_bigint, remove_with: '14.1', remove_after: '2021-07-22'
+    ignore_columns :id_convert_to_bigint, remove_with: '14.5', remove_after: '2021-10-22'
+    ignore_columns :stage_id_convert_to_bigint, remove_with: '14.5', remove_after: '2021-10-22'
 
     ##
     # Since Gitlab 11.5, deployments records started being created right after
@@ -192,7 +198,6 @@ module Ci
     add_authentication_token_field :token, encrypted: :required
 
     before_save :ensure_token
-    before_destroy { unscoped_project }
 
     after_save :stick_build_if_status_changed
 
@@ -308,8 +313,10 @@ module Ci
       end
 
       after_transition pending: :running do |build|
-        Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do
-          build.deployment&.run
+        unless build.update_deployment_after_transaction_commit?
+          Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do
+            build.deployment&.run
+          end
         end
 
         build.run_after_commit do
@@ -332,8 +339,10 @@ module Ci
       end
 
       after_transition any => [:success] do |build|
-        Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do
-          build.deployment&.succeed
+        unless build.update_deployment_after_transaction_commit?
+          Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do
+            build.deployment&.succeed
+          end
         end
 
         build.run_after_commit do
@@ -346,12 +355,14 @@ module Ci
         next unless build.project
         next unless build.deployment
 
-        begin
-          Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do
-            build.deployment.drop!
+        unless build.update_deployment_after_transaction_commit?
+          begin
+            Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do
+              build.deployment.drop!
+            end
+          rescue StandardError => e
+            Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, build_id: build.id)
           end
-        rescue StandardError => e
-          Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, build_id: build.id)
         end
 
         true
@@ -370,14 +381,29 @@ module Ci
       end
 
       after_transition any => [:skipped, :canceled] do |build, transition|
-        Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do
-          if transition.to_name == :skipped
-            build.deployment&.skip
-          else
-            build.deployment&.cancel
+        unless build.update_deployment_after_transaction_commit?
+          Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do
+            if transition.to_name == :skipped
+              build.deployment&.skip
+            else
+              build.deployment&.cancel
+            end
           end
         end
       end
+
+      # Synchronize Deployment Status
+      # Please note that the data integirty is not assured because we can't use
+      # a database transaction due to DB decomposition.
+      after_transition do |build, transition|
+        next if transition.loopback?
+        next unless build.project
+        next unless build.update_deployment_after_transaction_commit?
+
+        build.run_after_commit do
+          build.deployment&.sync_status_with(build)
+        end
+      end
     end
 
     def self.build_matchers(project)
@@ -1094,6 +1120,12 @@ module Ci
       runner&.instance_type?
     end
 
+    def update_deployment_after_transaction_commit?
+      strong_memoize(:update_deployment_after_transaction_commit) do
+        Feature.enabled?(:update_deployment_after_transaction_commit, project, default_enabled: :yaml)
+      end
+    end
+
     protected
 
     def run_status_commit_hooks!
@@ -1108,7 +1140,7 @@ module Ci
       return unless saved_change_to_status?
       return unless running?
 
-      ::Gitlab::Database::LoadBalancing::Sticking.stick(:build, id)
+      self.class.sticking.stick(:build, id)
     end
 
     def status_commit_hooks
@@ -1154,10 +1186,6 @@ module Ci
       self.update(erased_by: user, erased_at: Time.current, artifacts_expire_at: nil)
     end
 
-    def unscoped_project
-      @unscoped_project ||= Project.unscoped.find_by(id: project_id)
-    end
-
     def environment_url
       options&.dig(:environment, :url) || persisted_environment&.external_url
     end
diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb
index 90237a4be52..0d6d6f7a6a5 100644
--- a/app/models/ci/build_metadata.rb
+++ b/app/models/ci/build_metadata.rb
@@ -37,8 +37,8 @@ module Ci
         job_timeout_source: 4
     }
 
-    ignore_column :build_id_convert_to_bigint, remove_with: '14.2', remove_after: '2021-08-22'
-    ignore_columns :id_convert_to_bigint, remove_with: '14.3', remove_after: '2021-09-22'
+    ignore_column :build_id_convert_to_bigint, remove_with: '14.5', remove_after: '2021-10-22'
+    ignore_columns :id_convert_to_bigint, remove_with: '14.5', remove_after: '2021-10-22'
 
     def update_timeout_state
       timeout = timeout_with_highest_precedence
diff --git a/app/models/ci/build_need.rb b/app/models/ci/build_need.rb
index 003659570b3..bf1470ca20f 100644
--- a/app/models/ci/build_need.rb
+++ b/app/models/ci/build_need.rb
@@ -5,8 +5,6 @@ module Ci
     include BulkInsertSafe
     include IgnorableColumns
 
-    ignore_columns :build_id_convert_to_bigint, remove_with: '14.1', remove_after: '2021-07-22'
-
     belongs_to :build, class_name: "Ci::Processable", foreign_key: :build_id, inverse_of: :needs
 
     validates :build, presence: true
diff --git a/app/models/ci/build_runner_session.rb b/app/models/ci/build_runner_session.rb
index 45de47116cd..e12c0f82c99 100644
--- a/app/models/ci/build_runner_session.rb
+++ b/app/models/ci/build_runner_session.rb
@@ -6,8 +6,6 @@ module Ci
   class BuildRunnerSession < Ci::ApplicationRecord
     include IgnorableColumns
 
-    ignore_columns :build_id_convert_to_bigint, remove_with: '14.1', remove_after: '2021-07-22'
-
     TERMINAL_SUBPROTOCOL = 'terminal.gitlab.com'
     DEFAULT_SERVICE_NAME = 'build'
     DEFAULT_PORT_NAME = 'default_port'
diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb
index 7a15d7ba940..6edb5ef4579 100644
--- a/app/models/ci/build_trace_chunk.rb
+++ b/app/models/ci/build_trace_chunk.rb
@@ -9,8 +9,6 @@ module Ci
     include ::Gitlab::OptimisticLocking
     include IgnorableColumns
 
-    ignore_columns :build_id_convert_to_bigint, remove_with: '14.1', remove_after: '2021-07-22'
-
     belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id
 
     default_value_for :data_store, :redis_trace_chunks
diff --git a/app/models/ci/build_trace_metadata.rb b/app/models/ci/build_trace_metadata.rb
index 901b84ceec6..1ffa0e31f99 100644
--- a/app/models/ci/build_trace_metadata.rb
+++ b/app/models/ci/build_trace_metadata.rb
@@ -37,8 +37,10 @@ module Ci
       increment!(:archival_attempts, touch: :last_archival_attempt_at)
     end
 
-    def track_archival!(trace_artifact_id)
-      update!(trace_artifact_id: trace_artifact_id, archived_at: Time.current)
+    def track_archival!(trace_artifact_id, checksum)
+      update!(trace_artifact_id: trace_artifact_id,
+        checksum: checksum,
+        archived_at: Time.current)
     end
 
     def archival_attempts_message
@@ -49,6 +51,11 @@ module Ci
       end
     end
 
+    def remote_checksum_valid?
+      checksum.present? &&
+        checksum == remote_checksum
+    end
+
     private
 
     def backoff
diff --git a/app/models/ci/job_token/project_scope_link.rb b/app/models/ci/job_token/project_scope_link.rb
index 99118f8090b..c2ab8ca0929 100644
--- a/app/models/ci/job_token/project_scope_link.rb
+++ b/app/models/ci/job_token/project_scope_link.rb
@@ -5,7 +5,7 @@
 
 module Ci
   module JobToken
-    class ProjectScopeLink < ApplicationRecord
+    class ProjectScopeLink < Ci::ApplicationRecord
       self.table_name = 'ci_job_token_project_scope_links'
 
       belongs_to :source_project, class_name: 'Project'
diff --git a/app/models/ci/job_token/scope.rb b/app/models/ci/job_token/scope.rb
index 42cfdc21d66..3a5765aa00c 100644
--- a/app/models/ci/job_token/scope.rb
+++ b/app/models/ci/job_token/scope.rb
@@ -32,12 +32,15 @@ module Ci
       def all_projects
         Project.from_union([
           Project.id_in(source_project),
-          Project.where_exists(
-            Ci::JobToken::ProjectScopeLink
-              .from_project(source_project)
-              .where('projects.id = ci_job_token_project_scope_links.target_project_id'))
+          Project.id_in(target_project_ids)
         ], remove_duplicates: false)
       end
+
+      private
+
+      def target_project_ids
+        Ci::JobToken::ProjectScopeLink.from_project(source_project).pluck(:target_project_id)
+      end
     end
   end
 end
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index 1a0cec3c935..0041ec5135c 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -82,7 +82,8 @@ module Ci
     # Merge requests for which the current pipeline is running against
     # the merge request's latest commit.
     has_many :merge_requests_as_head_pipeline, foreign_key: "head_pipeline_id", class_name: 'MergeRequest'
-
+    has_many :package_build_infos, class_name: 'Packages::BuildInfo', dependent: :nullify, inverse_of: :pipeline # rubocop:disable Cop/ActiveRecordDependent
+    has_many :package_file_build_infos, class_name: 'Packages::PackageFileBuildInfo', dependent: :nullify, inverse_of: :pipeline # rubocop:disable Cop/ActiveRecordDependent
     has_many :pending_builds, -> { pending }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline
     has_many :failed_builds, -> { latest.failed }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline
     has_many :retryable_builds, -> { latest.failed_or_canceled.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline
@@ -861,11 +862,6 @@ module Ci
       self.duration = Gitlab::Ci::Pipeline::Duration.from_pipeline(self)
     end
 
-    def execute_hooks
-      project.execute_hooks(pipeline_data, :pipeline_hooks) if project.has_active_hooks?(:pipeline_hooks)
-      project.execute_integrations(pipeline_data, :pipeline_hooks) if project.has_active_integrations?(:pipeline_hooks)
-    end
-
     # All the merge requests for which the current pipeline runs/ran against
     def all_merge_requests
       @all_merge_requests ||=
@@ -929,9 +925,22 @@ module Ci
     end
 
     def environments_in_self_and_descendants
-      environment_ids = self_and_descendants.joins(:deployments).select(:'deployments.environment_id')
+      if ::Feature.enabled?(:avoid_cross_joins_environments_in_self_and_descendants, default_enabled: :yaml)
+        # We limit to 100 unique environments for application safety.
+        # See: https://gitlab.com/gitlab-org/gitlab/-/issues/340781#note_699114700
+        expanded_environment_names =
+          builds_in_self_and_descendants.joins(:metadata)
+                                        .where.not('ci_builds_metadata.expanded_environment_name' => nil)
+                                        .distinct('ci_builds_metadata.expanded_environment_name')
+                                        .limit(100)
+                                        .pluck(:expanded_environment_name)
+
+        Environment.where(project: project, name: expanded_environment_names)
+      else
+        environment_ids = self_and_descendants.joins(:deployments).select(:'deployments.environment_id')
 
-      Environment.where(id: environment_ids)
+        Environment.where(id: environment_ids)
+      end
     end
 
     # With multi-project and parent-child pipelines
@@ -1251,12 +1260,6 @@ module Ci
       messages.build(severity: severity, content: content)
     end
 
-    def pipeline_data
-      strong_memoize(:pipeline_data) do
-        Gitlab::DataBuilder::Pipeline.build(self)
-      end
-    end
-
     def merge_request_diff_sha
       return unless merge_request?
 
diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb
index 30d335fd7d5..372df8cc264 100644
--- a/app/models/ci/processable.rb
+++ b/app/models/ci/processable.rb
@@ -58,7 +58,8 @@ module Ci
 
       after_transition any => ::Ci::Processable.completed_statuses do |processable|
         next unless processable.with_resource_group?
-        next unless processable.resource_group.release_resource_from(processable)
+
+        processable.resource_group.release_resource_from(processable)
 
         processable.run_after_commit do
           Ci::ResourceGroups::AssignResourceFromResourceGroupWorker
diff --git a/app/models/ci/resource_group.rb b/app/models/ci/resource_group.rb
index 8a7456041e6..6d25f747a9d 100644
--- a/app/models/ci/resource_group.rb
+++ b/app/models/ci/resource_group.rb
@@ -14,6 +14,12 @@ module Ci
 
     before_create :ensure_resource
 
+    enum process_mode: {
+      unordered: 0,
+      oldest_first: 1,
+      newest_first: 2
+    }
+
     ##
     # NOTE: This is concurrency-safe method that the subquery in the `UPDATE`
     # works as explicit locking.
@@ -25,8 +31,34 @@ module Ci
       resources.retained_by(processable).update_all(build_id: nil) > 0
     end
 
+    def upcoming_processables
+      if unordered?
+        processables.waiting_for_resource
+      elsif oldest_first?
+        processables.waiting_for_resource_or_upcoming
+          .order(Arel.sql("commit_id ASC, #{sort_by_job_status}"))
+      elsif newest_first?
+        processables.waiting_for_resource_or_upcoming
+          .order(Arel.sql("commit_id DESC, #{sort_by_job_status}"))
+      else
+        Ci::Processable.none
+      end
+    end
+
     private
 
+    # In order to avoid deadlock, we do NOT specify the job execution order in the same pipeline.
+    # The system processes wherever ready to transition to `pending` status from `waiting_for_resource`.
+    # See https://gitlab.com/gitlab-org/gitlab/-/issues/202186 for more information.
+    def sort_by_job_status
+      <<~SQL
+        CASE status
+          WHEN 'waiting_for_resource' THEN 0
+          ELSE 1
+        END ASC
+      SQL
+    end
+
     def ensure_resource
       # Currently we only support one resource per group, which means
       # maximum one build can be set to the resource group, thus builds
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index 4aa232ad26b..2f718ad7582 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -51,7 +51,7 @@ module Ci
     has_many :runner_projects, inverse_of: :runner, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
     has_many :projects, through: :runner_projects
     has_many :runner_namespaces, inverse_of: :runner, autosave: true
-    has_many :groups, through: :runner_namespaces
+    has_many :groups, through: :runner_namespaces, disable_joins: true
 
     has_one :last_build, -> { order('id DESC') }, class_name: 'Ci::Build'
 
@@ -246,7 +246,7 @@ module Ci
 
       begin
         transaction do
-          self.projects << project
+          self.runner_projects << ::Ci::RunnerProject.new(project: project, runner: self)
           self.save!
         end
       rescue ActiveRecord::RecordInvalid => e
@@ -280,7 +280,7 @@ module Ci
     end
 
     def belongs_to_more_than_one_project?
-      self.projects.limit(2).count(:all) > 1
+      runner_projects.limit(2).count(:all) > 1
     end
 
     def assigned_to_group?
@@ -309,7 +309,9 @@ module Ci
     end
 
     def only_for?(project)
-      projects == [project]
+      ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338659') do
+        projects == [project]
+      end
     end
 
     def short_sha
@@ -344,7 +346,7 @@ module Ci
       # intention here is not to execute `Ci::RegisterJobService#execute` on
       # the primary database.
       #
-      ::Gitlab::Database::LoadBalancing::Sticking.stick(:runner, id)
+      ::Ci::Runner.sticking.stick(:runner, id)
 
       SecureRandom.hex.tap do |new_update|
         ::Gitlab::Workhorse.set_key_and_notify(runner_queue_key, new_update,
@@ -428,10 +430,8 @@ module Ci
     end
 
     def no_projects
-      ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338659') do
-        if projects.any?
-          errors.add(:runner, 'cannot have projects assigned')
-        end
+      if runner_projects.any?
+        errors.add(:runner, 'cannot have projects assigned')
       end
     end
 
@@ -444,14 +444,16 @@ module Ci
     end
 
     def any_project
-      unless projects.any?
+      unless runner_projects.any?
         errors.add(:runner, 'needs to be assigned to at least one project')
       end
     end
 
     def exactly_one_group
-      unless groups.one?
-        errors.add(:runner, 'needs to be assigned to exactly one group')
+      ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338659') do
+        unless groups.one?
+          errors.add(:runner, 'needs to be assigned to exactly one group')
+        end
       end
     end
 
diff --git a/app/models/ci/runner_namespace.rb b/app/models/ci/runner_namespace.rb
index d1353b97ed9..52a31863fb2 100644
--- a/app/models/ci/runner_namespace.rb
+++ b/app/models/ci/runner_namespace.rb
@@ -7,7 +7,6 @@ module Ci
     self.limit_name = 'ci_registered_group_runners'
     self.limit_scope = :group
     self.limit_relation = :recent_runners
-    self.limit_feature_flag = :ci_runner_limits
     self.limit_feature_flag_for_override = :ci_runner_limits_override
 
     belongs_to :runner, inverse_of: :runner_namespaces
diff --git a/app/models/ci/runner_project.rb b/app/models/ci/runner_project.rb
index e1c435e9b1f..148a29a0f8b 100644
--- a/app/models/ci/runner_project.rb
+++ b/app/models/ci/runner_project.rb
@@ -7,7 +7,6 @@ module Ci
     self.limit_name = 'ci_registered_project_runners'
     self.limit_scope = :project
     self.limit_relation = :recent_runners
-    self.limit_feature_flag = :ci_runner_limits
     self.limit_feature_flag_for_override = :ci_runner_limits_override
 
     belongs_to :runner, inverse_of: :runner_projects
diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb
index 39e26bf2785..131e18adf62 100644
--- a/app/models/ci/stage.rb
+++ b/app/models/ci/stage.rb
@@ -8,8 +8,6 @@ module Ci
     include Presentable
     include IgnorableColumns
 
-    ignore_column :id_convert_to_bigint, remove_with: '14.2', remove_after: '2021-08-22'
-
     enum status: Ci::HasStatus::STATUSES_ENUM
 
     belongs_to :project