Add latest changes from gitlab-org/gitlab@15-1-stable-eev15.1.0-rc42

7 files changed, 123 insertions, 77 deletions

diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb
index a06b920342c..13af5b1f8d1 100644
--- a/app/models/ci/bridge.rb
+++ b/app/models/ci/bridge.rb
@@ -215,14 +215,10 @@ module Ci
     end
 
     def downstream_variables
-      if ::Feature.enabled?(:ci_trigger_forward_variables, project)
-        calculate_downstream_variables
-          .reverse # variables priority
-          .uniq { |var| var[:key] } # only one variable key to pass
-          .reverse
-      else
-        legacy_downstream_variables
-      end
+      calculate_downstream_variables
+        .reverse # variables priority
+        .uniq { |var| var[:key] } # only one variable key to pass
+        .reverse
     end
 
     def target_revision_ref
@@ -268,16 +264,6 @@ module Ci
       }
     end
 
-    def legacy_downstream_variables
-      variables = scoped_variables.concat(pipeline.persisted_variables)
-
-      variables.to_runner_variables.yield_self do |all_variables|
-        yaml_variables.to_a.map do |hash|
-          { key: hash[:key], value: ::ExpandVariables.expand(hash[:value], all_variables) }
-        end
-      end
-    end
-
     def calculate_downstream_variables
       expand_variables = scoped_variables
                            .concat(pipeline.persisted_variables)
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index eea8086d71d..e35198ba31f 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -137,13 +137,14 @@ module Ci
       where('NOT EXISTS (?)', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').trace)
     end
 
-    scope :with_reports, ->(reports_scope) do
-      with_existing_job_artifacts(reports_scope)
+    scope :with_artifacts, ->(artifact_scope) do
+      with_existing_job_artifacts(artifact_scope)
         .eager_load_job_artifacts
     end
 
     scope :eager_load_job_artifacts, -> { includes(:job_artifacts) }
     scope :eager_load_tags, -> { includes(:tags) }
+    scope :eager_load_for_archiving_trace, -> { includes(:project, :pending_state) }
 
     scope :eager_load_everything, -> do
       includes(
@@ -424,10 +425,18 @@ module Ci
       pipeline.manual_actions.reject { |action| action.name == self.name }
     end
 
+    def environment_manual_actions
+      pipeline.manual_actions.filter { |action| action.expanded_environment_name == self.expanded_environment_name }
+    end
+
     def other_scheduled_actions
       pipeline.scheduled_actions.reject { |action| action.name == self.name }
     end
 
+    def environment_scheduled_actions
+      pipeline.scheduled_actions.filter { |action| action.expanded_environment_name == self.expanded_environment_name }
+    end
+
     def pages_generator?
       Gitlab.config.pages.enabled &&
         self.name == 'pages'
@@ -559,6 +568,10 @@ module Ci
       options&.dig(:environment, :on_stop)
     end
 
+    def stop_action_successful?
+      success?
+    end
+
     ##
     # All variables, including persisted environment variables.
     #
@@ -673,7 +686,7 @@ module Ci
     end
 
     def has_live_trace?
-      trace.live_trace_exist?
+      trace.live?
     end
 
     def has_archived_trace?
@@ -795,6 +808,7 @@ module Ci
 
     def execute_hooks
       return unless project
+      return if user&.blocked?
 
       project.execute_hooks(build_data.dup, :job_hooks) if project.has_active_hooks?(:job_hooks)
       project.execute_integrations(build_data.dup, :job_hooks) if project.has_active_integrations?(:job_hooks)
@@ -826,12 +840,26 @@ module Ci
     end
 
     def erase_erasable_artifacts!
+      if project.refreshing_build_artifacts_size?
+        Gitlab::ProjectStatsRefreshConflictsLogger.warn_artifact_deletion_during_stats_refresh(
+          method: 'Ci::Build#erase_erasable_artifacts!',
+          project_id: project_id
+        )
+      end
+
       job_artifacts.erasable.destroy_all # rubocop: disable Cop/DestroyAll
     end
 
     def erase(opts = {})
       return false unless erasable?
 
+      if project.refreshing_build_artifacts_size?
+        Gitlab::ProjectStatsRefreshConflictsLogger.warn_artifact_deletion_during_stats_refresh(
+          method: 'Ci::Build#erase',
+          project_id: project_id
+        )
+      end
+
       job_artifacts.destroy_all # rubocop: disable Cop/DestroyAll
       erase_trace!
       update_erased!(opts[:erased_by])
@@ -983,7 +1011,7 @@ module Ci
     end
 
     def collect_test_reports!(test_reports)
-      test_reports.get_suite(group_name).tap do |test_suite|
+      test_reports.get_suite(test_suite_name).tap do |test_suite|
         each_report(Ci::JobArtifact::TEST_REPORT_FILE_TYPES) do |file_type, blob|
           Gitlab::Ci::Parsers.fabricate!(file_type).parse!(
             blob,
@@ -1002,19 +1030,6 @@ module Ci
       accessibility_report
     end
 
-    def collect_coverage_reports!(coverage_report)
-      each_report(Ci::JobArtifact::COVERAGE_REPORT_FILE_TYPES) do |file_type, blob|
-        Gitlab::Ci::Parsers.fabricate!(file_type).parse!(
-          blob,
-          coverage_report,
-          project_path: project.full_path,
-          worktree_paths: pipeline.all_worktree_paths
-        )
-      end
-
-      coverage_report
-    end
-
     def collect_codequality_reports!(codequality_report)
       each_report(Ci::JobArtifact::CODEQUALITY_REPORT_FILE_TYPES) do |file_type, blob|
         Gitlab::Ci::Parsers.fabricate!(file_type).parse!(blob, codequality_report)
@@ -1032,7 +1047,7 @@ module Ci
     end
 
     def report_artifacts
-      job_artifacts.with_reports
+      job_artifacts.all_reports
     end
 
     # Virtual deployment status depending on the environment status.
@@ -1056,6 +1071,8 @@ module Ci
         all_runtime_metadata.delete_all
       end
 
+      deployment&.sync_status_with(self)
+
       Gitlab::AppLogger.info(
         message: 'Build doomed',
         class: self.class.name,
@@ -1145,6 +1162,14 @@ module Ci
       Gitlab::Utils::UsageData.track_usage_event('ci_users_executing_deployment_job', user_id) if user_id.present? && count_user_deployment?
     end
 
+    def each_report(report_types)
+      job_artifacts_for_types(report_types).each do |report_artifact|
+        report_artifact.each_blob do |blob|
+          yield report_artifact.file_type, blob, report_artifact
+        end
+      end
+    end
+
     protected
 
     def run_status_commit_hooks!
@@ -1155,6 +1180,18 @@ module Ci
 
     private
 
+    def test_suite_name
+      if matrix_build?
+        name
+      else
+        group_name
+      end
+    end
+
+    def matrix_build?
+      options.dig(:parallel, :matrix).present?
+    end
+
     def stick_build_if_status_changed
       return unless saved_change_to_status?
       return unless running?
@@ -1184,14 +1221,6 @@ module Ci
       end
     end
 
-    def each_report(report_types)
-      job_artifacts_for_types(report_types).each do |report_artifact|
-        report_artifact.each_blob do |blob|
-          yield report_artifact.file_type, blob, report_artifact
-        end
-      end
-    end
-
     def job_artifacts_for_types(report_types)
       # Use select to leverage cached associations and avoid N+1 queries
       job_artifacts.select { |artifact| artifact.file_type.in?(report_types) }
diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb
index c831ef12501..81943cfa651 100644
--- a/app/models/ci/job_artifact.rb
+++ b/app/models/ci/job_artifact.rb
@@ -124,10 +124,10 @@ module Ci
     # We will start using this column once we complete https://gitlab.com/gitlab-org/gitlab/-/issues/285597
     ignore_column :original_filename, remove_with: '14.7', remove_after: '2022-11-22'
 
-    mount_file_store_uploader JobArtifactUploader
+    mount_file_store_uploader JobArtifactUploader, skip_store_file: true
 
-    skip_callback :save, :after, :store_file!, if: :store_after_commit?
-    after_commit :store_file_after_commit!, on: [:create, :update], if: :store_after_commit?
+    after_save :store_file_in_transaction!, unless: :store_after_commit?
+    after_commit :store_file_after_transaction!, on: [:create, :update], if: :store_after_commit?
 
     validates :file_format, presence: true, unless: :trace?, on: :create
     validate :validate_file_format!, unless: :trace?, on: :create
@@ -139,6 +139,10 @@ module Ci
     scope :for_sha, ->(sha, project_id) { joins(job: :pipeline).where(ci_pipelines: { sha: sha, project_id: project_id }) }
     scope :for_job_ids, ->(job_ids) { where(job_id: job_ids) }
     scope :for_job_name, ->(name) { joins(:job).where(ci_builds: { name: name }) }
+    scope :created_at_before, ->(time) { where(arel_table[:created_at].lteq(time)) }
+    scope :id_before, ->(id) { where(arel_table[:id].lteq(id)) }
+    scope :id_after, ->(id) { where(arel_table[:id].gt(id)) }
+    scope :ordered_by_id, -> { order(:id) }
 
     scope :with_job, -> { joins(:job).includes(:job) }
 
@@ -148,7 +152,7 @@ module Ci
       where(file_type: types)
     end
 
-    scope :with_reports, -> do
+    scope :all_reports, -> do
       with_file_types(REPORT_TYPES.keys.map(&:to_s))
     end
 
@@ -187,7 +191,7 @@ module Ci
     scope :downloadable, -> { where(file_type: DOWNLOADABLE_TYPES) }
     scope :unlocked, -> { joins(job: :pipeline).merge(::Ci::Pipeline.unlocked) }
     scope :order_expired_asc, -> { order(expire_at: :asc) }
-    scope :with_destroy_preloads, -> { includes(project: [:route, :statistics]) }
+    scope :with_destroy_preloads, -> { includes(project: [:route, :statistics, :build_artifacts_size_refresh]) }
 
     scope :for_project, ->(project) { where(project_id: project) }
     scope :created_in_time_range, ->(from: nil, to: nil) { where(created_at: from..to) }
@@ -358,11 +362,24 @@ module Ci
 
     private
 
-    def store_file_after_commit!
-      return unless previous_changes.key?(:file)
+    def store_file_in_transaction!
+      store_file_now! if saved_change_to_file?
 
-      store_file!
-      update_file_store
+      file_stored_in_transaction_hooks
+    end
+
+    def store_file_after_transaction!
+      store_file_now! if previous_changes.key?(:file)
+
+      file_stored_after_transaction_hooks
+    end
+
+    # method overriden in EE
+    def file_stored_after_transaction_hooks
+    end
+
+    # method overriden in EE
+    def file_stored_in_transaction_hooks
     end
 
     def set_size
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index c10069382f2..5d316906bd3 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -81,6 +81,7 @@ module Ci
     has_many :downloadable_artifacts, -> do
       not_expired.or(where_exists(::Ci::Pipeline.artifacts_locked.where('ci_pipelines.id = ci_builds.commit_id'))).downloadable.with_job
     end, through: :latest_builds, source: :job_artifacts
+    has_many :latest_successful_builds, -> { latest.success.with_project_and_metadata }, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'Ci::Build'
 
     has_many :messages, class_name: 'Ci::PipelineMessage', inverse_of: :pipeline
 
@@ -239,7 +240,9 @@ module Ci
         next if transition.loopback?
 
         pipeline.run_after_commit do
-          PipelineHooksWorker.perform_async(pipeline.id)
+          unless pipeline.user&.blocked?
+            PipelineHooksWorker.perform_async(pipeline.id)
+          end
 
           if pipeline.project.jira_subscription_exists?
             # Passing the seq-id ensures this is idempotent
@@ -296,7 +299,12 @@ module Ci
         ref_status = pipeline.ci_ref&.update_status_by!(pipeline)
 
         pipeline.run_after_commit do
-          PipelineNotificationWorker.perform_async(pipeline.id, ref_status: ref_status)
+          # We don't send notifications for a pipeline dropped due to the
+          # user been blocked.
+          unless pipeline.user&.blocked?
+            PipelineNotificationWorker
+              .perform_async(pipeline.id, ref_status: ref_status)
+          end
         end
       end
 
@@ -327,14 +335,14 @@ module Ci
     scope :created_after, -> (time) { where('ci_pipelines.created_at > ?', time) }
     scope :created_before_id, -> (id) { where('ci_pipelines.id < ?', id) }
     scope :before_pipeline, -> (pipeline) { created_before_id(pipeline.id).outside_pipeline_family(pipeline) }
-    scope :with_pipeline_source, -> (source) { where(source: source)}
+    scope :with_pipeline_source, -> (source) { where(source: source) }
 
     scope :outside_pipeline_family, ->(pipeline) do
       where.not(id: pipeline.same_family_pipeline_ids)
     end
 
     scope :with_reports, -> (reports_scope) do
-      where('EXISTS (?)', ::Ci::Build.latest.with_reports(reports_scope).where('ci_pipelines.id=ci_builds.commit_id').select(1))
+      where('EXISTS (?)', ::Ci::Build.latest.with_artifacts(reports_scope).where('ci_pipelines.id=ci_builds.commit_id').select(1))
     end
 
     scope :with_only_interruptible_builds, -> do
@@ -688,7 +696,7 @@ module Ci
     def latest_report_artifacts
       ::Gitlab::SafeRequestStore.fetch("pipeline:#{self.id}:latest_report_artifacts") do
         ::Ci::JobArtifact.where(
-          id: job_artifacts.with_reports
+          id: job_artifacts.all_reports
             .select('max(ci_job_artifacts.id) as id')
             .group(:file_type)
         )
@@ -1049,12 +1057,16 @@ module Ci
       @latest_builds_with_artifacts ||= builds.latest.with_artifacts_not_expired.to_a
     end
 
-    def latest_report_builds(reports_scope = ::Ci::JobArtifact.with_reports)
-      builds.latest.with_reports(reports_scope)
+    def latest_report_builds(reports_scope = ::Ci::JobArtifact.all_reports)
+      builds.latest.with_artifacts(reports_scope)
     end
 
     def latest_test_report_builds
-      latest_report_builds(Ci::JobArtifact.test_reports).preload(:project)
+      latest_report_builds(Ci::JobArtifact.test_reports).preload(:project, :metadata)
+    end
+
+    def latest_report_builds_in_self_and_descendants(reports_scope = ::Ci::JobArtifact.all_reports)
+      builds_in_self_and_descendants.with_artifacts(reports_scope)
     end
 
     def builds_with_coverage
@@ -1073,10 +1085,6 @@ module Ci
       pipeline_artifacts&.report_exists?(:code_coverage)
     end
 
-    def can_generate_coverage_reports?
-      has_reports?(Ci::JobArtifact.coverage_reports)
-    end
-
     def has_codequality_mr_diff_report?
       pipeline_artifacts&.report_exists?(:code_quality_mr_diff)
     end
@@ -1107,14 +1115,6 @@ module Ci
       end
     end
 
-    def coverage_reports
-      Gitlab::Ci::Reports::CoverageReports.new.tap do |coverage_reports|
-        latest_report_builds(Ci::JobArtifact.coverage_reports).includes(:project).find_each do |build|
-          build.collect_coverage_reports!(coverage_reports)
-        end
-      end
-    end
-
     def codequality_reports
       Gitlab::Ci::Reports::CodequalityReports.new.tap do |codequality_reports|
         latest_report_builds(Ci::JobArtifact.codequality_reports).each do |build|
@@ -1308,8 +1308,8 @@ module Ci
     end
 
     def has_expired_test_reports?
-      strong_memoize(:artifacts_expired) do
-        !has_reports?(::Ci::JobArtifact.test_reports.not_expired)
+      strong_memoize(:has_expired_test_reports) do
+        has_reports?(::Ci::JobArtifact.test_reports.expired)
       end
     end
 
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index 7a1d52f5aea..61194c9b7d1 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -77,6 +77,7 @@ module Ci
     has_one :last_build, -> { order('id DESC') }, class_name: 'Ci::Build'
 
     before_save :ensure_token
+    before_save :update_semver, if: -> { version_changed? }
 
     scope :active, -> (value = true) { where(active: value) }
     scope :paused, -> { active(false) }
@@ -429,6 +430,7 @@ module Ci
         values = values&.slice(:version, :revision, :platform, :architecture, :ip_address, :config, :executor) || {}
         values[:contacted_at] = Time.current
         values[:executor_type] = EXECUTOR_NAME_TO_TYPES.fetch(values.delete(:executor), :unknown)
+        values[:semver] = semver_from_version(values[:version])
 
         cache_attributes(values)
 
@@ -449,6 +451,16 @@ module Ci
       read_attribute(:contacted_at)
     end
 
+    def semver_from_version(version)
+      parsed_runner_version = ::Gitlab::VersionInfo.parse(version)
+
+      parsed_runner_version.valid? ? parsed_runner_version.to_s : nil
+    end
+
+    def update_semver
+      self.semver = semver_from_version(self.version)
+    end
+
     def namespace_ids
       strong_memoize(:namespace_ids) do
         runner_namespaces.pluck(:namespace_id).compact
diff --git a/app/models/ci/secure_file.rb b/app/models/ci/secure_file.rb
index 9c82e106d6e..078b05ff779 100644
--- a/app/models/ci/secure_file.rb
+++ b/app/models/ci/secure_file.rb
@@ -23,6 +23,8 @@ module Ci
     after_initialize :generate_key_data
     before_validation :assign_checksum
 
+    scope :order_by_created_at, -> { order(created_at: :desc) }
+
     default_value_for(:file_store) { Ci::SecureFileUploader.default_store }
 
     mount_file_store_uploader Ci::SecureFileUploader
diff --git a/app/models/ci/sources/pipeline.rb b/app/models/ci/sources/pipeline.rb
index f78caf710a6..2df504cd3de 100644
--- a/app/models/ci/sources/pipeline.rb
+++ b/app/models/ci/sources/pipeline.rb
@@ -7,10 +7,10 @@ module Ci
 
       self.table_name = "ci_sources_pipelines"
 
-      belongs_to :project, class_name: "Project"
+      belongs_to :project, class_name: "::Project"
       belongs_to :pipeline, class_name: "Ci::Pipeline", inverse_of: :source_pipeline
 
-      belongs_to :source_project, class_name: "Project", foreign_key: :source_project_id
+      belongs_to :source_project, class_name: "::Project", foreign_key: :source_project_id
       belongs_to :source_job, class_name: "CommitStatus", foreign_key: :source_job_id
       belongs_to :source_bridge, class_name: "Ci::Bridge", foreign_key: :source_job_id
       belongs_to :source_pipeline, class_name: "Ci::Pipeline", foreign_key: :source_pipeline_id