Add latest changes from gitlab-org/gitlab@16-6-stable-eev16.6.0-rc42

17 files changed, 276 insertions, 60 deletions

diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb
index d0ccf5c543a..cf6401dc1da 100644
--- a/app/models/ci/bridge.rb
+++ b/app/models/ci/bridge.rb
@@ -114,7 +114,7 @@ module Ci
         project = options&.dig(:trigger, :project)
         next unless project
 
-        scoped_variables.to_runner_variables.yield_self do |all_variables|
+        scoped_variables.to_runner_variables.then do |all_variables|
           ::ExpandVariables.expand(project, all_variables)
         end
       end
@@ -199,7 +199,7 @@ module Ci
       branch = options&.dig(:trigger, :branch)
       return unless branch
 
-      scoped_variables.to_runner_variables.yield_self do |all_variables|
+      scoped_variables.to_runner_variables.then do |all_variables|
         ::ExpandVariables.expand(branch, all_variables)
       end
     end
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index d2cf9058976..0bb93a68470 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -392,8 +392,8 @@ module Ci
         name == 'pages'
     end
 
-    # overridden on EE
-    def pages_path_prefix; end
+    # Overriden on EE
+    def pages; end
 
     def runnable?
       true
@@ -729,7 +729,7 @@ module Ci
     end
 
     def artifacts_expired?
-      artifacts_expire_at && artifacts_expire_at < Time.current
+      artifacts_expire_at&.past?
     end
 
     def artifacts_expire_in
@@ -745,7 +745,7 @@ module Ci
 
     def has_expired_locked_archive_artifacts?
       locked_artifacts? &&
-        artifacts_expire_at.present? && artifacts_expire_at < Time.current
+        artifacts_expire_at&.past?
     end
 
     def has_expiring_archive_artifacts?
@@ -921,13 +921,25 @@ module Ci
     # Consider this object to have a structural integrity problems
     def doom!
       transaction do
-        update_columns(status: :failed, failure_reason: :data_integrity_failure)
+        now = Time.current
+        attributes = {
+          status: :failed,
+          failure_reason: :data_integrity_failure,
+          updated_at: now
+        }
+        attributes[:finished_at] = now unless finished_at.present?
+
+        update_columns(attributes)
         all_queuing_entries.delete_all
         all_runtime_metadata.delete_all
       end
 
       deployment&.sync_status_with(self)
 
+      ::Gitlab::Ci::Pipeline::Metrics
+        .job_failure_reason_counter
+        .increment(reason: :data_integrity_failure)
+
       Gitlab::AppLogger.info(
         message: 'Build doomed',
         class: self.class.name,
diff --git a/app/models/ci/build_trace_chunks/redis_base.rb b/app/models/ci/build_trace_chunks/redis_base.rb
index 3b7a844d122..5f6b5c30a6a 100644
--- a/app/models/ci/build_trace_chunks/redis_base.rb
+++ b/app/models/ci/build_trace_chunks/redis_base.rb
@@ -71,7 +71,11 @@ module Ci
         with_redis do |redis|
           # https://gitlab.com/gitlab-org/gitlab/-/issues/224171
           Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
-            redis.del(keys)
+            if Gitlab::Redis::ClusterUtil.cluster?(redis)
+              Gitlab::Redis::ClusterUtil.batch_unlink(keys, redis)
+            else
+              redis.del(keys)
+            end
           end
         end
       end
diff --git a/app/models/ci/build_trace_metadata.rb b/app/models/ci/build_trace_metadata.rb
index c5ad3d19425..525cb08f2ca 100644
--- a/app/models/ci/build_trace_metadata.rb
+++ b/app/models/ci/build_trace_metadata.rb
@@ -33,7 +33,7 @@ module Ci
       return false unless archival_attempts_available?
       return true unless last_archival_attempt_at
 
-      last_archival_attempt_at + backoff < Time.current
+      (last_archival_attempt_at + backoff).past?
     end
 
     def archival_attempts_available?
diff --git a/app/models/ci/catalog/components_project.rb b/app/models/ci/catalog/components_project.rb
index 2bc33a6f050..02593d41bc2 100644
--- a/app/models/ci/catalog/components_project.rb
+++ b/app/models/ci/catalog/components_project.rb
@@ -9,7 +9,8 @@ module Ci
 
       TEMPLATE_FILE = 'template.yml'
       TEMPLATES_DIR = 'templates'
-      TEMPLATE_PATH_REGEX = '^templates\/\w+\-?\w+(?:\/template)?\.yml$'
+      TEMPLATE_PATH_REGEX = '^templates\/[\w-]+(?:\/template)?\.yml$'
+      COMPONENTS_LIMIT = 10
 
       ComponentData = Struct.new(:content, :path, keyword_init: true)
 
@@ -18,8 +19,8 @@ module Ci
         @sha = sha
       end
 
-      def fetch_component_paths(sha)
-        project.repository.search_files_by_regexp(TEMPLATE_PATH_REGEX, sha)
+      def fetch_component_paths(sha, limit: COMPONENTS_LIMIT)
+        project.repository.search_files_by_regexp(TEMPLATE_PATH_REGEX, sha, limit: limit)
       end
 
       def extract_component_name(path)
diff --git a/app/models/ci/catalog/listing.rb b/app/models/ci/catalog/listing.rb
index c3b18af8c3f..51bd85016a5 100644
--- a/app/models/ci/catalog/listing.rb
+++ b/app/models/ci/catalog/listing.rb
@@ -3,42 +3,53 @@
 module Ci
   module Catalog
     class Listing
-      # This class is the SSoT to displaying the list of resources in the
-      # CI/CD Catalog given a namespace as a scope.
+      # This class is the SSoT to displaying the list of resources in the CI/CD Catalog.
       # This model is not directly backed by a table and joins catalog resources
       # with projects to return relevant data.
-      def initialize(namespace, current_user)
-        raise ArgumentError, 'Namespace is not a root namespace' unless namespace.root?
 
-        @namespace = namespace
+      MIN_SEARCH_LENGTH = 3
+
+      def initialize(current_user)
         @current_user = current_user
       end
 
-      def resources(sort: nil)
+      def resources(namespace: nil, sort: nil, search: nil)
+        relation = all_resources
+        relation = by_namespace(relation, namespace)
+        relation = by_search(relation, search)
+
         case sort.to_s
-        when 'name_desc' then all_resources.order_by_name_desc
-        when 'name_asc' then all_resources.order_by_name_asc
-        when 'latest_released_at_desc' then all_resources.order_by_latest_released_at_desc
-        when 'latest_released_at_asc' then all_resources.order_by_latest_released_at_asc
+        when 'name_desc' then relation.order_by_name_desc
+        when 'name_asc' then relation.order_by_name_asc
+        when 'latest_released_at_desc' then relation.order_by_latest_released_at_desc
+        when 'latest_released_at_asc' then relation.order_by_latest_released_at_asc
+        when 'created_at_asc' then relation.order_by_created_at_asc
         else
-          all_resources.order_by_created_at_desc
+          relation.order_by_created_at_desc
         end
       end
 
       private
 
-      attr_reader :namespace, :current_user
+      attr_reader :current_user
 
       def all_resources
-        Ci::Catalog::Resource
-          .joins(:project).includes(:project)
-          .merge(projects_in_namespace_visible_to_user)
+        Ci::Catalog::Resource.joins(:project).includes(:project)
+          .merge(Project.public_or_visible_to_user(current_user))
+      end
+
+      def by_namespace(relation, namespace)
+        return relation unless namespace
+        raise ArgumentError, 'Namespace is not a root namespace' unless namespace.root?
+
+        relation.merge(Project.in_namespace(namespace.self_and_descendant_ids))
       end
 
-      def projects_in_namespace_visible_to_user
-        Project
-          .in_namespace(namespace.self_and_descendant_ids)
-          .public_or_visible_to_user(current_user, ::Gitlab::Access::DEVELOPER)
+      def by_search(relation, search)
+        return relation unless search
+        return relation.none if search.length < MIN_SEARCH_LENGTH
+
+        relation.search(search)
       end
     end
   end
diff --git a/app/models/ci/catalog/resource.rb b/app/models/ci/catalog/resource.rb
index 8ffc0292a69..f947c5158cf 100644
--- a/app/models/ci/catalog/resource.rb
+++ b/app/models/ci/catalog/resource.rb
@@ -8,29 +8,55 @@ module Ci
     # dependency on the Project model and its need to join with that table
     # in order to generate the CI/CD catalog.
     class Resource < ::ApplicationRecord
+      include Gitlab::SQL::Pattern
+
       self.table_name = 'catalog_resources'
 
       belongs_to :project
-      has_many :components, class_name: 'Ci::Catalog::Resources::Component', inverse_of: :catalog_resource
-      has_many :versions, class_name: 'Ci::Catalog::Resources::Version', inverse_of: :catalog_resource
+      has_many :components, class_name: 'Ci::Catalog::Resources::Component', foreign_key: :catalog_resource_id,
+        inverse_of: :catalog_resource
+      has_many :versions, class_name: 'Ci::Catalog::Resources::Version', foreign_key: :catalog_resource_id,
+        inverse_of: :catalog_resource
 
       scope :for_projects, ->(project_ids) { where(project_id: project_ids) }
+      scope :search, ->(query) { fuzzy_search(query, [:name, :description], use_minimum_char_limit: false) }
+
       scope :order_by_created_at_desc, -> { reorder(created_at: :desc) }
-      scope :order_by_name_desc, -> { joins(:project).merge(Project.sorted_by_name_desc) }
-      scope :order_by_name_asc, -> { joins(:project).merge(Project.sorted_by_name_asc) }
+      scope :order_by_created_at_asc, -> { reorder(created_at: :asc) }
+      scope :order_by_name_desc, -> { reorder(arel_table[:name].desc.nulls_last) }
+      scope :order_by_name_asc, -> { reorder(arel_table[:name].asc.nulls_last) }
       scope :order_by_latest_released_at_desc, -> { reorder(arel_table[:latest_released_at].desc.nulls_last) }
       scope :order_by_latest_released_at_asc, -> { reorder(arel_table[:latest_released_at].asc.nulls_last) }
 
-      delegate :avatar_path, :description, :name, :star_count, :forks_count, to: :project
+      delegate :avatar_path, :star_count, :forks_count, to: :project
 
       enum state: { draft: 0, published: 1 }
 
-      def versions
-        project.releases.order_released_desc
+      before_create :sync_with_project
+
+      def unpublish!
+        update!(state: :draft)
+      end
+
+      def publish!
+        update!(state: :published)
+      end
+
+      def sync_with_project!
+        sync_with_project
+        save!
       end
 
-      def latest_version
-        project.releases.latest
+      private
+
+      # These columns are denormalized from the `projects` table. We first sync these
+      # columns when the catalog resource record is created. Then any updates to the
+      # `projects` columns will be synced to the `catalog_resources` table by a worker
+      # (to be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/429376.)
+      def sync_with_project
+        self.name = project.name
+        self.description = project.description
+        self.visibility_level = project.visibility_level
       end
     end
   end
diff --git a/app/models/ci/catalog/resources/component.rb b/app/models/ci/catalog/resources/component.rb
index 7b95c14ba7e..07d5404981b 100644
--- a/app/models/ci/catalog/resources/component.rb
+++ b/app/models/ci/catalog/resources/component.rb
@@ -6,6 +6,8 @@ module Ci
       # This class represents a CI/CD Catalog resource component.
       # The data will be used as metadata of a component.
       class Component < ::ApplicationRecord
+        include BulkInsertSafe
+
         self.table_name = 'catalog_resource_components'
 
         belongs_to :project, inverse_of: :ci_components
diff --git a/app/models/ci/catalog/resources/version.rb b/app/models/ci/catalog/resources/version.rb
index 68f60e6a965..bd0ebc77a6d 100644
--- a/app/models/ci/catalog/resources/version.rb
+++ b/app/models/ci/catalog/resources/version.rb
@@ -6,6 +6,8 @@ module Ci
       # This class represents a CI/CD Catalog resource version.
       # Only versions which contain valid CI components are included in this table.
       class Version < ::ApplicationRecord
+        include BulkInsertableAssociations
+
         self.table_name = 'catalog_resource_versions'
 
         belongs_to :release, inverse_of: :catalog_resource_version
@@ -14,6 +16,100 @@ module Ci
         has_many :components, class_name: 'Ci::Catalog::Resources::Component', inverse_of: :version
 
         validates :release, :catalog_resource, :project, presence: true
+
+        scope :for_catalog_resources, ->(catalog_resources) { where(catalog_resource_id: catalog_resources) }
+        scope :preloaded, -> { includes(:catalog_resource, project: [:route, { namespace: :route }], release: :author) }
+
+        scope :order_by_created_at_asc, -> { reorder(created_at: :asc) }
+        scope :order_by_created_at_desc, -> { reorder(created_at: :desc) }
+        # After we denormalize the `released_at` column, we won't need to use `joins(:release)` and keyset_order_*
+        scope :order_by_released_at_asc, -> { joins(:release).keyset_order_by_released_at_asc }
+        scope :order_by_released_at_desc, -> { joins(:release).keyset_order_by_released_at_desc }
+
+        delegate :name, :description, :tag, :sha, :released_at, :author_id, to: :release
+
+        class << self
+          # In the future, we should support semantic versioning.
+          # See https://gitlab.com/gitlab-org/gitlab/-/issues/427286
+          def latest
+            order_by_released_at_desc.first
+          end
+
+          # This query uses LATERAL JOIN to find the latest version for each catalog resource. To avoid
+          # joining the `catalog_resources` table, we build an in-memory table using the resource ids.
+          # Example:
+          # SELECT ...
+          # FROM (VALUES (CATALOG_RESOURCE_ID_1),(CATALOG_RESOURCE_ID_2)) catalog_resources (id)
+          # INNER JOIN LATERAL (...)
+          def latest_for_catalog_resources(catalog_resources)
+            return none if catalog_resources.empty?
+
+            catalog_resources_table = Ci::Catalog::Resource.arel_table
+            catalog_resources_id_list = catalog_resources.map { |resource| "(#{resource.id})" }.join(',')
+
+            # We need to use an alias for the `releases` table here so that it does not
+            # conflict with `joins(:release)` in the `order_by_released_at_*` scope.
+            join_query = Ci::Catalog::Resources::Version
+              .where(catalog_resources_table[:id].eq(arel_table[:catalog_resource_id]))
+              .joins("INNER JOIN releases AS rel ON rel.id = #{table_name}.release_id")
+              .order(Arel.sql('rel.released_at DESC'))
+              .limit(1)
+
+            Ci::Catalog::Resources::Version
+              .from("(VALUES #{catalog_resources_id_list}) #{catalog_resources_table.name} (id)")
+              .joins("INNER JOIN LATERAL (#{join_query.to_sql}) #{table_name} ON TRUE")
+          end
+
+          def keyset_order_by_released_at_asc
+            keyset_order = Gitlab::Pagination::Keyset::Order.build([
+              Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
+                attribute_name: :released_at,
+                column_expression: Release.arel_table[:released_at],
+                order_expression: Release.arel_table[:released_at].asc,
+                nullable: :not_nullable,
+                distinct: false
+              ),
+              Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
+                attribute_name: :id,
+                order_expression: Release.arel_table[:id].asc,
+                nullable: :not_nullable,
+                distinct: true
+              )
+            ])
+
+            reorder(keyset_order)
+          end
+
+          def keyset_order_by_released_at_desc
+            keyset_order = Gitlab::Pagination::Keyset::Order.build([
+              Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
+                attribute_name: :released_at,
+                column_expression: Release.arel_table[:released_at],
+                order_expression: Release.arel_table[:released_at].desc,
+                nullable: :not_nullable,
+                distinct: false
+              ),
+              Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
+                attribute_name: :id,
+                order_expression: Release.arel_table[:id].desc,
+                nullable: :not_nullable,
+                distinct: true
+              )
+            ])
+
+            reorder(keyset_order)
+          end
+
+          def order_by(order)
+            case order.to_s
+            when 'created_asc' then order_by_created_at_asc
+            when 'created_desc' then order_by_created_at_desc
+            when 'released_at_asc' then order_by_released_at_asc
+            else
+              order_by_released_at_desc
+            end
+          end
+        end
       end
     end
   end
diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb
index 2a346f97958..fe4437a4ad6 100644
--- a/app/models/ci/job_artifact.rb
+++ b/app/models/ci/job_artifact.rb
@@ -306,7 +306,7 @@ module Ci
     end
 
     def expired?
-      expire_at.present? && expire_at < Time.current
+      expire_at.present? && expire_at.past?
     end
 
     def expiring?
diff --git a/app/models/ci/job_token/scope.rb b/app/models/ci/job_token/scope.rb
index f389c642fd8..17809ba20d3 100644
--- a/app/models/ci/job_token/scope.rb
+++ b/app/models/ci/job_token/scope.rb
@@ -54,6 +54,11 @@ module Ci
         # if the setting is disabled any project is considered to be in scope.
         return true unless current_project.ci_outbound_job_token_scope_enabled?
 
+        if !accessed_project.private? &&
+            Feature.enabled?(:restrict_ci_job_token_for_public_and_internal_projects, accessed_project)
+          return true
+        end
+
         outbound_allowlist.includes?(accessed_project)
       end
 
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index 0a876d26cc9..cf3efc5998f 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -30,9 +30,11 @@ module Ci
     PROJECT_ROUTE_AND_NAMESPACE_ROUTE = {
       project: [:project_feature, :route, { namespace: :route }]
     }.freeze
-    CONFIG_EXTENSION = '.gitlab-ci.yml'
-    DEFAULT_CONFIG_PATH = CONFIG_EXTENSION
+
+    DEFAULT_CONFIG_PATH = '.gitlab-ci.yml'
+
     CANCELABLE_STATUSES = (Ci::HasStatus::CANCELABLE_STATUSES + ['manual']).freeze
+    UNLOCKABLE_STATUSES = (Ci::Pipeline.completed_statuses + [:manual]).freeze
 
     paginates_per 15
 
@@ -189,6 +191,7 @@ module Ci
 
         # this is needed to ensure tests to be covered
         transition [:running] => :running
+        transition [:waiting_for_callback] => :waiting_for_callback
       end
 
       event :request_resource do
@@ -203,6 +206,10 @@ module Ci
         transition any - [:running] => :running
       end
 
+      event :wait_for_callback do
+        transition any - [:waiting_for_callback] => :waiting_for_callback
+      end
+
       event :skip do
         transition any - [:skipped] => :skipped
       end
@@ -266,6 +273,32 @@ module Ci
         pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
       end
 
+      after_transition any => UNLOCKABLE_STATUSES do |pipeline|
+        # This is a temporary flag that we added just in case we need to totally
+        # stop unlocking pipelines due to unexpected issues during rollout.
+        next if Feature.enabled?(:ci_stop_unlock_pipelines, pipeline.project)
+
+        next unless Feature.enabled?(:ci_unlock_non_successful_pipelines, pipeline.project)
+
+        pipeline.run_after_commit do
+          Ci::Refs::UnlockPreviousPipelinesWorker.perform_async(pipeline.ci_ref_id)
+        end
+      end
+
+      # TODO: Remove this block once we've completed roll-out of ci_unlock_non_successful_pipelines
+      # https://gitlab.com/gitlab-org/gitlab/-/issues/428408
+      after_transition any => :success do |pipeline|
+        # This is a temporary flag that we added just in case we need to totally
+        # stop unlocking pipelines due to unexpected issues during rollout.
+        next if Feature.enabled?(:ci_stop_unlock_pipelines, pipeline.project)
+
+        next unless Feature.disabled?(:ci_unlock_non_successful_pipelines, pipeline.project)
+
+        pipeline.run_after_commit do
+          Ci::Refs::UnlockPreviousPipelinesWorker.perform_async(pipeline.ci_ref_id)
+        end
+      end
+
       after_transition [:created, :waiting_for_resource, :preparing, :pending, :running] => :success do |pipeline|
         # We wait a little bit to ensure that all Ci::BuildFinishedWorkers finish first
         # because this is where some metrics like code coverage is parsed and stored
@@ -380,7 +413,7 @@ module Ci
 
         pipeline.run_after_commit do
           next if pipeline.child?
-          next unless project.only_allow_merge_if_pipeline_succeeds?(inherit_group_setting: true)
+          next unless Feature.enabled?(:widget_pipeline_pass_subscription_update, project) || project.only_allow_merge_if_pipeline_succeeds?(inherit_group_setting: true)
 
           pipeline.all_merge_requests.opened.each do |merge_request|
             GraphqlTriggers.merge_request_merge_status_updated(merge_request)
@@ -389,6 +422,7 @@ module Ci
       end
     end
 
+    scope :with_unlockable_status, -> { with_status(*UNLOCKABLE_STATUSES) }
     scope :internal, -> { where(source: internal_sources) }
     scope :no_child, -> { where.not(source: :parent_pipeline) }
     scope :ci_sources, -> { where(source: Enums::Ci::Pipeline.ci_sources.values) }
@@ -554,7 +588,7 @@ module Ci
     end
 
     def self.bridgeable_statuses
-      ::Ci::Pipeline::AVAILABLE_STATUSES - %w[created waiting_for_resource preparing pending]
+      ::Ci::Pipeline::AVAILABLE_STATUSES - %w[created waiting_for_resource waiting_for_callback preparing pending]
     end
 
     def self.auto_devops_pipelines_completed_total
@@ -850,6 +884,7 @@ module Ci
         when 'created' then nil
         when 'waiting_for_resource' then request_resource
         when 'preparing' then prepare
+        when 'waiting_for_callback' then wait_for_callback
         when 'pending' then enqueue
         when 'running' then run
         when 'success' then succeed
@@ -1366,11 +1401,6 @@ module Ci
       merge_request.merge_request_diff_for(merge_request_diff_sha)
     end
 
-    def reduced_build_attributes_list_for_rules?
-      ::Feature.enabled?(:reduced_build_attributes_list_for_rules, project)
-    end
-    strong_memoize_attr :reduced_build_attributes_list_for_rules?
-
     private
 
     def add_message(severity, content)
diff --git a/app/models/ci/ref.rb b/app/models/ci/ref.rb
index 8655e8eb9b8..e8ce58f2de5 100644
--- a/app/models/ci/ref.rb
+++ b/app/models/ci/ref.rb
@@ -30,15 +30,6 @@ module Ci
       state :fixed, value: 3
       state :broken, value: 4
       state :still_failing, value: 5
-
-      after_transition any => [:fixed, :success] do |ci_ref|
-        # Do not try to unlock if no artifacts are locked
-        next unless ci_ref.artifacts_locked?
-
-        ci_ref.run_after_commit do
-          Ci::Refs::UnlockPreviousPipelinesWorker.perform_async(ci_ref.id)
-        end
-      end
     end
 
     class << self
@@ -75,5 +66,13 @@ module Ci
         self.status_name
       end
     end
+
+    def last_successful_ci_source_pipeline
+      pipelines.ci_sources.success.order(id: :desc).first
+    end
+
+    def last_unlockable_ci_source_pipeline
+      pipelines.ci_sources.with_unlockable_status.order(id: :desc).first
+    end
   end
 end
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index 91c919dc662..9c30beeeb59 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -123,6 +123,8 @@ module Ci
       joins(:runner_namespaces).where(ci_runner_namespaces: { namespace_id: group_id })
     }
 
+    scope :with_creator_id, -> (value) { where(creator_id: value) }
+
     scope :belonging_to_group_or_project_descendants, -> (group_id) {
       group_ids = Ci::NamespaceMirror.by_group_and_descendants(group_id).select(:namespace_id)
       project_ids = Ci::ProjectMirror.by_namespace_id(group_ids).select(:project_id)
@@ -217,6 +219,8 @@ module Ci
     validate :any_project, if: :project_type?
     validate :exactly_one_group, if: :group_type?
 
+    scope :with_version_prefix, ->(value) { joins(:runner_managers).merge(RunnerManager.with_version_prefix(value)) }
+
     acts_as_taggable
 
     after_destroy :cleanup_runner_queue
diff --git a/app/models/ci/runner_manager.rb b/app/models/ci/runner_manager.rb
index 7d8fc097f51..e6576859827 100644
--- a/app/models/ci/runner_manager.rb
+++ b/app/models/ci/runner_manager.rb
@@ -62,6 +62,16 @@ module Ci
 
     scope :order_id_desc, -> { order(id: :desc) }
 
+    scope :with_version_prefix, ->(value) do
+      regex = version_regex_expression_for_version(value)
+      value += '.' if regex.end_with?('\.') && !value.end_with?('.')
+      substring = Arel::Nodes::NamedFunction.new('substring', [
+        Ci::RunnerManager.arel_table[:version],
+        Arel.sql("'#{regex}'::text")
+      ])
+      where(substring.eq(sanitize_sql_like(value)))
+    end
+
     scope :with_upgrade_status, ->(upgrade_status) do
       joins(:runner_version).where(runner_version: { status: upgrade_status })
     end
@@ -137,5 +147,16 @@ module Ci
 
       Ci::Runners::ProcessRunnerVersionUpdateWorker.perform_async(new_version)
     end
+
+    def self.version_regex_expression_for_version(version)
+      case version
+      when /\d+\.\d+\.\d+/
+        '^\d+\.\d+\.\d+'
+      when /\d+\.\d+(\.)?/
+        '^\d+\.\d+\.'
+      else
+        '^\d+\.'
+      end
+    end
   end
 end
diff --git a/app/models/ci/sources/pipeline.rb b/app/models/ci/sources/pipeline.rb
index 5b6946b04fd..475d57ee4c8 100644
--- a/app/models/ci/sources/pipeline.rb
+++ b/app/models/ci/sources/pipeline.rb
@@ -12,7 +12,7 @@ module Ci
         :pipeline_id_convert_to_bigint, :source_pipeline_id_convert_to_bigint
       ], remove_with: '16.6', remove_after: '2023-10-22'
 
-      columns_changing_default :partition_id
+      columns_changing_default :partition_id, :source_partition_id
 
       self.table_name = "ci_sources_pipelines"
 
diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb
index 3a498972153..3d2df9a45ef 100644
--- a/app/models/ci/stage.rb
+++ b/app/models/ci/stage.rb
@@ -78,6 +78,10 @@ module Ci
         transition any - [:running] => :running
       end
 
+      event :wait_for_callback do
+        transition any - [:waiting_for_callback] => :waiting_for_callback
+      end
+
       event :skip do
         transition any - [:skipped] => :skipped
       end
@@ -109,6 +113,7 @@ module Ci
         when 'created' then nil
         when 'waiting_for_resource' then request_resource
         when 'preparing' then prepare
+        when 'waiting_for_callback' then wait_for_callback
         when 'pending' then enqueue
         when 'running' then run
         when 'success' then succeed