Add latest changes from gitlab-org/gitlab@master

12 files changed, 294 insertions, 114 deletions

diff --git a/app/graphql/queries/epic/epic_children.query.graphql b/app/graphql/queries/epic/epic_children.query.graphql
index 5ee27052f95..b0e55811b7d 100644
--- a/app/graphql/queries/epic/epic_children.query.graphql
+++ b/app/graphql/queries/epic/epic_children.query.graphql
@@ -42,6 +42,7 @@ fragment EpicNode on Epic {
   relationPath
   createdAt
   closedAt
+  confidential
   hasChildren
   hasIssues
   group {
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 1b0c27a8cbd..330b66c913a 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -11,7 +11,6 @@ module Ci
     include Importable
     include Ci::HasRef
     include IgnorableColumns
-    include TaggableQueries
 
     BuildArchivedError = Class.new(StandardError)
 
@@ -179,25 +178,6 @@ module Ci
       joins(:metadata).where("ci_builds_metadata.config_options -> 'artifacts' -> 'reports' ?| array[:job_types]", job_types: job_types)
     end
 
-    scope :matches_tag_ids, -> (tag_ids) do
-      matcher = ::ActsAsTaggableOn::Tagging
-        .where(taggable_type: CommitStatus.name)
-        .where(context: 'tags')
-        .where('taggable_id = ci_builds.id')
-        .where.not(tag_id: tag_ids).select('1')
-
-      where("NOT EXISTS (?)", matcher)
-    end
-
-    scope :with_any_tags, -> do
-      matcher = ::ActsAsTaggableOn::Tagging
-        .where(taggable_type: CommitStatus.name)
-        .where(context: 'tags')
-        .where('taggable_id = ci_builds.id').select('1')
-
-      where("EXISTS (?)", matcher)
-    end
-
     scope :queued_before, ->(time) { where(arel_table[:queued_at].lt(time)) }
 
     scope :preload_project_and_pipeline_project, -> do
diff --git a/app/models/ci/pending_build.rb b/app/models/ci/pending_build.rb
index b9a8a44bd6b..a1eaae8a21a 100644
--- a/app/models/ci/pending_build.rb
+++ b/app/models/ci/pending_build.rb
@@ -7,6 +7,9 @@ module Ci
     belongs_to :project
     belongs_to :build, class_name: 'Ci::Build'
 
+    scope :ref_protected, -> { where(protected: true) }
+    scope :queued_before, ->(time) { where(arel_table[:created_at].lt(time)) }
+
     def self.upsert_from_build!(build)
       entry = self.new(build: build, project: build.project, protected: build.protected?)
 
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index 23d73c4951b..e86abe9a11b 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -904,7 +904,7 @@ module Ci
 
     def same_family_pipeline_ids
       ::Gitlab::Ci::PipelineObjectHierarchy.new(
-        self.class.default_scoped.where(id: root_ancestor), options: { same_project: true }
+        self.class.default_scoped.where(id: root_ancestor), options: { project_condition: :same }
       ).base_and_descendants.select(:id)
     end
 
@@ -925,29 +925,34 @@ module Ci
       Environment.where(id: environment_ids)
     end
 
-    # Without using `unscoped`, caller scope is also included into the query.
-    # Using `unscoped` here will be redundant after Rails 6.1
+    # With multi-project and parent-child pipelines
+    def self_and_upstreams
+      object_hierarchy.base_and_ancestors
+    end
+
+    # With multi-project and parent-child pipelines
+    def self_with_upstreams_and_downstreams
+      object_hierarchy.all_objects
+    end
+
+    # With only parent-child pipelines
+    def self_and_ancestors
+      object_hierarchy(project_condition: :same).base_and_ancestors
+    end
+
+    # With only parent-child pipelines
     def self_and_descendants
-      ::Gitlab::Ci::PipelineObjectHierarchy
-        .new(self.class.unscoped.where(id: id), options: { same_project: true })
-        .base_and_descendants
+      object_hierarchy(project_condition: :same).base_and_descendants
     end
 
     def root_ancestor
       return self unless child?
 
-      Gitlab::Ci::PipelineObjectHierarchy
-        .new(self.class.unscoped.where(id: id), options: { same_project: true })
+      object_hierarchy(project_condition: :same)
         .base_and_ancestors(hierarchy_order: :desc)
         .first
     end
 
-    def self_with_ancestors_and_descendants(same_project: false)
-      ::Gitlab::Ci::PipelineObjectHierarchy
-        .new(self.class.unscoped.where(id: id), options: { same_project: same_project })
-        .all_objects
-    end
-
     def bridge_triggered?
       source_bridge.present?
     end
@@ -1207,14 +1212,6 @@ module Ci
       self.ci_ref = Ci::Ref.ensure_for(self)
     end
 
-    def base_and_ancestors(same_project: false)
-      # Without using `unscoped`, caller scope is also included into the query.
-      # Using `unscoped` here will be redundant after Rails 6.1
-      ::Gitlab::Ci::PipelineObjectHierarchy
-        .new(self.class.unscoped.where(id: id), options: { same_project: same_project })
-        .base_and_ancestors
-    end
-
     # We need `base_and_ancestors` in a specific order to "break" when needed.
     # If we use `find_each`, then the order is broken.
     # rubocop:disable Rails/FindEach
@@ -1225,7 +1222,7 @@ module Ci
         source_bridge.pending!
         Ci::AfterRequeueJobService.new(project, current_user).execute(source_bridge) # rubocop:disable CodeReuse/ServiceClass
       else
-        base_and_ancestors.includes(:source_bridge).each do |pipeline|
+        self_and_upstreams.includes(:source_bridge).each do |pipeline|
           break unless pipeline.bridge_waiting?
 
           pipeline.source_bridge.pending!
@@ -1308,6 +1305,13 @@ module Ci
 
       project.repository.keep_around(self.sha, self.before_sha)
     end
+
+    # Without using `unscoped`, caller scope is also included into the query.
+    # Using `unscoped` here will be redundant after Rails 6.1
+    def object_hierarchy(options = {})
+      ::Gitlab::Ci::PipelineObjectHierarchy
+        .new(self.class.unscoped.where(id: id), options: options)
+    end
   end
 end
 
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index 2db606898b9..cf23cd3be67 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -7,6 +7,7 @@ class CommitStatus < ApplicationRecord
   include Presentable
   include EnumWithNil
   include BulkInsertableAssociations
+  include TaggableQueries
 
   self.table_name = 'ci_builds'
 
diff --git a/app/models/concerns/taggable_queries.rb b/app/models/concerns/taggable_queries.rb
index 2897e5e6420..cba2e93a86d 100644
--- a/app/models/concerns/taggable_queries.rb
+++ b/app/models/concerns/taggable_queries.rb
@@ -12,5 +12,26 @@ module TaggableQueries
         .where(taggings: { context: context, taggable_type: polymorphic_name })
         .select('COALESCE(array_agg(tags.name ORDER BY name), ARRAY[]::text[])')
     end
+
+    def matches_tag_ids(tag_ids, table: quoted_table_name, column: 'id')
+      matcher = ::ActsAsTaggableOn::Tagging
+        .where(taggable_type: CommitStatus.name)
+        .where(context: 'tags')
+        .where("taggable_id = #{connection.quote_table_name(table)}.#{connection.quote_column_name(column)}") # rubocop:disable GitlabSecurity/SqlInjection
+        .where.not(tag_id: tag_ids)
+        .select('1')
+
+      where("NOT EXISTS (?)", matcher)
+    end
+
+    def with_any_tags(table: quoted_table_name, column: 'id')
+      matcher = ::ActsAsTaggableOn::Tagging
+        .where(taggable_type: CommitStatus.name)
+        .where(context: 'tags')
+        .where("taggable_id = #{connection.quote_table_name(table)}.#{connection.quote_column_name(column)}") # rubocop:disable GitlabSecurity/SqlInjection
+        .select('1')
+
+      where("EXISTS (?)", matcher)
+    end
   end
 end
diff --git a/app/services/ci/create_downstream_pipeline_service.rb b/app/services/ci/create_downstream_pipeline_service.rb
index 1eff76c2e5d..e9ec2338171 100644
--- a/app/services/ci/create_downstream_pipeline_service.rb
+++ b/app/services/ci/create_downstream_pipeline_service.rb
@@ -120,7 +120,7 @@ module Ci
       return false if @bridge.triggers_child_pipeline?
 
       if Feature.enabled?(:ci_drop_cyclical_triggered_pipelines, @bridge.project, default_enabled: :yaml)
-        pipeline_checksums = @bridge.pipeline.base_and_ancestors.filter_map do |pipeline|
+        pipeline_checksums = @bridge.pipeline.self_and_upstreams.filter_map do |pipeline|
           config_checksum(pipeline) unless pipeline.child?
         end
 
@@ -131,7 +131,7 @@ module Ci
     def has_max_descendants_depth?
       return false unless @bridge.triggers_child_pipeline?
 
-      ancestors_of_new_child = @bridge.pipeline.base_and_ancestors(same_project: true)
+      ancestors_of_new_child = @bridge.pipeline.self_and_ancestors
       ancestors_of_new_child.count > MAX_DESCENDANTS_DEPTH
     end
 
diff --git a/app/services/ci/expire_pipeline_cache_service.rb b/app/services/ci/expire_pipeline_cache_service.rb
index 80c83818d0b..48a6344f576 100644
--- a/app/services/ci/expire_pipeline_cache_service.rb
+++ b/app/services/ci/expire_pipeline_cache_service.rb
@@ -77,7 +77,7 @@ module Ci
         store.touch(path)
       end
 
-      pipeline.self_with_ancestors_and_descendants.each do |relative_pipeline|
+      pipeline.self_with_upstreams_and_downstreams.each do |relative_pipeline|
         store.touch(project_pipeline_path(relative_pipeline.project, relative_pipeline))
         store.touch(graphql_pipeline_path(relative_pipeline))
         store.touch(graphql_pipeline_sha_path(relative_pipeline.sha))
diff --git a/app/services/ci/queue/build_queue_service.rb b/app/services/ci/queue/build_queue_service.rb
new file mode 100644
index 00000000000..8190599fbb5
--- /dev/null
+++ b/app/services/ci/queue/build_queue_service.rb
@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Ci
+  module Queue
+    class BuildQueueService
+      include ::Gitlab::Utils::StrongMemoize
+
+      attr_reader :runner
+
+      def initialize(runner)
+        @runner = runner
+      end
+
+      def new_builds
+        strategy.new_builds
+      end
+
+      ##
+      # This is overridden in EE
+      #
+      def builds_for_shared_runner
+        strategy.builds_for_shared_runner
+      end
+
+      # rubocop:disable CodeReuse/ActiveRecord
+      def builds_for_group_runner
+        # Workaround for weird Rails bug, that makes `runner.groups.to_sql` to return `runner_id = NULL`
+        groups = ::Group.joins(:runner_namespaces).merge(runner.runner_namespaces)
+
+        hierarchy_groups = Gitlab::ObjectHierarchy
+          .new(groups, options: { use_distinct: ::Feature.enabled?(:use_distinct_in_register_job_object_hierarchy) })
+          .base_and_descendants
+
+        projects = Project.where(namespace_id: hierarchy_groups)
+          .with_group_runners_enabled
+          .with_builds_enabled
+          .without_deleted
+
+        relation = new_builds.where(project: projects)
+
+        order(relation)
+      end
+
+      def builds_for_project_runner
+        relation = new_builds
+          .where(project: runner.projects.without_deleted.with_builds_enabled)
+
+        order(relation)
+      end
+
+      def builds_queued_before(relation, time)
+        relation.queued_before(time)
+      end
+
+      def builds_for_protected_runner(relation)
+        relation.ref_protected
+      end
+
+      def builds_matching_tag_ids(relation, ids)
+        strategy.builds_matching_tag_ids(relation, ids)
+      end
+
+      def builds_with_any_tags(relation)
+        strategy.builds_with_any_tags(relation)
+      end
+
+      def order(relation)
+        strategy.order(relation)
+      end
+
+      def execute(relation)
+        strategy.build_ids(relation)
+      end
+
+      private
+
+      def strategy
+        strong_memoize(:strategy) do
+          if ::Feature.enabled?(:ci_pending_builds_queue_source, runner, default_enabled: :yaml)
+            Queue::PendingBuildsStrategy.new(runner)
+          else
+            Queue::BuildsTableStrategy.new(runner)
+          end
+        end
+      end
+    end
+  end
+end
+
+Ci::Queue::BuildQueueService.prepend_mod_with('Ci::Queue::BuildQueueService')
diff --git a/app/services/ci/queue/builds_table_strategy.rb b/app/services/ci/queue/builds_table_strategy.rb
new file mode 100644
index 00000000000..2039ece8281
--- /dev/null
+++ b/app/services/ci/queue/builds_table_strategy.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Ci
+  module Queue
+    class BuildsTableStrategy
+      attr_reader :runner
+
+      def initialize(runner)
+        @runner = runner
+      end
+
+      # rubocop:disable CodeReuse/ActiveRecord
+      def builds_for_shared_runner
+        relation = new_builds
+          # don't run projects which have not enabled shared runners and builds
+          .joins('INNER JOIN projects ON ci_builds.project_id = projects.id')
+          .where(projects: { shared_runners_enabled: true, pending_delete: false })
+          .joins('LEFT JOIN project_features ON ci_builds.project_id = project_features.project_id')
+          .where('project_features.builds_access_level IS NULL or project_features.builds_access_level > 0')
+
+        if Feature.enabled?(:ci_queueing_disaster_recovery, runner, type: :ops, default_enabled: :yaml)
+          # if disaster recovery is enabled, we fallback to FIFO scheduling
+          relation.order('ci_builds.id ASC')
+        else
+          # Implement fair scheduling
+          # this returns builds that are ordered by number of running builds
+          # we prefer projects that don't use shared runners at all
+          relation
+            .joins("LEFT JOIN (#{running_builds_for_shared_runners.to_sql}) AS project_builds ON ci_builds.project_id = project_builds.project_id")
+            .order(Arel.sql('COALESCE(project_builds.running_builds, 0) ASC'), 'ci_builds.id ASC')
+        end
+      end
+
+      def builds_matching_tag_ids(relation, ids)
+        # pick builds that does not have other tags than runner's one
+        relation.matches_tag_ids(ids)
+      end
+
+      def builds_with_any_tags(relation)
+        # pick builds that have at least one tag
+        relation.with_any_tags
+      end
+
+      def order(relation)
+        relation.order('id ASC')
+      end
+
+      def new_builds
+        ::Ci::Build.pending.unstarted
+      end
+
+      def build_ids(relation)
+        relation.pluck(:id)
+      end
+
+      private
+
+      def running_builds_for_shared_runners
+        ::Ci::Build.running
+          .where(runner: ::Ci::Runner.instance_type)
+          .group(:project_id)
+          .select(:project_id, 'COUNT(*) AS running_builds')
+      end
+      # rubocop:enable CodeReuse/ActiveRecord
+    end
+  end
+end
diff --git a/app/services/ci/queue/pending_builds_strategy.rb b/app/services/ci/queue/pending_builds_strategy.rb
new file mode 100644
index 00000000000..1c6007f0be8
--- /dev/null
+++ b/app/services/ci/queue/pending_builds_strategy.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Ci
+  module Queue
+    class PendingBuildsStrategy
+      attr_reader :runner
+
+      def initialize(runner)
+        @runner = runner
+      end
+
+      # rubocop:disable CodeReuse/ActiveRecord
+      def builds_for_shared_runner
+        relation = new_builds
+          # don't run projects which have not enabled shared runners and builds
+          .joins('INNER JOIN projects ON ci_pending_builds.project_id = projects.id')
+          .where(projects: { shared_runners_enabled: true, pending_delete: false })
+          .joins('LEFT JOIN project_features ON ci_pending_builds.project_id = project_features.project_id')
+          .where('project_features.builds_access_level IS NULL or project_features.builds_access_level > 0')
+
+        if Feature.enabled?(:ci_queueing_disaster_recovery, runner, type: :ops, default_enabled: :yaml)
+          # if disaster recovery is enabled, we fallback to FIFO scheduling
+          relation.order('ci_pending_builds.build_id ASC')
+        else
+          # Implement fair scheduling
+          # this returns builds that are ordered by number of running builds
+          # we prefer projects that don't use shared runners at all
+          relation
+            .joins("LEFT JOIN (#{running_builds_for_shared_runners.to_sql}) AS project_builds ON ci_pending_builds.project_id=project_builds.project_id")
+            .order(Arel.sql('COALESCE(project_builds.running_builds, 0) ASC'), 'ci_pending_builds.build_id ASC')
+        end
+      end
+
+      def builds_matching_tag_ids(relation, ids)
+        relation.merge(CommitStatus.matches_tag_ids(ids, table: 'ci_pending_builds', column: 'build_id'))
+      end
+
+      def builds_with_any_tags(relation)
+        relation.merge(CommitStatus.with_any_tags(table: 'ci_pending_builds', column: 'build_id'))
+      end
+
+      def order(relation)
+        relation.order('build_id ASC')
+      end
+
+      def new_builds
+        ::Ci::PendingBuild.all
+      end
+
+      def build_ids(relation)
+        relation.pluck(:build_id)
+      end
+
+      private
+
+      def running_builds_for_shared_runners
+        ::Ci::RunningBuild
+          .instance_type
+          .group(:project_id)
+          .select(:project_id, 'COUNT(*) AS running_builds')
+      end
+      # rubocop:enable CodeReuse/ActiveRecord
+    end
+  end
+end
diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb
index 6280bf4c986..ec50312c6d4 100644
--- a/app/services/ci/register_job_service.rb
+++ b/app/services/ci/register_job_service.rb
@@ -103,35 +103,40 @@ module Ci
 
     # rubocop: disable CodeReuse/ActiveRecord
     def each_build(params, &blk)
-      builds =
+      queue = ::Ci::Queue::BuildQueueService.new(runner)
+
+      builds = begin
         if runner.instance_type?
-          builds_for_shared_runner
+          queue.builds_for_shared_runner
         elsif runner.group_type?
-          builds_for_group_runner
+          queue.builds_for_group_runner
         else
-          builds_for_project_runner
+          queue.builds_for_project_runner
         end
+      end
+
+      if runner.ref_protected?
+        builds = queue.builds_for_protected_runner(builds)
+      end
 
       # pick builds that does not have other tags than runner's one
-      builds = builds.matches_tag_ids(runner.tags.ids)
+      builds = queue.builds_matching_tag_ids(builds, runner.tags.ids)
 
       # pick builds that have at least one tag
       unless runner.run_untagged?
-        builds = builds.with_any_tags
+        builds = queue.builds_with_any_tags(builds)
       end
 
       # pick builds that older than specified age
       if params.key?(:job_age)
-        builds = builds.queued_before(params[:job_age].seconds.ago)
+        builds = queue.builds_queued_before(builds, params[:job_age].seconds.ago)
       end
 
-      build_ids = retrieve_queue(-> { builds.pluck(:id) })
+      build_ids = retrieve_queue(-> { queue.execute(builds) })
 
       @metrics.observe_queue_size(-> { build_ids.size }, @runner.runner_type)
 
-      build_ids.each do |build_id|
-        yield Ci::Build.find(build_id)
-      end
+      build_ids.each { |build_id| yield Ci::Build.find(build_id) }
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
@@ -259,63 +264,6 @@ module Ci
       )
     end
 
-    # rubocop: disable CodeReuse/ActiveRecord
-    def builds_for_shared_runner
-      relation = new_builds.
-        # don't run projects which have not enabled shared runners and builds
-        joins(:project).where(projects: { shared_runners_enabled: true, pending_delete: false })
-        .joins('LEFT JOIN project_features ON ci_builds.project_id = project_features.project_id')
-        .where('project_features.builds_access_level IS NULL or project_features.builds_access_level > 0')
-
-      if Feature.enabled?(:ci_queueing_disaster_recovery, runner, type: :ops, default_enabled: :yaml)
-        # if disaster recovery is enabled, we fallback to FIFO scheduling
-        relation.order('ci_builds.id ASC')
-      else
-        # Implement fair scheduling
-        # this returns builds that are ordered by number of running builds
-        # we prefer projects that don't use shared runners at all
-        relation
-          .joins("LEFT JOIN (#{running_builds_for_shared_runners.to_sql}) AS project_builds ON ci_builds.project_id=project_builds.project_id")
-          .order(Arel.sql('COALESCE(project_builds.running_builds, 0) ASC'), 'ci_builds.id ASC')
-      end
-    end
-
-    def builds_for_project_runner
-      new_builds.where(project: runner.projects.without_deleted.with_builds_enabled).order('id ASC')
-    end
-
-    def builds_for_group_runner
-      # Workaround for weird Rails bug, that makes `runner.groups.to_sql` to return `runner_id = NULL`
-      groups = ::Group.joins(:runner_namespaces).merge(runner.runner_namespaces)
-
-      hierarchy_groups = Gitlab::ObjectHierarchy.new(groups, options: { use_distinct: Feature.enabled?(:use_distinct_in_register_job_object_hierarchy) }).base_and_descendants
-      projects = Project.where(namespace_id: hierarchy_groups)
-        .with_group_runners_enabled
-        .with_builds_enabled
-        .without_deleted
-      new_builds.where(project: projects).order('id ASC')
-    end
-
-    def running_builds_for_shared_runners
-      Ci::Build.running.where(runner: Ci::Runner.instance_type)
-        .group(:project_id).select(:project_id, 'count(*) AS running_builds')
-    end
-
-    def all_builds
-      if Feature.enabled?(:ci_pending_builds_queue_join, runner, default_enabled: :yaml)
-        Ci::Build.joins(:queuing_entry)
-      else
-        Ci::Build.all
-      end
-    end
-    # rubocop: enable CodeReuse/ActiveRecord
-
-    def new_builds
-      builds = all_builds.pending.unstarted
-      builds = builds.ref_protected if runner.ref_protected?
-      builds
-    end
-
     def pre_assign_runner_checks
       {
         missing_dependency_failure: -> (build, _) { !build.has_valid_build_dependencies? },