Add latest changes from gitlab-org/gitlab@master

4 files changed, 304 insertions, 7 deletions

diff --git a/lib/gitlab/ci/config/entry/bridge.rb b/lib/gitlab/ci/config/entry/bridge.rb
new file mode 100644
index 00000000000..7a6840218e1
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/bridge.rb
@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a CI/CD Bridge job that is responsible for
+        # defining a downstream project trigger.
+        #
+        class Bridge < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Configurable
+          include ::Gitlab::Config::Entry::Attributable
+          include ::Gitlab::Config::Entry::Inheritable
+
+          ALLOWED_KEYS = %i[trigger stage allow_failure only except
+                            when extends variables needs rules].freeze
+
+          validations do
+            validates :config, allowed_keys: ALLOWED_KEYS
+            validates :config, presence: true
+            validates :name, presence: true
+            validates :name, type: Symbol
+            validates :config, disallowed_keys: {
+                in: %i[only except when start_in],
+                message: 'key may not be used with `rules`'
+              },
+              if: :has_rules?
+
+            with_options allow_nil: true do
+              validates :when,
+                inclusion: { in: %w[on_success on_failure always],
+                              message: 'should be on_success, on_failure or always' }
+              validates :extends, type: String
+              validates :rules, array_of_hashes: true
+            end
+
+            validate on: :composed do
+              unless trigger.present? || bridge_needs.present?
+                errors.add(:config, 'should contain either a trigger or a needs:pipeline')
+              end
+            end
+
+            validate on: :composed do
+              next unless bridge_needs.present?
+              next if bridge_needs.one?
+
+              errors.add(:config, 'should contain at most one bridge need')
+            end
+          end
+
+          entry :trigger, ::Gitlab::Ci::Config::Entry::Trigger,
+            description: 'CI/CD Bridge downstream trigger definition.',
+            inherit: false
+
+          entry :needs, ::Gitlab::Ci::Config::Entry::Needs,
+            description: 'CI/CD Bridge needs dependency definition.',
+            inherit: false,
+            metadata: { allowed_needs: %i[job bridge] }
+
+          entry :stage, ::Gitlab::Ci::Config::Entry::Stage,
+            description: 'Pipeline stage this job will be executed into.',
+            inherit: false
+
+          entry :only, ::Gitlab::Ci::Config::Entry::Policy,
+            description: 'Refs policy this job will be executed for.',
+            default: ::Gitlab::Ci::Config::Entry::Policy::DEFAULT_ONLY,
+            inherit: false
+
+          entry :except, ::Gitlab::Ci::Config::Entry::Policy,
+            description: 'Refs policy this job will be executed for.',
+            inherit: false
+
+          entry :rules, ::Gitlab::Ci::Config::Entry::Rules,
+            description: 'List of evaluable Rules to determine job inclusion.',
+            inherit: false,
+            metadata: {
+              allowed_when: %w[on_success on_failure always never manual delayed].freeze
+            }
+
+          entry :variables, ::Gitlab::Ci::Config::Entry::Variables,
+            description: 'Environment variables available for this job.',
+            inherit: false
+
+          helpers(*ALLOWED_KEYS)
+          attributes(*ALLOWED_KEYS)
+
+          def self.matching?(name, config)
+            !name.to_s.start_with?('.') &&
+              config.is_a?(Hash) &&
+              (config.key?(:trigger) || config.key?(:needs))
+          end
+
+          def self.visible?
+            true
+          end
+
+          def compose!(deps = nil)
+            super do
+              has_workflow_rules = deps&.workflow&.has_rules?
+
+              # If workflow:rules: or rules: are used
+              # they are considered not compatible
+              # with `only/except` defaults
+              #
+              # Context: https://gitlab.com/gitlab-org/gitlab/merge_requests/21742
+              if has_rules? || has_workflow_rules
+                # Remove only/except defaults
+                # defaults are not considered as defined
+                @entries.delete(:only) unless only_defined?
+                @entries.delete(:except) unless except_defined?
+              end
+            end
+          end
+
+          def has_rules?
+            @config&.key?(:rules)
+          end
+
+          def name
+            @metadata[:name]
+          end
+
+          def value
+            { name: name,
+              trigger: (trigger_value if trigger_defined?),
+              needs: (needs_value if needs_defined?),
+              ignore: !!allow_failure,
+              stage: stage_value,
+              when: when_value,
+              extends: extends_value,
+              variables: (variables_value if variables_defined?),
+              rules: (rules_value if has_rules?),
+              only: only_value,
+              except: except_value }.compact
+          end
+
+          def bridge_needs
+            needs_value[:bridge] if needs_value
+          end
+
+          private
+
+          def overwrite_entry(deps, key, current_entry)
+            deps.default[key] unless current_entry.specified?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb
index b517dae4d2e..1d3036189b0 100644
--- a/lib/gitlab/ci/config/entry/jobs.rb
+++ b/lib/gitlab/ci/config/entry/jobs.rb
@@ -36,7 +36,7 @@ module Gitlab
             end
           end
 
-          TYPES = [Entry::Hidden, Entry::Job].freeze
+          TYPES = [Entry::Hidden, Entry::Job, Entry::Bridge].freeze
 
           private_constant :TYPES
 
@@ -77,5 +77,3 @@ module Gitlab
     end
   end
 end
-
-::Gitlab::Ci::Config::Entry::Jobs.prepend_if_ee('::EE::Gitlab::Ci::Config::Entry::Jobs')
diff --git a/lib/gitlab/ci/config/entry/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb
new file mode 100644
index 00000000000..7202784842a
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/trigger.rb
@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a cross-project downstream trigger.
+        #
+        class Trigger < ::Gitlab::Config::Entry::Simplifiable
+          strategy :SimpleTrigger, if: -> (config) { config.is_a?(String) }
+          strategy :ComplexTrigger, if: -> (config) { config.is_a?(Hash) }
+
+          class SimpleTrigger < ::Gitlab::Config::Entry::Node
+            include ::Gitlab::Config::Entry::Validatable
+
+            validations { validates :config, presence: true }
+
+            def value
+              { project: @config }
+            end
+          end
+
+          class ComplexTrigger < ::Gitlab::Config::Entry::Simplifiable
+            strategy :CrossProjectTrigger, if: -> (config) { !config.key?(:include) }
+
+            strategy :SameProjectTrigger, if: -> (config) do
+              ::Feature.enabled?(:ci_parent_child_pipeline, default_enabled: true) &&
+                config.key?(:include)
+            end
+
+            class CrossProjectTrigger < ::Gitlab::Config::Entry::Node
+              include ::Gitlab::Config::Entry::Validatable
+              include ::Gitlab::Config::Entry::Attributable
+
+              ALLOWED_KEYS = %i[project branch strategy].freeze
+              attributes :project, :branch, :strategy
+
+              validations do
+                validates :config, presence: true
+                validates :config, allowed_keys: ALLOWED_KEYS
+                validates :project, presence: true
+                validates :branch, type: String, allow_nil: true
+                validates :strategy, type: String, inclusion: { in: %w[depend], message: 'should be depend' }, allow_nil: true
+              end
+            end
+
+            class SameProjectTrigger < ::Gitlab::Config::Entry::Node
+              include ::Gitlab::Config::Entry::Validatable
+              include ::Gitlab::Config::Entry::Attributable
+              include ::Gitlab::Config::Entry::Configurable
+
+              INCLUDE_MAX_SIZE = 3
+              ALLOWED_KEYS = %i[strategy include].freeze
+              attributes :strategy
+
+              validations do
+                validates :config, presence: true
+                validates :config, allowed_keys: ALLOWED_KEYS
+                validates :strategy, type: String, inclusion: { in: %w[depend], message: 'should be depend' }, allow_nil: true
+              end
+
+              entry :include, ::Gitlab::Ci::Config::Entry::Includes,
+                description: 'List of external YAML files to include.',
+                reserved: true,
+                metadata: { max_size: INCLUDE_MAX_SIZE }
+
+              def value
+                @config
+              end
+            end
+
+            class UnknownStrategy < ::Gitlab::Config::Entry::Node
+              def errors
+                if ::Feature.enabled?(:ci_parent_child_pipeline, default_enabled: true)
+                  ['config must specify either project or include']
+                else
+                  ['config must specify project']
+                end
+              end
+            end
+          end
+
+          class UnknownStrategy < ::Gitlab::Config::Entry::Node
+            def errors
+              ["#{location} has to be either a string or a hash"]
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/import_export/import.rake b/lib/tasks/gitlab/import_export/import.rake
index 8fe61df605a..c832cba0287 100644
--- a/lib/tasks/gitlab/import_export/import.rake
+++ b/lib/tasks/gitlab/import_export/import.rake
@@ -7,12 +7,12 @@
 #   2. Performs Sidekiq job synchronously
 #
 # @example
-#   bundle exec rake "gitlab:import_export:import[root, root, imported_project, /path/to/file.tar.gz]"
+#   bundle exec rake "gitlab:import_export:import[root, root, imported_project, /path/to/file.tar.gz, true]"
 #
 namespace :gitlab do
   namespace :import_export do
     desc 'GitLab | Import/Export | EXPERIMENTAL | Import large project archives'
-    task :import, [:username, :namespace_path, :project_path, :archive_path] => :gitlab_environment do |_t, args|
+    task :import, [:username, :namespace_path, :project_path, :archive_path, :measurement_enabled] => :gitlab_environment do |_t, args|
       # Load it here to avoid polluting Rake tasks with Sidekiq test warnings
       require 'sidekiq/testing'
 
@@ -26,7 +26,8 @@ namespace :gitlab do
         namespace_path: args.namespace_path,
         project_path:   args.project_path,
         username:       args.username,
-        file_path:      args.archive_path
+        file_path:      args.archive_path,
+        measurement_enabled: args.measurement_enabled == 'true'
       ).import
     end
   end
@@ -38,6 +39,7 @@ class GitlabProjectImport
     @file_path    = opts.fetch(:file_path)
     @namespace    = Namespace.find_by_full_path(opts.fetch(:namespace_path))
     @current_user = User.find_by_username(opts.fetch(:username))
+    @measurement_enabled = opts.fetch(:measurement_enabled)
   end
 
   def import
@@ -72,6 +74,54 @@ class GitlabProjectImport
     RequestStore.clear!
   end
 
+  def with_count_queries(&block)
+    count = 0
+
+    counter_f = ->(name, started, finished, unique_id, payload) {
+      unless payload[:name].in? %w[CACHE SCHEMA]
+        count += 1
+      end
+    }
+
+    ActiveSupport::Notifications.subscribed(counter_f, "sql.active_record", &block)
+
+    puts "Number of sql calls: #{count}"
+  end
+
+  def with_gc_counter
+    gc_counts_before = GC.stat.select { |k, v| k =~ /count/ }
+    yield
+    gc_counts_after = GC.stat.select { |k, v| k =~ /count/ }
+    stats = gc_counts_before.merge(gc_counts_after) { |k, vb, va| va - vb }
+    puts "Total GC count: #{stats[:count]}"
+    puts "Minor GC count: #{stats[:minor_gc_count]}"
+    puts "Major GC count: #{stats[:major_gc_count]}"
+  end
+
+  def with_measure_time
+    timing = Benchmark.realtime do
+      yield
+    end
+
+    time = Time.at(timing).utc.strftime("%H:%M:%S")
+    puts "Time to finish: #{time}"
+  end
+
+  def with_measuring
+    puts "Measuring enabled..."
+    with_gc_counter do
+      with_count_queries do
+        with_measure_time do
+          yield
+        end
+      end
+    end
+  end
+
+  def measurement_enabled?
+    @measurement_enabled != false
+  end
+
   # We want to ensure that all Sidekiq jobs are executed
   # synchronously as part of that process.
   # This ensures that all expensive operations do not escape
@@ -79,8 +129,13 @@ class GitlabProjectImport
   def with_isolated_sidekiq_job
     Sidekiq::Testing.fake! do
       with_request_store do
+        # If you are attempting to import a large project into a development environment,
+        # you may see Gitaly throw an error about too many calls or invocations.
+        # This is due to a n+1 calls limit being set for development setups (not enforced in production)
+        # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24475#note_283090635
+        # For development setups, this code-path will be excluded from n+1 detection.
         ::Gitlab::GitalyClient.allow_n_plus_1_calls do
-          yield
+          measurement_enabled? ? with_measuring { yield } : yield
         end
       end