Add latest changes from gitlab-org/gitlab@master

8 files changed, 171 insertions, 77 deletions

diff --git a/lib/backup/tasks/task.rb b/lib/backup/tasks/task.rb
index 4727e19b550..6aadcb42648 100644
--- a/lib/backup/tasks/task.rb
+++ b/lib/backup/tasks/task.rb
@@ -6,7 +6,9 @@ module Backup
       attr_reader :progress, :options
 
       # Identifier used as parameter in the CLI to skip from executing
-      def self.id = raise NotImplementedError
+      def self.id
+        raise NotImplementedError
+      end
 
       def initialize(progress:, options:)
         @progress = progress
@@ -14,16 +16,24 @@ module Backup
       end
 
       # Key string that identifies the task
-      def key = raise NotImplementedError
+      def key
+        raise NotImplementedError
+      end
 
       # Name of the task used for logging.
-      def human_name = raise NotImplementedError
+      def human_name
+        raise NotImplementedError
+      end
 
       # Where the task should put its backup file/dir
-      def destination_path = raise NotImplementedError
+      def destination_path
+        raise NotImplementedError
+      end
 
       # The target factory method
-      def target = raise NotImplementedError
+      def target
+        raise NotImplementedError
+      end
 
       # Path to remove after a successful backup, uses #destination_path when not specified
       def cleanup_path
@@ -31,12 +41,18 @@ module Backup
       end
 
       # `true` if the destination might not exist on a successful backup
-      def destination_optional = false
+      def destination_optional
+        false
+      end
 
       # `true` if the task can be used
-      def enabled = true
+      def enabled
+        true
+      end
 
-      def enabled? = enabled
+      def enabled?
+        enabled
+      end
     end
   end
 end
diff --git a/lib/gitlab/background_migration/backfill_project_import_level.rb b/lib/gitlab/background_migration/backfill_project_import_level.rb
deleted file mode 100644
index 1a4b1e6731f..00000000000
--- a/lib/gitlab/background_migration/backfill_project_import_level.rb
+++ /dev/null
@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-# rubocop:disable Style/Documentation
-module Gitlab
-  module BackgroundMigration
-    class BackfillProjectImportLevel < BatchedMigrationJob
-      operation_name :update_import_level
-      feature_category :database
-
-      LEVEL = {
-        Gitlab::Access::NO_ACCESS => [0],
-        Gitlab::Access::DEVELOPER => [2],
-        Gitlab::Access::MAINTAINER => [1],
-        Gitlab::Access::OWNER => [nil]
-      }.freeze
-
-      def perform
-        each_sub_batch do |sub_batch|
-          update_import_level(sub_batch)
-        end
-      end
-
-      private
-
-      def update_import_level(relation)
-        LEVEL.each do |import_level, creation_level|
-          namespace_ids = relation
-            .where(type: 'Group', project_creation_level: creation_level)
-
-          NamespaceSetting.where(
-            namespace_id: namespace_ids
-          ).update_all(project_import_level: import_level)
-        end
-      end
-    end
-  end
-end
-
-# rubocop:enable Style/Documentation
diff --git a/lib/gitlab/ci/config/interpolation/text_interpolator.rb b/lib/gitlab/ci/config/interpolation/text_interpolator.rb
index f5c83023f92..773defbfa37 100644
--- a/lib/gitlab/ci/config/interpolation/text_interpolator.rb
+++ b/lib/gitlab/ci/config/interpolation/text_interpolator.rb
@@ -10,8 +10,8 @@ module Gitlab
         class TextInterpolator
           attr_reader :errors
 
-          def initialize(config, input_args, variables)
-            @config = config
+          def initialize(yaml_documents, input_args, variables)
+            @yaml_documents = yaml_documents
             @input_args = input_args.to_h
             @variables = variables
             @errors = []
@@ -37,14 +37,12 @@ module Gitlab
           end
 
           def interpolate!
-            return errors.concat(config.errors) unless config.valid?
-
             if inputs_without_header?
               return errors.push(
                 _('Given inputs not defined in the `spec` section of the included configuration file'))
             end
 
-            return @result ||= config.content unless config.header
+            return @result ||= yaml_documents.content unless yaml_documents.header
 
             return errors.concat(header.errors) unless header.valid?
             return errors.concat(inputs.errors) unless inputs.valid?
@@ -62,14 +60,14 @@ module Gitlab
 
           private
 
-          attr_reader :config, :input_args, :variables
+          attr_reader :yaml_documents, :input_args, :variables
 
           def inputs_without_header?
-            input_args.any? && !config.header
+            input_args.any? && !yaml_documents.header
           end
 
           def header
-            @header ||= Header::Root.new(config.header).tap do |header|
+            @header ||= Header::Root.new(yaml_documents.header).tap do |header|
               header.key = 'header'
 
               header.compose!
@@ -77,7 +75,7 @@ module Gitlab
           end
 
           def content
-            @content ||= config.content
+            @content ||= yaml_documents.content
           end
 
           def spec
diff --git a/lib/gitlab/ci/config/yaml/documents.rb b/lib/gitlab/ci/config/yaml/documents.rb
index 04a31da8a2e..37d6e37b792 100644
--- a/lib/gitlab/ci/config/yaml/documents.rb
+++ b/lib/gitlab/ci/config/yaml/documents.rb
@@ -7,19 +7,12 @@ module Gitlab
         class Documents
           include Gitlab::Utils::StrongMemoize
 
-          attr_reader :errors
-
           def initialize(documents)
             @documents = documents
-            @errors = []
 
             parsed_first_document
           end
 
-          def valid?
-            errors.none?
-          end
-
           def header
             return unless has_header?
 
@@ -46,8 +39,6 @@ module Gitlab
             return {} if documents.count == 0
 
             documents.first.load!
-          rescue ::Gitlab::Config::Loader::FormatError => e
-            errors << e.message
           end
           strong_memoize_attr :parsed_first_document
         end
diff --git a/lib/gitlab/ci/config/yaml/loader.rb b/lib/gitlab/ci/config/yaml/loader.rb
index 1e9ac2b3dd5..bf20cd9c027 100644
--- a/lib/gitlab/ci/config/yaml/loader.rb
+++ b/lib/gitlab/ci/config/yaml/loader.rb
@@ -17,20 +17,23 @@ module Gitlab
           end
 
           def load
-            yaml_result = load_uninterpolated_yaml
-
-            return yaml_result unless yaml_result.valid?
+            if Feature.disabled?(:ci_text_interpolation, Feature.current_request, type: :gitlab_com_derisk)
+              return legacy_load
+            end
 
-            interpolator = Interpolation::Interpolator.new(yaml_result, inputs, variables)
+            interpolator = Interpolation::TextInterpolator.new(yaml_documents, inputs, variables)
 
             interpolator.interpolate!
 
             if interpolator.valid?
-              # This Result contains only the interpolated config and does not have a header
-              Yaml::Result.new(config: interpolator.to_hash, error: nil, interpolated: interpolator.interpolated?)
+              loaded_yaml = yaml(interpolator.to_result).load!
+
+              Yaml::Result.new(config: loaded_yaml, error: nil, interpolated: interpolator.interpolated?)
             else
               Yaml::Result.new(error: interpolator.error_message, interpolated: interpolator.interpolated?)
             end
+          rescue ::Gitlab::Config::Loader::FormatError => e
+            Yaml::Result.new(error: e.message, error_class: e)
           end
 
           def load_uninterpolated_yaml
@@ -43,6 +46,38 @@ module Gitlab
 
           attr_reader :content, :inputs, :variables
 
+          def yaml(content)
+            ensure_custom_tags
+
+            ::Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS)
+          end
+
+          def yaml_documents
+            docs = content
+              .split(::Gitlab::Config::Loader::MultiDocYaml::MULTI_DOC_DIVIDER, MAX_DOCUMENTS + 1)
+              .map { |d| yaml(d) }
+
+            docs.reject!(&:blank?)
+
+            Yaml::Documents.new(docs)
+          end
+
+          def legacy_load
+            yaml_result = load_uninterpolated_yaml
+
+            return yaml_result unless yaml_result.valid?
+
+            interpolator = Interpolation::Interpolator.new(yaml_result, inputs, variables)
+
+            interpolator.interpolate!
+
+            if interpolator.valid?
+              Yaml::Result.new(config: interpolator.to_hash, error: nil, interpolated: interpolator.interpolated?)
+            else
+              Yaml::Result.new(error: interpolator.error_message, interpolated: interpolator.interpolated?)
+            end
+          end
+
           def load_yaml!
             ensure_custom_tags
 
diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb
index 915e48828d2..fdc57f7f9dc 100644
--- a/lib/gitlab/ci/pipeline/chain/validate/external.rb
+++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb
@@ -72,7 +72,12 @@ module Gitlab
             end
 
             def validation_service_url
-              Gitlab::CurrentSettings.external_pipeline_validation_service_url || ENV['EXTERNAL_VALIDATION_SERVICE_URL']
+              if migration_enabled?
+                # After derisking, feature flag will be removed in favor of existing external_pipeline_validation_service_url setting
+                ENV['EXTERNAL_VALIDATION_SERVICE_RUNWAY_URL']
+              else
+                Gitlab::CurrentSettings.external_pipeline_validation_service_url || ENV['EXTERNAL_VALIDATION_SERVICE_URL']
+              end
             end
 
             def validation_service_token
@@ -141,6 +146,10 @@ module Gitlab
             def stages_attributes
               command.yaml_processor_result.stages_attributes
             end
+
+            def migration_enabled?
+              ENV['EXTERNAL_VALIDATION_SERVICE_RUNWAY_URL'].present? && Feature.enabled?(:external_pipeline_validation_migration, project, type: :gitlab_com_derisk)
+            end
           end
         end
       end
diff --git a/lib/gitlab/database/query_analyzer.rb b/lib/gitlab/database/query_analyzer.rb
index 6f64d04270f..b2a49a0f722 100644
--- a/lib/gitlab/database/query_analyzer.rb
+++ b/lib/gitlab/database/query_analyzer.rb
@@ -11,7 +11,7 @@ module Gitlab
       include ::Singleton
 
       Parsed = Struct.new(
-        :sql, :connection, :pg
+        :sql, :connection, :pg, :event_name
       )
 
       attr_reader :all_analyzers
@@ -20,12 +20,15 @@ module Gitlab
         @all_analyzers = []
       end
 
+      # @info most common event names are:
+      #       Model Load, Model Create, Model Update, Model Pluck, Model Destroy, Model Insert, Model Delete All
+      #       Model Exists?, nil, TRANSACTION, SCHEMA
       def hook!
         @subscriber = ActiveSupport::Notifications.subscribe('sql.active_record') do |event|
           # In some cases analyzer code might trigger another SQL call
           # to avoid stack too deep this detects recursive call of subscriber
           with_ignored_recursive_calls do
-            process_sql(event.payload[:sql], event.payload[:connection])
+            process_sql(event.payload[:sql], event.payload[:connection], event.payload[:name].to_s)
           end
         end
       end
@@ -76,11 +79,11 @@ module Gitlab
         Thread.current[:query_analyzer_enabled_analyzers] ||= []
       end
 
-      def process_sql(sql, connection)
+      def process_sql(sql, connection, event_name)
         analyzers = enabled_analyzers
         return unless analyzers&.any?
 
-        parsed = parse(sql, connection)
+        parsed = parse(sql, connection, event_name)
         return unless parsed
 
         analyzers.each do |analyzer|
@@ -93,12 +96,12 @@ module Gitlab
         end
       end
 
-      def parse(sql, connection)
+      def parse(sql, connection, event_name)
         parsed = PgQuery.parse(sql)
         return unless parsed
 
         normalized = PgQuery.normalize(sql)
-        Parsed.new(normalized, connection, parsed)
+        Parsed.new(normalized, connection, parsed, normalize_event_name(event_name))
       rescue PgQuery::ParseError => e
         # Ignore PgQuery parse errors (due to depth limit or other reasons)
         Gitlab::ErrorTracking.track_exception(e)
@@ -116,6 +119,12 @@ module Gitlab
           Thread.current[:query_analyzer_recursive] = nil
         end
       end
+
+      def normalize_event_name(event_name)
+        split_event_name = event_name.to_s.downcase.split(' ')
+
+        split_event_name.size > 1 ? split_event_name.from(1).join('_') : split_event_name.join('_')
+      end
     end
   end
 end
diff --git a/lib/gitlab/database/query_analyzers/log_large_in_lists.rb b/lib/gitlab/database/query_analyzers/log_large_in_lists.rb
new file mode 100644
index 00000000000..6cf82e0e3cd
--- /dev/null
+++ b/lib/gitlab/database/query_analyzers/log_large_in_lists.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module QueryAnalyzers
+      # The purpose of this analyzer is to log query activity that contains `IN` clauses having more that 2500 items
+      # as this type of query can cause performance degradation in the database.
+      #
+      # The feature flag should prevent sampling going above 1% or 0.01% of queries hitting
+      # to avoid performance issues
+      class LogLargeInLists < Base
+        REGEX = /\bIN\s*\(([$?\d\s*,]*)\)+/i
+        MIN_QUERY_SIZE = 10_000
+        IN_SIZE_LIMIT = 2_500
+        EVENT_NAMES = %w[load pluck].freeze
+
+        EXCLUDE_FROM_TRACE = %w[
+          lib/gitlab/database/query_analyzer.rb
+          lib/gitlab/database/query_analyzers/log_large_in_lists.rb
+        ].freeze
+
+        class << self
+          def enabled?
+            ::Feature::FlipperFeature.table_exists? &&
+              Feature.enabled?(:log_large_in_list_queries, type: :ops)
+          end
+
+          # Skips queries containing less than 10000 chars or any other events than +load+ and +pluck+
+          def requires_tracking?(parsed)
+            return false if parsed.sql.size < MIN_QUERY_SIZE
+
+            EVENT_NAMES.include?(parsed.event_name)
+          end
+
+          def analyze(parsed)
+            result = check_argument_size(parsed.sql)
+
+            log(result, parsed.event_name) if result.any?
+          end
+
+          private
+
+          def check_argument_size(sql)
+            matches = sql.scan(REGEX).flatten
+
+            return [] if matches.empty?
+
+            matches.filter_map do |match|
+              match_size = match.split(',').size
+
+              match_size if match_size > IN_SIZE_LIMIT
+            end
+          end
+
+          def log(result, event_name)
+            Gitlab::AppLogger.warn(
+              message: 'large_in_list_found',
+              matches: result.size,
+              event_name: event_name,
+              in_list_size: result.join(', '),
+              stacktrace: backtrace.first(5)
+            )
+          end
+
+          def backtrace
+            Gitlab::BacktraceCleaner.clean_backtrace(caller).reject do |line|
+              EXCLUDE_FROM_TRACE.any? { |exclusion| line.include?(exclusion) }
+            end
+          end
+        end
+      end
+    end
+  end
+end