Add latest changes from gitlab-org/gitlab@master

14 files changed, 202 insertions, 71 deletions

diff --git a/lib/gitlab/background_migration/job_coordinator.rb b/lib/gitlab/background_migration/job_coordinator.rb
index acbb5f76ad8..c440db58b94 100644
--- a/lib/gitlab/background_migration/job_coordinator.rb
+++ b/lib/gitlab/background_migration/job_coordinator.rb
@@ -14,7 +14,7 @@ module Gitlab
           worker_class = worker_for_tracking_database[tracking_database]
 
           if worker_class.nil?
-            raise ArgumentError, "tracking_database must be one of [#{worker_for_tracking_database.keys.join(', ')}]"
+            raise ArgumentError, "The '#{tracking_database}' must be one of #{worker_for_tracking_database.keys.to_a}"
           end
 
           new(worker_class)
diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
index 2bc3285def3..04b1c4a6f73 100644
--- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
 variables:
-  DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.26.0'
+  DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.25.0'
 
 .dast-auto-deploy:
   image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}"
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
index 6bf4d3e15ee..c29b5b74bfc 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
 variables:
-  AUTO_DEPLOY_IMAGE_VERSION: 'v2.26.0'
+  AUTO_DEPLOY_IMAGE_VERSION: 'v2.25.0'
 
 .auto-deploy:
   image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}"
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
index ae5051b517c..d09bb53a5b1 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
@@ -1,5 +1,5 @@
 variables:
-  AUTO_DEPLOY_IMAGE_VERSION: 'v2.26.0'
+  AUTO_DEPLOY_IMAGE_VERSION: 'v2.25.0'
 
 .auto-deploy:
   image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}"
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 80f675c825d..f5e4aa1b19d 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -3,6 +3,7 @@
 module Gitlab
   module Database
     module MigrationHelpers
+      include Migrations::ReestablishedConnectionStack
       include Migrations::BackgroundMigrationHelpers
       include Migrations::BatchedBackgroundMigrationHelpers
       include DynamicModelHelpers
diff --git a/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb
index 5a25128f3a9..d8d07fcaf2d 100644
--- a/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb
+++ b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb
@@ -27,7 +27,7 @@ module Gitlab
             return
           end
 
-          Gitlab::Database::QueryAnalyzer.instance.within([validator_class]) do
+          Gitlab::Database::QueryAnalyzer.instance.within([validator_class, connection_validator_class]) do
             validator_class.allowed_gitlab_schemas = self.allowed_gitlab_schemas
 
             super
@@ -45,6 +45,10 @@ module Gitlab
           Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas
         end
 
+        def connection_validator_class
+          Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection
+        end
+
         def unmatched_schemas
           (self.allowed_gitlab_schemas || []) - allowed_schemas_for_connection
         end
diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb
index 44aa25413e1..9bffed43077 100644
--- a/lib/gitlab/database/migrations/background_migration_helpers.rb
+++ b/lib/gitlab/database/migrations/background_migration_helpers.rb
@@ -47,6 +47,19 @@ module Gitlab
               'in the body of your migration class'
           end
 
+          # Background Migrations do not work well for in cases requiring to update `gitlab_shared`
+          # Once the decomposition is done, enqueued jobs for `gitlab_shared` tables (on CI database)
+          # will not be executed since the queue (which is stored in Redis) is tied to main database, not to schema.
+          # The batched background migrations do not have those limitations since the tracking tables
+          # are properly database-only.
+          if background_migration_restrict_gitlab_migration_schemas&.include?(:gitlab_shared)
+            raise 'The `#queue_background_migration_jobs_by_range_at_intervals` cannot " \
+              "use `restrict_gitlab_migration:` " with `:gitlab_shared`. ' \
+              'Background migrations do encode migration worker which is tied to a given database. ' \
+              'After split this worker will not be properly duplicated into decomposed database. ' \
+              'Use batched background migrations instead that do support well working across all databases.'
+          end
+
           raise "#{model_class} does not have an ID column of #{primary_column_name} to use for batch ranges" unless model_class.column_names.include?(primary_column_name.to_s)
           raise "#{primary_column_name} is not an integer or string column" unless [:integer, :string].include?(model_class.columns_hash[primary_column_name.to_s].type)
 
@@ -96,14 +109,20 @@ module Gitlab
         # delay_interval - The duration between each job's scheduled time
         # batch_size - The maximum number of jobs to fetch to memory from the database.
         def requeue_background_migration_jobs_by_range_at_intervals(job_class_name, delay_interval, batch_size: BATCH_SIZE, initial_delay: 0)
-          job_coordinator = coordinator_for_tracking_database
-
           if transaction_open?
             raise 'The `#requeue_background_migration_jobs_by_range_at_intervals` can not be run inside a transaction, ' \
               'you can disable transactions by calling disable_ddl_transaction! ' \
               'in the body of your migration class'
           end
 
+          if background_migration_restrict_gitlab_migration_schemas&.any?
+            raise 'The `#requeue_background_migration_jobs_by_range_at_intervals` cannot use `restrict_gitlab_migration:`. ' \
+              'The `#requeue_background_migration_jobs_by_range_at_intervals` needs to be executed on all databases since ' \
+              'each database has its own queue of background migrations.'
+          end
+
+          job_coordinator = coordinator_for_tracking_database
+
           # To not overload the worker too much we enforce a minimum interval both
           # when scheduling and performing jobs.
           delay_interval = [delay_interval, job_coordinator.minimum_interval].max
@@ -145,34 +164,40 @@ module Gitlab
         # This method does not garauntee that all jobs completed successfully.
         # It can only be used if the previous background migration used the queue_background_migration_jobs_by_range_at_intervals helper.
         def finalize_background_migration(class_name, delete_tracking_jobs: ['succeeded'])
-          if self.is_a?(::Gitlab::Database::MigrationHelpers::RestrictGitlabSchema)
-            raise 'The `#finalize_background_migration` is currently not supported with `Migration[2.0]`. Use `Migration[1.0]`. ' \
-              'For more information visit: https://docs.gitlab.com/ee/development/database/migrations_for_multiple_databases.html'
-          end
-
           if transaction_open?
             raise 'The `#finalize_background_migration` can not be run inside a transaction, ' \
               'you can disable transactions by calling disable_ddl_transaction! ' \
               'in the body of your migration class'
           end
 
+          if background_migration_restrict_gitlab_migration_schemas&.any?
+            raise 'The `#finalize_background_migration` cannot use `restrict_gitlab_migration:`. ' \
+              'The `#finalize_background_migration` needs to be executed on all databases since ' \
+              'each database has its own queue of background migrations.'
+          end
+
           job_coordinator = coordinator_for_tracking_database
 
-          # Empty the sidekiq queue.
-          job_coordinator.steal(class_name)
+          with_restored_connection_stack do
+            # Since we are running trusted code (background migration class) allow to execute any type of finalize
+            Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do
+              # Empty the sidekiq queue.
+              job_coordinator.steal(class_name)
 
-          # Process pending tracked jobs.
-          jobs = Gitlab::Database::BackgroundMigrationJob.pending.for_migration_class(class_name)
+              # Process pending tracked jobs.
+              jobs = Gitlab::Database::BackgroundMigrationJob.pending.for_migration_class(class_name)
 
-          jobs.find_each do |job|
-            job_coordinator.perform(job.class_name, job.arguments)
-          end
+              jobs.find_each do |job|
+                job_coordinator.perform(job.class_name, job.arguments)
+              end
 
-          # Empty the sidekiq queue.
-          job_coordinator.steal(class_name)
+              # Empty the sidekiq queue.
+              job_coordinator.steal(class_name)
 
-          # Delete job tracking rows.
-          delete_job_tracking(class_name, status: delete_tracking_jobs) if delete_tracking_jobs
+              # Delete job tracking rows.
+              delete_job_tracking(class_name, status: delete_tracking_jobs) if delete_tracking_jobs
+            end
+          end
         end
 
         def migrate_in(*args, coordinator: coordinator_for_tracking_database)
@@ -197,6 +222,10 @@ module Gitlab
 
         private
 
+        def background_migration_restrict_gitlab_migration_schemas
+          self.allowed_gitlab_schemas if self.respond_to?(:allowed_gitlab_schemas)
+        end
+
         def with_migration_context(&block)
           Gitlab::ApplicationContext.with_context(caller_id: self.class.to_s, &block)
         end
@@ -206,11 +235,9 @@ module Gitlab
         end
 
         def coordinator_for_tracking_database
-          Gitlab::BackgroundMigration.coordinator_for_database(tracking_database)
-        end
+          tracking_database = Gitlab::Database.db_config_name(connection)
 
-        def tracking_database
-          Gitlab::BackgroundMigration::DEFAULT_TRACKING_DATABASE
+          Gitlab::BackgroundMigration.coordinator_for_database(tracking_database)
         end
       end
     end
diff --git a/lib/gitlab/database/migrations/reestablished_connection_stack.rb b/lib/gitlab/database/migrations/reestablished_connection_stack.rb
new file mode 100644
index 00000000000..d7cf482c32a
--- /dev/null
+++ b/lib/gitlab/database/migrations/reestablished_connection_stack.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module Migrations
+      module ReestablishedConnectionStack
+        # This is workaround for `db:migrate` that switches `ActiveRecord::Base.connection`
+        # depending on execution. This is subject to be removed once proper fix is implemented:
+        # https://gitlab.com/gitlab-org/gitlab/-/issues/362341
+        #
+        # In some cases when we run application code we need to restore application connection stack:
+        # - ApplicationRecord (in fact ActiveRecord::Base): points to main
+        # - Ci::ApplicationRecord: points to ci
+        #
+        # rubocop:disable Database/MultipleDatabases
+        def with_restored_connection_stack(&block)
+          original_handler = ActiveRecord::Base.connection_handler
+
+          original_db_config = ActiveRecord::Base.connection_db_config
+          return yield if ActiveRecord::Base.configurations.primary?(original_db_config.name)
+
+          # If the `ActiveRecord::Base` connection is different than `:main`
+          # re-establish and configure `SharedModel` context accordingly
+          # to previously established `ActiveRecord::Base` to allow the application
+          # code to use `ApplicationRecord` and `Ci::ApplicationRecord` usual way.
+          # We swap a connection handler as migration context does hold an actual
+          # connection which we cannot close.
+          base_model = Gitlab::Database.database_base_models.fetch(original_db_config.name.to_sym)
+
+          # copy connections over to new connection handler
+          db_configs = original_handler.connection_pool_names.map do |connection_pool_name|
+            [connection_pool_name.constantize, connection_pool_name.constantize.connection_db_config]
+          end
+
+          new_handler = ActiveRecord::ConnectionAdapters::ConnectionHandler.new
+          ActiveRecord::Base.connection_handler = new_handler
+
+          db_configs.each do |klass, db_config|
+            new_handler.establish_connection(db_config, owner_name: klass)
+          end
+
+          # re-establish ActiveRecord::Base to main
+          ActiveRecord::Base.establish_connection :main # rubocop:disable Database/EstablishConnection
+
+          Gitlab::Database::SharedModel.using_connection(base_model.connection) do
+            yield
+          end
+        ensure
+          ActiveRecord::Base.connection_handler = original_handler
+          new_handler&.clear_all_connections!
+        end
+        # rubocop:enable Database/MultipleDatabases
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
index 034e18ec9f4..a541ecf5316 100644
--- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
+++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
@@ -6,7 +6,6 @@ module Gitlab
       module TableManagementHelpers
         include ::Gitlab::Database::SchemaHelpers
         include ::Gitlab::Database::MigrationHelpers
-        include ::Gitlab::Database::Migrations::BackgroundMigrationHelpers
 
         ALLOWED_TABLES = %w[audit_events web_hook_logs].freeze
         ERROR_SCOPE = 'table partitioning'
diff --git a/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb b/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb
new file mode 100644
index 00000000000..3de9e8011fb
--- /dev/null
+++ b/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module QueryAnalyzers
+      # The purpose of this analyzer is to validate if tables observed
+      # are properly used according to schema used by current connection
+      class GitlabSchemasValidateConnection < Base
+        CrossSchemaAccessError = Class.new(QueryAnalyzerError)
+
+        class << self
+          def enabled?
+            true
+          end
+
+          def analyze(parsed)
+            tables = parsed.pg.select_tables + parsed.pg.dml_tables
+            table_schemas = ::Gitlab::Database::GitlabSchema.table_schemas(tables)
+            return if table_schemas.empty?
+
+            allowed_schemas = ::Gitlab::Database.gitlab_schemas_for_connection(parsed.connection)
+            return unless allowed_schemas
+
+            invalid_schemas = table_schemas - allowed_schemas
+            if invalid_schemas.any?
+              message = "The query tried to access #{tables} (of #{table_schemas.to_a}) "
+              message += "which is outside of allowed schemas (#{allowed_schemas}) "
+              message += "for the current connection '#{Gitlab::Database.db_config_name(parsed.connection)}'"
+
+              raise CrossSchemaAccessError, message
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/shared_model.rb b/lib/gitlab/database/shared_model.rb
index 563fab692ef..f4c8fca8fa2 100644
--- a/lib/gitlab/database/shared_model.rb
+++ b/lib/gitlab/database/shared_model.rb
@@ -15,14 +15,16 @@ module Gitlab
           previous_connection = self.overriding_connection
 
           unless previous_connection.nil? || previous_connection.equal?(connection)
-            raise 'cannot nest connection overrides for shared models with different connections'
+            raise "Cannot change connection for Gitlab::Database::SharedModel "\
+              "from '#{Gitlab::Database.db_config_name(previous_connection)}' "\
+              "to '#{Gitlab::Database.db_config_name(connection)}'"
           end
 
           self.overriding_connection = connection
 
           yield
         ensure
-          self.overriding_connection = nil unless previous_connection.equal?(self.overriding_connection)
+          self.overriding_connection = previous_connection
         end
 
         def connection
diff --git a/lib/gitlab/graphql/global_id_compatibility.rb b/lib/gitlab/graphql/global_id_compatibility.rb
deleted file mode 100644
index a96e4c4b976..00000000000
--- a/lib/gitlab/graphql/global_id_compatibility.rb
+++ /dev/null
@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Graphql
-    module GlobalIDCompatibility
-      # TODO: remove this module once the compatibility layer is no longer needed.
-      # See: https://gitlab.com/gitlab-org/gitlab/-/issues/257883
-      def coerce_global_id_arguments!(args)
-        global_id_arguments = self.class.arguments.values.select do |arg|
-          arg.type.is_a?(Class) && arg.type <= ::Types::GlobalIDType
-        end
-
-        global_id_arguments.each do |arg|
-          k = arg.keyword
-          args[k] &&= arg.type.coerce_isolated_input(args[k])
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/metrics/rails_slis.rb b/lib/gitlab/metrics/rails_slis.rb
index c4f305dbdc4..71da0085c8c 100644
--- a/lib/gitlab/metrics/rails_slis.rb
+++ b/lib/gitlab/metrics/rails_slis.rb
@@ -5,16 +5,16 @@ module Gitlab
     module RailsSlis
       class << self
         def initialize_request_slis!
-          Gitlab::Metrics::Sli.initialize_sli(:rails_request_apdex, possible_request_labels) unless Gitlab::Metrics::Sli.initialized?(:rails_request_apdex)
-          Gitlab::Metrics::Sli.initialize_sli(:graphql_query_apdex, possible_graphql_query_labels) unless Gitlab::Metrics::Sli.initialized?(:graphql_query_apdex)
+          Gitlab::Metrics::Sli::Apdex.initialize_sli(:rails_request, possible_request_labels)
+          Gitlab::Metrics::Sli::Apdex.initialize_sli(:graphql_query, possible_graphql_query_labels)
         end
 
         def request_apdex
-          Gitlab::Metrics::Sli[:rails_request_apdex]
+          Gitlab::Metrics::Sli::Apdex[:rails_request]
         end
 
         def graphql_query_apdex
-          Gitlab::Metrics::Sli[:graphql_query_apdex]
+          Gitlab::Metrics::Sli::Apdex[:graphql_query]
         end
 
         private
diff --git a/lib/gitlab/metrics/sli.rb b/lib/gitlab/metrics/sli.rb
index de73db0755d..fcd893b675f 100644
--- a/lib/gitlab/metrics/sli.rb
+++ b/lib/gitlab/metrics/sli.rb
@@ -2,12 +2,10 @@
 
 module Gitlab
   module Metrics
-    class Sli
-      SliNotInitializedError = Class.new(StandardError)
-
+    module Sli
       COUNTER_PREFIX = 'gitlab_sli'
 
-      class << self
+      module ClassMethods
         INITIALIZATION_MUTEX = Mutex.new
 
         def [](name)
@@ -16,6 +14,8 @@ module Gitlab
 
         def initialize_sli(name, possible_label_combinations)
           INITIALIZATION_MUTEX.synchronize do
+            next known_slis[name] if initialized?(name)
+
             sli = new(name)
             sli.initialize_counters(possible_label_combinations)
             known_slis[name] = sli
@@ -33,6 +33,10 @@ module Gitlab
         end
       end
 
+      def self.included(mod)
+        mod.extend(ClassMethods)
+      end
+
       attr_reader :name
 
       def initialize(name)
@@ -41,16 +45,17 @@ module Gitlab
       end
 
       def initialize_counters(possible_label_combinations)
-        @initialized_with_combinations = possible_label_combinations.any?
+        # This module is effectively an abstract class
+        @initialized_with_combinations = possible_label_combinations.any? # rubocop:disable Gitlab/ModuleWithInstanceVariables
         possible_label_combinations.each do |label_combination|
           total_counter.get(label_combination)
-          success_counter.get(label_combination)
+          numerator_counter.get(label_combination)
         end
       end
 
-      def increment(labels:, success:)
+      def increment(labels:, increment_numerator:)
         total_counter.increment(labels)
-        success_counter.increment(labels) if success
+        numerator_counter.increment(labels) if increment_numerator
       end
 
       def initialized?
@@ -60,23 +65,43 @@ module Gitlab
       private
 
       def total_counter
-        prometheus.counter(total_counter_name.to_sym, "Total number of measurements for #{name}")
+        prometheus.counter(counter_name('total'), "Total number of measurements for #{name}")
       end
 
-      def success_counter
-        prometheus.counter(success_counter_name.to_sym, "Number of successful measurements for #{name}")
+      def counter_name(suffix)
+        :"#{COUNTER_PREFIX}:#{name}_#{self.class.name.demodulize.underscore}:#{suffix}"
       end
 
-      def total_counter_name
-        "#{COUNTER_PREFIX}:#{name}:total"
+      def prometheus
+        Gitlab::Metrics
       end
 
-      def success_counter_name
-        "#{COUNTER_PREFIX}:#{name}:success_total"
+      class Apdex
+        include Sli
+
+        def increment(labels:, success:)
+          super(labels: labels, increment_numerator: success)
+        end
+
+        private
+
+        def numerator_counter
+          prometheus.counter(counter_name('success_total'), "Number of successful measurements for #{name}")
+        end
       end
 
-      def prometheus
-        Gitlab::Metrics
+      class ErrorRate
+        include Sli
+
+        def increment(labels:, error:)
+          super(labels: labels, increment_numerator: error)
+        end
+
+        private
+
+        def numerator_counter
+          prometheus.counter(counter_name('error_total'), "Number of error measurements for #{name}")
+        end
       end
     end
   end