Add latest changes from gitlab-org/gitlab@16-3-stable-eev16.3.0-rc42

24 files changed, 341 insertions, 382 deletions

diff --git a/lib/gitlab/database/batch_counter.rb b/lib/gitlab/database/batch_counter.rb
index abb62140503..7abcef5c93b 100644
--- a/lib/gitlab/database/batch_counter.rb
+++ b/lib/gitlab/database/batch_counter.rb
@@ -16,17 +16,18 @@ module Gitlab
       DEFAULT_DISTINCT_BATCH_SIZE = 10_000
       DEFAULT_BATCH_SIZE = 100_000
 
-      def initialize(relation, column: nil, operation: :count, operation_args: nil)
+      def initialize(relation, column: nil, operation: :count, operation_args: nil, max_allowed_loops: nil)
         @relation = relation
         @column = column || relation.primary_key
         @operation = operation
         @operation_args = operation_args
+        @max_allowed_loops = max_allowed_loops || MAX_ALLOWED_LOOPS
       end
 
       def unwanted_configuration?(finish, batch_size, start)
         (@operation == :count && batch_size <= MIN_REQUIRED_BATCH_SIZE) ||
           (@operation == :sum && batch_size < DEFAULT_SUM_BATCH_SIZE) ||
-          (finish - start) / batch_size >= MAX_ALLOWED_LOOPS ||
+          (finish - start) / batch_size >= @max_allowed_loops ||
           start >= finish
       end
 
@@ -139,6 +140,7 @@ module Gitlab
             relation: @relation.table_name,
             operation: @operation,
             operation_args: @operation_args,
+            max_allowed_loops: @max_allowed_loops,
             start: batch_start,
             mode: mode,
             query: query,
diff --git a/lib/gitlab/database/bump_sequences.rb b/lib/gitlab/database/bump_sequences.rb
new file mode 100644
index 00000000000..f1b725b2de9
--- /dev/null
+++ b/lib/gitlab/database/bump_sequences.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    class BumpSequences
+      SEQUENCE_NAME_MATCHER = /nextval\('([a-z_]+)'::regclass\)/
+
+      # gitlab_schema: can be 'gitlab_main', 'gitlab_ci', 'gitlab_main_cell', 'gitlab_shared'
+      # increase_by: positive number, to increase the sequence by
+      # base_model: is to choose which connection to use to query the tables
+      def initialize(gitlab_schema, increase_by, base_model = ApplicationRecord)
+        @base_model = base_model
+        @gitlab_schema = gitlab_schema
+        @increase_by = increase_by
+      end
+
+      def execute
+        sequences_by_gitlab_schema(base_model, gitlab_schema).each do |sequence_name|
+          increment_sequence_by(base_model.connection, sequence_name, increase_by)
+        end
+      end
+
+      private
+
+      attr_reader :base_model, :gitlab_schema, :increase_by
+
+      def sequences_by_gitlab_schema(base_model, gitlab_schema)
+        tables = Gitlab::Database::GitlabSchema.tables_to_schema.select do |_table_name, schema_name|
+          schema_name == gitlab_schema
+        end.keys
+
+        sequences = []
+
+        tables.each do |table|
+          model = Class.new(base_model) do
+            self.table_name = table
+          end
+
+          model.columns.each do |column|
+            match_result = column.default_function&.match(SEQUENCE_NAME_MATCHER)
+            next unless match_result
+
+            sequences << match_result[1]
+          end
+        end
+
+        sequences
+      end
+
+      # This method is going to increase the sequence next_value by:
+      #  - increment_by + 1 if the sequence has the attribute is_called = True (which is the common case)
+      #  - increment_by if the sequence has the attribute is_called = False (for example, a newly created sequence)
+      # It uses ALTER SEQUENCE as a safety mechanism to avoid that no concurrent insertions
+      # will cause conflicts on the sequence.
+      # This is because ALTER SEQUENCE blocks concurrent nextval, currval, lastval, and setval calls.
+      def increment_sequence_by(connection, sequence_name, increment_by)
+        connection.transaction do
+          # The first call is to make sure that the sequence's is_called value is set to `true`
+          # This guarantees that the next call to `nextval` will increase the sequence by `increment_by`
+          connection.select_value("SELECT nextval($1)", nil, [sequence_name])
+          connection.execute("ALTER SEQUENCE #{sequence_name} INCREMENT BY #{increment_by}")
+          connection.select_value("select nextval($1)", nil, [sequence_name])
+          connection.execute("ALTER SEQUENCE #{sequence_name} INCREMENT BY 1")
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/ci_builds_partitioning.rb b/lib/gitlab/database/ci_builds_partitioning.rb
deleted file mode 100644
index 9f8b19f2d23..00000000000
--- a/lib/gitlab/database/ci_builds_partitioning.rb
+++ /dev/null
@@ -1,224 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Database
-    class CiBuildsPartitioning
-      include AsyncDdlExclusiveLeaseGuard
-
-      ATTEMPTS = 5
-      LOCK_TIMEOUT = 10.seconds
-      LEASE_TIMEOUT = 30.minutes
-
-      FK_NAME = :fk_e20479742e_p
-      TEMP_FK_NAME = :temp_fk_e20479742e_p
-      NEXT_PARTITION_ID = 101
-      BUILDS_PARTITION_NAME = 'gitlab_partitions_dynamic.ci_builds_101'
-      ANNOTATION_PARTITION_NAME = 'gitlab_partitions_dynamic.ci_job_annotations_101'
-      RUNNER_MACHINE_PARTITION_NAME = 'gitlab_partitions_dynamic.ci_runner_machine_builds_101'
-
-      def initialize(logger: Gitlab::AppLogger)
-        @connection = ::Ci::ApplicationRecord.connection
-        @timing_configuration = Array.new(ATTEMPTS) { [LOCK_TIMEOUT, 3.minutes] }
-        @logger = logger
-      end
-
-      def execute
-        return unless can_execute?
-
-        try_obtain_lease do
-          swap_foreign_keys
-          create_new_ci_builds_partition
-          create_new_job_annotations_partition
-          create_new_runner_machine_partition
-        end
-
-      rescue StandardError => e
-        log_info("Failed to execute: #{e.message}")
-      end
-
-      private
-
-      attr_reader :connection, :timing_configuration, :logger
-
-      delegate :quote_table_name, :quote_column_name, to: :connection
-
-      def swap_foreign_keys
-        if new_foreign_key_exists?
-          log_info('Foreign key already renamed, nothing to do')
-
-          return
-        end
-
-        with_lock_retries do
-          connection.execute drop_old_foreign_key_sql
-
-          rename_constraint :p_ci_builds_metadata, TEMP_FK_NAME, FK_NAME
-
-          each_partition do |partition|
-            rename_constraint partition.identifier, TEMP_FK_NAME, FK_NAME
-          end
-        end
-
-        log_info('Foreign key successfully renamed')
-      end
-
-      def create_new_ci_builds_partition
-        if connection.table_exists?(BUILDS_PARTITION_NAME)
-          log_info('p_ci_builds partition exists, nothing to do')
-          return
-        end
-
-        with_lock_retries do
-          connection.execute new_ci_builds_partition_sql
-        end
-
-        log_info('Partition for p_ci_builds successfully created')
-      end
-
-      def create_new_job_annotations_partition
-        if connection.table_exists?(ANNOTATION_PARTITION_NAME)
-          log_info('p_ci_job_annotations partition exists, nothing to do')
-          return
-        end
-
-        with_lock_retries do
-          connection.execute new_job_annotations_partition_sql
-        end
-
-        log_info('Partition for p_ci_job_annotations successfully created')
-      end
-
-      def create_new_runner_machine_partition
-        if connection.table_exists?(RUNNER_MACHINE_PARTITION_NAME)
-          log_info('p_ci_runner_machine_builds partition exists, nothing to do')
-          return
-        end
-
-        with_lock_retries do
-          connection.execute new_runner_machine_partition_sql
-        end
-
-        log_info('Partition for p_ci_runner_machine_builds successfully created')
-      end
-
-      def can_execute?
-        return false if process_disabled?
-        return false unless Gitlab.com?
-
-        if vacuum_running?
-          log_info('Autovacuum detected')
-
-          return false
-        end
-
-        true
-      end
-
-      def process_disabled?
-        ::Feature.disabled?(:complete_p_ci_builds_partitioning)
-      end
-
-      def new_foreign_key_exists?
-        Gitlab::Database::SharedModel.using_connection(connection) do
-          Gitlab::Database::PostgresForeignKey
-            .by_constrained_table_name_or_identifier(:p_ci_builds_metadata)
-            .by_referenced_table_name(:p_ci_builds)
-            .by_name(FK_NAME)
-            .exists?
-        end
-      end
-
-      def vacuum_running?
-        Gitlab::Database::SharedModel.using_connection(connection) do
-          Gitlab::Database::PostgresAutovacuumActivity
-            .wraparound_prevention
-            .for_tables(%i[ci_builds ci_builds_metadata])
-            .any?
-        end
-      end
-
-      def drop_old_foreign_key_sql
-        <<~SQL.squish
-          SET LOCAL statement_timeout TO '11s';
-
-          LOCK TABLE ci_builds, p_ci_builds_metadata IN ACCESS EXCLUSIVE MODE;
-
-          ALTER TABLE p_ci_builds_metadata DROP CONSTRAINT #{FK_NAME};
-        SQL
-      end
-
-      def rename_constraint(table_name, old_name, new_name)
-        connection.execute <<~SQL
-          ALTER TABLE #{quote_table_name(table_name)}
-          RENAME CONSTRAINT #{quote_column_name(old_name)} TO #{quote_column_name(new_name)}
-        SQL
-      end
-
-      def new_ci_builds_partition_sql
-        <<~SQL
-          SET LOCAL statement_timeout TO '11s';
-
-          LOCK ci_pipelines, ci_stages IN SHARE ROW EXCLUSIVE MODE;
-          LOCK TABLE ONLY p_ci_builds IN ACCESS EXCLUSIVE MODE;
-
-          CREATE TABLE IF NOT EXISTS #{BUILDS_PARTITION_NAME}
-            PARTITION OF p_ci_builds
-            FOR VALUES IN (#{NEXT_PARTITION_ID});
-        SQL
-      end
-
-      def new_job_annotations_partition_sql
-        <<~SQL
-          SET LOCAL statement_timeout TO '11s';
-
-          LOCK TABLE p_ci_builds IN SHARE ROW EXCLUSIVE MODE;
-          LOCK TABLE ONLY p_ci_job_annotations IN ACCESS EXCLUSIVE MODE;
-
-          CREATE TABLE IF NOT EXISTS #{ANNOTATION_PARTITION_NAME}
-            PARTITION OF p_ci_job_annotations
-            FOR VALUES IN (#{NEXT_PARTITION_ID});
-        SQL
-      end
-
-      def new_runner_machine_partition_sql
-        <<~SQL
-          SET LOCAL statement_timeout TO '11s';
-
-          LOCK TABLE p_ci_builds IN SHARE ROW EXCLUSIVE MODE;
-          LOCK TABLE ONLY p_ci_runner_machine_builds IN ACCESS EXCLUSIVE MODE;
-
-          CREATE TABLE IF NOT EXISTS #{RUNNER_MACHINE_PARTITION_NAME}
-            PARTITION OF p_ci_runner_machine_builds
-            FOR VALUES IN (#{NEXT_PARTITION_ID});
-        SQL
-      end
-
-      def with_lock_retries(&block)
-        Gitlab::Database::WithLockRetries.new(
-          timing_configuration: timing_configuration,
-          connection: connection,
-          logger: logger,
-          klass: self.class
-        ).run(raise_on_exhaustion: true, &block)
-      end
-
-      def each_partition(&block)
-        Gitlab::Database::SharedModel.using_connection(connection) do
-          Gitlab::Database::PostgresPartitionedTable.each_partition(:p_ci_builds_metadata, &block)
-        end
-      end
-
-      def log_info(message)
-        logger.info(message: message, class: self.class.to_s)
-      end
-
-      def connection_db_config
-        ::Ci::ApplicationRecord.connection_db_config
-      end
-
-      def lease_timeout
-        LEASE_TIMEOUT
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/database/health_status.rb b/lib/gitlab/database/health_status.rb
index 69bb8a70afd..b8dba802b56 100644
--- a/lib/gitlab/database/health_status.rb
+++ b/lib/gitlab/database/health_status.rb
@@ -6,7 +6,8 @@ module Gitlab
       DEFAULT_INIDICATORS = [
         Indicators::AutovacuumActiveOnTable,
         Indicators::WriteAheadLog,
-        Indicators::PatroniApdex
+        Indicators::PatroniApdex,
+        Indicators::WalRate
       ].freeze
 
       class << self
diff --git a/lib/gitlab/database/health_status/indicators/patroni_apdex.rb b/lib/gitlab/database/health_status/indicators/patroni_apdex.rb
index 680c86cf7b2..5631b6db6ff 100644
--- a/lib/gitlab/database/health_status/indicators/patroni_apdex.rb
+++ b/lib/gitlab/database/health_status/indicators/patroni_apdex.rb
@@ -4,82 +4,19 @@ module Gitlab
   module Database
     module HealthStatus
       module Indicators
-        class PatroniApdex
-          include Gitlab::Utils::StrongMemoize
-
-          def initialize(context)
-            @gitlab_schema = context.gitlab_schema.to_sym
-          end
-
-          def evaluate
-            return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled?
-
-            connection_error_message = fetch_connection_error_message
-            return unknown_signal(connection_error_message) if connection_error_message.present?
-
-            apdex_sli = fetch_sli(apdex_sli_query)
-            return unknown_signal('Patroni service apdex can not be calculated') unless apdex_sli.present?
-
-            if apdex_sli.to_f > apdex_slo.to_f
-              Signals::Normal.new(self.class, reason: 'Patroni service apdex is above SLO')
-            else
-              Signals::Stop.new(self.class, reason: 'Patroni service apdex is below SLO')
-            end
-          end
-
+        class PatroniApdex < PrometheusAlertIndicator
           private
 
-          attr_reader :gitlab_schema
-
           def enabled?
             Feature.enabled?(:batched_migrations_health_status_patroni_apdex, type: :ops)
           end
 
-          def unknown_signal(reason)
-            Signals::Unknown.new(self.class, reason: reason)
-          end
-
-          def fetch_connection_error_message
-            return 'Patroni Apdex Settings not configured' unless database_apdex_settings.present?
-            return 'Prometheus client is not ready' unless client.ready?
-            return 'Apdex SLI query is not configured' unless apdex_sli_query
-            return 'Apdex SLO is not configured' unless apdex_slo
-          end
-
-          def client
-            @client ||= Gitlab::PrometheusClient.new(
-              database_apdex_settings[:prometheus_api_url],
-              allow_local_requests: true,
-              verify: true
-            )
-          end
-
-          def database_apdex_settings
-            @database_apdex_settings ||= Gitlab::CurrentSettings.database_apdex_settings&.with_indifferent_access
+          def sli_query_key
+            :apdex_sli_query
           end
 
-          def apdex_sli_query
-            {
-              gitlab_main: database_apdex_settings[:apdex_sli_query][:main],
-              gitlab_ci: database_apdex_settings[:apdex_sli_query][:ci]
-            }.fetch(gitlab_schema)
-          end
-          strong_memoize_attr :apdex_sli_query
-
-          def apdex_slo
-            {
-              gitlab_main: database_apdex_settings[:apdex_slo][:main],
-              gitlab_ci: database_apdex_settings[:apdex_slo][:ci]
-            }.fetch(gitlab_schema)
-          end
-          strong_memoize_attr :apdex_slo
-
-          def fetch_sli(query)
-            response = client.query(query)
-            metric = response&.first || {}
-            value = metric.fetch('value', [])
-
-            Array.wrap(value).second
+          def slo_key
+            :apdex_slo
           end
         end
       end
diff --git a/lib/gitlab/database/health_status/indicators/prometheus_alert_indicator.rb b/lib/gitlab/database/health_status/indicators/prometheus_alert_indicator.rb
new file mode 100644
index 00000000000..3d630d21d4c
--- /dev/null
+++ b/lib/gitlab/database/health_status/indicators/prometheus_alert_indicator.rb
@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module HealthStatus
+      module Indicators
+        class PrometheusAlertIndicator
+          include Gitlab::Utils::StrongMemoize
+
+          ALERT_CONDITIONS = {
+            above: :above,
+            below: :below
+          }.freeze
+
+          def initialize(context)
+            @gitlab_schema = context.gitlab_schema.to_sym
+          end
+
+          def evaluate
+            return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled?
+
+            connection_error_message = fetch_connection_error_message
+            return unknown_signal(connection_error_message) if connection_error_message.present?
+
+            sli = fetch_sli(sli_query)
+            return unknown_signal("#{indicator_name} can not be calculated") unless sli.present?
+
+            if alert_condition == ALERT_CONDITIONS[:above] ? sli.to_f > slo.to_f : sli.to_f < slo.to_f
+              Signals::Normal.new(self.class, reason: "#{indicator_name} SLI condition met")
+            else
+              Signals::Stop.new(self.class, reason: "#{indicator_name} SLI condition not met")
+            end
+          end
+
+          private
+
+          attr_reader :gitlab_schema
+
+          def indicator_name
+            self.class.name.demodulize
+          end
+
+          # By default SLIs are expected to be above SLOs, but there can be cases
+          # where we want it to be below SLO (eg: WAL rate). For such indicators
+          # the sub-class should override this default alert_condition.
+          def alert_condition
+            ALERT_CONDITIONS[:above]
+          end
+
+          def enabled?
+            raise NotImplementedError, "prometheus alert based indicators must implement #{__method__}"
+          end
+
+          def slo_key
+            raise NotImplementedError, "prometheus alert based indicators must implement #{__method__}"
+          end
+
+          def sli_key
+            raise NotImplementedError, "prometheus alert based indicators must implement #{__method__}"
+          end
+
+          def fetch_connection_error_message
+            return 'Prometheus Settings not configured' unless prometheus_alert_db_indicators_settings.present?
+            return 'Prometheus client is not ready' unless client.ready?
+            return "#{indicator_name} SLI query is not configured" unless sli_query
+            return "#{indicator_name} SLO is not configured" unless slo
+          end
+
+          def prometheus_alert_db_indicators_settings
+            @prometheus_alert_db_indicators_settings ||= Gitlab::CurrentSettings
+              .prometheus_alert_db_indicators_settings&.with_indifferent_access
+          end
+
+          def client
+            @client ||= Gitlab::PrometheusClient.new(
+              prometheus_alert_db_indicators_settings[:prometheus_api_url],
+              allow_local_requests: true,
+              verify: true
+            )
+          end
+
+          def sli_query
+            {
+              gitlab_main: prometheus_alert_db_indicators_settings[sli_query_key][:main],
+              gitlab_ci: prometheus_alert_db_indicators_settings[sli_query_key][:ci]
+            }.fetch(gitlab_schema)
+          end
+          strong_memoize_attr :sli_query
+
+          def slo
+            {
+              gitlab_main: prometheus_alert_db_indicators_settings[slo_key][:main],
+              gitlab_ci: prometheus_alert_db_indicators_settings[slo_key][:ci]
+            }.fetch(gitlab_schema)
+          end
+          strong_memoize_attr :slo
+
+          def fetch_sli(query)
+            response = client.query(query)
+            metric = response&.first || {}
+            value = metric.fetch('value', [])
+
+            Array.wrap(value).second
+          end
+
+          def unknown_signal(reason)
+            Signals::Unknown.new(self.class, reason: reason)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/health_status/indicators/wal_rate.rb b/lib/gitlab/database/health_status/indicators/wal_rate.rb
new file mode 100644
index 00000000000..de31b5899eb
--- /dev/null
+++ b/lib/gitlab/database/health_status/indicators/wal_rate.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module HealthStatus
+      module Indicators
+        class WalRate < PrometheusAlertIndicator
+          private
+
+          def enabled?
+            Feature.enabled?(:db_health_check_wal_rate, type: :ops)
+          end
+
+          def sli_query_key
+            :wal_rate_sli_query
+          end
+
+          def slo_key
+            :wal_rate_slo
+          end
+
+          def alert_condition
+            ALERT_CONDITIONS[:below]
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 256c524e989..60cec12b4b5 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -1191,6 +1191,19 @@ into similar problems in the future (e.g. when new tables are created).
           .present?
       end
 
+      # While it is safe to call `change_column_default` on a column without
+      # default it would still require access exclusive lock on the table
+      # and for tables with high autovacuum(wraparound prevention) it will
+      # fail if their executions overlap.
+      #
+      def remove_column_default(table_name, column_name)
+        column = connection.columns(table_name).find { |col| col.name == column_name.to_s }
+
+        if column.default || column.default_function
+          change_column_default(table_name, column_name, to: nil)
+        end
+      end
+
       private
 
       def multiple_columns(columns, separator: ', ')
diff --git a/lib/gitlab/database/migration_helpers/convert_to_bigint.rb b/lib/gitlab/database/migration_helpers/convert_to_bigint.rb
index 63928d7dc09..11f1e62e8b9 100644
--- a/lib/gitlab/database/migration_helpers/convert_to_bigint.rb
+++ b/lib/gitlab/database/migration_helpers/convert_to_bigint.rb
@@ -15,6 +15,20 @@ module Gitlab
 
           Gitlab.com? && !Gitlab.jh?
         end
+
+        def temp_column_removed?(table_name, column_name)
+          !column_exists?(table_name.to_s, convert_to_bigint_column(column_name))
+        end
+
+        def columns_swapped?(table_name, column_name)
+          table_columns = columns(table_name.to_s)
+          temp_column_name = convert_to_bigint_column(column_name)
+
+          column = table_columns.find { |c| c.name == column_name.to_s }
+          temp_column = table_columns.find { |c| c.name == temp_column_name }
+
+          column.sql_type == 'bigint' && temp_column.sql_type == 'integer'
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb
index cb2a98b553f..efb1957d5e7 100644
--- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb
+++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb
@@ -242,7 +242,7 @@ module Gitlab
             "\n\n" \
             "For more information, check the documentation" \
             "\n\n" \
-            "\thttps://docs.gitlab.com/ee/user/admin_area/monitoring/background_migrations.html#database-migrations-failing-because-of-batched-background-migration-not-finished"
+            "\thttps://docs.gitlab.com/ee/update/background_migrations.html#database-migrations-failing-because-of-batched-background-migration-not-finished"
         end
       end
     end
diff --git a/lib/gitlab/database/migrations/squasher.rb b/lib/gitlab/database/migrations/squasher.rb
new file mode 100644
index 00000000000..98fdf873aa5
--- /dev/null
+++ b/lib/gitlab/database/migrations/squasher.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module Gitlab
+  module Database
+    module Migrations
+      class Squasher
+        RSPEC_FILENAME_REGEXP = /\A([0-9]+_)?([_a-z0-9]*)\.rb\z/
+
+        def initialize(git_output)
+          @migration_data = migration_files_from_git(git_output).filter_map do |mf|
+            basename = Pathname(mf).basename.to_s
+            file_name_match = ActiveRecord::Migration::MigrationFilenameRegexp.match(basename)
+            slug = file_name_match[2]
+            unless slug == 'init_schema'
+              {
+                path: mf,
+                basename: basename,
+                timestamp: file_name_match[1],
+                slug: slug
+              }
+            end
+          end
+        end
+
+        def files_to_delete
+          @migration_data.pluck(:path) + schema_migrations + find_migration_specs
+        end
+
+        private
+
+        def schema_migrations
+          @migration_data.map { |m| "db/schema_migrations/#{m[:timestamp]}" }
+        end
+
+        def find_migration_specs
+          @file_slugs = Set.new @migration_data.pluck(:slug)
+          (migration_specs + ee_migration_specs).select { |f| file_has_slug?(f) }
+        end
+
+        def migration_files_from_git(body)
+          body.chomp
+              .split("\n")
+              .select { |fn| fn.end_with?('.rb') }
+        end
+
+        def match_file_slug(filename)
+          m = RSPEC_FILENAME_REGEXP.match(filename)
+          return if m.nil?
+
+          m[2].sub(/_spec$/, '')
+        end
+
+        def file_has_slug?(filename)
+          spec_slug = match_file_slug(Pathname(filename).basename.to_s)
+          return false if spec_slug.nil?
+
+          @file_slugs.include?(spec_slug)
+        end
+
+        def migration_specs
+          Dir.glob(Rails.root.join('spec/migrations/*.rb'))
+        end
+
+        def ee_migration_specs
+          Dir.glob(Rails.root.join('ee/spec/migrations/*.rb'))
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/postgres_constraint.rb b/lib/gitlab/database/postgres_constraint.rb
index fa3870cb9c7..9c49251664b 100644
--- a/lib/gitlab/database/postgres_constraint.rb
+++ b/lib/gitlab/database/postgres_constraint.rb
@@ -17,7 +17,7 @@ module Gitlab
       scope :valid, -> { where(constraint_valid: true) }
 
       scope :by_table_identifier, ->(identifier) do
-        unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER
+        unless Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER.match?(identifier)
           raise ArgumentError, "Table name is not fully qualified with a schema: #{identifier}"
         end
 
diff --git a/lib/gitlab/database/postgres_foreign_key.rb b/lib/gitlab/database/postgres_foreign_key.rb
index bb3e1d45f15..9fb3098efe0 100644
--- a/lib/gitlab/database/postgres_foreign_key.rb
+++ b/lib/gitlab/database/postgres_foreign_key.rb
@@ -21,7 +21,7 @@ module Gitlab
       enum on_update_action: ACTION_TYPES, _prefix: :on_update
 
       scope :by_referenced_table_identifier, ->(identifier) do
-        unless identifier =~ Database::FULLY_QUALIFIED_IDENTIFIER
+        unless Database::FULLY_QUALIFIED_IDENTIFIER.match?(identifier)
           raise ArgumentError, "Referenced table name is not fully qualified with a schema: #{identifier}"
         end
 
@@ -31,7 +31,7 @@ module Gitlab
       scope :by_referenced_table_name, ->(name) { where(referenced_table_name: name) }
 
       scope :by_constrained_table_identifier, ->(identifier) do
-        unless identifier =~ Database::FULLY_QUALIFIED_IDENTIFIER
+        unless Database::FULLY_QUALIFIED_IDENTIFIER.match?(identifier)
           raise ArgumentError, "Constrained table name is not fully qualified with a schema: #{identifier}"
         end
 
@@ -41,7 +41,7 @@ module Gitlab
       scope :by_constrained_table_name, ->(name) { where(constrained_table_name: name) }
 
       scope :by_constrained_table_name_or_identifier, ->(name) do
-        if name =~ Database::FULLY_QUALIFIED_IDENTIFIER
+        if Database::FULLY_QUALIFIED_IDENTIFIER.match?(name)
           by_constrained_table_identifier(name)
         else
           by_constrained_table_name(name)
diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb
index 50009cadf5d..1c775482e7e 100644
--- a/lib/gitlab/database/postgres_index.rb
+++ b/lib/gitlab/database/postgres_index.rb
@@ -14,7 +14,7 @@ module Gitlab
       has_many :queued_reindexing_actions, class_name: 'Gitlab::Database::Reindexing::QueuedAction', foreign_key: :index_identifier
 
       scope :by_identifier, ->(identifier) do
-        unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER
+        unless Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER.match?(identifier)
           raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}"
         end
 
diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb
index e63c6fc86ea..f79b8b5e32c 100644
--- a/lib/gitlab/database/postgres_partition.rb
+++ b/lib/gitlab/database/postgres_partition.rb
@@ -10,7 +10,7 @@ module Gitlab
       # identifier includes the partition schema.
       # For example 'gitlab_partitions_static.events_03', or 'gitlab_partitions_dynamic.logs_03'
       scope :for_identifier, ->(identifier) do
-        unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER
+        unless Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER.match?(identifier)
           raise ArgumentError, "Partition name is not fully qualified with a schema: #{identifier}"
         end
 
@@ -22,7 +22,7 @@ module Gitlab
       end
 
       scope :for_parent_table, ->(parent_table) do
-        if parent_table =~ Database::FULLY_QUALIFIED_IDENTIFIER
+        if Database::FULLY_QUALIFIED_IDENTIFIER.match?(parent_table)
           where(parent_identifier: parent_table).order(:name)
         else
           where("parent_identifier = concat(current_schema(), '.', ?)", parent_table).order(:name)
diff --git a/lib/gitlab/database/postgres_partitioned_table.rb b/lib/gitlab/database/postgres_partitioned_table.rb
index fead7379e43..76e2cd48f80 100644
--- a/lib/gitlab/database/postgres_partitioned_table.rb
+++ b/lib/gitlab/database/postgres_partitioned_table.rb
@@ -10,7 +10,7 @@ module Gitlab
       has_many :postgres_partitions, foreign_key: 'parent_identifier', primary_key: 'identifier'
 
       scope :by_identifier, ->(identifier) do
-        unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER
+        unless Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER.match?(identifier)
           raise ArgumentError, "Table name is not fully qualified with a schema: #{identifier}"
         end
 
diff --git a/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb b/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb
index 71d2554844e..21392283ccf 100644
--- a/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb
+++ b/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb
@@ -11,6 +11,8 @@ module Gitlab
         end
 
         def force_disconnect_if_old!
+          return if Rails.env.test? && transaction_open?
+
           if force_disconnect_timer.expired?
             disconnect!
             reset_force_disconnect_timer!
diff --git a/lib/gitlab/database/query_analyzers/query_recorder.rb b/lib/gitlab/database/query_analyzers/query_recorder.rb
deleted file mode 100644
index 63b4fbb8c1d..00000000000
--- a/lib/gitlab/database/query_analyzers/query_recorder.rb
+++ /dev/null
@@ -1,53 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Database
-    module QueryAnalyzers
-      class QueryRecorder < Base
-        LOG_PATH = 'query_recorder/'
-        LIST_PARAMETER_REGEX = %r{\$\d+(?:\s*,\s*\$\d+)+}.freeze
-        SINGLE_PARAMETER_REGEX = %r{\$\d+}.freeze
-
-        class << self
-          def enabled?
-            # Only enable QueryRecorder in CI on database MRs or default branch
-            ENV['CI_MERGE_REQUEST_LABELS']&.include?('database') ||
-              (ENV['CI_COMMIT_REF_NAME'].present? && ENV['CI_COMMIT_REF_NAME'] == ENV['CI_DEFAULT_BRANCH'])
-          end
-
-          def analyze(parsed)
-            payload = {
-              normalized: normalize_query(parsed.sql)
-            }
-
-            log_query(payload)
-          end
-
-          def log_file
-            Rails.root.join(LOG_PATH, "#{ENV.fetch('CI_JOB_NAME_SLUG', 'rspec')}.ndjson")
-          end
-
-          private
-
-          def log_query(payload)
-            log_dir = Rails.root.join(LOG_PATH)
-
-            # Create log directory if it does not exist since it is only created
-            # ahead of time by certain CI jobs
-            FileUtils.mkdir_p(log_dir) unless Dir.exist?(log_dir)
-
-            log_line = "#{Gitlab::Json.dump(payload)}\n"
-
-            File.write(log_file, log_line, mode: 'a')
-          end
-
-          def normalize_query(query)
-            query
-              .gsub(LIST_PARAMETER_REGEX, '?,?,?') # Replace list parameters with ?,?,?
-              .gsub(SINGLE_PARAMETER_REGEX, '?')   # Replace single parameters with ?
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb
index 9c860ebc6aa..4a1b0be848e 100644
--- a/lib/gitlab/database/reindexing.rb
+++ b/lib/gitlab/database/reindexing.rb
@@ -59,7 +59,6 @@ module Gitlab
       # most bloated indexes for reindexing.
       def self.perform_with_heuristic(candidate_indexes = Gitlab::Database::PostgresIndex.reindexing_support, maximum_records: DEFAULT_INDEXES_PER_INVOCATION)
         IndexSelection.new(candidate_indexes).take(maximum_records).each do |index|
-          Gitlab::Database::CiBuildsPartitioning.new.execute
           Coordinator.new(index).perform
         end
       end
diff --git a/lib/gitlab/database/reindexing/reindex_concurrently.rb b/lib/gitlab/database/reindexing/reindex_concurrently.rb
index 60fa4deda39..aa445082aa9 100644
--- a/lib/gitlab/database/reindexing/reindex_concurrently.rb
+++ b/lib/gitlab/database/reindexing/reindex_concurrently.rb
@@ -20,7 +20,7 @@ module Gitlab
 
         def perform
           raise ReindexError, 'indexes serving an exclusion constraint are currently not supported' if index.exclusion?
-          raise ReindexError, 'index is a left-over temporary index from a previous reindexing run' if index.name =~ /#{TEMPORARY_INDEX_PATTERN}/o
+          raise ReindexError, 'index is a left-over temporary index from a previous reindexing run' if /#{TEMPORARY_INDEX_PATTERN}/o.match?(index.name)
 
           # Expression indexes require additional statistics in `pg_statistic`:
           # select * from pg_statistic where starelid = (select oid from pg_class where relname = 'some_index');
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
index 562e651cabc..72ae2849911 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
@@ -23,6 +23,7 @@ module Gitlab
             with_paths = MigrationClasses::Route.arel_table[:path]
                            .matches_any(path_patterns)
             namespaces.joins(:route).where(with_paths)
+              .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046")
           end
 
           def rename_namespace(namespace)
@@ -45,6 +46,7 @@ module Gitlab
             reverts_for_type('namespace') do |path_before_rename, current_path|
               matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path)
               namespace = MigrationClasses::Namespace.joins(:route)
+                            .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046")
                             .find_by(matches_path)&.becomes(MigrationClasses::Namespace) # rubocop: disable Cop/AvoidBecomes
 
               if namespace
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
index 5dbf30bad4e..155e35b64f4 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb
@@ -37,6 +37,8 @@ module Gitlab
             reverts_for_type('project') do |path_before_rename, current_path|
               matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path)
               project = MigrationClasses::Project.joins(:route)
+                          .allow_cross_joins_across_databases(url:
+                            'https://gitlab.com/gitlab-org/gitlab/-/issues/421843')
                           .find_by(matches_path)
 
               if project
@@ -67,6 +69,7 @@ module Gitlab
                            .matches_any(path_patterns)
 
             @projects_for_paths = MigrationClasses::Project.joins(:route).where(with_paths)
+              .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843')
           end
         end
       end
diff --git a/lib/gitlab/database/schema_validation/schema_inconsistency.rb b/lib/gitlab/database/schema_validation/schema_inconsistency.rb
deleted file mode 100644
index 9f39db5b4c0..00000000000
--- a/lib/gitlab/database/schema_validation/schema_inconsistency.rb
+++ /dev/null
@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Database
-    module SchemaValidation
-      class SchemaInconsistency < ApplicationRecord
-        self.table_name = :schema_inconsistencies
-
-        belongs_to :issue
-
-        validates :object_name, :valitador_name, :table_name, :diff, presence: true
-
-        scope :with_open_issues, -> { joins(:issue).where('issue.state_id': Issue.available_states[:opened]) }
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/database/tables_sorted_by_foreign_keys.rb b/lib/gitlab/database/tables_sorted_by_foreign_keys.rb
index b2a7f5442e9..9593cb45947 100644
--- a/lib/gitlab/database/tables_sorted_by_foreign_keys.rb
+++ b/lib/gitlab/database/tables_sorted_by_foreign_keys.rb
@@ -34,7 +34,7 @@ module Gitlab
       def all_foreign_keys
         @all_foreign_keys ||= @tables.each_with_object(Hash.new { |h, k| h[k] = [] }) do |table, hash|
           foreign_keys_for(table).each do |fk|
-            hash[fk.to_table] << table
+            hash[fk.referenced_table_name] << table
           end
         end
       end
@@ -45,12 +45,10 @@ module Gitlab
         #
         # See spec/lib/gitlab/database/tables_sorted_by_foreign_keys_spec.rb
         # for an example
-        name = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils.extract_schema_qualified_name(table)
+        table = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils.extract_schema_qualified_name(table)
 
-        if name.schema == ::Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA.to_s
-          @connection.foreign_keys(name.identifier)
-        else
-          @connection.foreign_keys(table)
+        Gitlab::Database::SharedModel.using_connection(@connection) do
+          Gitlab::Database::PostgresForeignKey.by_constrained_table_name_or_identifier(table.identifier).load
         end
       end
     end