Add latest changes from gitlab-org/gitlab@15-2-stable-eev15.2.0-rc42

14 files changed, 242 insertions, 25 deletions

diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb
index ebc3ee240bd..72aa1cfe00b 100644
--- a/lib/gitlab/database/background_migration/batched_job.rb
+++ b/lib/gitlab/database/background_migration/batched_job.rb
@@ -26,6 +26,7 @@ module Gitlab
         scope :successful_in_execution_order, -> { where.not(finished_at: nil).with_status(:succeeded).order(:finished_at) }
         scope :with_preloads, -> { preload(:batched_migration) }
         scope :created_since, ->(date_time) { where('created_at >= ?', date_time) }
+        scope :blocked_by_max_attempts, -> { where('attempts >= ?', MAX_ATTEMPTS) }
 
         state_machine :status, initial: :pending do
           state :pending, value: 0
@@ -128,7 +129,8 @@ module Gitlab
               batched_migration.column_name,
               batch_min_value: min_value,
               batch_size: new_batch_size,
-              job_arguments: batched_migration.job_arguments
+              job_arguments: batched_migration.job_arguments,
+              job_class: batched_migration.job_class
             )
             midpoint = next_batch_bounds.last
 
diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb
index d052d5adc4c..9c8db2243f9 100644
--- a/lib/gitlab/database/background_migration/batched_migration.rb
+++ b/lib/gitlab/database/background_migration/batched_migration.rb
@@ -104,6 +104,12 @@ module Gitlab
             .sum(:batch_size)
         end
 
+        def reset_attempts_of_blocked_jobs!
+          batched_jobs.blocked_by_max_attempts.each_batch(of: 100) do |batch|
+            batch.update_all(attempts: 0)
+          end
+        end
+
         def interval_elapsed?(variance: 0)
           return true unless last_job
 
@@ -199,10 +205,32 @@ module Gitlab
           BatchOptimizer.new(self).optimize!
         end
 
+        def health_context
+          HealthStatus::Context.new([table_name])
+        end
+
         def hold!(until_time: 10.minutes.from_now)
+          duration_s = (until_time - Time.current).round
+          Gitlab::AppLogger.info(
+            message: "#{self} put on hold until #{until_time}",
+            migration_id: id,
+            job_class_name: job_class_name,
+            duration_s: duration_s
+          )
+
           update!(on_hold_until: until_time)
         end
 
+        def on_hold?
+          return false unless on_hold_until
+
+          on_hold_until > Time.zone.now
+        end
+
+        def to_s
+          "BatchedMigration[id: #{id}]"
+        end
+
         private
 
         def validate_batched_jobs_status
diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb
index 388eb596ce2..1bc2e931391 100644
--- a/lib/gitlab/database/background_migration/batched_migration_runner.rb
+++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb
@@ -29,7 +29,8 @@ module Gitlab
           if next_batched_job = find_or_create_next_batched_job(active_migration)
             migration_wrapper.perform(next_batched_job)
 
-            active_migration.optimize!
+            adjust_migration(active_migration)
+
             active_migration.failure! if next_batched_job.failed? && active_migration.should_stop?
           else
             finish_active_migration(active_migration)
@@ -71,12 +72,17 @@ module Gitlab
           elsif migration.finished?
             Gitlab::AppLogger.warn "Batched background migration for the given configuration is already finished: #{configuration}"
           else
+            migration.reset_attempts_of_blocked_jobs!
+
             migration.finalize!
             migration.batched_jobs.with_status(:pending).each { |job| migration_wrapper.perform(job) }
 
             run_migration_while(migration, :finalizing)
 
-            raise FailedToFinalize unless migration.finished?
+            error_message = "Batched migration #{migration.job_class_name} could not be completed and a manual action is required."\
+                            "Check the admin panel at (`/admin/background_migrations`) for more details."
+
+            raise FailedToFinalize, error_message unless migration.finished?
           end
         end
 
@@ -101,7 +107,8 @@ module Gitlab
             active_migration.column_name,
             batch_min_value: batch_min_value,
             batch_size: active_migration.batch_size,
-            job_arguments: active_migration.job_arguments)
+            job_arguments: active_migration.job_arguments,
+            job_class: active_migration.job_class)
 
           return if next_batch_bounds.nil?
 
@@ -135,6 +142,16 @@ module Gitlab
             migration.reload_last_job
           end
         end
+
+        def adjust_migration(active_migration)
+          signal = HealthStatus.evaluate(active_migration)
+
+          if signal.is_a?(HealthStatus::Signals::Stop)
+            active_migration.hold!
+          else
+            active_migration.optimize!
+          end
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/background_migration/health_status.rb b/lib/gitlab/database/background_migration/health_status.rb
new file mode 100644
index 00000000000..01f9c5eb5fd
--- /dev/null
+++ b/lib/gitlab/database/background_migration/health_status.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module BackgroundMigration
+      module HealthStatus
+        # Rather than passing along the migration, we use a more explicitly defined context
+        Context = Struct.new(:tables)
+
+        def self.evaluate(migration, indicator = Indicators::AutovacuumActiveOnTable)
+          signal = begin
+            indicator.new(migration.health_context).evaluate
+          rescue StandardError => e
+            Gitlab::ErrorTracking.track_exception(e, migration_id: migration.id,
+                                                  job_class_name: migration.job_class_name)
+            Signals::Unknown.new(indicator, reason: "unexpected error: #{e.message} (#{e.class})")
+          end
+
+          log_signal(signal, migration) if signal.log_info?
+
+          signal
+        end
+
+        def self.log_signal(signal, migration)
+          Gitlab::AppLogger.info(
+            message: "#{migration} signaled: #{signal}",
+            migration_id: migration.id
+          )
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/background_migration/health_status/indicators.rb b/lib/gitlab/database/background_migration/health_status/indicators.rb
new file mode 100644
index 00000000000..69503e5b61f
--- /dev/null
+++ b/lib/gitlab/database/background_migration/health_status/indicators.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module BackgroundMigration
+      module HealthStatus
+        module Indicators
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table.rb b/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table.rb
new file mode 100644
index 00000000000..48e12609a13
--- /dev/null
+++ b/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module BackgroundMigration
+      module HealthStatus
+        module Indicators
+          class AutovacuumActiveOnTable
+            def initialize(context)
+              @context = context
+            end
+
+            def evaluate
+              return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled?
+
+              autovacuum_active_on = active_autovacuums_for(context.tables)
+
+              if autovacuum_active_on.empty?
+                Signals::Normal.new(self.class, reason: 'no autovacuum running on any relevant tables')
+              else
+                Signals::Stop.new(self.class, reason: "autovacuum running on: #{autovacuum_active_on.join(', ')}")
+              end
+            end
+
+            private
+
+            attr_reader :context
+
+            def enabled?
+              Feature.enabled?(:batched_migrations_health_status_autovacuum, type: :ops)
+            end
+
+            def active_autovacuums_for(tables)
+              Gitlab::Database::PostgresAutovacuumActivity.for_tables(tables)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/background_migration/health_status/signals.rb b/lib/gitlab/database/background_migration/health_status/signals.rb
new file mode 100644
index 00000000000..6cd0ebd1bd0
--- /dev/null
+++ b/lib/gitlab/database/background_migration/health_status/signals.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module BackgroundMigration
+      module HealthStatus
+        module Signals
+          # Base class for a signal
+          class Base
+            attr_reader :indicator_class, :reason
+
+            def initialize(indicator_class, reason:)
+              @indicator_class = indicator_class
+              @reason = reason
+            end
+
+            def to_s
+              "#{short_name} (indicator: #{indicator_class}; reason: #{reason})"
+            end
+
+            # :nocov:
+            def log_info?
+              false
+            end
+            # :nocov:
+
+            private
+
+            def short_name
+              self.class.name.demodulize
+            end
+          end
+
+          # A Signals::Stop is an indication to put a migration on hold or stop it entirely:
+          # In general, we want to slow down or pause the migration.
+          class Stop < Base
+            # :nocov:
+            def log_info?
+              true
+            end
+            # :nocov:
+          end
+
+          # A Signals::Normal indicates normal system state: We carry on with the migration
+          # and may even attempt to optimize its throughput etc.
+          class Normal < Base; end
+
+          # When given an Signals::Unknown, something unexpected happened while
+          # we evaluated system indicators.
+          class Unknown < Base
+            # :nocov:
+            def log_info?
+              true
+            end
+            # :nocov:
+          end
+
+          # No signal could be determined, e.g. because the indicator
+          # was disabled.
+          class NotAvailable < Base; end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/each_database.rb b/lib/gitlab/database/each_database.rb
index 0d876f5124f..02f008abf85 100644
--- a/lib/gitlab/database/each_database.rb
+++ b/lib/gitlab/database/each_database.rb
@@ -36,8 +36,7 @@ module Gitlab
         private
 
         def select_base_models(names)
-          base_models = Gitlab::Database.database_base_models
-
+          base_models = Gitlab::Database.database_base_models_with_gitlab_shared
           return base_models if names.empty?
 
           names.each_with_object(HashWithIndifferentAccess.new) do |name, hash|
@@ -48,7 +47,7 @@ module Gitlab
         end
 
         def with_shared_model_connections(shared_model, selected_databases, &blk)
-          Gitlab::Database.database_base_models.each_pair do |connection_name, connection_model|
+          Gitlab::Database.database_base_models_with_gitlab_shared.each_pair do |connection_name, connection_model|
             if shared_model.limit_connection_names
               next unless shared_model.limit_connection_names.include?(connection_name.to_sym)
             end
diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb
index baf4cc48424..365a4283d4c 100644
--- a/lib/gitlab/database/gitlab_schema.rb
+++ b/lib/gitlab/database/gitlab_schema.rb
@@ -13,6 +13,8 @@
 module Gitlab
   module Database
     module GitlabSchema
+      GITLAB_SCHEMAS_FILE = 'lib/gitlab/database/gitlab_schemas.yml'
+
       # These tables are deleted/renamed, but still referenced by migrations.
       # This is needed for now, but should be removed in the future
       DELETED_TABLES = {
@@ -93,7 +95,7 @@ module Gitlab
       end
 
       def self.tables_to_schema
-        @tables_to_schema ||= YAML.load_file(Rails.root.join('lib/gitlab/database/gitlab_schemas.yml'))
+        @tables_to_schema ||= YAML.load_file(Rails.root.join(GITLAB_SCHEMAS_FILE))
       end
 
       def self.schema_names
@@ -102,3 +104,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::Database::GitlabSchema.prepend_mod
diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml
index 71c323cb393..4a467d18f0a 100644
--- a/lib/gitlab/database/gitlab_schemas.yml
+++ b/lib/gitlab/database/gitlab_schemas.yml
@@ -108,10 +108,12 @@ ci_resource_groups: :gitlab_ci
 ci_resources: :gitlab_ci
 ci_runner_namespaces: :gitlab_ci
 ci_runner_projects: :gitlab_ci
+ci_runner_versions: :gitlab_ci
 ci_runners: :gitlab_ci
 ci_running_builds: :gitlab_ci
 ci_sources_pipelines: :gitlab_ci
 ci_secure_files: :gitlab_ci
+ci_secure_file_states: :gitlab_ci
 ci_sources_projects: :gitlab_ci
 ci_stages: :gitlab_ci
 ci_subscriptions_projects: :gitlab_ci
@@ -131,7 +133,6 @@ cluster_providers_gcp: :gitlab_main
 clusters_applications_cert_managers: :gitlab_main
 clusters_applications_cilium: :gitlab_main
 clusters_applications_crossplane: :gitlab_main
-clusters_applications_elastic_stacks: :gitlab_main
 clusters_applications_helm: :gitlab_main
 clusters_applications_ingress: :gitlab_main
 clusters_applications_jupyter: :gitlab_main
@@ -139,7 +140,6 @@ clusters_applications_knative: :gitlab_main
 clusters_applications_prometheus: :gitlab_main
 clusters_applications_runners: :gitlab_main
 clusters: :gitlab_main
-clusters_integration_elasticstack: :gitlab_main
 clusters_integration_prometheus: :gitlab_main
 clusters_kubernetes_namespaces: :gitlab_main
 commit_user_mentions: :gitlab_main
@@ -325,6 +325,7 @@ milestone_releases: :gitlab_main
 milestones: :gitlab_main
 namespace_admin_notes: :gitlab_main
 namespace_aggregation_schedules: :gitlab_main
+namespace_bans: :gitlab_main
 namespace_limits: :gitlab_main
 namespace_package_settings: :gitlab_main
 namespace_root_storage_statistics: :gitlab_main
@@ -423,6 +424,8 @@ project_incident_management_settings: :gitlab_main
 project_metrics_settings: :gitlab_main
 project_mirror_data: :gitlab_main
 project_pages_metadata: :gitlab_main
+project_relation_export_uploads: :gitlab_main
+project_relation_exports: :gitlab_main
 project_repositories: :gitlab_main
 project_repository_states: :gitlab_main
 project_repository_storage_moves: :gitlab_main
@@ -432,7 +435,6 @@ projects: :gitlab_main
 projects_sync_events: :gitlab_main
 project_statistics: :gitlab_main
 project_topics: :gitlab_main
-project_tracing_settings: :gitlab_main
 prometheus_alert_events: :gitlab_main
 prometheus_alerts: :gitlab_main
 prometheus_metrics: :gitlab_main
@@ -467,6 +469,10 @@ routes: :gitlab_main
 saml_group_links: :gitlab_main
 saml_providers: :gitlab_main
 saved_replies: :gitlab_main
+sbom_components: :gitlab_main
+sbom_occurrences: :gitlab_main
+sbom_component_versions: :gitlab_main
+sbom_sources: :gitlab_main
 schema_migrations: :gitlab_internal
 scim_identities: :gitlab_main
 scim_oauth_access_tokens: :gitlab_main
@@ -547,6 +553,7 @@ vulnerability_flags: :gitlab_main
 vulnerability_historical_statistics: :gitlab_main
 vulnerability_identifiers: :gitlab_main
 vulnerability_issue_links: :gitlab_main
+vulnerability_merge_request_links: :gitlab_main
 vulnerability_occurrence_identifiers: :gitlab_main
 vulnerability_occurrence_pipelines: :gitlab_main
 vulnerability_occurrences: :gitlab_main
@@ -571,3 +578,4 @@ zentao_tracker_data: :gitlab_main
 dingtalk_tracker_data: :gitlab_main
 zoom_meetings: :gitlab_main
 batched_background_migration_job_transition_logs: :gitlab_shared
+user_namespace_callouts: :gitlab_main
diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb
index 6517923d23e..5f9416fb4db 100644
--- a/lib/gitlab/database/load_balancing.rb
+++ b/lib/gitlab/database/load_balancing.rb
@@ -19,7 +19,7 @@ module Gitlab
       ].freeze
 
       def self.base_models
-        @base_models ||= ::Gitlab::Database.database_base_models.values.freeze
+        @base_models ||= ::Gitlab::Database.database_base_models_using_load_balancing.values.freeze
       end
 
       def self.each_load_balancer
diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb
index 4aaeaa7b365..936b986ea07 100644
--- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb
+++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb
@@ -72,12 +72,6 @@ module Gitlab
         )
           Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode!
 
-          if transaction_open?
-            raise 'The `queue_batched_background_migration` cannot be run inside a transaction. ' \
-              'You can disable transactions by calling `disable_ddl_transaction!` in the body of ' \
-              'your migration class.'
-          end
-
           gitlab_schema ||= gitlab_schema_from_context
 
           Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information
@@ -182,12 +176,6 @@ module Gitlab
         def delete_batched_background_migration(job_class_name, table_name, column_name, job_arguments)
           Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode!
 
-          if transaction_open?
-            raise 'The `#delete_batched_background_migration` cannot be run inside a transaction. ' \
-              'You can disable transactions by calling `disable_ddl_transaction!` in the body of ' \
-              'your migration class.'
-          end
-
           Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information
 
           Gitlab::Database::BackgroundMigration::BatchedMigration
diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb
index 0c6a8d3d856..f38d847b0e8 100644
--- a/lib/gitlab/database/migrations/test_batched_background_runner.rb
+++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb
@@ -14,7 +14,7 @@ module Gitlab
         def jobs_by_migration_name
           Gitlab::Database::BackgroundMigration::BatchedMigration
             .executable
-            .created_after(2.days.ago) # Simple way to exclude migrations already running before migration testing
+            .created_after(3.hours.ago) # Simple way to exclude migrations already running before migration testing
             .to_h do |migration|
             batching_strategy = migration.batch_class.new(connection: connection)
 
diff --git a/lib/gitlab/database/postgres_autovacuum_activity.rb b/lib/gitlab/database/postgres_autovacuum_activity.rb
new file mode 100644
index 00000000000..a4dc199c259
--- /dev/null
+++ b/lib/gitlab/database/postgres_autovacuum_activity.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    class PostgresAutovacuumActivity < SharedModel
+      self.table_name = 'postgres_autovacuum_activity'
+      self.primary_key = 'table_identifier'
+
+      def self.for_tables(tables)
+        Gitlab::Database::LoadBalancing::Session.current.use_primary do
+          where('schema = current_schema()').where(table: tables)
+        end
+      end
+
+      def to_s
+        "table #{table_identifier} (started: #{vacuum_start})"
+      end
+    end
+  end
+end