Add latest changes from gitlab-org/gitlab@15-3-stable-eev15.3.0-rc42

7 files changed, 114 insertions, 17 deletions

diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb
index 72aa1cfe00b..81898a59da7 100644
--- a/lib/gitlab/database/background_migration/batched_job.rb
+++ b/lib/gitlab/database/background_migration/batched_job.rb
@@ -112,7 +112,7 @@ module Gitlab
         end
 
         def can_split?(exception)
-          attempts >= MAX_ATTEMPTS && TIMEOUT_EXCEPTIONS.include?(exception&.class) && batch_size > sub_batch_size
+          attempts >= MAX_ATTEMPTS && TIMEOUT_EXCEPTIONS.include?(exception&.class) && batch_size > sub_batch_size && batch_size > 1
         end
 
         def split_and_retry!
@@ -121,7 +121,7 @@ module Gitlab
 
             new_batch_size = batch_size / 2
 
-            raise SplitAndRetryError, 'Job cannot be split further' if new_batch_size < 1
+            break update!(attempts: 0) if new_batch_size < 1
 
             batching_strategy = batched_migration.batch_class.new(connection: self.class.connection)
             next_batch_bounds = batching_strategy.next_batch(
diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb
index 9c8db2243f9..6aed1eed994 100644
--- a/lib/gitlab/database/background_migration/batched_migration.rb
+++ b/lib/gitlab/database/background_migration/batched_migration.rb
@@ -206,7 +206,7 @@ module Gitlab
         end
 
         def health_context
-          HealthStatus::Context.new([table_name])
+          HealthStatus::Context.new(connection, [table_name])
         end
 
         def hold!(until_time: 10.minutes.from_now)
@@ -231,6 +231,12 @@ module Gitlab
           "BatchedMigration[id: #{id}]"
         end
 
+        def progress
+          return unless total_tuple_count.to_i > 0
+
+          100 * migrated_tuple_count / total_tuple_count
+        end
+
         private
 
         def validate_batched_jobs_status
diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb
index 1bc2e931391..7224ff2b517 100644
--- a/lib/gitlab/database/background_migration/batched_migration_runner.rb
+++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb
@@ -144,9 +144,9 @@ module Gitlab
         end
 
         def adjust_migration(active_migration)
-          signal = HealthStatus.evaluate(active_migration)
+          signals = HealthStatus.evaluate(active_migration)
 
-          if signal.is_a?(HealthStatus::Signals::Stop)
+          if signals.any?(&:stop?)
             active_migration.hold!
           else
             active_migration.optimize!
diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
index 5f4b2be3da8..ad747a8131d 100644
--- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
+++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
@@ -64,9 +64,10 @@ module Gitlab
             batch_column: tracking_record.migration_column_name,
             sub_batch_size: tracking_record.sub_batch_size,
             pause_ms: tracking_record.pause_ms,
+            job_arguments: tracking_record.migration_job_arguments,
             connection: connection)
 
-          job_instance.perform(*tracking_record.migration_job_arguments)
+          job_instance.perform
 
           job_instance
         end
diff --git a/lib/gitlab/database/background_migration/health_status.rb b/lib/gitlab/database/background_migration/health_status.rb
index 01f9c5eb5fd..9a283074b32 100644
--- a/lib/gitlab/database/background_migration/health_status.rb
+++ b/lib/gitlab/database/background_migration/health_status.rb
@@ -4,21 +4,29 @@ module Gitlab
   module Database
     module BackgroundMigration
       module HealthStatus
+        DEFAULT_INIDICATORS = [
+          Indicators::AutovacuumActiveOnTable,
+          Indicators::WriteAheadLog
+        ].freeze
+
         # Rather than passing along the migration, we use a more explicitly defined context
-        Context = Struct.new(:tables)
+        Context = Struct.new(:connection, :tables)
 
-        def self.evaluate(migration, indicator = Indicators::AutovacuumActiveOnTable)
-          signal = begin
-            indicator.new(migration.health_context).evaluate
-          rescue StandardError => e
-            Gitlab::ErrorTracking.track_exception(e, migration_id: migration.id,
-                                                  job_class_name: migration.job_class_name)
-            Signals::Unknown.new(indicator, reason: "unexpected error: #{e.message} (#{e.class})")
-          end
+        def self.evaluate(migration, indicators = DEFAULT_INIDICATORS)
+          indicators.map do |indicator|
+            signal = begin
+              indicator.new(migration.health_context).evaluate
+            rescue StandardError => e
+              Gitlab::ErrorTracking.track_exception(e, migration_id: migration.id,
+                                                    job_class_name: migration.job_class_name)
 
-          log_signal(signal, migration) if signal.log_info?
+              Signals::Unknown.new(indicator, reason: "unexpected error: #{e.message} (#{e.class})")
+            end
 
-          signal
+            log_signal(signal, migration) if signal.log_info?
+
+            signal
+          end
         end
 
         def self.log_signal(signal, migration)
diff --git a/lib/gitlab/database/background_migration/health_status/indicators/write_ahead_log.rb b/lib/gitlab/database/background_migration/health_status/indicators/write_ahead_log.rb
new file mode 100644
index 00000000000..d2fb0a8b751
--- /dev/null
+++ b/lib/gitlab/database/background_migration/health_status/indicators/write_ahead_log.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module BackgroundMigration
+      module HealthStatus
+        module Indicators
+          class WriteAheadLog
+            include Gitlab::Utils::StrongMemoize
+
+            LIMIT = 42
+            PENDING_WAL_COUNT_SQL = <<~SQL
+              WITH
+              current_wal_file AS (
+                SELECT pg_walfile_name(pg_current_wal_insert_lsn()) AS pg_walfile_name
+              ),
+              current_wal AS (
+                SELECT
+                  ('x' || substring(pg_walfile_name, 9, 8))::bit(32)::int AS log,
+                  ('x' || substring(pg_walfile_name, 17, 8))::bit(32)::int AS seg,
+                  pg_walfile_name
+                FROM current_wal_file
+              ),
+              archive_wal AS (
+                SELECT
+                  ('x' || substring(last_archived_wal, 9, 8))::bit(32)::int AS log,
+                  ('x' || substring(last_archived_wal, 17, 8))::bit(32)::int AS seg,
+                  last_archived_wal
+                FROM pg_stat_archiver
+              )
+              SELECT ((current_wal.log - archive_wal.log) * 256) + (current_wal.seg - archive_wal.seg) AS pending_wal_count
+              FROM current_wal, archive_wal
+            SQL
+
+            def initialize(context)
+              @connection = context.connection
+            end
+
+            def evaluate
+              return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled?
+
+              unless pending_wal_count
+                return Signals::NotAvailable.new(self.class, reason: 'WAL archive queue can not be calculated')
+              end
+
+              if pending_wal_count > LIMIT
+                Signals::Stop.new(self.class, reason: "WAL archive queue is too big")
+              else
+                Signals::Normal.new(self.class, reason: 'WAL archive queue is within limit')
+              end
+            end
+
+            private
+
+            attr_reader :connection
+
+            def enabled?
+              Feature.enabled?(:batched_migrations_health_status_wal, type: :ops)
+            end
+
+            # Returns number of WAL segments pending archival
+            def pending_wal_count
+              strong_memoize(:pending_wal_count) do
+                Gitlab::Database::LoadBalancing::Session.current.use_primary do
+                  connection.execute(PENDING_WAL_COUNT_SQL).to_a.first&.fetch('pending_wal_count')
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/background_migration/health_status/signals.rb b/lib/gitlab/database/background_migration/health_status/signals.rb
index 6cd0ebd1bd0..be741a9d91b 100644
--- a/lib/gitlab/database/background_migration/health_status/signals.rb
+++ b/lib/gitlab/database/background_migration/health_status/signals.rb
@@ -22,6 +22,10 @@ module Gitlab
             def log_info?
               false
             end
+
+            def stop?
+              false
+            end
             # :nocov:
 
             private
@@ -38,6 +42,10 @@ module Gitlab
             def log_info?
               true
             end
+
+            def stop?
+              true
+            end
             # :nocov:
           end