Add latest changes from gitlab-org/gitlab@master

14 files changed, 296 insertions, 88 deletions

diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb
index 5f922d87c6f..5147ea92291 100644
--- a/lib/gitlab/database/background_migration/batched_job.rb
+++ b/lib/gitlab/database/background_migration/batched_job.rb
@@ -4,6 +4,7 @@ module Gitlab
   module Database
     module BackgroundMigration
       SplitAndRetryError = Class.new(StandardError)
+      ReduceSubBatchSizeError = Class.new(StandardError)
 
       class BatchedJob < SharedModel
         include EachBatch
@@ -12,6 +13,9 @@ module Gitlab
         self.table_name = :batched_background_migration_jobs
 
         MAX_ATTEMPTS = 3
+        MIN_BATCH_SIZE = 1
+        SUB_BATCH_SIZE_REDUCE_FACTOR = 0.75
+        SUB_BATCH_SIZE_THRESHOLD = 65
         STUCK_JOBS_TIMEOUT = 1.hour.freeze
         TIMEOUT_EXCEPTIONS = [ActiveRecord::StatementTimeout, ActiveRecord::ConnectionTimeoutError,
                               ActiveRecord::AdapterTimeout, ActiveRecord::LockWaitTimeout,
@@ -59,12 +63,12 @@ module Gitlab
           end
 
           after_transition any => :failed do |job, transition|
-            error_hash = transition.args.find { |arg| arg[:error].present? }
+            exception, from_sub_batch = job.class.extract_transition_options(transition.args)
 
-            exception = error_hash&.fetch(:error)
+            job.reduce_sub_batch_size! if from_sub_batch && job.can_reduce_sub_batch_size?
 
             job.split_and_retry! if job.can_split?(exception)
-          rescue SplitAndRetryError => error
+          rescue SplitAndRetryError, ReduceSubBatchSizeError => error
             Gitlab::AppLogger.error(
               message: error.message,
               batched_job_id: job.id,
@@ -75,9 +79,7 @@ module Gitlab
           end
 
           after_transition do |job, transition|
-            error_hash = transition.args.find { |arg| arg[:error].present? }
-
-            exception = error_hash&.fetch(:error)
+            exception, _ = job.class.extract_transition_options(transition.args)
 
             job.batched_job_transition_logs.create(previous_status: transition.from, next_status: transition.to, exception_class: exception&.class, exception_message: exception&.message)
 
@@ -100,6 +102,17 @@ module Gitlab
         delegate :job_class, :table_name, :column_name, :job_arguments, :job_class_name,
           to: :batched_migration, prefix: :migration
 
+        def self.extract_transition_options(args)
+          error_hash = args.find { |arg| arg[:error].present? }
+
+          return [] unless error_hash
+
+          exception = error_hash.fetch(:error)
+          from_sub_batch = error_hash[:from_sub_batch]
+
+          [exception, from_sub_batch]
+        end
+
         def time_efficiency
           return unless succeeded?
           return unless finished_at && started_at
@@ -111,10 +124,15 @@ module Gitlab
         end
 
         def can_split?(exception)
-          attempts >= MAX_ATTEMPTS &&
-            exception&.class&.in?(TIMEOUT_EXCEPTIONS) &&
-            batch_size > sub_batch_size &&
-            batch_size > 1
+          return if still_retryable?
+
+          exception.class.in?(TIMEOUT_EXCEPTIONS) && within_batch_size_boundaries?
+        end
+
+        def can_reduce_sub_batch_size?
+          return false unless Feature.enabled?(:reduce_sub_batch_size_on_timeouts)
+
+          still_retryable? && within_batch_size_boundaries?
         end
 
         def split_and_retry!
@@ -163,6 +181,51 @@ module Gitlab
             end
           end
         end
+
+        # It reduces the size of +sub_batch_size+ by 25%
+        def reduce_sub_batch_size!
+          raise ReduceSubBatchSizeError, 'Only sub_batch_size of failed jobs can be reduced' unless failed?
+
+          return if sub_batch_exceeds_threshold?
+
+          with_lock do
+            actual_sub_batch_size = sub_batch_size
+            reduced_sub_batch_size = (sub_batch_size * SUB_BATCH_SIZE_REDUCE_FACTOR).to_i.clamp(1, batch_size)
+
+            update!(sub_batch_size: reduced_sub_batch_size)
+
+            Gitlab::AppLogger.warn(
+              message: 'Sub batch size reduced due to timeout',
+              batched_job_id: id,
+              sub_batch_size: actual_sub_batch_size,
+              reduced_sub_batch_size: reduced_sub_batch_size,
+              attempts: attempts,
+              batched_migration_id: batched_migration.id,
+              job_class_name: migration_job_class_name,
+              job_arguments: migration_job_arguments
+            )
+          end
+        end
+
+        def still_retryable?
+          attempts < MAX_ATTEMPTS
+        end
+
+        def within_batch_size_boundaries?
+          batch_size > MIN_BATCH_SIZE && batch_size > sub_batch_size
+        end
+
+        # It doesn't allow sub-batch size to be reduced lower than the threshold
+        #
+        # @info It will prevent the next iteration to reduce the +sub_batch_size+ lower
+        #       than the +SUB_BATCH_SIZE_THRESHOLD+ or 65% of its original size.
+        def sub_batch_exceeds_threshold?
+          initial_sub_batch_size = batched_migration.sub_batch_size
+          reduced_sub_batch_size = (sub_batch_size * SUB_BATCH_SIZE_REDUCE_FACTOR).to_i
+          diff = initial_sub_batch_size - reduced_sub_batch_size
+
+          (1.0 * diff / initial_sub_batch_size * 100).round(2) > SUB_BATCH_SIZE_THRESHOLD
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
index f1fc3efae9e..8fdaa685ba9 100644
--- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
+++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
@@ -15,13 +15,21 @@ module Gitlab
         # when starting and finishing execution, and optionally saves batch_metrics
         # the migration provides, if any are given.
         #
-        # The job's batch_metrics are serialized to JSON for storage.
+        # @info The job's batch_metrics are serialized to JSON for storage.
+        #
+        # @info Track exceptions that could happen when processing sub-batches
+        #       through +Gitlab::BackgroundMigration::SubBatchTimeoutException+
         def perform(batch_tracking_record)
           start_tracking_execution(batch_tracking_record)
 
           execute_batch(batch_tracking_record)
 
           batch_tracking_record.succeed!
+        rescue SubBatchTimeoutError => exception
+          caused_by = exception.caused_by
+          batch_tracking_record.failure!(error: caused_by, from_sub_batch: true)
+
+          raise caused_by
         rescue Exception => error # rubocop:disable Lint/RescueException
           batch_tracking_record.failure!(error: error)
 
@@ -67,7 +75,8 @@ module Gitlab
             sub_batch_size: tracking_record.sub_batch_size,
             pause_ms: tracking_record.pause_ms,
             job_arguments: tracking_record.migration_job_arguments,
-            connection: connection)
+            connection: connection,
+            sub_batch_exception: ::Gitlab::Database::BackgroundMigration::SubBatchTimeoutError)
 
           job_instance.perform
 
diff --git a/lib/gitlab/database/background_migration/sub_batch_timeout_error.rb b/lib/gitlab/database/background_migration/sub_batch_timeout_error.rb
new file mode 100644
index 00000000000..815dff11e1f
--- /dev/null
+++ b/lib/gitlab/database/background_migration/sub_batch_timeout_error.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module BackgroundMigration
+      class SubBatchTimeoutError < StandardError
+        def initialize(caused_by)
+          @caused_by = caused_by
+
+          super(caused_by)
+        end
+
+        attr_reader :caused_by
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/schema_validation/database.rb b/lib/gitlab/database/schema_validation/database.rb
index 15f003a1b0c..07bd02e58e1 100644
--- a/lib/gitlab/database/schema_validation/database.rb
+++ b/lib/gitlab/database/schema_validation/database.rb
@@ -4,6 +4,8 @@ module Gitlab
   module Database
     module SchemaValidation
       class Database
+        STATIC_PARTITIONS_SCHEMA = 'gitlab_partitions_static'
+
         def initialize(connection)
           @connection = connection
         end
@@ -12,33 +14,69 @@ module Gitlab
           index_map[index_name]
         end
 
-        def indexes
-          index_map.values
+        def fetch_trigger_by_name(trigger_name)
+          trigger_map[trigger_name]
         end
 
         def index_exists?(index_name)
           index_map[index_name].present?
         end
 
+        def trigger_exists?(trigger_name)
+          trigger_map[trigger_name].present?
+        end
+
+        def indexes
+          index_map.values
+        end
+
+        def triggers
+          trigger_map.values
+        end
+
         private
 
+        attr_reader :connection
+
+        def schemas
+          @schemas ||= [STATIC_PARTITIONS_SCHEMA, connection.current_schema]
+        end
+
         def index_map
           @index_map ||=
             fetch_indexes.transform_values! do |index_stmt|
-              Index.new(PgQuery.parse(index_stmt).tree.stmts.first.stmt.index_stmt)
+              SchemaObjects::Index.new(PgQuery.parse(index_stmt).tree.stmts.first.stmt.index_stmt)
             end
         end
 
-        attr_reader :connection
+        def trigger_map
+          @trigger_map ||=
+            fetch_triggers.transform_values! do |trigger_stmt|
+              SchemaObjects::Trigger.new(PgQuery.parse(trigger_stmt).tree.stmts.first.stmt.create_trig_stmt)
+            end
+        end
 
         def fetch_indexes
           sql = <<~SQL
             SELECT indexname, indexdef
             FROM pg_indexes
-            WHERE indexname NOT LIKE '%_pkey' AND schemaname IN ('public', 'gitlab_partitions_static');
+            WHERE indexname NOT LIKE '%_pkey' AND schemaname IN ($1, $2);
+          SQL
+
+          connection.select_rows(sql, nil, schemas).to_h
+        end
+
+        def fetch_triggers
+          sql = <<~SQL
+            SELECT triggers.tgname, pg_get_triggerdef(triggers.oid)
+            FROM pg_catalog.pg_trigger triggers
+            INNER JOIN pg_catalog.pg_class rel ON triggers.tgrelid = rel.oid
+            INNER JOIN pg_catalog.pg_namespace nsp ON nsp.oid = rel.relnamespace
+            WHERE triggers.tgisinternal IS FALSE
+            AND nsp.nspname IN ($1, $2)
           SQL
 
-          @fetch_indexes ||= connection.exec_query(sql).rows.to_h
+          connection.select_rows(sql, nil, schemas).to_h
         end
       end
     end
diff --git a/lib/gitlab/database/schema_validation/index.rb b/lib/gitlab/database/schema_validation/index.rb
deleted file mode 100644
index af0d5f31f4e..00000000000
--- a/lib/gitlab/database/schema_validation/index.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Database
-    module SchemaValidation
-      class Index
-        def initialize(parsed_stmt)
-          @parsed_stmt = parsed_stmt
-        end
-
-        def name
-          parsed_stmt.idxname
-        end
-
-        def statement
-          @statement ||= PgQuery.deparse_stmt(parsed_stmt)
-        end
-
-        private
-
-        attr_reader :parsed_stmt
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/database/schema_validation/indexes.rb b/lib/gitlab/database/schema_validation/indexes.rb
deleted file mode 100644
index b7c3705bde9..00000000000
--- a/lib/gitlab/database/schema_validation/indexes.rb
+++ /dev/null
@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Database
-    module SchemaValidation
-      class Indexes
-        def initialize(structure_sql, database)
-          @structure_sql = structure_sql
-          @database = database
-        end
-
-        def missing_indexes
-          structure_sql.indexes.map(&:name) - database.indexes.map(&:name)
-        end
-
-        def extra_indexes
-          database.indexes.map(&:name) - structure_sql.indexes.map(&:name)
-        end
-
-        def wrong_indexes
-          structure_sql.indexes.filter_map do |structure_sql_index|
-            database_index = database.fetch_index_by_name(structure_sql_index.name)
-
-            next if database_index.nil?
-            next if database_index.statement == structure_sql_index.statement
-
-            structure_sql_index.name
-          end
-        end
-
-        private
-
-        attr_reader :structure_sql, :database
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/database/schema_validation/schema_objects/base.rb b/lib/gitlab/database/schema_validation/schema_objects/base.rb
new file mode 100644
index 00000000000..b0c8eb087dd
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/schema_objects/base.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module SchemaValidation
+      module SchemaObjects
+        class Base
+          def initialize(parsed_stmt)
+            @parsed_stmt = parsed_stmt
+          end
+
+          def name
+            raise NoMethodError, "subclasses of #{self.class.name} must implement #{__method__}"
+          end
+
+          def statement
+            @statement ||= PgQuery.deparse_stmt(parsed_stmt)
+          end
+
+          private
+
+          attr_reader :parsed_stmt
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/schema_validation/schema_objects/index.rb b/lib/gitlab/database/schema_validation/schema_objects/index.rb
new file mode 100644
index 00000000000..28d61b18266
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/schema_objects/index.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module SchemaValidation
+      module SchemaObjects
+        class Index < Base
+          def name
+            parsed_stmt.idxname
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/schema_validation/schema_objects/trigger.rb b/lib/gitlab/database/schema_validation/schema_objects/trigger.rb
new file mode 100644
index 00000000000..508e6b27ed3
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/schema_objects/trigger.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module SchemaValidation
+      module SchemaObjects
+        class Trigger < Base
+          def name
+            parsed_stmt.trigname
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/schema_validation/structure_sql.rb b/lib/gitlab/database/schema_validation/structure_sql.rb
index 306cc22f9f5..cb62af8d8b8 100644
--- a/lib/gitlab/database/schema_validation/structure_sql.rb
+++ b/lib/gitlab/database/schema_validation/structure_sql.rb
@@ -4,33 +4,56 @@ module Gitlab
   module Database
     module SchemaValidation
       class StructureSql
-        def initialize(structure_file_path)
+        DEFAULT_SCHEMA = 'public'
+
+        def initialize(structure_file_path, schema_name = DEFAULT_SCHEMA)
           @structure_file_path = structure_file_path
+          @schema_name = schema_name
         end
 
         def index_exists?(index_name)
           indexes.find { |index| index.name == index_name }.present?
         end
 
+        def trigger_exists?(trigger_name)
+          triggers.find { |trigger| trigger.name == trigger_name }.present?
+        end
+
         def indexes
-          @indexes ||= index_statements.map do |index_statement|
-            index_statement.relation.schemaname = "public" if index_statement.relation.schemaname == ''
+          @indexes ||= map_with_default_schema(index_statements, SchemaObjects::Index)
+        end
 
-            Index.new(index_statement)
-          end
+        def triggers
+          @triggers ||= map_with_default_schema(trigger_statements, SchemaObjects::Trigger)
         end
 
         private
 
-        attr_reader :structure_file_path
+        attr_reader :structure_file_path, :schema_name
 
         def index_statements
-          parsed_structure_file.tree.stmts.filter_map { |s| s.stmt.index_stmt }
+          statements.filter_map { |s| s.stmt.index_stmt }
+        end
+
+        def trigger_statements
+          statements.filter_map { |s| s.stmt.create_trig_stmt }
+        end
+
+        def statements
+          @statements ||= parsed_structure_file.tree.stmts
         end
 
         def parsed_structure_file
           PgQuery.parse(File.read(structure_file_path))
         end
+
+        def map_with_default_schema(statements, validation_class)
+          statements.map do |statement|
+            statement.relation.schemaname = schema_name if statement.relation.schemaname == ''
+
+            validation_class.new(statement)
+          end
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/schema_validation/validators/base_validator.rb b/lib/gitlab/database/schema_validation/validators/base_validator.rb
index 1fd3d7c85c4..14995b5f378 100644
--- a/lib/gitlab/database/schema_validation/validators/base_validator.rb
+++ b/lib/gitlab/database/schema_validation/validators/base_validator.rb
@@ -15,8 +15,11 @@ module Gitlab
           def self.all_validators
             [
               ExtraIndexes,
+              ExtraTriggers,
               MissingIndexes,
-              DifferentDefinitionIndexes
+              MissingTriggers,
+              DifferentDefinitionIndexes,
+              DifferentDefinitionTriggers
             ]
           end
 
diff --git a/lib/gitlab/database/schema_validation/validators/different_definition_triggers.rb b/lib/gitlab/database/schema_validation/validators/different_definition_triggers.rb
new file mode 100644
index 00000000000..efb87a70ca8
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/validators/different_definition_triggers.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module SchemaValidation
+      module Validators
+        class DifferentDefinitionTriggers < BaseValidator
+          def execute
+            structure_sql.triggers.filter_map do |structure_sql_trigger|
+              database_trigger = database.fetch_trigger_by_name(structure_sql_trigger.name)
+
+              next if database_trigger.nil?
+              next if database_trigger.statement == structure_sql_trigger.statement
+
+              build_inconsistency(self.class, structure_sql_trigger)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/schema_validation/validators/extra_triggers.rb b/lib/gitlab/database/schema_validation/validators/extra_triggers.rb
new file mode 100644
index 00000000000..f03bb49526c
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/validators/extra_triggers.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module SchemaValidation
+      module Validators
+        class ExtraTriggers < BaseValidator
+          def execute
+            database.triggers.filter_map do |trigger|
+              next if structure_sql.trigger_exists?(trigger.name)
+
+              build_inconsistency(self.class, trigger)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/schema_validation/validators/missing_triggers.rb b/lib/gitlab/database/schema_validation/validators/missing_triggers.rb
new file mode 100644
index 00000000000..c7137c68c1c
--- /dev/null
+++ b/lib/gitlab/database/schema_validation/validators/missing_triggers.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module SchemaValidation
+      module Validators
+        class MissingTriggers < BaseValidator
+          def execute
+            structure_sql.triggers.filter_map do |index|
+              next if database.trigger_exists?(index.name)
+
+              build_inconsistency(self.class, index)
+            end
+          end
+        end
+      end
+    end
+  end
+end