Add latest changes from gitlab-org/gitlab@master

2 files changed, 198 insertions, 0 deletions

diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index b7d510c19f9..5077143e15e 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -280,6 +280,46 @@ module Gitlab
         end
       end
 
+      # Executes the block with a retry mechanism that alters the +lock_timeout+ and +sleep_time+ between attempts.
+      # The timings can be controlled via the +timing_configuration+ parameter.
+      # If the lock was not acquired within the retry period, a last attempt is made without using +lock_timeout+.
+      #
+      # ==== Examples
+      #   # Invoking without parameters
+      #   with_lock_retries do
+      #     drop_table :my_table
+      #   end
+      #
+      #   # Invoking with custom +timing_configuration+
+      #   t = [
+      #     [1.second, 1.second],
+      #     [2.seconds, 2.seconds]
+      #   ]
+      #
+      #   with_lock_retries(timing_configuration: t) do
+      #     drop_table :my_table # this will be retried twice
+      #   end
+      #
+      #   # Disabling the retries using an environment variable
+      #   > export DISABLE_LOCK_RETRIES=true
+      #
+      #   with_lock_retries do
+      #     drop_table :my_table # one invocation, it will not retry at all
+      #   end
+      #
+      # ==== Parameters
+      # * +timing_configuration+ - [[ActiveSupport::Duration, ActiveSupport::Duration], ...] lock timeout for the block, sleep time before the next iteration, defaults to `Gitlab::Database::WithLockRetries::DEFAULT_TIMING_CONFIGURATION`
+      # * +logger+ - [Gitlab::JsonLogger]
+      # * +env+ - [Hash] custom environment hash, see the example with `DISABLE_LOCK_RETRIES`
+      def with_lock_retries(**args, &block)
+        merged_args = {
+          klass: self.class,
+          logger: Gitlab::BackgroundMigration::Logger
+        }.merge(args)
+
+        Gitlab::Database::WithLockRetries.new(merged_args).run(&block)
+      end
+
       def true_value
         Database.true_value
       end
diff --git a/lib/gitlab/database/with_lock_retries.rb b/lib/gitlab/database/with_lock_retries.rb
new file mode 100644
index 00000000000..37f7e8fbdac
--- /dev/null
+++ b/lib/gitlab/database/with_lock_retries.rb
@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    class WithLockRetries
+      NULL_LOGGER = Gitlab::JsonLogger.new('/dev/null')
+
+      # Each element of the array represents a retry iteration.
+      # - DEFAULT_TIMING_CONFIGURATION.size provides the iteration count.
+      # - First element: DB lock_timeout
+      # - Second element: Sleep time after unsuccessful lock attempt (LockWaitTimeout error raised)
+      # - Worst case, this configuration would retry for about 40 minutes.
+      DEFAULT_TIMING_CONFIGURATION = [
+        [0.1.seconds, 0.05.seconds], # short timings, lock_timeout: 100ms, sleep after LockWaitTimeout: 50ms
+        [0.1.seconds, 0.05.seconds],
+        [0.2.seconds, 0.05.seconds],
+        [0.3.seconds, 0.10.seconds],
+        [0.4.seconds, 0.15.seconds],
+        [0.5.seconds, 2.seconds],
+        [0.5.seconds, 2.seconds],
+        [0.5.seconds, 2.seconds],
+        [0.5.seconds, 2.seconds],
+        [1.second, 5.seconds], # probably high traffic, increase timings
+        [1.second, 1.minute],
+        [0.1.seconds, 0.05.seconds],
+        [0.1.seconds, 0.05.seconds],
+        [0.2.seconds, 0.05.seconds],
+        [0.3.seconds, 0.10.seconds],
+        [0.4.seconds, 0.15.seconds],
+        [0.5.seconds, 2.seconds],
+        [0.5.seconds, 2.seconds],
+        [0.5.seconds, 2.seconds],
+        [3.seconds, 3.minutes], # probably high traffic or long locks, increase timings
+        [0.1.seconds, 0.05.seconds],
+        [0.1.seconds, 0.05.seconds],
+        [0.5.seconds, 2.seconds],
+        [0.5.seconds, 2.seconds],
+        [5.seconds, 2.minutes],
+        [0.5.seconds, 0.5.seconds],
+        [0.5.seconds, 0.5.seconds],
+        [7.seconds, 5.minutes],
+        [0.5.seconds, 0.5.seconds],
+        [0.5.seconds, 0.5.seconds],
+        [7.seconds, 5.minutes],
+        [0.5.seconds, 0.5.seconds],
+        [0.5.seconds, 0.5.seconds],
+        [7.seconds, 5.minutes],
+        [0.1.seconds, 0.05.seconds],
+        [0.1.seconds, 0.05.seconds],
+        [0.5.seconds, 2.seconds],
+        [10.seconds, 10.minutes],
+        [0.1.seconds, 0.05.seconds],
+        [0.5.seconds, 2.seconds],
+        [10.seconds, 10.minutes]
+      ].freeze
+
+      def initialize(logger: NULL_LOGGER, timing_configuration: DEFAULT_TIMING_CONFIGURATION, klass: nil, env: ENV)
+        @logger = logger
+        @klass = klass
+        @timing_configuration = timing_configuration
+        @env = env
+        @current_iteration = 1
+        @log_params = { method: 'with_lock_retries', class: klass.to_s }
+      end
+
+      def run(&block)
+        raise 'no block given' unless block_given?
+
+        @block = block
+
+        if lock_retries_disabled?
+          log(message: 'DISABLE_LOCK_RETRIES environment variable is true, executing the block without retry')
+
+          return run_block
+        end
+
+        begin
+          run_block_with_transaction
+        rescue ActiveRecord::LockWaitTimeout
+          if retry_with_lock_timeout?
+            wait_until_next_retry
+
+            retry
+          else
+            run_block_without_lock_timeout
+          end
+        end
+      end
+
+      private
+
+      attr_reader :logger, :env, :block, :current_iteration, :log_params, :timing_configuration
+
+      def run_block
+        block.call
+      end
+
+      def run_block_with_transaction
+        ActiveRecord::Base.transaction(requires_new: true) do
+          execute("SET LOCAL lock_timeout TO '#{current_lock_timeout_in_ms}ms'")
+
+          log(message: 'Lock timeout is set', current_iteration: current_iteration, lock_timeout_in_ms: current_lock_timeout_in_ms)
+
+          run_block
+
+          log(message: 'Migration finished', current_iteration: current_iteration, lock_timeout_in_ms: current_lock_timeout_in_ms)
+        end
+      end
+
+      def retry_with_lock_timeout?
+        current_iteration != retry_count
+      end
+
+      def wait_until_next_retry
+        log(message: 'ActiveRecord::LockWaitTimeout error, retrying after sleep', current_iteration: current_iteration, sleep_time_in_seconds: current_sleep_time_in_seconds)
+
+        sleep(current_sleep_time_in_seconds)
+
+        @current_iteration += 1
+      end
+
+      def run_block_without_lock_timeout
+        log(message: "Couldn't acquire lock to perform the migration", current_iteration: current_iteration)
+        log(message: "Executing the migration without lock timeout", current_iteration: current_iteration)
+
+        execute("SET LOCAL lock_timeout TO '0'")
+
+        run_block
+
+        log(message: 'Migration finished', current_iteration: current_iteration)
+      end
+
+      def lock_retries_disabled?
+        Gitlab::Utils.to_boolean(env['DISABLE_LOCK_RETRIES'])
+      end
+
+      def log(params)
+        logger.info(log_params.merge(params))
+      end
+
+      def execute(statement)
+        ActiveRecord::Base.connection.execute(statement)
+      end
+
+      def retry_count
+        timing_configuration.size
+      end
+
+      def current_lock_timeout_in_ms
+        timing_configuration[current_iteration - 1][0].in_milliseconds
+      end
+
+      def current_sleep_time_in_seconds
+        timing_configuration[current_iteration - 1][1].to_i
+      end
+    end
+  end
+end