diff options
Diffstat (limited to 'spec/lib/gitlab/database/with_lock_retries_outside_transaction_spec.rb')
| -rw-r--r-- | spec/lib/gitlab/database/with_lock_retries_outside_transaction_spec.rb | 244 |
1 files changed, 244 insertions, 0 deletions
diff --git a/spec/lib/gitlab/database/with_lock_retries_outside_transaction_spec.rb b/spec/lib/gitlab/database/with_lock_retries_outside_transaction_spec.rb new file mode 100644 index 00000000000..e93d8ab590d --- /dev/null +++ b/spec/lib/gitlab/database/with_lock_retries_outside_transaction_spec.rb @@ -0,0 +1,244 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::WithLockRetriesOutsideTransaction do + let(:env) { {} } + let(:logger) { Gitlab::Database::WithLockRetries::NULL_LOGGER } + let(:subject) { described_class.new(env: env, logger: logger, timing_configuration: timing_configuration) } + + let(:timing_configuration) do + [ + [1.second, 1.second], + [1.second, 1.second], + [1.second, 1.second], + [1.second, 1.second], + [1.second, 1.second] + ] + end + + describe '#run' do + it 'requires block' do + expect { subject.run }.to raise_error(StandardError, 'no block given') + end + + context 'when DISABLE_LOCK_RETRIES is set' do + let(:env) { { 'DISABLE_LOCK_RETRIES' => 'true' } } + + it 'executes the passed block without retrying' do + object = double + + expect(object).to receive(:method).once + + subject.run { object.method } + end + end + + context 'when lock retry is enabled' do + let(:lock_fiber) do + Fiber.new do + # Initiating a second DB connection for the lock + conn = ActiveRecordSecond.establish_connection(Rails.configuration.database_configuration[Rails.env]).connection + conn.transaction do + conn.execute("LOCK TABLE #{Project.table_name} in exclusive mode") + + Fiber.yield + end + ActiveRecordSecond.remove_connection # force disconnect + end + end + + before do + stub_const('ActiveRecordSecond', Class.new(ActiveRecord::Base)) + + lock_fiber.resume # start the transaction and lock the table + end + + after do + lock_fiber.resume if lock_fiber.alive? + end + + context 'lock_fiber' do + it 'acquires lock successfully' do + check_exclusive_lock_query = """ + SELECT 1 + FROM pg_locks l + JOIN pg_class t ON l.relation = t.oid + WHERE t.relkind = 'r' AND l.mode = 'ExclusiveLock' AND t.relname = '#{Project.table_name}' + """ + + expect(ActiveRecord::Base.connection.execute(check_exclusive_lock_query).to_a).to be_present + end + end + + shared_examples 'retriable exclusive lock on `projects`' do + it 'succeeds executing the given block' do + lock_attempts = 0 + lock_acquired = false + + # the actual number of attempts to run_block_with_lock_timeout can never exceed the number of + # timings_configurations, so here we limit the retry_count if it exceeds that value + # + # also, there is no call to sleep after the final attempt, which is why it will always be one less + expected_runs_with_timeout = [retry_count, timing_configuration.size].min + expect(subject).to receive(:sleep).exactly(expected_runs_with_timeout - 1).times + + expect(subject).to receive(:run_block_with_lock_timeout).exactly(expected_runs_with_timeout).times.and_wrap_original do |method| + lock_fiber.resume if lock_attempts == retry_count + + method.call + end + + subject.run do + lock_attempts += 1 + + if lock_attempts == retry_count # we reached the last retry iteration, if we kill the thread, the last try (no lock_timeout) will succeed + lock_fiber.resume + end + + ActiveRecord::Base.transaction do + ActiveRecord::Base.connection.execute("LOCK TABLE #{Project.table_name} in exclusive mode") + lock_acquired = true + end + end + + expect(lock_attempts).to eq(retry_count) + expect(lock_acquired).to eq(true) + end + end + + context 'after 3 iterations' do + it_behaves_like 'retriable exclusive lock on `projects`' do + let(:retry_count) { 4 } + end + + context 'setting the idle transaction timeout' do + context 'when there is no outer transaction: disable_ddl_transaction! is set in the migration' do + it 'does not disable the idle transaction timeout' do + allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(false) + allow(subject).to receive(:run_block_with_lock_timeout).once.and_raise(ActiveRecord::LockWaitTimeout) + allow(subject).to receive(:run_block_with_lock_timeout).once + + expect(subject).not_to receive(:disable_idle_in_transaction_timeout) + + subject.run {} + end + end + + context 'when there is outer transaction: disable_ddl_transaction! is not set in the migration' do + it 'disables the idle transaction timeout so the code can sleep and retry' do + allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(true) + + n = 0 + allow(subject).to receive(:run_block_with_lock_timeout).twice do + n += 1 + raise(ActiveRecord::LockWaitTimeout) if n == 1 + end + + expect(subject).to receive(:disable_idle_in_transaction_timeout).once + + subject.run {} + end + end + end + end + + context 'after the retries are exhausted' do + let(:timing_configuration) do + [ + [1.second, 1.second] + ] + end + + it 'disables the lock_timeout' do + allow(subject).to receive(:run_block_with_lock_timeout).once.and_raise(ActiveRecord::LockWaitTimeout) + + expect(subject).to receive(:disable_lock_timeout) + + subject.run {} + end + end + + context 'after the retries, without setting lock_timeout' do + let(:retry_count) { timing_configuration.size + 1 } + + it_behaves_like 'retriable exclusive lock on `projects`' do + before do + expect(subject).to receive(:run_block_without_lock_timeout).and_call_original + end + end + end + + context 'after the retries, when requested to raise an error' do + let(:expected_attempts_with_timeout) { timing_configuration.size } + let(:retry_count) { timing_configuration.size + 1 } + + it 'raises an error instead of waiting indefinitely for the lock' do + lock_attempts = 0 + lock_acquired = false + + expect(subject).to receive(:sleep).exactly(expected_attempts_with_timeout - 1).times + expect(subject).to receive(:run_block_with_lock_timeout).exactly(expected_attempts_with_timeout).times.and_call_original + + expect do + subject.run(raise_on_exhaustion: true) do + lock_attempts += 1 + + ActiveRecord::Base.transaction do + ActiveRecord::Base.connection.execute("LOCK TABLE #{Project.table_name} in exclusive mode") + lock_acquired = true + end + end + end.to raise_error(described_class::AttemptsExhaustedError) + + expect(lock_attempts).to eq(retry_count - 1) + expect(lock_acquired).to eq(false) + end + end + + context 'when statement timeout is reached' do + it 'raises StatementInvalid error' do + lock_acquired = false + ActiveRecord::Base.connection.execute("SET statement_timeout='100ms'") + + expect do + subject.run do + ActiveRecord::Base.connection.execute("SELECT 1 FROM pg_sleep(0.11)") # 110ms + lock_acquired = true + end + end.to raise_error(ActiveRecord::StatementInvalid) + + expect(lock_acquired).to eq(false) + end + end + end + end + + context 'restore local database variables' do + it do + expect { subject.run {} }.not_to change { ActiveRecord::Base.connection.execute("SHOW lock_timeout").to_a } + end + + it do + expect { subject.run {} }.not_to change { ActiveRecord::Base.connection.execute("SHOW idle_in_transaction_session_timeout").to_a } + end + end + + context 'casting durations correctly' do + let(:timing_configuration) { [[0.015.seconds, 0.025.seconds], [0.015.seconds, 0.025.seconds]] } # 15ms, 25ms + + it 'executes `SET lock_timeout` using the configured timeout value in milliseconds' do + expect(ActiveRecord::Base.connection).to receive(:execute).with('RESET idle_in_transaction_session_timeout; RESET lock_timeout').and_call_original + expect(ActiveRecord::Base.connection).to receive(:execute).with("SET lock_timeout TO '15ms'").and_call_original + + subject.run { } + end + + it 'calls `sleep` after the first iteration fails, using the configured sleep time' do + expect(subject).to receive(:run_block_with_lock_timeout).and_raise(ActiveRecord::LockWaitTimeout).twice + expect(subject).to receive(:sleep).with(0.025) + + subject.run { } + end + end +end |