diff options
Diffstat (limited to 'spec/lib/gitlab/database/async_foreign_keys')
3 files changed, 371 insertions, 0 deletions
diff --git a/spec/lib/gitlab/database/async_foreign_keys/foreign_key_validator_spec.rb b/spec/lib/gitlab/database/async_foreign_keys/foreign_key_validator_spec.rb new file mode 100644 index 00000000000..90137e259f5 --- /dev/null +++ b/spec/lib/gitlab/database/async_foreign_keys/foreign_key_validator_spec.rb @@ -0,0 +1,152 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::AsyncForeignKeys::ForeignKeyValidator, feature_category: :database do + include ExclusiveLeaseHelpers + + describe '#perform' do + let!(:lease) { stub_exclusive_lease(lease_key, :uuid, timeout: lease_timeout) } + let(:lease_key) { "gitlab/database/asyncddl/actions/#{Gitlab::Database::PRIMARY_DATABASE_NAME}" } + let(:lease_timeout) { described_class::TIMEOUT_PER_ACTION } + + let(:fk_model) { Gitlab::Database::AsyncForeignKeys::PostgresAsyncForeignKeyValidation } + let(:table_name) { '_test_async_fks' } + let(:fk_name) { 'fk_parent_id' } + let(:validation) { create(:postgres_async_foreign_key_validation, table_name: table_name, name: fk_name) } + let(:connection) { validation.connection } + + subject { described_class.new(validation) } + + before do + connection.create_table(table_name) do |t| + t.references :parent, foreign_key: { to_table: table_name, validate: false, name: fk_name } + end + end + + it 'validates the FK while controlling statement timeout' do + allow(connection).to receive(:execute).and_call_original + expect(connection).to receive(:execute) + .with("SET statement_timeout TO '43200s'").ordered.and_call_original + expect(connection).to receive(:execute) + .with('ALTER TABLE "_test_async_fks" VALIDATE CONSTRAINT "fk_parent_id";').ordered.and_call_original + expect(connection).to receive(:execute) + .with("RESET statement_timeout").ordered.and_call_original + + subject.perform + end + + context 'with fully qualified table names' do + let(:validation) do + create(:postgres_async_foreign_key_validation, + table_name: "public.#{table_name}", + name: fk_name + ) + end + + it 'validates the FK' do + allow(connection).to receive(:execute).and_call_original + + expect(connection).to receive(:execute) + .with('ALTER TABLE "public"."_test_async_fks" VALIDATE CONSTRAINT "fk_parent_id";').ordered.and_call_original + + subject.perform + end + end + + it 'removes the FK validation record from table' do + expect(validation).to receive(:destroy!).and_call_original + + expect { subject.perform }.to change { fk_model.count }.by(-1) + end + + it 'skips logic if not able to acquire exclusive lease' do + expect(lease).to receive(:try_obtain).ordered.and_return(false) + expect(connection).not_to receive(:execute).with(/ALTER TABLE/) + expect(validation).not_to receive(:destroy!) + + expect { subject.perform }.not_to change { fk_model.count } + end + + it 'logs messages around execution' do + allow(Gitlab::AppLogger).to receive(:info).and_call_original + + subject.perform + + expect(Gitlab::AppLogger) + .to have_received(:info) + .with(a_hash_including(message: 'Starting to validate foreign key')) + + expect(Gitlab::AppLogger) + .to have_received(:info) + .with(a_hash_including(message: 'Finished validating foreign key')) + end + + context 'when the FK does not exist' do + before do + connection.create_table(table_name, force: true) + end + + it 'skips validation and removes the record' do + expect(connection).not_to receive(:execute).with(/ALTER TABLE/) + + expect { subject.perform }.to change { fk_model.count }.by(-1) + end + + it 'logs an appropriate message' do + expected_message = "Skipping #{fk_name} validation since it does not exist. The queuing entry will be deleted" + + allow(Gitlab::AppLogger).to receive(:info).and_call_original + + subject.perform + + expect(Gitlab::AppLogger) + .to have_received(:info) + .with(a_hash_including(message: expected_message)) + end + end + + context 'with error handling' do + before do + allow(connection).to receive(:execute).and_call_original + + allow(connection).to receive(:execute) + .with('ALTER TABLE "_test_async_fks" VALIDATE CONSTRAINT "fk_parent_id";') + .and_raise(ActiveRecord::StatementInvalid) + end + + context 'on production' do + before do + allow(Gitlab::ErrorTracking).to receive(:should_raise_for_dev?).and_return(false) + end + + it 'increases execution attempts' do + expect { subject.perform }.to change { validation.attempts }.by(1) + + expect(validation.last_error).to be_present + expect(validation).not_to be_destroyed + end + + it 'logs an error message including the fk_name' do + expect(Gitlab::AppLogger) + .to receive(:error) + .with(a_hash_including(:message, :fk_name)) + .and_call_original + + subject.perform + end + end + + context 'on development' do + it 'also raises errors' do + expect { subject.perform } + .to raise_error(ActiveRecord::StatementInvalid) + .and change { validation.attempts }.by(1) + + expect(validation.last_error).to be_present + expect(validation).not_to be_destroyed + end + end + end + end +end diff --git a/spec/lib/gitlab/database/async_foreign_keys/migration_helpers_spec.rb b/spec/lib/gitlab/database/async_foreign_keys/migration_helpers_spec.rb new file mode 100644 index 00000000000..0bd0e8045ff --- /dev/null +++ b/spec/lib/gitlab/database/async_foreign_keys/migration_helpers_spec.rb @@ -0,0 +1,167 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::AsyncForeignKeys::MigrationHelpers, feature_category: :database do + let(:migration) { Gitlab::Database::Migration[2.1].new } + let(:connection) { ApplicationRecord.connection } + let(:fk_model) { Gitlab::Database::AsyncForeignKeys::PostgresAsyncForeignKeyValidation } + let(:table_name) { '_test_async_fks' } + let(:column_name) { 'parent_id' } + let(:fk_name) { nil } + + context 'with regular tables' do + before do + allow(migration).to receive(:puts) + allow(migration.connection).to receive(:transaction_open?).and_return(false) + + connection.create_table(table_name) do |t| + t.integer column_name + end + + migration.add_concurrent_foreign_key( + table_name, table_name, + column: column_name, validate: false, name: fk_name) + end + + describe '#prepare_async_foreign_key_validation' do + it 'creates the record for the async FK validation' do + expect do + migration.prepare_async_foreign_key_validation(table_name, column_name) + end.to change { fk_model.where(table_name: table_name).count }.by(1) + + record = fk_model.find_by(table_name: table_name) + + expect(record.name).to start_with('fk_') + end + + context 'when an explicit name is given' do + let(:fk_name) { 'my_fk_name' } + + it 'creates the record with the given name' do + expect do + migration.prepare_async_foreign_key_validation(table_name, name: fk_name) + end.to change { fk_model.where(name: fk_name).count }.by(1) + + record = fk_model.find_by(name: fk_name) + + expect(record.table_name).to eq(table_name) + end + end + + context 'when the FK does not exist' do + it 'returns an error' do + expect do + migration.prepare_async_foreign_key_validation(table_name, name: 'no_fk') + end.to raise_error RuntimeError, /Could not find foreign key "no_fk" on table "_test_async_fks"/ + end + end + + context 'when the record already exists' do + let(:fk_name) { 'my_fk_name' } + + it 'does attempt to create the record' do + create(:postgres_async_foreign_key_validation, table_name: table_name, name: fk_name) + + expect do + migration.prepare_async_foreign_key_validation(table_name, name: fk_name) + end.not_to change { fk_model.where(name: fk_name).count } + end + end + + context 'when the async FK validation table does not exist' do + it 'does not raise an error' do + connection.drop_table(:postgres_async_foreign_key_validations) + + expect(fk_model).not_to receive(:safe_find_or_create_by!) + + expect { migration.prepare_async_foreign_key_validation(table_name, column_name) }.not_to raise_error + end + end + end + + describe '#unprepare_async_foreign_key_validation' do + before do + migration.prepare_async_foreign_key_validation(table_name, column_name, name: fk_name) + end + + it 'destroys the record' do + expect do + migration.unprepare_async_foreign_key_validation(table_name, column_name) + end.to change { fk_model.where(table_name: table_name).count }.by(-1) + end + + context 'when an explicit name is given' do + let(:fk_name) { 'my_test_async_fk' } + + it 'destroys the record' do + expect do + migration.unprepare_async_foreign_key_validation(table_name, name: fk_name) + end.to change { fk_model.where(name: fk_name).count }.by(-1) + end + end + + context 'when the async fk validation table does not exist' do + it 'does not raise an error' do + connection.drop_table(:postgres_async_foreign_key_validations) + + expect(fk_model).not_to receive(:find_by) + + expect { migration.unprepare_async_foreign_key_validation(table_name, column_name) }.not_to raise_error + end + end + end + end + + context 'with partitioned tables' do + let(:partition_schema) { 'gitlab_partitions_dynamic' } + let(:partition1_name) { "#{partition_schema}.#{table_name}_202001" } + let(:partition2_name) { "#{partition_schema}.#{table_name}_202002" } + let(:fk_name) { 'my_partitioned_fk_name' } + + before do + connection.execute(<<~SQL) + CREATE TABLE #{table_name} ( + id serial NOT NULL, + #{column_name} int NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (id, created_at) + ) PARTITION BY RANGE (created_at); + + CREATE TABLE #{partition1_name} PARTITION OF #{table_name} + FOR VALUES FROM ('2020-01-01') TO ('2020-02-01'); + + CREATE TABLE #{partition2_name} PARTITION OF #{table_name} + FOR VALUES FROM ('2020-02-01') TO ('2020-03-01'); + SQL + end + + describe '#prepare_partitioned_async_foreign_key_validation' do + it 'delegates to prepare_async_foreign_key_validation for each partition' do + expect(migration) + .to receive(:prepare_async_foreign_key_validation) + .with(partition1_name, column_name, name: fk_name) + + expect(migration) + .to receive(:prepare_async_foreign_key_validation) + .with(partition2_name, column_name, name: fk_name) + + migration.prepare_partitioned_async_foreign_key_validation(table_name, column_name, name: fk_name) + end + end + + describe '#unprepare_partitioned_async_foreign_key_validation' do + it 'delegates to unprepare_async_foreign_key_validation for each partition' do + expect(migration) + .to receive(:unprepare_async_foreign_key_validation) + .with(partition1_name, column_name, name: fk_name) + + expect(migration) + .to receive(:unprepare_async_foreign_key_validation) + .with(partition2_name, column_name, name: fk_name) + + migration.unprepare_partitioned_async_foreign_key_validation(table_name, column_name, name: fk_name) + end + end + end +end diff --git a/spec/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation_spec.rb b/spec/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation_spec.rb new file mode 100644 index 00000000000..ba201d93f52 --- /dev/null +++ b/spec/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation_spec.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::AsyncForeignKeys::PostgresAsyncForeignKeyValidation, type: :model, + feature_category: :database do + it { is_expected.to be_a Gitlab::Database::SharedModel } + + describe 'validations' do + let_it_be(:fk_validation) { create(:postgres_async_foreign_key_validation) } + let(:identifier_limit) { described_class::MAX_IDENTIFIER_LENGTH } + let(:last_error_limit) { described_class::MAX_LAST_ERROR_LENGTH } + + subject { fk_validation } + + it { is_expected.to validate_presence_of(:name) } + it { is_expected.to validate_uniqueness_of(:name) } + it { is_expected.to validate_length_of(:name).is_at_most(identifier_limit) } + it { is_expected.to validate_presence_of(:table_name) } + it { is_expected.to validate_length_of(:table_name).is_at_most(identifier_limit) } + it { is_expected.to validate_length_of(:last_error).is_at_most(last_error_limit) } + end + + describe 'scopes' do + let!(:failed_validation) { create(:postgres_async_foreign_key_validation, attempts: 1) } + let!(:new_validation) { create(:postgres_async_foreign_key_validation) } + + describe '.ordered' do + subject { described_class.ordered } + + it { is_expected.to eq([new_validation, failed_validation]) } + end + end + + describe '#handle_exception!' do + let_it_be_with_reload(:fk_validation) { create(:postgres_async_foreign_key_validation) } + + let(:error) { instance_double(StandardError, message: 'Oups', backtrace: %w[this that]) } + + subject { fk_validation.handle_exception!(error) } + + it 'increases the attempts number' do + expect { subject }.to change { fk_validation.reload.attempts }.by(1) + end + + it 'saves error details' do + subject + + expect(fk_validation.reload.last_error).to eq("Oups\nthis\nthat") + end + end +end |