diff options
Diffstat (limited to 'spec/lib/gitlab/database/partitioning')
7 files changed, 105 insertions, 115 deletions
diff --git a/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb b/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb index 8c406c90e36..b2c4e4b54a4 100644 --- a/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb +++ b/spec/lib/gitlab/database/partitioning/detached_partition_dropper_spec.rb @@ -5,6 +5,8 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do include Database::TableSchemaHelpers + subject(:dropper) { described_class.new } + let(:connection) { ActiveRecord::Base.connection } def expect_partition_present(name) @@ -23,10 +25,18 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do before do connection.execute(<<~SQL) + CREATE TABLE referenced_table ( + id bigserial primary key not null + ) + SQL + connection.execute(<<~SQL) + CREATE TABLE parent_table ( id bigserial not null, + referenced_id bigint not null, created_at timestamptz not null, - primary key (id, created_at) + primary key (id, created_at), + constraint fk_referenced foreign key (referenced_id) references referenced_table(id) ) PARTITION BY RANGE(created_at) SQL end @@ -59,7 +69,7 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do attached: false, drop_after: 1.day.from_now) - subject.perform + dropper.perform expect_partition_present('test_partition') end @@ -75,7 +85,7 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do end it 'drops the partition' do - subject.perform + dropper.perform expect(table_oid('test_partition')).to be_nil end @@ -86,16 +96,62 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do end it 'does not drop the partition' do - subject.perform + dropper.perform expect(table_oid('test_partition')).not_to be_nil end end + context 'removing foreign keys' do + it 'removes foreign keys from the table before dropping it' do + expect(dropper).to receive(:drop_detached_partition).and_wrap_original do |drop_method, partition_name| + expect(partition_name).to eq('test_partition') + expect(foreign_key_exists_by_name(partition_name, 'fk_referenced', schema: Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA)).to be_falsey + + drop_method.call(partition_name) + end + + expect(foreign_key_exists_by_name('test_partition', 'fk_referenced', schema: Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA)).to be_truthy + + dropper.perform + end + + it 'does not remove foreign keys from the parent table' do + expect { dropper.perform }.not_to change { foreign_key_exists_by_name('parent_table', 'fk_referenced') }.from(true) + end + + context 'when another process drops the foreign key' do + it 'skips dropping that foreign key' do + expect(dropper).to receive(:drop_foreign_key_if_present).and_wrap_original do |drop_meth, *args| + connection.execute('alter table gitlab_partitions_dynamic.test_partition drop constraint fk_referenced;') + drop_meth.call(*args) + end + + dropper.perform + + expect_partition_removed('test_partition') + end + end + + context 'when another process drops the partition' do + it 'skips dropping the foreign key' do + expect(dropper).to receive(:drop_foreign_key_if_present).and_wrap_original do |drop_meth, *args| + connection.execute('drop table gitlab_partitions_dynamic.test_partition') + Postgresql::DetachedPartition.where(table_name: 'test_partition').delete_all + end + + expect(Gitlab::AppLogger).not_to receive(:error) + dropper.perform + end + end + end + context 'when another process drops the table while the first waits for a lock' do it 'skips the table' do + # First call to .lock is for removing foreign keys + expect(Postgresql::DetachedPartition).to receive(:lock).once.ordered.and_call_original # Rspec's receive_method_chain does not support .and_wrap_original, so we need to nest here. - expect(Postgresql::DetachedPartition).to receive(:lock).and_wrap_original do |lock_meth| + expect(Postgresql::DetachedPartition).to receive(:lock).once.ordered.and_wrap_original do |lock_meth| locked = lock_meth.call expect(locked).to receive(:find_by).and_wrap_original do |find_meth, *find_args| # Another process drops the table then deletes this entry @@ -106,9 +162,9 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do locked end - expect(subject).not_to receive(:drop_one) + expect(dropper).not_to receive(:drop_one) - subject.perform + dropper.perform end end end @@ -123,19 +179,26 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do end it 'does not drop the partition, but does remove the DetachedPartition entry' do - subject.perform + dropper.perform aggregate_failures do expect(table_oid('test_partition')).not_to be_nil expect(Postgresql::DetachedPartition.find_by(table_name: 'test_partition')).to be_nil end end - it 'removes the detached_partition entry' do - detached_partition = Postgresql::DetachedPartition.find_by!(table_name: 'test_partition') + context 'when another process removes the entry before this process' do + it 'does nothing' do + expect(Postgresql::DetachedPartition).to receive(:lock).and_wrap_original do |lock_meth| + Postgresql::DetachedPartition.delete_all + lock_meth.call + end - subject.perform + expect(Gitlab::AppLogger).not_to receive(:error) - expect(Postgresql::DetachedPartition.exists?(id: detached_partition.id)).to be_falsey + dropper.perform + + expect(table_oid('test_partition')).not_to be_nil + end end end @@ -155,7 +218,7 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do end it 'drops both partitions' do - subject.perform + dropper.perform expect_partition_removed('partition_1') expect_partition_removed('partition_2') @@ -163,10 +226,10 @@ RSpec.describe Gitlab::Database::Partitioning::DetachedPartitionDropper do context 'when the first drop returns an error' do it 'still drops the second partition' do - expect(subject).to receive(:drop_detached_partition).ordered.and_raise('injected error') - expect(subject).to receive(:drop_detached_partition).ordered.and_call_original + expect(dropper).to receive(:drop_detached_partition).ordered.and_raise('injected error') + expect(dropper).to receive(:drop_detached_partition).ordered.and_call_original - subject.perform + dropper.perform # We don't know which partition we tried to drop first, so the tests here have to work with either one expect(Postgresql::DetachedPartition.count).to eq(1) diff --git a/spec/lib/gitlab/database/partitioning/monthly_strategy_spec.rb b/spec/lib/gitlab/database/partitioning/monthly_strategy_spec.rb index 27ada12b067..67d80d71e2a 100644 --- a/spec/lib/gitlab/database/partitioning/monthly_strategy_spec.rb +++ b/spec/lib/gitlab/database/partitioning/monthly_strategy_spec.rb @@ -10,7 +10,7 @@ RSpec.describe Gitlab::Database::Partitioning::MonthlyStrategy do let(:model) { double('model', table_name: table_name) } let(:partitioning_key) { double } - let(:table_name) { :partitioned_test } + let(:table_name) { :_test_partitioned_test } before do connection.execute(<<~SQL) @@ -18,11 +18,11 @@ RSpec.describe Gitlab::Database::Partitioning::MonthlyStrategy do (id serial not null, created_at timestamptz not null, PRIMARY KEY (id, created_at)) PARTITION BY RANGE (created_at); - CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.partitioned_test_000000 + CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_partitioned_test_000000 PARTITION OF #{table_name} FOR VALUES FROM (MINVALUE) TO ('2020-05-01'); - CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.partitioned_test_202005 + CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_partitioned_test_202005 PARTITION OF #{table_name} FOR VALUES FROM ('2020-05-01') TO ('2020-06-01'); SQL @@ -30,8 +30,8 @@ RSpec.describe Gitlab::Database::Partitioning::MonthlyStrategy do it 'detects both partitions' do expect(subject).to eq([ - Gitlab::Database::Partitioning::TimePartition.new(table_name, nil, '2020-05-01', partition_name: 'partitioned_test_000000'), - Gitlab::Database::Partitioning::TimePartition.new(table_name, '2020-05-01', '2020-06-01', partition_name: 'partitioned_test_202005') + Gitlab::Database::Partitioning::TimePartition.new(table_name, nil, '2020-05-01', partition_name: '_test_partitioned_test_000000'), + Gitlab::Database::Partitioning::TimePartition.new(table_name, '2020-05-01', '2020-06-01', partition_name: '_test_partitioned_test_202005') ]) end end @@ -41,7 +41,7 @@ RSpec.describe Gitlab::Database::Partitioning::MonthlyStrategy do let(:model) do Class.new(ActiveRecord::Base) do - self.table_name = 'partitioned_test' + self.table_name = '_test_partitioned_test' self.primary_key = :id end end @@ -59,11 +59,11 @@ RSpec.describe Gitlab::Database::Partitioning::MonthlyStrategy do (id serial not null, created_at timestamptz not null, PRIMARY KEY (id, created_at)) PARTITION BY RANGE (created_at); - CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.partitioned_test_000000 + CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_partitioned_test_000000 PARTITION OF #{model.table_name} FOR VALUES FROM (MINVALUE) TO ('2020-05-01'); - CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.partitioned_test_202006 + CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_partitioned_test_202006 PARTITION OF #{model.table_name} FOR VALUES FROM ('2020-06-01') TO ('2020-07-01'); SQL @@ -166,7 +166,7 @@ RSpec.describe Gitlab::Database::Partitioning::MonthlyStrategy do (id serial not null, created_at timestamptz not null, PRIMARY KEY (id, created_at)) PARTITION BY RANGE (created_at); - CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.partitioned_test_202006 + CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_partitioned_test_202006 PARTITION OF #{model.table_name} FOR VALUES FROM ('2020-06-01') TO ('2020-07-01'); SQL @@ -181,13 +181,13 @@ RSpec.describe Gitlab::Database::Partitioning::MonthlyStrategy do describe '#extra_partitions' do let(:model) do Class.new(ActiveRecord::Base) do - self.table_name = 'partitioned_test' + self.table_name = '_test_partitioned_test' self.primary_key = :id end end let(:partitioning_key) { :created_at } - let(:table_name) { :partitioned_test } + let(:table_name) { :_test_partitioned_test } around do |example| travel_to(Date.parse('2020-08-22')) { example.run } @@ -200,15 +200,15 @@ RSpec.describe Gitlab::Database::Partitioning::MonthlyStrategy do (id serial not null, created_at timestamptz not null, PRIMARY KEY (id, created_at)) PARTITION BY RANGE (created_at); - CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.partitioned_test_000000 + CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_partitioned_test_000000 PARTITION OF #{table_name} FOR VALUES FROM (MINVALUE) TO ('2020-05-01'); - CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.partitioned_test_202005 + CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_partitioned_test_202005 PARTITION OF #{table_name} FOR VALUES FROM ('2020-05-01') TO ('2020-06-01'); - CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.partitioned_test_202006 + CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_partitioned_test_202006 PARTITION OF #{table_name} FOR VALUES FROM ('2020-06-01') TO ('2020-07-01') SQL @@ -235,7 +235,7 @@ RSpec.describe Gitlab::Database::Partitioning::MonthlyStrategy do it 'prunes the unbounded partition ending 2020-05-01' do min_value_to_may = Gitlab::Database::Partitioning::TimePartition.new(model.table_name, nil, '2020-05-01', - partition_name: 'partitioned_test_000000') + partition_name: '_test_partitioned_test_000000') expect(subject).to contain_exactly(min_value_to_may) end @@ -246,8 +246,8 @@ RSpec.describe Gitlab::Database::Partitioning::MonthlyStrategy do it 'prunes the unbounded partition and the partition for May-June' do expect(subject).to contain_exactly( - Gitlab::Database::Partitioning::TimePartition.new(model.table_name, nil, '2020-05-01', partition_name: 'partitioned_test_000000'), - Gitlab::Database::Partitioning::TimePartition.new(model.table_name, '2020-05-01', '2020-06-01', partition_name: 'partitioned_test_202005') + Gitlab::Database::Partitioning::TimePartition.new(model.table_name, nil, '2020-05-01', partition_name: '_test_partitioned_test_000000'), + Gitlab::Database::Partitioning::TimePartition.new(model.table_name, '2020-05-01', '2020-06-01', partition_name: '_test_partitioned_test_202005') ) end @@ -256,16 +256,16 @@ RSpec.describe Gitlab::Database::Partitioning::MonthlyStrategy do it 'prunes empty partitions' do expect(subject).to contain_exactly( - Gitlab::Database::Partitioning::TimePartition.new(model.table_name, nil, '2020-05-01', partition_name: 'partitioned_test_000000'), - Gitlab::Database::Partitioning::TimePartition.new(model.table_name, '2020-05-01', '2020-06-01', partition_name: 'partitioned_test_202005') + Gitlab::Database::Partitioning::TimePartition.new(model.table_name, nil, '2020-05-01', partition_name: '_test_partitioned_test_000000'), + Gitlab::Database::Partitioning::TimePartition.new(model.table_name, '2020-05-01', '2020-06-01', partition_name: '_test_partitioned_test_202005') ) end it 'does not prune non-empty partitions' do - connection.execute("INSERT INTO #{table_name} (created_at) VALUES (('2020-05-15'))") # inserting one record into partitioned_test_202005 + connection.execute("INSERT INTO #{table_name} (created_at) VALUES (('2020-05-15'))") # inserting one record into _test_partitioned_test_202005 expect(subject).to contain_exactly( - Gitlab::Database::Partitioning::TimePartition.new(model.table_name, nil, '2020-05-01', partition_name: 'partitioned_test_000000') + Gitlab::Database::Partitioning::TimePartition.new(model.table_name, nil, '2020-05-01', partition_name: '_test_partitioned_test_000000') ) end end diff --git a/spec/lib/gitlab/database/partitioning/multi_database_partition_dropper_spec.rb b/spec/lib/gitlab/database/partitioning/multi_database_partition_dropper_spec.rb deleted file mode 100644 index 56d6ebb7aff..00000000000 --- a/spec/lib/gitlab/database/partitioning/multi_database_partition_dropper_spec.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::Partitioning::MultiDatabasePartitionDropper, '#drop_detached_partitions' do - subject(:drop_detached_partitions) { multi_db_dropper.drop_detached_partitions } - - let(:multi_db_dropper) { described_class.new } - - let(:connection_wrapper1) { double(scope: scope1) } - let(:connection_wrapper2) { double(scope: scope2) } - - let(:scope1) { double(connection: connection1) } - let(:scope2) { double(connection: connection2) } - - let(:connection1) { double('connection') } - let(:connection2) { double('connection') } - - let(:dropper_class) { Gitlab::Database::Partitioning::DetachedPartitionDropper } - let(:dropper1) { double('partition dropper') } - let(:dropper2) { double('partition dropper') } - - before do - allow(multi_db_dropper).to receive(:databases).and_return({ db1: connection_wrapper1, db2: connection_wrapper2 }) - end - - it 'drops detached partitions for each database' do - expect(Gitlab::Database::SharedModel).to receive(:using_connection).with(connection1).and_yield.ordered - expect(dropper_class).to receive(:new).and_return(dropper1).ordered - expect(dropper1).to receive(:perform) - - expect(Gitlab::Database::SharedModel).to receive(:using_connection).with(connection2).and_yield.ordered - expect(dropper_class).to receive(:new).and_return(dropper2).ordered - expect(dropper2).to receive(:perform) - - drop_detached_partitions - end -end diff --git a/spec/lib/gitlab/database/partitioning/multi_database_partition_manager_spec.rb b/spec/lib/gitlab/database/partitioning/multi_database_partition_manager_spec.rb deleted file mode 100644 index 3c94c1bf4ea..00000000000 --- a/spec/lib/gitlab/database/partitioning/multi_database_partition_manager_spec.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::Partitioning::MultiDatabasePartitionManager, '#sync_partitions' do - subject(:sync_partitions) { manager.sync_partitions } - - let(:manager) { described_class.new(models) } - let(:models) { [model1, model2] } - - let(:model1) { double('model1', connection: connection1, table_name: 'table1') } - let(:model2) { double('model2', connection: connection1, table_name: 'table2') } - - let(:connection1) { double('connection1') } - let(:connection2) { double('connection2') } - - let(:target_manager_class) { Gitlab::Database::Partitioning::PartitionManager } - let(:target_manager1) { double('partition manager') } - let(:target_manager2) { double('partition manager') } - - before do - allow(manager).to receive(:connection_name).and_return('name') - end - - it 'syncs model partitions, setting up the appropriate connection for each', :aggregate_failures do - expect(Gitlab::Database::SharedModel).to receive(:using_connection).with(model1.connection).and_yield.ordered - expect(target_manager_class).to receive(:new).with(model1).and_return(target_manager1).ordered - expect(target_manager1).to receive(:sync_partitions) - - expect(Gitlab::Database::SharedModel).to receive(:using_connection).with(model2.connection).and_yield.ordered - expect(target_manager_class).to receive(:new).with(model2).and_return(target_manager2).ordered - expect(target_manager2).to receive(:sync_partitions) - - sync_partitions - end -end diff --git a/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb b/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb index 7c4cfcfb3a9..1c6f5c5c694 100644 --- a/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb +++ b/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb @@ -195,7 +195,7 @@ RSpec.describe Gitlab::Database::Partitioning::PartitionManager do end # Postgres 11 does not support foreign keys to partitioned tables - if Gitlab::Database.main.version.to_f >= 12 + if ApplicationRecord.database.version.to_f >= 12 context 'when the model is the target of a foreign key' do before do connection.execute(<<~SQL) diff --git a/spec/lib/gitlab/database/partitioning/partition_monitoring_spec.rb b/spec/lib/gitlab/database/partitioning/partition_monitoring_spec.rb index 7024cbd55ff..006ce8a7f48 100644 --- a/spec/lib/gitlab/database/partitioning/partition_monitoring_spec.rb +++ b/spec/lib/gitlab/database/partitioning/partition_monitoring_spec.rb @@ -4,9 +4,8 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Partitioning::PartitionMonitoring do describe '#report_metrics' do - subject { described_class.new(models).report_metrics } + subject { described_class.new.report_metrics_for_model(model) } - let(:models) { [model] } let(:model) { double(partitioning_strategy: partitioning_strategy, table_name: table) } let(:partitioning_strategy) { double(missing_partitions: missing_partitions, current_partitions: current_partitions, extra_partitions: extra_partitions) } let(:table) { "some_table" } diff --git a/spec/lib/gitlab/database/partitioning/replace_table_spec.rb b/spec/lib/gitlab/database/partitioning/replace_table_spec.rb index 8e27797208c..fdf514b519f 100644 --- a/spec/lib/gitlab/database/partitioning/replace_table_spec.rb +++ b/spec/lib/gitlab/database/partitioning/replace_table_spec.rb @@ -5,7 +5,9 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Partitioning::ReplaceTable, '#perform' do include Database::TableSchemaHelpers - subject(:replace_table) { described_class.new(original_table, replacement_table, archived_table, 'id').perform } + subject(:replace_table) do + described_class.new(connection, original_table, replacement_table, archived_table, 'id').perform + end let(:original_table) { '_test_original_table' } let(:replacement_table) { '_test_replacement_table' } |