diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-06-24 03:08:43 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-06-24 03:08:43 +0300 |
commit | 082b24b03bbb9dca7edf1341aa7b0a51c9aeb18b (patch) | |
tree | 5a0b89d06053474c7565de01d9a2500713bda235 /spec/lib/gitlab | |
parent | 3a06969fe871598ead8b99ce42a1152f03d4ef91 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'spec/lib/gitlab')
5 files changed, 480 insertions, 245 deletions
diff --git a/spec/lib/gitlab/database/dynamic_model_helpers_spec.rb b/spec/lib/gitlab/database/dynamic_model_helpers_spec.rb new file mode 100644 index 00000000000..c0dc14a3f4f --- /dev/null +++ b/spec/lib/gitlab/database/dynamic_model_helpers_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Database::DynamicModelHelpers do + describe '#define_batchable_model' do + subject { including_class.new.define_batchable_model(table_name) } + + let(:including_class) { Class.new.include(described_class) } + let(:table_name) { 'projects' } + + it 'is an ActiveRecord model' do + expect(subject.ancestors).to include(ActiveRecord::Base) + end + + it 'includes EachBatch' do + expect(subject.included_modules).to include(EachBatch) + end + + it 'has the correct table name' do + expect(subject.table_name).to eq(table_name) + end + + it 'has the inheritance type column disable' do + expect(subject.inheritance_column).to eq('_type_disabled') + end + end +end diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb index bed444ee7c7..1f725687bdc 100644 --- a/spec/lib/gitlab/database/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers_spec.rb @@ -1215,166 +1215,6 @@ describe Gitlab::Database::MigrationHelpers do end end - describe '#bulk_queue_background_migration_jobs_by_range' do - context 'when the model has an ID column' do - let!(:id1) { create(:user).id } - let!(:id2) { create(:user).id } - let!(:id3) { create(:user).id } - - before do - User.class_eval do - include EachBatch - end - end - - context 'with enough rows to bulk queue jobs more than once' do - before do - stub_const('Gitlab::Database::MigrationHelpers::BACKGROUND_MIGRATION_JOB_BUFFER_SIZE', 1) - end - - it 'queues jobs correctly' do - Sidekiq::Testing.fake! do - model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2) - - expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id2]]) - expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['FooJob', [id3, id3]]) - end - end - - it 'queues jobs in groups of buffer size 1' do - expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id1, id2]]]) - expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id3, id3]]]) - - model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2) - end - end - - context 'with not enough rows to bulk queue jobs more than once' do - it 'queues jobs correctly' do - Sidekiq::Testing.fake! do - model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2) - - expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id2]]) - expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['FooJob', [id3, id3]]) - end - end - - it 'queues jobs in bulk all at once (big buffer size)' do - expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id1, id2]], - ['FooJob', [id3, id3]]]) - - model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2) - end - end - - context 'without specifying batch_size' do - it 'queues jobs correctly' do - Sidekiq::Testing.fake! do - model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob') - - expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3]]) - end - end - end - end - - context "when the model doesn't have an ID column" do - it 'raises error (for now)' do - expect do - model.bulk_queue_background_migration_jobs_by_range(ProjectAuthorization, 'FooJob') - end.to raise_error(StandardError, /does not have an ID/) - end - end - end - - describe '#queue_background_migration_jobs_by_range_at_intervals' do - context 'when the model has an ID column' do - let!(:id1) { create(:user).id } - let!(:id2) { create(:user).id } - let!(:id3) { create(:user).id } - - around do |example| - Timecop.freeze { example.run } - end - - before do - User.class_eval do - include EachBatch - end - end - - it 'returns the final expected delay' do - Sidekiq::Testing.fake! do - final_delay = model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, batch_size: 2) - - expect(final_delay.to_f).to eq(20.minutes.to_f) - end - end - - it 'returns zero when nothing gets queued' do - Sidekiq::Testing.fake! do - final_delay = model.queue_background_migration_jobs_by_range_at_intervals(User.none, 'FooJob', 10.minutes) - - expect(final_delay).to eq(0) - end - end - - context 'with batch_size option' do - it 'queues jobs correctly' do - Sidekiq::Testing.fake! do - model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, batch_size: 2) - - expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id2]]) - expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(10.minutes.from_now.to_f) - expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['FooJob', [id3, id3]]) - expect(BackgroundMigrationWorker.jobs[1]['at']).to eq(20.minutes.from_now.to_f) - end - end - end - - context 'without batch_size option' do - it 'queues jobs correctly' do - Sidekiq::Testing.fake! do - model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes) - - expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3]]) - expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(10.minutes.from_now.to_f) - end - end - end - - context 'with other_job_arguments option' do - it 'queues jobs correctly' do - Sidekiq::Testing.fake! do - model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, other_job_arguments: [1, 2]) - - expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3, 1, 2]]) - expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(10.minutes.from_now.to_f) - end - end - end - - context 'with initial_delay option' do - it 'queues jobs correctly' do - Sidekiq::Testing.fake! do - model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, other_job_arguments: [1, 2], initial_delay: 10.minutes) - - expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3, 1, 2]]) - expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(20.minutes.from_now.to_f) - end - end - end - end - - context "when the model doesn't have an ID column" do - it 'raises error (for now)' do - expect do - model.queue_background_migration_jobs_by_range_at_intervals(ProjectAuthorization, 'FooJob', 10.seconds) - end.to raise_error(StandardError, /does not have an ID/) - end - end - end - describe '#change_column_type_using_background_migration' do let!(:issue) { create(:issue, :closed, closed_at: Time.zone.now) } @@ -1485,26 +1325,6 @@ describe Gitlab::Database::MigrationHelpers do end end - describe '#perform_background_migration_inline?' do - it 'returns true in a test environment' do - stub_rails_env('test') - - expect(model.perform_background_migration_inline?).to eq(true) - end - - it 'returns true in a development environment' do - stub_rails_env('development') - - expect(model.perform_background_migration_inline?).to eq(true) - end - - it 'returns false in a production environment' do - stub_rails_env('production') - - expect(model.perform_background_migration_inline?).to eq(false) - end - end - describe '#index_exists_by_name?' do it 'returns true if an index exists' do ActiveRecord::Base.connection.execute( @@ -1973,62 +1793,6 @@ describe Gitlab::Database::MigrationHelpers do end end - describe '#migrate_async' do - it 'calls BackgroundMigrationWorker.perform_async' do - expect(BackgroundMigrationWorker).to receive(:perform_async).with("Class", "hello", "world") - - model.migrate_async("Class", "hello", "world") - end - - it 'pushes a context with the current class name as caller_id' do - expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s) - - model.migrate_async('Class', 'hello', 'world') - end - end - - describe '#migrate_in' do - it 'calls BackgroundMigrationWorker.perform_in' do - expect(BackgroundMigrationWorker).to receive(:perform_in).with(10.minutes, 'Class', 'Hello', 'World') - - model.migrate_in(10.minutes, 'Class', 'Hello', 'World') - end - - it 'pushes a context with the current class name as caller_id' do - expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s) - - model.migrate_in(10.minutes, 'Class', 'Hello', 'World') - end - end - - describe '#bulk_migrate_async' do - it 'calls BackgroundMigrationWorker.bulk_perform_async' do - expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([%w(Class hello world)]) - - model.bulk_migrate_async([%w(Class hello world)]) - end - - it 'pushes a context with the current class name as caller_id' do - expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s) - - model.bulk_migrate_async([%w(Class hello world)]) - end - end - - describe '#bulk_migrate_in' do - it 'calls BackgroundMigrationWorker.bulk_perform_in_' do - expect(BackgroundMigrationWorker).to receive(:bulk_perform_in).with(10.minutes, [%w(Class hello world)]) - - model.bulk_migrate_in(10.minutes, [%w(Class hello world)]) - end - - it 'pushes a context with the current class name as caller_id' do - expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s) - - model.bulk_migrate_in(10.minutes, [%w(Class hello world)]) - end - end - describe '#check_constraint_name' do it 'returns a valid constraint name' do name = model.check_constraint_name(:this_is_a_very_long_table_name, diff --git a/spec/lib/gitlab/database/migrations/background_migration_helpers_spec.rb b/spec/lib/gitlab/database/migrations/background_migration_helpers_spec.rb new file mode 100644 index 00000000000..1b1ac0cb387 --- /dev/null +++ b/spec/lib/gitlab/database/migrations/background_migration_helpers_spec.rb @@ -0,0 +1,245 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Database::Migrations::BackgroundMigrationHelpers do + let(:model) do + ActiveRecord::Migration.new.extend(described_class) + end + + describe '#bulk_queue_background_migration_jobs_by_range' do + context 'when the model has an ID column' do + let!(:id1) { create(:user).id } + let!(:id2) { create(:user).id } + let!(:id3) { create(:user).id } + + before do + User.class_eval do + include EachBatch + end + end + + context 'with enough rows to bulk queue jobs more than once' do + before do + stub_const('Gitlab::Database::Migrations::BackgroundMigrationHelpers::BACKGROUND_MIGRATION_JOB_BUFFER_SIZE', 1) + end + + it 'queues jobs correctly' do + Sidekiq::Testing.fake! do + model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2) + + expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id2]]) + expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['FooJob', [id3, id3]]) + end + end + + it 'queues jobs in groups of buffer size 1' do + expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id1, id2]]]) + expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id3, id3]]]) + + model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2) + end + end + + context 'with not enough rows to bulk queue jobs more than once' do + it 'queues jobs correctly' do + Sidekiq::Testing.fake! do + model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2) + + expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id2]]) + expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['FooJob', [id3, id3]]) + end + end + + it 'queues jobs in bulk all at once (big buffer size)' do + expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id1, id2]], + ['FooJob', [id3, id3]]]) + + model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2) + end + end + + context 'without specifying batch_size' do + it 'queues jobs correctly' do + Sidekiq::Testing.fake! do + model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob') + + expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3]]) + end + end + end + end + + context "when the model doesn't have an ID column" do + it 'raises error (for now)' do + expect do + model.bulk_queue_background_migration_jobs_by_range(ProjectAuthorization, 'FooJob') + end.to raise_error(StandardError, /does not have an ID/) + end + end + end + + describe '#queue_background_migration_jobs_by_range_at_intervals' do + context 'when the model has an ID column' do + let!(:id1) { create(:user).id } + let!(:id2) { create(:user).id } + let!(:id3) { create(:user).id } + + around do |example| + Timecop.freeze { example.run } + end + + before do + User.class_eval do + include EachBatch + end + end + + it 'returns the final expected delay' do + Sidekiq::Testing.fake! do + final_delay = model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, batch_size: 2) + + expect(final_delay.to_f).to eq(20.minutes.to_f) + end + end + + it 'returns zero when nothing gets queued' do + Sidekiq::Testing.fake! do + final_delay = model.queue_background_migration_jobs_by_range_at_intervals(User.none, 'FooJob', 10.minutes) + + expect(final_delay).to eq(0) + end + end + + context 'with batch_size option' do + it 'queues jobs correctly' do + Sidekiq::Testing.fake! do + model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, batch_size: 2) + + expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id2]]) + expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(10.minutes.from_now.to_f) + expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['FooJob', [id3, id3]]) + expect(BackgroundMigrationWorker.jobs[1]['at']).to eq(20.minutes.from_now.to_f) + end + end + end + + context 'without batch_size option' do + it 'queues jobs correctly' do + Sidekiq::Testing.fake! do + model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes) + + expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3]]) + expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(10.minutes.from_now.to_f) + end + end + end + + context 'with other_job_arguments option' do + it 'queues jobs correctly' do + Sidekiq::Testing.fake! do + model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, other_job_arguments: [1, 2]) + + expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3, 1, 2]]) + expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(10.minutes.from_now.to_f) + end + end + end + + context 'with initial_delay option' do + it 'queues jobs correctly' do + Sidekiq::Testing.fake! do + model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, other_job_arguments: [1, 2], initial_delay: 10.minutes) + + expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3, 1, 2]]) + expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(20.minutes.from_now.to_f) + end + end + end + end + + context "when the model doesn't have an ID column" do + it 'raises error (for now)' do + expect do + model.queue_background_migration_jobs_by_range_at_intervals(ProjectAuthorization, 'FooJob', 10.seconds) + end.to raise_error(StandardError, /does not have an ID/) + end + end + end + + describe '#perform_background_migration_inline?' do + it 'returns true in a test environment' do + stub_rails_env('test') + + expect(model.perform_background_migration_inline?).to eq(true) + end + + it 'returns true in a development environment' do + stub_rails_env('development') + + expect(model.perform_background_migration_inline?).to eq(true) + end + + it 'returns false in a production environment' do + stub_rails_env('production') + + expect(model.perform_background_migration_inline?).to eq(false) + end + end + + describe '#migrate_async' do + it 'calls BackgroundMigrationWorker.perform_async' do + expect(BackgroundMigrationWorker).to receive(:perform_async).with("Class", "hello", "world") + + model.migrate_async("Class", "hello", "world") + end + + it 'pushes a context with the current class name as caller_id' do + expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s) + + model.migrate_async('Class', 'hello', 'world') + end + end + + describe '#migrate_in' do + it 'calls BackgroundMigrationWorker.perform_in' do + expect(BackgroundMigrationWorker).to receive(:perform_in).with(10.minutes, 'Class', 'Hello', 'World') + + model.migrate_in(10.minutes, 'Class', 'Hello', 'World') + end + + it 'pushes a context with the current class name as caller_id' do + expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s) + + model.migrate_in(10.minutes, 'Class', 'Hello', 'World') + end + end + + describe '#bulk_migrate_async' do + it 'calls BackgroundMigrationWorker.bulk_perform_async' do + expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([%w(Class hello world)]) + + model.bulk_migrate_async([%w(Class hello world)]) + end + + it 'pushes a context with the current class name as caller_id' do + expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s) + + model.bulk_migrate_async([%w(Class hello world)]) + end + end + + describe '#bulk_migrate_in' do + it 'calls BackgroundMigrationWorker.bulk_perform_in_' do + expect(BackgroundMigrationWorker).to receive(:bulk_perform_in).with(10.minutes, [%w(Class hello world)]) + + model.bulk_migrate_in(10.minutes, [%w(Class hello world)]) + end + + it 'pushes a context with the current class name as caller_id' do + expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s) + + model.bulk_migrate_in(10.minutes, [%w(Class hello world)]) + end + end +end diff --git a/spec/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table_spec.rb b/spec/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table_spec.rb new file mode 100644 index 00000000000..2016c31b891 --- /dev/null +++ b/spec/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table_spec.rb @@ -0,0 +1,164 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Database::PartitioningMigrationHelpers::BackfillPartitionedTable, '#perform' do + subject { described_class.new } + + let(:source_table) { '_test_partitioning_backfills' } + let(:destination_table) { "#{source_table}_part" } + let(:unique_key) { 'id' } + + before do + allow(subject).to receive(:transaction_open?).and_return(false) + end + + context 'when the destination table exists' do + before do + connection.execute(<<~SQL) + CREATE TABLE #{source_table} ( + id serial NOT NULL PRIMARY KEY, + col1 int NOT NULL, + col2 text NOT NULL, + created_at timestamptz NOT NULL + ) + SQL + + connection.execute(<<~SQL) + CREATE TABLE #{destination_table} ( + id serial NOT NULL, + col1 int NOT NULL, + col2 text NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (id, created_at) + ) PARTITION BY RANGE (created_at) + SQL + + connection.execute(<<~SQL) + CREATE TABLE #{destination_table}_202001 PARTITION OF #{destination_table} + FOR VALUES FROM ('2020-01-01') TO ('2020-02-01') + SQL + + connection.execute(<<~SQL) + CREATE TABLE #{destination_table}_202002 PARTITION OF #{destination_table} + FOR VALUES FROM ('2020-02-01') TO ('2020-03-01') + SQL + + source_model.table_name = source_table + destination_model.table_name = destination_table + + stub_const("#{described_class}::SUB_BATCH_SIZE", 2) + stub_const("#{described_class}::PAUSE_SECONDS", pause_seconds) + + allow(subject).to receive(:sleep) + end + + let(:connection) { ActiveRecord::Base.connection } + let(:source_model) { Class.new(ActiveRecord::Base) } + let(:destination_model) { Class.new(ActiveRecord::Base) } + let(:timestamp) { Time.utc(2020, 1, 2).round } + let(:pause_seconds) { 1 } + + let!(:source1) { create_source_record(timestamp) } + let!(:source2) { create_source_record(timestamp + 1.day) } + let!(:source3) { create_source_record(timestamp + 1.month) } + + it 'copies data into the destination table idempotently' do + expect(destination_model.count).to eq(0) + + subject.perform(source1.id, source3.id, source_table, destination_table, unique_key) + + expect(destination_model.count).to eq(3) + + source_model.find_each do |source_record| + destination_record = destination_model.find_by_id(source_record.id) + + expect(destination_record.attributes).to eq(source_record.attributes) + end + + subject.perform(source1.id, source3.id, source_table, destination_table, unique_key) + + expect(destination_model.count).to eq(3) + end + + it 'breaks the assigned batch into smaller batches' do + expect_next_instance_of(described_class::BulkCopy) do |bulk_copy| + expect(bulk_copy).to receive(:copy_between).with(source1.id, source2.id) + expect(bulk_copy).to receive(:copy_between).with(source3.id, source3.id) + end + + subject.perform(source1.id, source3.id, source_table, destination_table, unique_key) + end + + it 'pauses after copying each sub-batch' do + expect(subject).to receive(:sleep).with(pause_seconds).twice + + subject.perform(source1.id, source3.id, source_table, destination_table, unique_key) + end + + context 'when the feature flag is disabled' do + let(:mock_connection) { double('connection') } + + before do + allow(subject).to receive(:connection).and_return(mock_connection) + stub_feature_flags(backfill_partitioned_audit_events: false) + end + + it 'exits without attempting to copy data' do + expect(mock_connection).not_to receive(:execute) + + subject.perform(1, 100, source_table, destination_table, unique_key) + + expect(destination_model.count).to eq(0) + end + end + + context 'when the job is run within an explicit transaction block' do + let(:mock_connection) { double('connection') } + + before do + allow(subject).to receive(:connection).and_return(mock_connection) + allow(subject).to receive(:transaction_open?).and_return(true) + end + + it 'raises an error before copying data' do + expect(mock_connection).not_to receive(:execute) + + expect do + subject.perform(1, 100, source_table, destination_table, unique_key) + end.to raise_error(/Aborting job to backfill partitioned #{source_table}/) + + expect(destination_model.count).to eq(0) + end + end + end + + context 'when the destination table does not exist' do + let(:mock_connection) { double('connection') } + let(:mock_logger) { double('logger') } + + before do + allow(subject).to receive(:connection).and_return(mock_connection) + allow(subject).to receive(:logger).and_return(mock_logger) + + expect(mock_connection).to receive(:table_exists?).with(destination_table).and_return(false) + allow(mock_logger).to receive(:warn) + end + + it 'exits without attempting to copy data' do + expect(mock_connection).not_to receive(:execute) + + subject.perform(1, 100, source_table, destination_table, unique_key) + end + + it 'logs a warning message that the job was skipped' do + expect(mock_logger).to receive(:warn).with(/#{destination_table} does not exist/) + + subject.perform(1, 100, source_table, destination_table, unique_key) + end + end + + def create_source_record(timestamp) + source_model.create!(col1: 123, col2: 'original value', created_at: timestamp) + end +end diff --git a/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb b/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb index 586b57d2002..c78c143f2f8 100644 --- a/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb +++ b/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb @@ -25,7 +25,7 @@ describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers allow(migration).to receive(:partitioned_table_name).and_return(partitioned_table) allow(migration).to receive(:sync_function_name).and_return(function_name) allow(migration).to receive(:sync_trigger_name).and_return(trigger_name) - allow(migration).to receive(:assert_table_is_whitelisted) + allow(migration).to receive(:assert_table_is_allowed) end describe '#partition_table_by_date' do @@ -33,15 +33,19 @@ describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers let(:old_primary_key) { 'id' } let(:new_primary_key) { [old_primary_key, partition_column] } - context 'when the table is not whitelisted' do - let(:template_table) { :this_table_is_not_whitelisted } + before do + allow(migration).to receive(:queue_background_migration_jobs_by_range_at_intervals) + end + + context 'when the table is not allowed' do + let(:template_table) { :this_table_is_not_allowed } it 'raises an error' do - expect(migration).to receive(:assert_table_is_whitelisted).with(template_table).and_call_original + expect(migration).to receive(:assert_table_is_allowed).with(template_table).and_call_original expect do migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date - end.to raise_error(/#{template_table} is not whitelisted for use/) + end.to raise_error(/#{template_table} is not allowed for use/) end end @@ -237,6 +241,36 @@ describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers expect(model.find(second_todo.id).attributes).to eq(second_todo.attributes) end end + + describe 'copying historic data to the partitioned table' do + let(:template_table) { 'todos' } + let(:migration_class) { '::Gitlab::Database::PartitioningMigrationHelpers::BackfillPartitionedTable' } + let(:sub_batch_size) { described_class::SUB_BATCH_SIZE } + let(:pause_seconds) { described_class::PAUSE_SECONDS } + let!(:first_id) { create(:todo).id } + let!(:second_id) { create(:todo).id } + let!(:third_id) { create(:todo).id } + + before do + stub_const("#{described_class.name}::BATCH_SIZE", 2) + + expect(migration).to receive(:queue_background_migration_jobs_by_range_at_intervals).and_call_original + end + + it 'enqueues jobs to copy each batch of data' do + Sidekiq::Testing.fake! do + migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date + + expect(BackgroundMigrationWorker.jobs.size).to eq(2) + + first_job_arguments = [first_id, second_id, template_table, partitioned_table, 'id'] + expect(BackgroundMigrationWorker.jobs[0]['args']).to eq([migration_class, first_job_arguments]) + + second_job_arguments = [third_id, third_id, template_table, partitioned_table, 'id'] + expect(BackgroundMigrationWorker.jobs[1]['args']).to eq([migration_class, second_job_arguments]) + end + end + end end describe '#drop_partitioned_table_for' do @@ -244,15 +278,15 @@ describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers %w[000000 201912 202001 202002].map { |suffix| "#{partitioned_table}_#{suffix}" }.unshift(partitioned_table) end - context 'when the table is not whitelisted' do - let(:template_table) { :this_table_is_not_whitelisted } + context 'when the table is not allowed' do + let(:template_table) { :this_table_is_not_allowed } it 'raises an error' do - expect(migration).to receive(:assert_table_is_whitelisted).with(template_table).and_call_original + expect(migration).to receive(:assert_table_is_allowed).with(template_table).and_call_original expect do migration.drop_partitioned_table_for template_table - end.to raise_error(/#{template_table} is not whitelisted for use/) + end.to raise_error(/#{template_table} is not allowed for use/) end end |