diff options
Diffstat (limited to 'spec/lib/gitlab/database/background_migration')
4 files changed, 462 insertions, 0 deletions
diff --git a/spec/lib/gitlab/database/background_migration/batched_job_spec.rb b/spec/lib/gitlab/database/background_migration/batched_job_spec.rb new file mode 100644 index 00000000000..1020aafcf08 --- /dev/null +++ b/spec/lib/gitlab/database/background_migration/batched_job_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model do + it_behaves_like 'having unique enum values' + + describe 'associations' do + it { is_expected.to belong_to(:batched_migration).with_foreign_key(:batched_background_migration_id) } + end + + describe 'delegated batched_migration attributes' do + let(:batched_job) { build(:batched_background_migration_job) } + let(:batched_migration) { batched_job.batched_migration } + + describe '#migration_aborted?' do + before do + batched_migration.status = :aborted + end + + it 'returns the migration aborted?' do + expect(batched_job.migration_aborted?).to eq(batched_migration.aborted?) + end + end + + describe '#migration_job_class' do + it 'returns the migration job_class' do + expect(batched_job.migration_job_class).to eq(batched_migration.job_class) + end + end + + describe '#migration_table_name' do + it 'returns the migration table_name' do + expect(batched_job.migration_table_name).to eq(batched_migration.table_name) + end + end + + describe '#migration_column_name' do + it 'returns the migration column_name' do + expect(batched_job.migration_column_name).to eq(batched_migration.column_name) + end + end + + describe '#migration_job_arguments' do + it 'returns the migration job_arguments' do + expect(batched_job.migration_job_arguments).to eq(batched_migration.job_arguments) + end + end + end +end diff --git a/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb b/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb new file mode 100644 index 00000000000..f4a939e7c1f --- /dev/null +++ b/spec/lib/gitlab/database/background_migration/batched_migration_spec.rb @@ -0,0 +1,160 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :model do + it_behaves_like 'having unique enum values' + + describe 'associations' do + it { is_expected.to have_many(:batched_jobs).with_foreign_key(:batched_background_migration_id) } + + describe '#last_job' do + let!(:batched_migration) { create(:batched_background_migration) } + let!(:batched_job1) { create(:batched_background_migration_job, batched_migration: batched_migration) } + let!(:batched_job2) { create(:batched_background_migration_job, batched_migration: batched_migration) } + + it 'returns the most recent (in order of id) batched job' do + expect(batched_migration.last_job).to eq(batched_job2) + end + end + end + + describe '.queue_order' do + let!(:migration1) { create(:batched_background_migration) } + let!(:migration2) { create(:batched_background_migration) } + let!(:migration3) { create(:batched_background_migration) } + + it 'returns batched migrations ordered by their id' do + expect(described_class.queue_order.all).to eq([migration1, migration2, migration3]) + end + end + + describe '#interval_elapsed?' do + context 'when the migration has no last_job' do + let(:batched_migration) { build(:batched_background_migration) } + + it 'returns true' do + expect(batched_migration.interval_elapsed?).to eq(true) + end + end + + context 'when the migration has a last_job' do + let(:interval) { 2.minutes } + let(:batched_migration) { create(:batched_background_migration, interval: interval) } + + context 'when the last_job is less than an interval old' do + it 'returns false' do + freeze_time do + create(:batched_background_migration_job, + batched_migration: batched_migration, + created_at: Time.current - 1.minute) + + expect(batched_migration.interval_elapsed?).to eq(false) + end + end + end + + context 'when the last_job is exactly an interval old' do + it 'returns true' do + freeze_time do + create(:batched_background_migration_job, + batched_migration: batched_migration, + created_at: Time.current - 2.minutes) + + expect(batched_migration.interval_elapsed?).to eq(true) + end + end + end + + context 'when the last_job is more than an interval old' do + it 'returns true' do + freeze_time do + create(:batched_background_migration_job, + batched_migration: batched_migration, + created_at: Time.current - 3.minutes) + + expect(batched_migration.interval_elapsed?).to eq(true) + end + end + end + end + end + + describe '#create_batched_job!' do + let(:batched_migration) { create(:batched_background_migration) } + + it 'creates a batched_job with the correct batch configuration' do + batched_job = batched_migration.create_batched_job!(1, 5) + + expect(batched_job).to have_attributes( + min_value: 1, + max_value: 5, + batch_size: batched_migration.batch_size, + sub_batch_size: batched_migration.sub_batch_size) + end + end + + describe '#next_min_value' do + let!(:batched_migration) { create(:batched_background_migration) } + + context 'when a previous job exists' do + let!(:batched_job) { create(:batched_background_migration_job, batched_migration: batched_migration) } + + it 'returns the next value after the previous maximum' do + expect(batched_migration.next_min_value).to eq(batched_job.max_value + 1) + end + end + + context 'when a previous job does not exist' do + it 'returns the migration minimum value' do + expect(batched_migration.next_min_value).to eq(batched_migration.min_value) + end + end + end + + describe '#job_class' do + let(:job_class) { Gitlab::BackgroundMigration::CopyColumnUsingBackgroundMigrationJob } + let(:batched_migration) { build(:batched_background_migration) } + + it 'returns the class of the job for the migration' do + expect(batched_migration.job_class).to eq(job_class) + end + end + + describe '#batch_class' do + let(:batch_class) { Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchingStrategy} + let(:batched_migration) { build(:batched_background_migration) } + + it 'returns the class of the batch strategy for the migration' do + expect(batched_migration.batch_class).to eq(batch_class) + end + end + + shared_examples_for 'an attr_writer that demodulizes assigned class names' do |attribute_name| + let(:batched_migration) { build(:batched_background_migration) } + + context 'when a module name exists' do + it 'removes the module name' do + batched_migration.public_send(:"#{attribute_name}=", '::Foo::Bar') + + expect(batched_migration[attribute_name]).to eq('Bar') + end + end + + context 'when a module name does not exist' do + it 'does not change the given class name' do + batched_migration.public_send(:"#{attribute_name}=", 'Bar') + + expect(batched_migration[attribute_name]).to eq('Bar') + end + end + end + + describe '#job_class_name=' do + it_behaves_like 'an attr_writer that demodulizes assigned class names', :job_class_name + end + + describe '#batch_class_name=' do + it_behaves_like 'an attr_writer that demodulizes assigned class names', :batch_class_name + end +end diff --git a/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb b/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb new file mode 100644 index 00000000000..17cceb35ff7 --- /dev/null +++ b/spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, '#perform' do + let(:migration_wrapper) { described_class.new } + let(:job_class) { Gitlab::BackgroundMigration::CopyColumnUsingBackgroundMigrationJob } + + let_it_be(:active_migration) { create(:batched_background_migration, :active, job_arguments: [:id, :other_id]) } + + let!(:job_record) { create(:batched_background_migration_job, batched_migration: active_migration) } + + it 'runs the migration job' do + expect_next_instance_of(job_class) do |job_instance| + expect(job_instance).to receive(:perform).with(1, 10, 'events', 'id', 1, 'id', 'other_id') + end + + migration_wrapper.perform(job_record) + end + + it 'updates the the tracking record in the database' do + expect(job_record).to receive(:update!).with(hash_including(attempts: 1, status: :running)).and_call_original + + freeze_time do + migration_wrapper.perform(job_record) + + reloaded_job_record = job_record.reload + + expect(reloaded_job_record).not_to be_pending + expect(reloaded_job_record.attempts).to eq(1) + expect(reloaded_job_record.started_at).to eq(Time.current) + end + end + + context 'when the migration job does not raise an error' do + it 'marks the tracking record as succeeded' do + expect_next_instance_of(job_class) do |job_instance| + expect(job_instance).to receive(:perform).with(1, 10, 'events', 'id', 1, 'id', 'other_id') + end + + freeze_time do + migration_wrapper.perform(job_record) + + reloaded_job_record = job_record.reload + + expect(reloaded_job_record).to be_succeeded + expect(reloaded_job_record.finished_at).to eq(Time.current) + end + end + end + + context 'when the migration job raises an error' do + it 'marks the tracking record as failed before raising the error' do + expect_next_instance_of(job_class) do |job_instance| + expect(job_instance).to receive(:perform) + .with(1, 10, 'events', 'id', 1, 'id', 'other_id') + .and_raise(RuntimeError, 'Something broke!') + end + + freeze_time do + expect { migration_wrapper.perform(job_record) }.to raise_error(RuntimeError, 'Something broke!') + + reloaded_job_record = job_record.reload + + expect(reloaded_job_record).to be_failed + expect(reloaded_job_record.finished_at).to eq(Time.current) + end + end + end +end diff --git a/spec/lib/gitlab/database/background_migration/scheduler_spec.rb b/spec/lib/gitlab/database/background_migration/scheduler_spec.rb new file mode 100644 index 00000000000..ba745acdf8a --- /dev/null +++ b/spec/lib/gitlab/database/background_migration/scheduler_spec.rb @@ -0,0 +1,182 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::BackgroundMigration::Scheduler, '#perform' do + let(:scheduler) { described_class.new } + + shared_examples_for 'it has no jobs to run' do + it 'does not create and run a migration job' do + test_wrapper = double('test wrapper') + + expect(test_wrapper).not_to receive(:perform) + + expect do + scheduler.perform(migration_wrapper: test_wrapper) + end.not_to change { Gitlab::Database::BackgroundMigration::BatchedJob.count } + end + end + + context 'when there are no active migrations' do + let!(:migration) { create(:batched_background_migration, :finished) } + + it_behaves_like 'it has no jobs to run' + end + + shared_examples_for 'it has completed the migration' do + it 'marks the migration as finished' do + relation = Gitlab::Database::BackgroundMigration::BatchedMigration.finished.where(id: first_migration.id) + + expect { scheduler.perform }.to change { relation.count }.by(1) + end + end + + context 'when there are active migrations' do + let!(:first_migration) { create(:batched_background_migration, :active, batch_size: 2) } + let!(:last_migration) { create(:batched_background_migration, :active) } + + let(:job_relation) do + Gitlab::Database::BackgroundMigration::BatchedJob.where(batched_background_migration_id: first_migration.id) + end + + context 'when the migration interval has not elapsed' do + before do + expect_next_found_instance_of(Gitlab::Database::BackgroundMigration::BatchedMigration) do |migration| + expect(migration).to receive(:interval_elapsed?).and_return(false) + end + end + + it_behaves_like 'it has no jobs to run' + end + + context 'when the interval has elapsed' do + before do + expect_next_found_instance_of(Gitlab::Database::BackgroundMigration::BatchedMigration) do |migration| + expect(migration).to receive(:interval_elapsed?).and_return(true) + end + end + + context 'when the first migration has no previous jobs' do + context 'when the migration has batches to process' do + let!(:event1) { create(:event) } + let!(:event2) { create(:event) } + let!(:event3) { create(:event) } + + it 'runs the job for the first batch' do + first_migration.update!(min_value: event1.id, max_value: event3.id) + + expect_next_instance_of(Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper) do |wrapper| + expect(wrapper).to receive(:perform).and_wrap_original do |_, job_record| + expect(job_record).to eq(job_relation.first) + end + end + + expect { scheduler.perform }.to change { job_relation.count }.by(1) + + expect(job_relation.first).to have_attributes( + min_value: event1.id, + max_value: event2.id, + batch_size: first_migration.batch_size, + sub_batch_size: first_migration.sub_batch_size) + end + end + + context 'when the migration has no batches to process' do + it_behaves_like 'it has no jobs to run' + it_behaves_like 'it has completed the migration' + end + end + + context 'when the first migration has previous jobs' do + let!(:event1) { create(:event) } + let!(:event2) { create(:event) } + let!(:event3) { create(:event) } + + let!(:previous_job) do + create(:batched_background_migration_job, + batched_migration: first_migration, + min_value: event1.id, + max_value: event2.id, + batch_size: 2, + sub_batch_size: 1) + end + + context 'when the migration is ready to process another job' do + it 'runs the migration job for the next batch' do + first_migration.update!(min_value: event1.id, max_value: event3.id) + + expect_next_instance_of(Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper) do |wrapper| + expect(wrapper).to receive(:perform).and_wrap_original do |_, job_record| + expect(job_record).to eq(job_relation.last) + end + end + + expect { scheduler.perform }.to change { job_relation.count }.by(1) + + expect(job_relation.last).to have_attributes( + min_value: event3.id, + max_value: event3.id, + batch_size: first_migration.batch_size, + sub_batch_size: first_migration.sub_batch_size) + end + end + + context 'when the migration has no batches remaining' do + let!(:final_job) do + create(:batched_background_migration_job, + batched_migration: first_migration, + min_value: event3.id, + max_value: event3.id, + batch_size: 2, + sub_batch_size: 1) + end + + it_behaves_like 'it has no jobs to run' + it_behaves_like 'it has completed the migration' + end + end + + context 'when the bounds of the next batch exceed the migration maximum value' do + let!(:events) { create_list(:event, 3) } + let(:event1) { events[0] } + let(:event2) { events[1] } + + context 'when the batch maximum exceeds the migration maximum' do + it 'clamps the batch maximum to the migration maximum' do + first_migration.update!(batch_size: 5, min_value: event1.id, max_value: event2.id) + + expect_next_instance_of(Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper) do |wrapper| + expect(wrapper).to receive(:perform) + end + + expect { scheduler.perform }.to change { job_relation.count }.by(1) + + expect(job_relation.first).to have_attributes( + min_value: event1.id, + max_value: event2.id, + batch_size: first_migration.batch_size, + sub_batch_size: first_migration.sub_batch_size) + end + end + + context 'when the batch minimum exceeds the migration maximum' do + let!(:previous_job) do + create(:batched_background_migration_job, + batched_migration: first_migration, + min_value: event1.id, + max_value: event2.id, + batch_size: 5, + sub_batch_size: 1) + end + + before do + first_migration.update!(batch_size: 5, min_value: 1, max_value: event2.id) + end + + it_behaves_like 'it has no jobs to run' + it_behaves_like 'it has completed the migration' + end + end + end + end +end |