# frozen_string_literal: true require 'spec_helper' RSpec.describe Gitlab::Database::Migrations::BackgroundMigrationHelpers do let(:model) do ActiveRecord::Migration.new.extend(described_class) end describe '#bulk_queue_background_migration_jobs_by_range' do context 'when the model has an ID column' do let!(:id1) { create(:user).id } let!(:id2) { create(:user).id } let!(:id3) { create(:user).id } before do User.class_eval do include EachBatch end end context 'with enough rows to bulk queue jobs more than once' do before do stub_const('Gitlab::Database::Migrations::BackgroundMigrationHelpers::JOB_BUFFER_SIZE', 1) end it 'queues jobs correctly' do Sidekiq::Testing.fake! do model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2) expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id2]]) expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['FooJob', [id3, id3]]) end end it 'queues jobs in groups of buffer size 1' do expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id1, id2]]]) expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id3, id3]]]) model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2) end end context 'with not enough rows to bulk queue jobs more than once' do it 'queues jobs correctly' do Sidekiq::Testing.fake! do model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2) expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id2]]) expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['FooJob', [id3, id3]]) end end it 'queues jobs in bulk all at once (big buffer size)' do expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id1, id2]], ['FooJob', [id3, id3]]]) model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2) end end context 'without specifying batch_size' do it 'queues jobs correctly' do Sidekiq::Testing.fake! do model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob') expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3]]) end end end end context "when the model doesn't have an ID column" do it 'raises error (for now)' do expect do model.bulk_queue_background_migration_jobs_by_range(ProjectAuthorization, 'FooJob') end.to raise_error(StandardError, /does not have an ID/) end end end describe '#queue_background_migration_jobs_by_range_at_intervals' do context 'when the model has an ID column' do let!(:id1) { create(:user).id } let!(:id2) { create(:user).id } let!(:id3) { create(:user).id } around do |example| freeze_time { example.run } end before do User.class_eval do include EachBatch end end it 'returns the final expected delay' do Sidekiq::Testing.fake! do final_delay = model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, batch_size: 2) expect(final_delay.to_f).to eq(20.minutes.to_f) end end it 'returns zero when nothing gets queued' do Sidekiq::Testing.fake! do final_delay = model.queue_background_migration_jobs_by_range_at_intervals(User.none, 'FooJob', 10.minutes) expect(final_delay).to eq(0) end end context 'with batch_size option' do it 'queues jobs correctly' do Sidekiq::Testing.fake! do model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, batch_size: 2) expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id2]]) expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(10.minutes.from_now.to_f) expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['FooJob', [id3, id3]]) expect(BackgroundMigrationWorker.jobs[1]['at']).to eq(20.minutes.from_now.to_f) end end end context 'without batch_size option' do it 'queues jobs correctly' do Sidekiq::Testing.fake! do model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes) expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3]]) expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(10.minutes.from_now.to_f) end end end context 'with other_job_arguments option' do it 'queues jobs correctly' do Sidekiq::Testing.fake! do model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, other_job_arguments: [1, 2]) expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3, 1, 2]]) expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(10.minutes.from_now.to_f) end end end context 'with initial_delay option' do it 'queues jobs correctly' do Sidekiq::Testing.fake! do model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, other_job_arguments: [1, 2], initial_delay: 10.minutes) expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3, 1, 2]]) expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(20.minutes.from_now.to_f) end end end context 'with track_jobs option' do it 'creates a record for each job in the database' do Sidekiq::Testing.fake! do expect do model.queue_background_migration_jobs_by_range_at_intervals(User, '::FooJob', 10.minutes, other_job_arguments: [1, 2], track_jobs: true) end.to change { Gitlab::Database::BackgroundMigrationJob.count }.from(0).to(1) expect(BackgroundMigrationWorker.jobs.size).to eq(1) tracked_job = Gitlab::Database::BackgroundMigrationJob.first expect(tracked_job.class_name).to eq('FooJob') expect(tracked_job.arguments).to eq([id1, id3, 1, 2]) expect(tracked_job).to be_pending end end end context 'without track_jobs option' do it 'does not create records in the database' do Sidekiq::Testing.fake! do expect do model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, other_job_arguments: [1, 2]) end.not_to change { Gitlab::Database::BackgroundMigrationJob.count } expect(BackgroundMigrationWorker.jobs.size).to eq(1) end end end end context 'when the model specifies a primary_column_name' do let!(:id1) { create(:container_expiration_policy).id } let!(:id2) { create(:container_expiration_policy).id } let!(:id3) { create(:container_expiration_policy).id } around do |example| freeze_time { example.run } end before do ContainerExpirationPolicy.class_eval do include EachBatch end end it 'returns the final expected delay', :aggregate_failures do Sidekiq::Testing.fake! do final_delay = model.queue_background_migration_jobs_by_range_at_intervals(ContainerExpirationPolicy, 'FooJob', 10.minutes, batch_size: 2, primary_column_name: :project_id) expect(final_delay.to_f).to eq(20.minutes.to_f) expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id2]]) expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(10.minutes.from_now.to_f) expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['FooJob', [id3, id3]]) expect(BackgroundMigrationWorker.jobs[1]['at']).to eq(20.minutes.from_now.to_f) end end context "when the primary_column_name is not an integer" do it 'raises error' do expect do model.queue_background_migration_jobs_by_range_at_intervals(ContainerExpirationPolicy, 'FooJob', 10.minutes, primary_column_name: :enabled) end.to raise_error(StandardError, /is not an integer column/) end end context "when the primary_column_name does not exist" do it 'raises error' do expect do model.queue_background_migration_jobs_by_range_at_intervals(ContainerExpirationPolicy, 'FooJob', 10.minutes, primary_column_name: :foo) end.to raise_error(StandardError, /does not have an ID column of foo/) end end end context "when the model doesn't have an ID or primary_column_name column" do it 'raises error (for now)' do expect do model.queue_background_migration_jobs_by_range_at_intervals(ProjectAuthorization, 'FooJob', 10.seconds) end.to raise_error(StandardError, /does not have an ID/) end end end describe '#perform_background_migration_inline?' do it 'returns true in a test environment' do stub_rails_env('test') expect(model.perform_background_migration_inline?).to eq(true) end it 'returns true in a development environment' do stub_rails_env('development') expect(model.perform_background_migration_inline?).to eq(true) end it 'returns false in a production environment' do stub_rails_env('production') expect(model.perform_background_migration_inline?).to eq(false) end end describe '#queue_batched_background_migration' do let(:pgclass_info) { instance_double('Gitlab::Database::PgClass', cardinality_estimate: 42) } before do allow(Gitlab::Database::PgClass).to receive(:for_table).and_call_original end it 'creates the database record for the migration' do expect(Gitlab::Database::PgClass).to receive(:for_table).with(:projects).and_return(pgclass_info) expect do model.queue_batched_background_migration( 'MyJobClass', :projects, :id, job_interval: 5.minutes, batch_min_value: 5, batch_max_value: 1000, batch_class_name: 'MyBatchClass', batch_size: 100, sub_batch_size: 10) end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) expect(Gitlab::Database::BackgroundMigration::BatchedMigration.last).to have_attributes( job_class_name: 'MyJobClass', table_name: 'projects', column_name: 'id', interval: 300, min_value: 5, max_value: 1000, batch_class_name: 'MyBatchClass', batch_size: 100, sub_batch_size: 10, job_arguments: %w[], status: 'active', total_tuple_count: pgclass_info.cardinality_estimate) end context 'when the job interval is lower than the minimum' do let(:minimum_delay) { described_class::BATCH_MIN_DELAY } it 'sets the job interval to the minimum value' do expect do model.queue_batched_background_migration('MyJobClass', :events, :id, job_interval: minimum_delay - 1.minute) end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) created_migration = Gitlab::Database::BackgroundMigration::BatchedMigration.last expect(created_migration.interval).to eq(minimum_delay) end end context 'when additional arguments are passed to the method' do it 'saves the arguments on the database record' do expect do model.queue_batched_background_migration( 'MyJobClass', :projects, :id, 'my', 'arguments', job_interval: 5.minutes, batch_max_value: 1000) end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) expect(Gitlab::Database::BackgroundMigration::BatchedMigration.last).to have_attributes( job_class_name: 'MyJobClass', table_name: 'projects', column_name: 'id', interval: 300, min_value: 1, max_value: 1000, job_arguments: %w[my arguments]) end end context 'when the max_value is not given' do context 'when records exist in the database' do let!(:event1) { create(:event) } let!(:event2) { create(:event) } let!(:event3) { create(:event) } it 'creates the record with the current max value' do expect do model.queue_batched_background_migration('MyJobClass', :events, :id, job_interval: 5.minutes) end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) created_migration = Gitlab::Database::BackgroundMigration::BatchedMigration.last expect(created_migration.max_value).to eq(event3.id) end it 'creates the record with an active status' do expect do model.queue_batched_background_migration('MyJobClass', :events, :id, job_interval: 5.minutes) end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) expect(Gitlab::Database::BackgroundMigration::BatchedMigration.last).to be_active end end context 'when the database is empty' do it 'sets the max value to the min value' do expect do model.queue_batched_background_migration('MyJobClass', :events, :id, job_interval: 5.minutes) end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) created_migration = Gitlab::Database::BackgroundMigration::BatchedMigration.last expect(created_migration.max_value).to eq(created_migration.min_value) end it 'creates the record with a finished status' do expect do model.queue_batched_background_migration('MyJobClass', :projects, :id, job_interval: 5.minutes) end.to change { Gitlab::Database::BackgroundMigration::BatchedMigration.count }.by(1) expect(Gitlab::Database::BackgroundMigration::BatchedMigration.last).to be_finished end end end end describe '#migrate_async' do it 'calls BackgroundMigrationWorker.perform_async' do expect(BackgroundMigrationWorker).to receive(:perform_async).with("Class", "hello", "world") model.migrate_async("Class", "hello", "world") end it 'pushes a context with the current class name as caller_id' do expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s) model.migrate_async('Class', 'hello', 'world') end end describe '#migrate_in' do it 'calls BackgroundMigrationWorker.perform_in' do expect(BackgroundMigrationWorker).to receive(:perform_in).with(10.minutes, 'Class', 'Hello', 'World') model.migrate_in(10.minutes, 'Class', 'Hello', 'World') end it 'pushes a context with the current class name as caller_id' do expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s) model.migrate_in(10.minutes, 'Class', 'Hello', 'World') end end describe '#bulk_migrate_async' do it 'calls BackgroundMigrationWorker.bulk_perform_async' do expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([%w(Class hello world)]) model.bulk_migrate_async([%w(Class hello world)]) end it 'pushes a context with the current class name as caller_id' do expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s) model.bulk_migrate_async([%w(Class hello world)]) end end describe '#bulk_migrate_in' do it 'calls BackgroundMigrationWorker.bulk_perform_in_' do expect(BackgroundMigrationWorker).to receive(:bulk_perform_in).with(10.minutes, [%w(Class hello world)]) model.bulk_migrate_in(10.minutes, [%w(Class hello world)]) end it 'pushes a context with the current class name as caller_id' do expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s) model.bulk_migrate_in(10.minutes, [%w(Class hello world)]) end end describe '#delete_queued_jobs' do let(:job1) { double } let(:job2) { double } it 'deletes all queued jobs for the given background migration' do expect(Gitlab::BackgroundMigration).to receive(:steal).with('BackgroundMigrationClassName') do |&block| expect(block.call(job1)).to be(false) expect(block.call(job2)).to be(false) end expect(job1).to receive(:delete) expect(job2).to receive(:delete) model.delete_queued_jobs('BackgroundMigrationClassName') end end end