Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2020-06-24 03:08:43 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2020-06-24 03:08:43 +0300
commit082b24b03bbb9dca7edf1341aa7b0a51c9aeb18b (patch)
tree5a0b89d06053474c7565de01d9a2500713bda235 /spec/lib/gitlab
parent3a06969fe871598ead8b99ce42a1152f03d4ef91 (diff)
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'spec/lib/gitlab')
-rw-r--r--spec/lib/gitlab/database/dynamic_model_helpers_spec.rb28
-rw-r--r--spec/lib/gitlab/database/migration_helpers_spec.rb236
-rw-r--r--spec/lib/gitlab/database/migrations/background_migration_helpers_spec.rb245
-rw-r--r--spec/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table_spec.rb164
-rw-r--r--spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb52
5 files changed, 480 insertions, 245 deletions
diff --git a/spec/lib/gitlab/database/dynamic_model_helpers_spec.rb b/spec/lib/gitlab/database/dynamic_model_helpers_spec.rb
new file mode 100644
index 00000000000..c0dc14a3f4f
--- /dev/null
+++ b/spec/lib/gitlab/database/dynamic_model_helpers_spec.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Database::DynamicModelHelpers do
+ describe '#define_batchable_model' do
+ subject { including_class.new.define_batchable_model(table_name) }
+
+ let(:including_class) { Class.new.include(described_class) }
+ let(:table_name) { 'projects' }
+
+ it 'is an ActiveRecord model' do
+ expect(subject.ancestors).to include(ActiveRecord::Base)
+ end
+
+ it 'includes EachBatch' do
+ expect(subject.included_modules).to include(EachBatch)
+ end
+
+ it 'has the correct table name' do
+ expect(subject.table_name).to eq(table_name)
+ end
+
+ it 'has the inheritance type column disable' do
+ expect(subject.inheritance_column).to eq('_type_disabled')
+ end
+ end
+end
diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb
index bed444ee7c7..1f725687bdc 100644
--- a/spec/lib/gitlab/database/migration_helpers_spec.rb
+++ b/spec/lib/gitlab/database/migration_helpers_spec.rb
@@ -1215,166 +1215,6 @@ describe Gitlab::Database::MigrationHelpers do
end
end
- describe '#bulk_queue_background_migration_jobs_by_range' do
- context 'when the model has an ID column' do
- let!(:id1) { create(:user).id }
- let!(:id2) { create(:user).id }
- let!(:id3) { create(:user).id }
-
- before do
- User.class_eval do
- include EachBatch
- end
- end
-
- context 'with enough rows to bulk queue jobs more than once' do
- before do
- stub_const('Gitlab::Database::MigrationHelpers::BACKGROUND_MIGRATION_JOB_BUFFER_SIZE', 1)
- end
-
- it 'queues jobs correctly' do
- Sidekiq::Testing.fake! do
- model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2)
-
- expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id2]])
- expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['FooJob', [id3, id3]])
- end
- end
-
- it 'queues jobs in groups of buffer size 1' do
- expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id1, id2]]])
- expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id3, id3]]])
-
- model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2)
- end
- end
-
- context 'with not enough rows to bulk queue jobs more than once' do
- it 'queues jobs correctly' do
- Sidekiq::Testing.fake! do
- model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2)
-
- expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id2]])
- expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['FooJob', [id3, id3]])
- end
- end
-
- it 'queues jobs in bulk all at once (big buffer size)' do
- expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id1, id2]],
- ['FooJob', [id3, id3]]])
-
- model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2)
- end
- end
-
- context 'without specifying batch_size' do
- it 'queues jobs correctly' do
- Sidekiq::Testing.fake! do
- model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob')
-
- expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3]])
- end
- end
- end
- end
-
- context "when the model doesn't have an ID column" do
- it 'raises error (for now)' do
- expect do
- model.bulk_queue_background_migration_jobs_by_range(ProjectAuthorization, 'FooJob')
- end.to raise_error(StandardError, /does not have an ID/)
- end
- end
- end
-
- describe '#queue_background_migration_jobs_by_range_at_intervals' do
- context 'when the model has an ID column' do
- let!(:id1) { create(:user).id }
- let!(:id2) { create(:user).id }
- let!(:id3) { create(:user).id }
-
- around do |example|
- Timecop.freeze { example.run }
- end
-
- before do
- User.class_eval do
- include EachBatch
- end
- end
-
- it 'returns the final expected delay' do
- Sidekiq::Testing.fake! do
- final_delay = model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, batch_size: 2)
-
- expect(final_delay.to_f).to eq(20.minutes.to_f)
- end
- end
-
- it 'returns zero when nothing gets queued' do
- Sidekiq::Testing.fake! do
- final_delay = model.queue_background_migration_jobs_by_range_at_intervals(User.none, 'FooJob', 10.minutes)
-
- expect(final_delay).to eq(0)
- end
- end
-
- context 'with batch_size option' do
- it 'queues jobs correctly' do
- Sidekiq::Testing.fake! do
- model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, batch_size: 2)
-
- expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id2]])
- expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(10.minutes.from_now.to_f)
- expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['FooJob', [id3, id3]])
- expect(BackgroundMigrationWorker.jobs[1]['at']).to eq(20.minutes.from_now.to_f)
- end
- end
- end
-
- context 'without batch_size option' do
- it 'queues jobs correctly' do
- Sidekiq::Testing.fake! do
- model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes)
-
- expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3]])
- expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(10.minutes.from_now.to_f)
- end
- end
- end
-
- context 'with other_job_arguments option' do
- it 'queues jobs correctly' do
- Sidekiq::Testing.fake! do
- model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, other_job_arguments: [1, 2])
-
- expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3, 1, 2]])
- expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(10.minutes.from_now.to_f)
- end
- end
- end
-
- context 'with initial_delay option' do
- it 'queues jobs correctly' do
- Sidekiq::Testing.fake! do
- model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, other_job_arguments: [1, 2], initial_delay: 10.minutes)
-
- expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3, 1, 2]])
- expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(20.minutes.from_now.to_f)
- end
- end
- end
- end
-
- context "when the model doesn't have an ID column" do
- it 'raises error (for now)' do
- expect do
- model.queue_background_migration_jobs_by_range_at_intervals(ProjectAuthorization, 'FooJob', 10.seconds)
- end.to raise_error(StandardError, /does not have an ID/)
- end
- end
- end
-
describe '#change_column_type_using_background_migration' do
let!(:issue) { create(:issue, :closed, closed_at: Time.zone.now) }
@@ -1485,26 +1325,6 @@ describe Gitlab::Database::MigrationHelpers do
end
end
- describe '#perform_background_migration_inline?' do
- it 'returns true in a test environment' do
- stub_rails_env('test')
-
- expect(model.perform_background_migration_inline?).to eq(true)
- end
-
- it 'returns true in a development environment' do
- stub_rails_env('development')
-
- expect(model.perform_background_migration_inline?).to eq(true)
- end
-
- it 'returns false in a production environment' do
- stub_rails_env('production')
-
- expect(model.perform_background_migration_inline?).to eq(false)
- end
- end
-
describe '#index_exists_by_name?' do
it 'returns true if an index exists' do
ActiveRecord::Base.connection.execute(
@@ -1973,62 +1793,6 @@ describe Gitlab::Database::MigrationHelpers do
end
end
- describe '#migrate_async' do
- it 'calls BackgroundMigrationWorker.perform_async' do
- expect(BackgroundMigrationWorker).to receive(:perform_async).with("Class", "hello", "world")
-
- model.migrate_async("Class", "hello", "world")
- end
-
- it 'pushes a context with the current class name as caller_id' do
- expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s)
-
- model.migrate_async('Class', 'hello', 'world')
- end
- end
-
- describe '#migrate_in' do
- it 'calls BackgroundMigrationWorker.perform_in' do
- expect(BackgroundMigrationWorker).to receive(:perform_in).with(10.minutes, 'Class', 'Hello', 'World')
-
- model.migrate_in(10.minutes, 'Class', 'Hello', 'World')
- end
-
- it 'pushes a context with the current class name as caller_id' do
- expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s)
-
- model.migrate_in(10.minutes, 'Class', 'Hello', 'World')
- end
- end
-
- describe '#bulk_migrate_async' do
- it 'calls BackgroundMigrationWorker.bulk_perform_async' do
- expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([%w(Class hello world)])
-
- model.bulk_migrate_async([%w(Class hello world)])
- end
-
- it 'pushes a context with the current class name as caller_id' do
- expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s)
-
- model.bulk_migrate_async([%w(Class hello world)])
- end
- end
-
- describe '#bulk_migrate_in' do
- it 'calls BackgroundMigrationWorker.bulk_perform_in_' do
- expect(BackgroundMigrationWorker).to receive(:bulk_perform_in).with(10.minutes, [%w(Class hello world)])
-
- model.bulk_migrate_in(10.minutes, [%w(Class hello world)])
- end
-
- it 'pushes a context with the current class name as caller_id' do
- expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s)
-
- model.bulk_migrate_in(10.minutes, [%w(Class hello world)])
- end
- end
-
describe '#check_constraint_name' do
it 'returns a valid constraint name' do
name = model.check_constraint_name(:this_is_a_very_long_table_name,
diff --git a/spec/lib/gitlab/database/migrations/background_migration_helpers_spec.rb b/spec/lib/gitlab/database/migrations/background_migration_helpers_spec.rb
new file mode 100644
index 00000000000..1b1ac0cb387
--- /dev/null
+++ b/spec/lib/gitlab/database/migrations/background_migration_helpers_spec.rb
@@ -0,0 +1,245 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Database::Migrations::BackgroundMigrationHelpers do
+ let(:model) do
+ ActiveRecord::Migration.new.extend(described_class)
+ end
+
+ describe '#bulk_queue_background_migration_jobs_by_range' do
+ context 'when the model has an ID column' do
+ let!(:id1) { create(:user).id }
+ let!(:id2) { create(:user).id }
+ let!(:id3) { create(:user).id }
+
+ before do
+ User.class_eval do
+ include EachBatch
+ end
+ end
+
+ context 'with enough rows to bulk queue jobs more than once' do
+ before do
+ stub_const('Gitlab::Database::Migrations::BackgroundMigrationHelpers::BACKGROUND_MIGRATION_JOB_BUFFER_SIZE', 1)
+ end
+
+ it 'queues jobs correctly' do
+ Sidekiq::Testing.fake! do
+ model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2)
+
+ expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id2]])
+ expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['FooJob', [id3, id3]])
+ end
+ end
+
+ it 'queues jobs in groups of buffer size 1' do
+ expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id1, id2]]])
+ expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id3, id3]]])
+
+ model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2)
+ end
+ end
+
+ context 'with not enough rows to bulk queue jobs more than once' do
+ it 'queues jobs correctly' do
+ Sidekiq::Testing.fake! do
+ model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2)
+
+ expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id2]])
+ expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['FooJob', [id3, id3]])
+ end
+ end
+
+ it 'queues jobs in bulk all at once (big buffer size)' do
+ expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([['FooJob', [id1, id2]],
+ ['FooJob', [id3, id3]]])
+
+ model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob', batch_size: 2)
+ end
+ end
+
+ context 'without specifying batch_size' do
+ it 'queues jobs correctly' do
+ Sidekiq::Testing.fake! do
+ model.bulk_queue_background_migration_jobs_by_range(User, 'FooJob')
+
+ expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3]])
+ end
+ end
+ end
+ end
+
+ context "when the model doesn't have an ID column" do
+ it 'raises error (for now)' do
+ expect do
+ model.bulk_queue_background_migration_jobs_by_range(ProjectAuthorization, 'FooJob')
+ end.to raise_error(StandardError, /does not have an ID/)
+ end
+ end
+ end
+
+ describe '#queue_background_migration_jobs_by_range_at_intervals' do
+ context 'when the model has an ID column' do
+ let!(:id1) { create(:user).id }
+ let!(:id2) { create(:user).id }
+ let!(:id3) { create(:user).id }
+
+ around do |example|
+ Timecop.freeze { example.run }
+ end
+
+ before do
+ User.class_eval do
+ include EachBatch
+ end
+ end
+
+ it 'returns the final expected delay' do
+ Sidekiq::Testing.fake! do
+ final_delay = model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, batch_size: 2)
+
+ expect(final_delay.to_f).to eq(20.minutes.to_f)
+ end
+ end
+
+ it 'returns zero when nothing gets queued' do
+ Sidekiq::Testing.fake! do
+ final_delay = model.queue_background_migration_jobs_by_range_at_intervals(User.none, 'FooJob', 10.minutes)
+
+ expect(final_delay).to eq(0)
+ end
+ end
+
+ context 'with batch_size option' do
+ it 'queues jobs correctly' do
+ Sidekiq::Testing.fake! do
+ model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, batch_size: 2)
+
+ expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id2]])
+ expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(10.minutes.from_now.to_f)
+ expect(BackgroundMigrationWorker.jobs[1]['args']).to eq(['FooJob', [id3, id3]])
+ expect(BackgroundMigrationWorker.jobs[1]['at']).to eq(20.minutes.from_now.to_f)
+ end
+ end
+ end
+
+ context 'without batch_size option' do
+ it 'queues jobs correctly' do
+ Sidekiq::Testing.fake! do
+ model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes)
+
+ expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3]])
+ expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(10.minutes.from_now.to_f)
+ end
+ end
+ end
+
+ context 'with other_job_arguments option' do
+ it 'queues jobs correctly' do
+ Sidekiq::Testing.fake! do
+ model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, other_job_arguments: [1, 2])
+
+ expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3, 1, 2]])
+ expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(10.minutes.from_now.to_f)
+ end
+ end
+ end
+
+ context 'with initial_delay option' do
+ it 'queues jobs correctly' do
+ Sidekiq::Testing.fake! do
+ model.queue_background_migration_jobs_by_range_at_intervals(User, 'FooJob', 10.minutes, other_job_arguments: [1, 2], initial_delay: 10.minutes)
+
+ expect(BackgroundMigrationWorker.jobs[0]['args']).to eq(['FooJob', [id1, id3, 1, 2]])
+ expect(BackgroundMigrationWorker.jobs[0]['at']).to eq(20.minutes.from_now.to_f)
+ end
+ end
+ end
+ end
+
+ context "when the model doesn't have an ID column" do
+ it 'raises error (for now)' do
+ expect do
+ model.queue_background_migration_jobs_by_range_at_intervals(ProjectAuthorization, 'FooJob', 10.seconds)
+ end.to raise_error(StandardError, /does not have an ID/)
+ end
+ end
+ end
+
+ describe '#perform_background_migration_inline?' do
+ it 'returns true in a test environment' do
+ stub_rails_env('test')
+
+ expect(model.perform_background_migration_inline?).to eq(true)
+ end
+
+ it 'returns true in a development environment' do
+ stub_rails_env('development')
+
+ expect(model.perform_background_migration_inline?).to eq(true)
+ end
+
+ it 'returns false in a production environment' do
+ stub_rails_env('production')
+
+ expect(model.perform_background_migration_inline?).to eq(false)
+ end
+ end
+
+ describe '#migrate_async' do
+ it 'calls BackgroundMigrationWorker.perform_async' do
+ expect(BackgroundMigrationWorker).to receive(:perform_async).with("Class", "hello", "world")
+
+ model.migrate_async("Class", "hello", "world")
+ end
+
+ it 'pushes a context with the current class name as caller_id' do
+ expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s)
+
+ model.migrate_async('Class', 'hello', 'world')
+ end
+ end
+
+ describe '#migrate_in' do
+ it 'calls BackgroundMigrationWorker.perform_in' do
+ expect(BackgroundMigrationWorker).to receive(:perform_in).with(10.minutes, 'Class', 'Hello', 'World')
+
+ model.migrate_in(10.minutes, 'Class', 'Hello', 'World')
+ end
+
+ it 'pushes a context with the current class name as caller_id' do
+ expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s)
+
+ model.migrate_in(10.minutes, 'Class', 'Hello', 'World')
+ end
+ end
+
+ describe '#bulk_migrate_async' do
+ it 'calls BackgroundMigrationWorker.bulk_perform_async' do
+ expect(BackgroundMigrationWorker).to receive(:bulk_perform_async).with([%w(Class hello world)])
+
+ model.bulk_migrate_async([%w(Class hello world)])
+ end
+
+ it 'pushes a context with the current class name as caller_id' do
+ expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s)
+
+ model.bulk_migrate_async([%w(Class hello world)])
+ end
+ end
+
+ describe '#bulk_migrate_in' do
+ it 'calls BackgroundMigrationWorker.bulk_perform_in_' do
+ expect(BackgroundMigrationWorker).to receive(:bulk_perform_in).with(10.minutes, [%w(Class hello world)])
+
+ model.bulk_migrate_in(10.minutes, [%w(Class hello world)])
+ end
+
+ it 'pushes a context with the current class name as caller_id' do
+ expect(Gitlab::ApplicationContext).to receive(:with_context).with(caller_id: model.class.to_s)
+
+ model.bulk_migrate_in(10.minutes, [%w(Class hello world)])
+ end
+ end
+end
diff --git a/spec/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table_spec.rb b/spec/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table_spec.rb
new file mode 100644
index 00000000000..2016c31b891
--- /dev/null
+++ b/spec/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table_spec.rb
@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Database::PartitioningMigrationHelpers::BackfillPartitionedTable, '#perform' do
+ subject { described_class.new }
+
+ let(:source_table) { '_test_partitioning_backfills' }
+ let(:destination_table) { "#{source_table}_part" }
+ let(:unique_key) { 'id' }
+
+ before do
+ allow(subject).to receive(:transaction_open?).and_return(false)
+ end
+
+ context 'when the destination table exists' do
+ before do
+ connection.execute(<<~SQL)
+ CREATE TABLE #{source_table} (
+ id serial NOT NULL PRIMARY KEY,
+ col1 int NOT NULL,
+ col2 text NOT NULL,
+ created_at timestamptz NOT NULL
+ )
+ SQL
+
+ connection.execute(<<~SQL)
+ CREATE TABLE #{destination_table} (
+ id serial NOT NULL,
+ col1 int NOT NULL,
+ col2 text NOT NULL,
+ created_at timestamptz NOT NULL,
+ PRIMARY KEY (id, created_at)
+ ) PARTITION BY RANGE (created_at)
+ SQL
+
+ connection.execute(<<~SQL)
+ CREATE TABLE #{destination_table}_202001 PARTITION OF #{destination_table}
+ FOR VALUES FROM ('2020-01-01') TO ('2020-02-01')
+ SQL
+
+ connection.execute(<<~SQL)
+ CREATE TABLE #{destination_table}_202002 PARTITION OF #{destination_table}
+ FOR VALUES FROM ('2020-02-01') TO ('2020-03-01')
+ SQL
+
+ source_model.table_name = source_table
+ destination_model.table_name = destination_table
+
+ stub_const("#{described_class}::SUB_BATCH_SIZE", 2)
+ stub_const("#{described_class}::PAUSE_SECONDS", pause_seconds)
+
+ allow(subject).to receive(:sleep)
+ end
+
+ let(:connection) { ActiveRecord::Base.connection }
+ let(:source_model) { Class.new(ActiveRecord::Base) }
+ let(:destination_model) { Class.new(ActiveRecord::Base) }
+ let(:timestamp) { Time.utc(2020, 1, 2).round }
+ let(:pause_seconds) { 1 }
+
+ let!(:source1) { create_source_record(timestamp) }
+ let!(:source2) { create_source_record(timestamp + 1.day) }
+ let!(:source3) { create_source_record(timestamp + 1.month) }
+
+ it 'copies data into the destination table idempotently' do
+ expect(destination_model.count).to eq(0)
+
+ subject.perform(source1.id, source3.id, source_table, destination_table, unique_key)
+
+ expect(destination_model.count).to eq(3)
+
+ source_model.find_each do |source_record|
+ destination_record = destination_model.find_by_id(source_record.id)
+
+ expect(destination_record.attributes).to eq(source_record.attributes)
+ end
+
+ subject.perform(source1.id, source3.id, source_table, destination_table, unique_key)
+
+ expect(destination_model.count).to eq(3)
+ end
+
+ it 'breaks the assigned batch into smaller batches' do
+ expect_next_instance_of(described_class::BulkCopy) do |bulk_copy|
+ expect(bulk_copy).to receive(:copy_between).with(source1.id, source2.id)
+ expect(bulk_copy).to receive(:copy_between).with(source3.id, source3.id)
+ end
+
+ subject.perform(source1.id, source3.id, source_table, destination_table, unique_key)
+ end
+
+ it 'pauses after copying each sub-batch' do
+ expect(subject).to receive(:sleep).with(pause_seconds).twice
+
+ subject.perform(source1.id, source3.id, source_table, destination_table, unique_key)
+ end
+
+ context 'when the feature flag is disabled' do
+ let(:mock_connection) { double('connection') }
+
+ before do
+ allow(subject).to receive(:connection).and_return(mock_connection)
+ stub_feature_flags(backfill_partitioned_audit_events: false)
+ end
+
+ it 'exits without attempting to copy data' do
+ expect(mock_connection).not_to receive(:execute)
+
+ subject.perform(1, 100, source_table, destination_table, unique_key)
+
+ expect(destination_model.count).to eq(0)
+ end
+ end
+
+ context 'when the job is run within an explicit transaction block' do
+ let(:mock_connection) { double('connection') }
+
+ before do
+ allow(subject).to receive(:connection).and_return(mock_connection)
+ allow(subject).to receive(:transaction_open?).and_return(true)
+ end
+
+ it 'raises an error before copying data' do
+ expect(mock_connection).not_to receive(:execute)
+
+ expect do
+ subject.perform(1, 100, source_table, destination_table, unique_key)
+ end.to raise_error(/Aborting job to backfill partitioned #{source_table}/)
+
+ expect(destination_model.count).to eq(0)
+ end
+ end
+ end
+
+ context 'when the destination table does not exist' do
+ let(:mock_connection) { double('connection') }
+ let(:mock_logger) { double('logger') }
+
+ before do
+ allow(subject).to receive(:connection).and_return(mock_connection)
+ allow(subject).to receive(:logger).and_return(mock_logger)
+
+ expect(mock_connection).to receive(:table_exists?).with(destination_table).and_return(false)
+ allow(mock_logger).to receive(:warn)
+ end
+
+ it 'exits without attempting to copy data' do
+ expect(mock_connection).not_to receive(:execute)
+
+ subject.perform(1, 100, source_table, destination_table, unique_key)
+ end
+
+ it 'logs a warning message that the job was skipped' do
+ expect(mock_logger).to receive(:warn).with(/#{destination_table} does not exist/)
+
+ subject.perform(1, 100, source_table, destination_table, unique_key)
+ end
+ end
+
+ def create_source_record(timestamp)
+ source_model.create!(col1: 123, col2: 'original value', created_at: timestamp)
+ end
+end
diff --git a/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb b/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb
index 586b57d2002..c78c143f2f8 100644
--- a/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb
+++ b/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb
@@ -25,7 +25,7 @@ describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers
allow(migration).to receive(:partitioned_table_name).and_return(partitioned_table)
allow(migration).to receive(:sync_function_name).and_return(function_name)
allow(migration).to receive(:sync_trigger_name).and_return(trigger_name)
- allow(migration).to receive(:assert_table_is_whitelisted)
+ allow(migration).to receive(:assert_table_is_allowed)
end
describe '#partition_table_by_date' do
@@ -33,15 +33,19 @@ describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers
let(:old_primary_key) { 'id' }
let(:new_primary_key) { [old_primary_key, partition_column] }
- context 'when the table is not whitelisted' do
- let(:template_table) { :this_table_is_not_whitelisted }
+ before do
+ allow(migration).to receive(:queue_background_migration_jobs_by_range_at_intervals)
+ end
+
+ context 'when the table is not allowed' do
+ let(:template_table) { :this_table_is_not_allowed }
it 'raises an error' do
- expect(migration).to receive(:assert_table_is_whitelisted).with(template_table).and_call_original
+ expect(migration).to receive(:assert_table_is_allowed).with(template_table).and_call_original
expect do
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date
- end.to raise_error(/#{template_table} is not whitelisted for use/)
+ end.to raise_error(/#{template_table} is not allowed for use/)
end
end
@@ -237,6 +241,36 @@ describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers
expect(model.find(second_todo.id).attributes).to eq(second_todo.attributes)
end
end
+
+ describe 'copying historic data to the partitioned table' do
+ let(:template_table) { 'todos' }
+ let(:migration_class) { '::Gitlab::Database::PartitioningMigrationHelpers::BackfillPartitionedTable' }
+ let(:sub_batch_size) { described_class::SUB_BATCH_SIZE }
+ let(:pause_seconds) { described_class::PAUSE_SECONDS }
+ let!(:first_id) { create(:todo).id }
+ let!(:second_id) { create(:todo).id }
+ let!(:third_id) { create(:todo).id }
+
+ before do
+ stub_const("#{described_class.name}::BATCH_SIZE", 2)
+
+ expect(migration).to receive(:queue_background_migration_jobs_by_range_at_intervals).and_call_original
+ end
+
+ it 'enqueues jobs to copy each batch of data' do
+ Sidekiq::Testing.fake! do
+ migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date
+
+ expect(BackgroundMigrationWorker.jobs.size).to eq(2)
+
+ first_job_arguments = [first_id, second_id, template_table, partitioned_table, 'id']
+ expect(BackgroundMigrationWorker.jobs[0]['args']).to eq([migration_class, first_job_arguments])
+
+ second_job_arguments = [third_id, third_id, template_table, partitioned_table, 'id']
+ expect(BackgroundMigrationWorker.jobs[1]['args']).to eq([migration_class, second_job_arguments])
+ end
+ end
+ end
end
describe '#drop_partitioned_table_for' do
@@ -244,15 +278,15 @@ describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers
%w[000000 201912 202001 202002].map { |suffix| "#{partitioned_table}_#{suffix}" }.unshift(partitioned_table)
end
- context 'when the table is not whitelisted' do
- let(:template_table) { :this_table_is_not_whitelisted }
+ context 'when the table is not allowed' do
+ let(:template_table) { :this_table_is_not_allowed }
it 'raises an error' do
- expect(migration).to receive(:assert_table_is_whitelisted).with(template_table).and_call_original
+ expect(migration).to receive(:assert_table_is_allowed).with(template_table).and_call_original
expect do
migration.drop_partitioned_table_for template_table
- end.to raise_error(/#{template_table} is not whitelisted for use/)
+ end.to raise_error(/#{template_table} is not allowed for use/)
end
end