diff options
Diffstat (limited to 'spec/lib/click_house/migration_support')
3 files changed, 404 insertions, 0 deletions
diff --git a/spec/lib/click_house/migration_support/exclusive_lock_spec.rb b/spec/lib/click_house/migration_support/exclusive_lock_spec.rb new file mode 100644 index 00000000000..5176cc75266 --- /dev/null +++ b/spec/lib/click_house/migration_support/exclusive_lock_spec.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ClickHouse::MigrationSupport::ExclusiveLock, feature_category: :database do + include ExclusiveLeaseHelpers + + let(:worker_class) do + # This worker will be active longer than the ClickHouse worker TTL + Class.new do + def self.name + 'TestWorker' + end + + include ::ApplicationWorker + include ::ClickHouseWorker + + def perform(*); end + end + end + + before do + stub_const('TestWorker', worker_class) + end + + describe '.register_running_worker' do + before do + TestWorker.click_house_migration_lock(10.seconds) + end + + it 'yields without arguments' do + expect { |b| described_class.register_running_worker(worker_class, 'test', &b) }.to yield_with_no_args + end + + it 'registers worker for a limited period of time', :freeze_time, :aggregate_failures do + expect(described_class.active_sidekiq_workers?).to eq false + + described_class.register_running_worker(worker_class, 'test') do + expect(described_class.active_sidekiq_workers?).to eq true + travel 9.seconds + expect(described_class.active_sidekiq_workers?).to eq true + travel 2.seconds + expect(described_class.active_sidekiq_workers?).to eq false + end + end + end + + describe '.pause_workers?' do + subject(:pause_workers?) { described_class.pause_workers? } + + it { is_expected.to eq false } + + context 'with lock taken' do + let!(:lease) { stub_exclusive_lease_taken(described_class::MIGRATION_LEASE_KEY) } + + it { is_expected.to eq true } + end + end + + describe '.execute_migration' do + it 'yields without raising error' do + expect { |b| described_class.execute_migration(&b) }.to yield_with_no_args + end + + context 'when migration lock is taken' do + let!(:lease) { stub_exclusive_lease_taken(described_class::MIGRATION_LEASE_KEY) } + + it 'raises LockError' do + expect do + expect { |b| described_class.execute_migration(&b) }.not_to yield_control + end.to raise_error ::ClickHouse::MigrationSupport::Errors::LockError + end + end + + context 'when ClickHouse workers are still active', :freeze_time do + let(:sleep_time) { described_class::WORKERS_WAIT_SLEEP } + let!(:started_at) { Time.current } + + def migration + expect { |b| described_class.execute_migration(&b) }.to yield_with_no_args + end + + around do |example| + described_class.register_running_worker(worker_class, anything) do + example.run + end + end + + it 'waits for workers and raises ClickHouse::MigrationSupport::LockError if workers do not stop in time' do + expect(described_class).to receive(:sleep).at_least(1).with(sleep_time) { travel(sleep_time) } + + expect { migration }.to raise_error(ClickHouse::MigrationSupport::Errors::LockError, + /Timed out waiting for active workers/) + expect(Time.current - started_at).to eq(described_class::DEFAULT_CLICKHOUSE_WORKER_TTL) + end + + context 'when wait_for_clickhouse_workers_during_migration FF is disabled' do + before do + stub_feature_flags(wait_for_clickhouse_workers_during_migration: false) + end + + it 'runs migration without waiting for workers' do + expect { migration }.not_to raise_error + expect(Time.current - started_at).to eq(0.0) + end + end + + it 'ignores expired workers' do + travel(described_class::DEFAULT_CLICKHOUSE_WORKER_TTL + 1.second) + + migration + end + + context 'when worker registration is almost expiring' do + let(:worker_class) do + # This worker will be active for less than the ClickHouse worker TTL + Class.new do + def self.name + 'TestWorker' + end + + include ::ApplicationWorker + include ::ClickHouseWorker + + click_house_migration_lock( + ClickHouse::MigrationSupport::ExclusiveLock::DEFAULT_CLICKHOUSE_WORKER_TTL - 1.second) + + def perform(*); end + end + end + + it 'completes migration' do + expect(described_class).to receive(:sleep).at_least(1).with(sleep_time) { travel(sleep_time) } + + expect { migration }.not_to raise_error + end + end + end + end +end diff --git a/spec/lib/click_house/migration_support/migration_context_spec.rb b/spec/lib/click_house/migration_support/migration_context_spec.rb new file mode 100644 index 00000000000..0f70e1e3f94 --- /dev/null +++ b/spec/lib/click_house/migration_support/migration_context_spec.rb @@ -0,0 +1,203 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ClickHouse::MigrationSupport::MigrationContext, + click_house: :without_migrations, feature_category: :database do + include ClickHouseTestHelpers + + # We don't need to delete data since we don't modify Postgres data + self.use_transactional_tests = false + + let(:connection) { ::ClickHouse::Connection.new(:main) } + let(:schema_migration) { ClickHouse::MigrationSupport::SchemaMigration.new(connection) } + + let(:migrations_base_dir) { 'click_house/migrations' } + let(:migrations_dir) { expand_fixture_path("#{migrations_base_dir}/#{migrations_dirname}") } + let(:migration_context) { described_class.new(connection, migrations_dir, schema_migration) } + let(:target_version) { nil } + + after do + unload_click_house_migration_classes(expand_fixture_path(migrations_base_dir)) + end + + describe 'performs migrations' do + include ExclusiveLeaseHelpers + + subject(:migration) { migrate(migration_context, target_version) } + + describe 'when creating a table' do + let(:migrations_dirname) { 'plain_table_creation' } + let(:lease_key) { 'click_house:migrations' } + let(:lease_timeout) { 1.hour } + + it 'executes migration through ClickHouse::MigrationSupport::ExclusiveLock.execute_migration' do + expect(ClickHouse::MigrationSupport::ExclusiveLock).to receive(:execute_migration) + + # Test that not running execute_migration will not execute migrations + expect { migration }.not_to change { active_schema_migrations_count } + end + + it 'creates a table' do + expect(ClickHouse::MigrationSupport::ExclusiveLock).to receive(:execute_migration).and_call_original + expect_to_obtain_exclusive_lease(lease_key, timeout: lease_timeout) + + expect { migration }.to change { active_schema_migrations_count }.from(0).to(1) + + table_schema = describe_table('some') + expect(schema_migrations).to contain_exactly(a_hash_including(version: '1', active: 1)) + expect(table_schema).to match({ + id: a_hash_including(type: 'UInt64'), + date: a_hash_including(type: 'Date') + }) + end + + context 'when a migration is already running' do + let(:migration_name) { 'create_some_table' } + + before do + stub_exclusive_lease_taken(lease_key) + end + + it 'raises error after timeout when migration is executing concurrently' do + expect { migration }.to raise_error(ClickHouse::MigrationSupport::Errors::LockError) + .and not_change { active_schema_migrations_count } + end + end + end + + describe 'when dropping a table' do + let(:migrations_dirname) { 'drop_table' } + let(:target_version) { 2 } + + it 'drops table' do + migrate(migration_context, 1) + expect(table_names).to include('some') + + migration + expect(table_names).not_to include('some') + end + end + + context 'when a migration raises an error' do + let(:migrations_dirname) { 'migration_with_error' } + + it 'passes the error to caller as a StandardError' do + expect { migration }.to raise_error StandardError, + "An error has occurred, all later migrations canceled:\n\nA migration error happened" + expect(schema_migrations).to be_empty + end + end + + context 'when connecting to not-existing database' do + let(:migrations_dirname) { 'plain_table_creation' } + let(:connection) { ::ClickHouse::Connection.new(:unknown_database) } + + it 'raises ConfigurationError' do + expect { migration }.to raise_error ClickHouse::Client::ConfigurationError, + "The database 'unknown_database' is not configured" + end + end + + context 'when target_version is incorrect' do + let(:target_version) { 2 } + let(:migrations_dirname) { 'plain_table_creation' } + + it 'raises UnknownMigrationVersionError' do + expect { migration }.to raise_error ClickHouse::MigrationSupport::Errors::UnknownMigrationVersionError + + expect(active_schema_migrations_count).to eq 0 + end + end + + context 'when migrations with duplicate name exist' do + let(:migrations_dirname) { 'duplicate_name' } + + it 'raises DuplicateMigrationNameError' do + expect { migration }.to raise_error ClickHouse::MigrationSupport::Errors::DuplicateMigrationNameError + + expect(active_schema_migrations_count).to eq 0 + end + end + + context 'when migrations with duplicate version exist' do + let(:migrations_dirname) { 'duplicate_version' } + + it 'raises DuplicateMigrationVersionError' do + expect { migration }.to raise_error ClickHouse::MigrationSupport::Errors::DuplicateMigrationVersionError + + expect(active_schema_migrations_count).to eq 0 + end + end + end + + describe 'performs rollbacks' do + subject(:migration) { rollback(migration_context, target_version) } + + before do + # Ensure that all migrations are up + migrate(migration_context, nil) + end + + context 'when down method is present' do + let(:migrations_dirname) { 'table_creation_with_down_method' } + + context 'when specifying target_version' do + it 'removes migrations and performs down method' do + expect(table_names).to include('some', 'another') + + # test that target_version is prioritized over step + expect { rollback(migration_context, 1, 10000) }.to change { active_schema_migrations_count }.from(2).to(1) + expect(table_names).not_to include('another') + expect(table_names).to include('some') + expect(schema_migrations).to contain_exactly( + a_hash_including(version: '1', active: 1), + a_hash_including(version: '2', active: 0) + ) + + expect { rollback(migration_context, nil) }.to change { active_schema_migrations_count }.to(0) + expect(table_names).not_to include('some', 'another') + + expect(schema_migrations).to contain_exactly( + a_hash_including(version: '1', active: 0), + a_hash_including(version: '2', active: 0) + ) + end + end + + context 'when specifying step' do + it 'removes migrations and performs down method' do + expect(table_names).to include('some', 'another') + + expect { rollback(migration_context, nil, 1) }.to change { active_schema_migrations_count }.from(2).to(1) + expect(table_names).not_to include('another') + expect(table_names).to include('some') + + expect { rollback(migration_context, nil, 2) }.to change { active_schema_migrations_count }.to(0) + expect(table_names).not_to include('some', 'another') + end + end + end + + context 'when down method is missing' do + let(:migrations_dirname) { 'plain_table_creation' } + let(:target_version) { 0 } + + it 'removes migration ignoring missing down method' do + expect { migration }.to change { active_schema_migrations_count }.from(1).to(0) + .and not_change { table_names & %w[some] }.from(%w[some]) + end + end + + context 'when target_version is incorrect' do + let(:target_version) { -1 } + let(:migrations_dirname) { 'plain_table_creation' } + + it 'raises UnknownMigrationVersionError' do + expect { migration }.to raise_error ClickHouse::MigrationSupport::Errors::UnknownMigrationVersionError + + expect(active_schema_migrations_count).to eq 1 + end + end + end +end diff --git a/spec/lib/click_house/migration_support/sidekiq_middleware_spec.rb b/spec/lib/click_house/migration_support/sidekiq_middleware_spec.rb new file mode 100644 index 00000000000..03c9edfabaa --- /dev/null +++ b/spec/lib/click_house/migration_support/sidekiq_middleware_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ClickHouse::MigrationSupport::SidekiqMiddleware, feature_category: :database do + let(:worker_with_click_house_worker) do + Class.new do + def self.name + 'TestWorker' + end + include ApplicationWorker + include ClickHouseWorker + end + end + + let(:worker_without_click_house_worker) do + Class.new do + def self.name + 'TestWorkerWithoutClickHouseWorker' + end + include ApplicationWorker + end + end + + subject(:middleware) { described_class.new } + + before do + stub_const('TestWorker', worker_with_click_house_worker) + stub_const('TestWorkerWithoutClickHouseWorker', worker_without_click_house_worker) + end + + describe '#call' do + let(:worker) { worker_class.new } + let(:job) { { 'jid' => 123, 'class' => worker_class.name } } + let(:queue) { 'test_queue' } + + context 'when worker does not include ClickHouseWorker' do + let(:worker_class) { worker_without_click_house_worker } + + it 'yields control without registering running worker' do + expect(ClickHouse::MigrationSupport::ExclusiveLock).not_to receive(:register_running_worker) + expect { |b| middleware.call(worker, job, queue, &b) }.to yield_with_no_args + end + end + + context 'when worker includes ClickHouseWorker' do + let(:worker_class) { worker_with_click_house_worker } + + it 'registers running worker and yields control' do + expect(ClickHouse::MigrationSupport::ExclusiveLock) + .to receive(:register_running_worker) + .with(worker_class, 'test_queue:123') + .and_wrap_original do |method, worker_class, worker_id| + expect { |b| method.call(worker_class, worker_id, &b) }.to yield_with_no_args + end + + middleware.call(worker, job, queue) + end + end + end +end |