diff options
Diffstat (limited to 'spec')
-rw-r--r-- | spec/lib/gitlab/hashed_storage/migrator_spec.rb | 114 | ||||
-rw-r--r-- | spec/models/project_spec.rb | 52 | ||||
-rw-r--r-- | spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb | 6 | ||||
-rw-r--r-- | spec/services/projects/hashed_storage/migrate_repository_service_spec.rb | 12 | ||||
-rw-r--r-- | spec/services/projects/hashed_storage/rollback_attachments_service_spec.rb | 100 | ||||
-rw-r--r-- | spec/services/projects/hashed_storage/rollback_repository_service_spec.rb | 111 | ||||
-rw-r--r-- | spec/services/projects/hashed_storage/rollback_service_spec.rb | 57 | ||||
-rw-r--r-- | spec/workers/hashed_storage/project_migrate_worker_spec.rb (renamed from spec/workers/project_migrate_hashed_storage_worker_spec.rb) | 2 | ||||
-rw-r--r-- | spec/workers/hashed_storage/project_rollback_worker_spec.rb | 50 | ||||
-rw-r--r-- | spec/workers/hashed_storage/rollbacker_worker_spec.rb | 27 |
10 files changed, 510 insertions, 21 deletions
diff --git a/spec/lib/gitlab/hashed_storage/migrator_spec.rb b/spec/lib/gitlab/hashed_storage/migrator_spec.rb index 3942f168ceb..6154b3e2f76 100644 --- a/spec/lib/gitlab/hashed_storage/migrator_spec.rb +++ b/spec/lib/gitlab/hashed_storage/migrator_spec.rb @@ -1,21 +1,29 @@ require 'spec_helper' describe Gitlab::HashedStorage::Migrator do - describe '#bulk_schedule' do - it 'schedules job to StorageMigratorWorker' do + describe '#bulk_schedule_migration' do + it 'schedules job to HashedStorage::MigratorWorker' do Sidekiq::Testing.fake! do - expect { subject.bulk_schedule(start: 1, finish: 5) }.to change(HashedStorage::MigratorWorker.jobs, :size).by(1) + expect { subject.bulk_schedule_migration(start: 1, finish: 5) }.to change(HashedStorage::MigratorWorker.jobs, :size).by(1) + end + end + end + + describe '#bulk_schedule_rollback' do + it 'schedules job to HashedStorage::RollbackerWorker' do + Sidekiq::Testing.fake! do + expect { subject.bulk_schedule_rollback(start: 1, finish: 5) }.to change(HashedStorage::RollbackerWorker.jobs, :size).by(1) end end end describe '#bulk_migrate' do - let(:projects) { create_list(:project, 2, :legacy_storage) } + let(:projects) { create_list(:project, 2, :legacy_storage, :empty_repo) } let(:ids) { projects.map(&:id) } - it 'enqueue jobs to ProjectMigrateHashedStorageWorker' do + it 'enqueue jobs to HashedStorage::ProjectMigrateWorker' do Sidekiq::Testing.fake! do - expect { subject.bulk_migrate(start: ids.min, finish: ids.max) }.to change(ProjectMigrateHashedStorageWorker.jobs, :size).by(2) + expect { subject.bulk_migrate(start: ids.min, finish: ids.max) }.to change(HashedStorage::ProjectMigrateWorker.jobs, :size).by(2) end end @@ -32,13 +40,53 @@ describe Gitlab::HashedStorage::Migrator do subject.bulk_migrate(start: ids.min, finish: ids.max) end - it 'has migrated projects set as writable' do + it 'has all projects migrated and set as writable' do perform_enqueued_jobs do subject.bulk_migrate(start: ids.min, finish: ids.max) end projects.each do |project| - expect(project.reload.repository_read_only?).to be_falsey + project.reload + + expect(project.hashed_storage?(:repository)).to be_truthy + expect(project.repository_read_only?).to be_falsey + end + end + end + + describe '#bulk_rollback' do + let(:projects) { create_list(:project, 2, :empty_repo) } + let(:ids) { projects.map(&:id) } + + it 'enqueue jobs to HashedStorage::ProjectRollbackWorker' do + Sidekiq::Testing.fake! do + expect { subject.bulk_rollback(start: ids.min, finish: ids.max) }.to change(HashedStorage::ProjectRollbackWorker.jobs, :size).by(2) + end + end + + it 'rescues and log exceptions' do + allow_any_instance_of(Project).to receive(:rollback_to_legacy_storage!).and_raise(StandardError) + expect { subject.bulk_rollback(start: ids.min, finish: ids.max) }.not_to raise_error + end + + it 'delegates each project in specified range to #rollback' do + projects.each do |project| + expect(subject).to receive(:rollback).with(project) + end + + subject.bulk_rollback(start: ids.min, finish: ids.max) + end + + it 'has all projects rolledback and set as writable' do + perform_enqueued_jobs do + subject.bulk_rollback(start: ids.min, finish: ids.max) + end + + projects.each do |project| + project.reload + + expect(project.legacy_storage?).to be_truthy + expect(project.repository_read_only?).to be_falsey end end end @@ -48,7 +96,7 @@ describe Gitlab::HashedStorage::Migrator do it 'enqueues project migration job' do Sidekiq::Testing.fake! do - expect { subject.migrate(project) }.to change(ProjectMigrateHashedStorageWorker.jobs, :size).by(1) + expect { subject.migrate(project) }.to change(HashedStorage::ProjectMigrateWorker.jobs, :size).by(1) end end @@ -79,7 +127,7 @@ describe Gitlab::HashedStorage::Migrator do it 'doesnt enqueue any migration job' do Sidekiq::Testing.fake! do - expect { subject.migrate(project) }.not_to change(ProjectMigrateHashedStorageWorker.jobs, :size) + expect { subject.migrate(project) }.not_to change(HashedStorage::ProjectMigrateWorker.jobs, :size) end end @@ -88,4 +136,50 @@ describe Gitlab::HashedStorage::Migrator do end end end + + describe '#rollback' do + let(:project) { create(:project, :empty_repo) } + + it 'enqueues project rollback job' do + Sidekiq::Testing.fake! do + expect { subject.rollback(project) }.to change(HashedStorage::ProjectRollbackWorker.jobs, :size).by(1) + end + end + + it 'rescues and log exceptions' do + allow(project).to receive(:rollback_to_hashed_storage!).and_raise(StandardError) + + expect { subject.rollback(project) }.not_to raise_error + end + + it 'rolls-back project storage' do + perform_enqueued_jobs do + subject.rollback(project) + end + + expect(project.reload.legacy_storage?).to be_truthy + end + + it 'has rolled-back project set as writable' do + perform_enqueued_jobs do + subject.rollback(project) + end + + expect(project.reload.repository_read_only?).to be_falsey + end + + context 'when project is already on legacy storage' do + let(:project) { create(:project, :legacy_storage, :empty_repo) } + + it 'doesnt enqueue any rollback job' do + Sidekiq::Testing.fake! do + expect { subject.rollback(project) }.not_to change(HashedStorage::ProjectRollbackWorker.jobs, :size) + end + end + + it 'returns false' do + expect(subject.rollback(project)).to be_falsey + end + end + end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 9fb0d04ca9e..b2392f9521f 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -3430,28 +3430,42 @@ describe Project do project.migrate_to_hashed_storage! end - it 'schedules ProjectMigrateHashedStorageWorker with delayed start when the project repo is in use' do + it 'schedules HashedStorage::ProjectMigrateWorker with delayed start when the project repo is in use' do Gitlab::ReferenceCounter.new(project.gl_repository(is_wiki: false)).increase - expect(ProjectMigrateHashedStorageWorker).to receive(:perform_in) + expect(HashedStorage::ProjectMigrateWorker).to receive(:perform_in) project.migrate_to_hashed_storage! end - it 'schedules ProjectMigrateHashedStorageWorker with delayed start when the wiki repo is in use' do + it 'schedules HashedStorage::ProjectMigrateWorker with delayed start when the wiki repo is in use' do Gitlab::ReferenceCounter.new(project.gl_repository(is_wiki: true)).increase - expect(ProjectMigrateHashedStorageWorker).to receive(:perform_in) + expect(HashedStorage::ProjectMigrateWorker).to receive(:perform_in) project.migrate_to_hashed_storage! end - it 'schedules ProjectMigrateHashedStorageWorker' do - expect(ProjectMigrateHashedStorageWorker).to receive(:perform_async).with(project.id) + it 'schedules HashedStorage::ProjectMigrateWorker' do + expect(HashedStorage::ProjectMigrateWorker).to receive(:perform_async).with(project.id) project.migrate_to_hashed_storage! end end + + describe '#rollback_to_legacy_storage!' do + let(:project) { create(:project, :empty_repo, :legacy_storage) } + + it 'returns nil' do + expect(project.rollback_to_legacy_storage!).to be_nil + end + + it 'does not run validations' do + expect(project).not_to receive(:valid?) + + project.rollback_to_legacy_storage! + end + end end context 'hashed storage' do @@ -3527,11 +3541,35 @@ describe Project do project = create(:project, storage_version: 1, skip_disk_validation: true) Sidekiq::Testing.fake! do - expect { project.migrate_to_hashed_storage! }.to change(ProjectMigrateHashedStorageWorker.jobs, :size).by(1) + expect { project.migrate_to_hashed_storage! }.to change(HashedStorage::ProjectMigrateWorker.jobs, :size).by(1) end end end end + + describe '#rollback_to_legacy_storage!' do + let(:project) { create(:project, :repository, skip_disk_validation: true) } + + it 'returns true' do + expect(project.rollback_to_legacy_storage!).to be_truthy + end + + it 'does not run validations' do + expect(project).not_to receive(:valid?) + + project.rollback_to_legacy_storage! + end + + it 'does not flag as read-only' do + expect { project.rollback_to_legacy_storage! }.not_to change { project.repository_read_only } + end + + it 'enqueues a job' do + Sidekiq::Testing.fake! do + expect { project.rollback_to_legacy_storage! }.to change(HashedStorage::ProjectRollbackWorker.jobs, :size).by(1) + end + end + end end describe '#gl_repository' do diff --git a/spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb b/spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb index 61dbb57ec08..639dd930618 100644 --- a/spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb +++ b/spec/services/projects/hashed_storage/migrate_attachments_service_spec.rb @@ -70,10 +70,10 @@ describe Projects::HashedStorage::MigrateAttachmentsService do FileUtils.mkdir_p(base_path(hashed_storage)) end - it 'raises AttachmentMigrationError' do + it 'raises AttachmentCannotMoveError' do expect(FileUtils).not_to receive(:mv).with(base_path(legacy_storage), base_path(hashed_storage)) - expect { service.execute }.to raise_error(Projects::HashedStorage::AttachmentMigrationError) + expect { service.execute }.to raise_error(Projects::HashedStorage::AttachmentCannotMoveError) end end end @@ -86,6 +86,8 @@ describe Projects::HashedStorage::MigrateAttachmentsService do context '#new_disk_path' do it 'returns new disk_path for project' do + service.execute + expect(service.new_disk_path).to eq(project.disk_path) end end diff --git a/spec/services/projects/hashed_storage/migrate_repository_service_spec.rb b/spec/services/projects/hashed_storage/migrate_repository_service_spec.rb index 0772dc4b85b..e77e2198439 100644 --- a/spec/services/projects/hashed_storage/migrate_repository_service_spec.rb +++ b/spec/services/projects/hashed_storage/migrate_repository_service_spec.rb @@ -28,7 +28,17 @@ describe Projects::HashedStorage::MigrateRepositoryService do it 'fails when a git operation is in progress' do allow(project).to receive(:repo_reference_count) { 1 } - expect { service.execute }.to raise_error(Projects::HashedStorage::RepositoryMigrationError) + expect { service.execute }.to raise_error(Projects::HashedStorage::RepositoryInUseError) + end + end + + context 'when repository doesnt exist on disk' do + let(:project) { create(:project, :legacy_storage) } + + it 'skips the disk change but increase the version' do + service.execute + + expect(project.hashed_storage?(:repository)).to be_truthy end end diff --git a/spec/services/projects/hashed_storage/rollback_attachments_service_spec.rb b/spec/services/projects/hashed_storage/rollback_attachments_service_spec.rb new file mode 100644 index 00000000000..6f4154d6011 --- /dev/null +++ b/spec/services/projects/hashed_storage/rollback_attachments_service_spec.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Projects::HashedStorage::RollbackAttachmentsService do + subject(:service) { described_class.new(project, logger: nil) } + + let(:project) { create(:project, :repository, skip_disk_validation: true) } + let(:legacy_storage) { Storage::LegacyProject.new(project) } + let(:hashed_storage) { Storage::HashedProject.new(project) } + + let!(:upload) { Upload.find_by(path: file_uploader.upload_path) } + let(:file_uploader) { build(:file_uploader, project: project) } + let(:old_disk_path) { File.join(base_path(hashed_storage), upload.path) } + let(:new_disk_path) { File.join(base_path(legacy_storage), upload.path) } + + context '#execute' do + context 'when succeeds' do + it 'moves attachments to legacy storage layout' do + expect(File.file?(old_disk_path)).to be_truthy + expect(File.file?(new_disk_path)).to be_falsey + expect(File.exist?(base_path(hashed_storage))).to be_truthy + expect(File.exist?(base_path(legacy_storage))).to be_falsey + expect(FileUtils).to receive(:mv).with(base_path(hashed_storage), base_path(legacy_storage)).and_call_original + + service.execute + + expect(File.exist?(base_path(legacy_storage))).to be_truthy + expect(File.exist?(base_path(hashed_storage))).to be_falsey + expect(File.file?(old_disk_path)).to be_falsey + expect(File.file?(new_disk_path)).to be_truthy + end + + it 'returns true' do + expect(service.execute).to be_truthy + end + + it 'sets skipped to false' do + service.execute + + expect(service.skipped?).to be_falsey + end + end + + context 'when original folder does not exist anymore' do + before do + FileUtils.rm_rf(base_path(hashed_storage)) + end + + it 'skips moving folders and go to next' do + expect(FileUtils).not_to receive(:mv).with(base_path(hashed_storage), base_path(legacy_storage)) + + service.execute + + expect(File.exist?(base_path(legacy_storage))).to be_falsey + expect(File.file?(new_disk_path)).to be_falsey + end + + it 'returns true' do + expect(service.execute).to be_truthy + end + + it 'sets skipped to true' do + service.execute + + expect(service.skipped?).to be_truthy + end + end + + context 'when target folder already exists' do + before do + FileUtils.mkdir_p(base_path(legacy_storage)) + end + + it 'raises AttachmentCannotMoveError' do + expect(FileUtils).not_to receive(:mv).with(base_path(legacy_storage), base_path(hashed_storage)) + + expect { service.execute }.to raise_error(Projects::HashedStorage::AttachmentCannotMoveError) + end + end + end + + context '#old_disk_path' do + it 'returns old disk_path for project' do + expect(service.old_disk_path).to eq(project.disk_path) + end + end + + context '#new_disk_path' do + it 'returns new disk_path for project' do + service.execute + + expect(service.new_disk_path).to eq(project.full_path) + end + end + + def base_path(storage) + File.join(FileUploader.root, storage.disk_path) + end +end diff --git a/spec/services/projects/hashed_storage/rollback_repository_service_spec.rb b/spec/services/projects/hashed_storage/rollback_repository_service_spec.rb new file mode 100644 index 00000000000..41927934501 --- /dev/null +++ b/spec/services/projects/hashed_storage/rollback_repository_service_spec.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Projects::HashedStorage::RollbackRepositoryService, :clean_gitlab_redis_shared_state do + include GitHelpers + + let(:gitlab_shell) { Gitlab::Shell.new } + let(:project) { create(:project, :repository, :wiki_repo, storage_version: ::Project::HASHED_STORAGE_FEATURES[:repository]) } + let(:legacy_storage) { Storage::LegacyProject.new(project) } + let(:hashed_storage) { Storage::HashedProject.new(project) } + + subject(:service) { described_class.new(project, project.disk_path) } + + describe '#execute' do + let(:old_disk_path) { hashed_storage.disk_path } + let(:new_disk_path) { legacy_storage.disk_path } + + before do + allow(service).to receive(:gitlab_shell) { gitlab_shell } + end + + context 'repository lock' do + it 'tries to lock the repository' do + expect(service).to receive(:try_to_set_repository_read_only!) + + service.execute + end + + it 'fails when a git operation is in progress' do + allow(project).to receive(:repo_reference_count) { 1 } + + expect { service.execute }.to raise_error(Projects::HashedStorage::RepositoryInUseError) + end + end + + context 'when repository doesnt exist on disk' do + let(:project) { create(:project) } + + it 'skips the disk change but decrease the version' do + service.execute + + expect(project.legacy_storage?).to be_truthy + end + end + + context 'when succeeds' do + it 'renames project and wiki repositories' do + service.execute + + expect(gitlab_shell.exists?(project.repository_storage, "#{new_disk_path}.git")).to be_truthy + expect(gitlab_shell.exists?(project.repository_storage, "#{new_disk_path}.wiki.git")).to be_truthy + end + + it 'updates project to be legacy and not read-only' do + service.execute + + expect(project.legacy_storage?).to be_truthy + expect(project.repository_read_only).to be_falsey + end + + it 'move operation is called for both repositories' do + expect_move_repository(old_disk_path, new_disk_path) + expect_move_repository("#{old_disk_path}.wiki", "#{new_disk_path}.wiki") + + service.execute + end + + it 'writes project full path to .git/config' do + service.execute + + rugged_config = rugged_repo(project.repository).config['gitlab.fullpath'] + + expect(rugged_config).to eq project.full_path + end + end + + context 'when one move fails' do + it 'rolls repositories back to original name' do + allow(service).to receive(:move_repository).and_call_original + allow(service).to receive(:move_repository).with(old_disk_path, new_disk_path).once { false } # will disable first move only + + expect(service).to receive(:rollback_folder_move).and_call_original + + service.execute + + expect(gitlab_shell.exists?(project.repository_storage, "#{new_disk_path}.git")).to be_falsey + expect(gitlab_shell.exists?(project.repository_storage, "#{new_disk_path}.wiki.git")).to be_falsey + expect(project.repository_read_only?).to be_falsey + end + + context 'when rollback fails' do + before do + legacy_storage.ensure_storage_path_exists + gitlab_shell.mv_repository(project.repository_storage, old_disk_path, new_disk_path) + end + + it 'does not try to move nil repository over existing' do + expect(gitlab_shell).not_to receive(:mv_repository).with(project.repository_storage, old_disk_path, new_disk_path) + expect_move_repository("#{old_disk_path}.wiki", "#{new_disk_path}.wiki") + + service.execute + end + end + end + + def expect_move_repository(from_name, to_name) + expect(gitlab_shell).to receive(:mv_repository).with(project.repository_storage, from_name, to_name).and_call_original + end + end +end diff --git a/spec/services/projects/hashed_storage/rollback_service_spec.rb b/spec/services/projects/hashed_storage/rollback_service_spec.rb new file mode 100644 index 00000000000..427d1535559 --- /dev/null +++ b/spec/services/projects/hashed_storage/rollback_service_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Projects::HashedStorage::RollbackService do + let(:project) { create(:project, :empty_repo, :wiki_repo) } + let(:logger) { double } + + subject(:service) { described_class.new(project, project.full_path, logger: logger) } + + describe '#execute' do + context 'attachments rollback' do + let(:attachments_service_class) { Projects::HashedStorage::RollbackAttachmentsService } + let(:attachments_service) { attachments_service_class.new(project, logger: logger) } + + it 'delegates rollback to Projects::HashedStorage::RollbackAttachmentsService' do + expect(attachments_service_class).to receive(:new) + .with(project, logger: logger) + .and_return(attachments_service) + expect(attachments_service).to receive(:execute) + + service.execute + end + + it 'does not delegate rollback if repository is in legacy storage already' do + project.storage_version = nil + expect(attachments_service_class).not_to receive(:new) + + service.execute + end + end + + context 'repository rollback' do + let(:repository_service_class) { Projects::HashedStorage::RollbackRepositoryService } + let(:repository_service) { repository_service_class.new(project, project.full_path, logger: logger) } + + it 'delegates rollback to RollbackRepositoryService' do + project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:repository] + + expect(repository_service_class).to receive(:new) + .with(project, project.full_path, logger: logger) + .and_return(repository_service) + expect(repository_service).to receive(:execute) + + service.execute + end + + it 'does not delegate rollback if repository is in legacy storage already' do + project.storage_version = nil + + expect(repository_service_class).not_to receive(:new) + + service.execute + end + end + end +end diff --git a/spec/workers/project_migrate_hashed_storage_worker_spec.rb b/spec/workers/hashed_storage/project_migrate_worker_spec.rb index 333eb6a0569..340e722aa7e 100644 --- a/spec/workers/project_migrate_hashed_storage_worker_spec.rb +++ b/spec/workers/hashed_storage/project_migrate_worker_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe ProjectMigrateHashedStorageWorker, :clean_gitlab_redis_shared_state do +describe HashedStorage::ProjectMigrateWorker, :clean_gitlab_redis_shared_state do include ExclusiveLeaseHelpers describe '#perform' do diff --git a/spec/workers/hashed_storage/project_rollback_worker_spec.rb b/spec/workers/hashed_storage/project_rollback_worker_spec.rb new file mode 100644 index 00000000000..d833553c0ec --- /dev/null +++ b/spec/workers/hashed_storage/project_rollback_worker_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe HashedStorage::ProjectRollbackWorker, :clean_gitlab_redis_shared_state do + include ExclusiveLeaseHelpers + + describe '#perform' do + let(:project) { create(:project, :empty_repo) } + let(:lease_key) { "project_migrate_hashed_storage_worker:#{project.id}" } + let(:lease_timeout) { described_class::LEASE_TIMEOUT } + let(:rollback_service) { ::Projects::HashedStorage::RollbackService } + + it 'skips when project no longer exists' do + expect(rollback_service).not_to receive(:new) + + subject.perform(-1) + end + + it 'skips when project is pending delete' do + pending_delete_project = create(:project, :empty_repo, pending_delete: true) + + expect(rollback_service).not_to receive(:new) + + subject.perform(pending_delete_project.id) + end + + it 'delegates rollback to service class when have exclusive lease' do + stub_exclusive_lease(lease_key, 'uuid', timeout: lease_timeout) + + service_spy = spy + + allow(rollback_service) + .to receive(:new).with(project, project.disk_path, logger: subject.logger) + .and_return(service_spy) + + subject.perform(project.id) + + expect(service_spy).to have_received(:execute) + end + + it 'skips when it cant acquire the exclusive lease' do + stub_exclusive_lease_taken(lease_key, timeout: lease_timeout) + + expect(rollback_service).not_to receive(:new) + + subject.perform(project.id) + end + end +end diff --git a/spec/workers/hashed_storage/rollbacker_worker_spec.rb b/spec/workers/hashed_storage/rollbacker_worker_spec.rb new file mode 100644 index 00000000000..4055f380978 --- /dev/null +++ b/spec/workers/hashed_storage/rollbacker_worker_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe HashedStorage::RollbackerWorker do + subject(:worker) { described_class.new } + let(:projects) { create_list(:project, 2, :empty_repo) } + let(:ids) { projects.map(&:id) } + + describe '#perform' do + it 'delegates to MigratorService' do + expect_any_instance_of(Gitlab::HashedStorage::Migrator).to receive(:bulk_rollback).with(start: 5, finish: 10) + + worker.perform(5, 10) + end + + it 'rollsback projects in the specified range' do + perform_enqueued_jobs do + worker.perform(ids.min, ids.max) + end + + projects.each do |project| + expect(project.reload.legacy_storage?).to be_truthy + end + end + end +end |