diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-04-20 13:00:54 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-04-20 13:00:54 +0300 |
commit | 3cccd102ba543e02725d247893729e5c73b38295 (patch) | |
tree | f36a04ec38517f5deaaacb5acc7d949688d1e187 /spec/lib/backup | |
parent | 205943281328046ef7b4528031b90fbda70c75ac (diff) |
Add latest changes from gitlab-org/gitlab@14-10-stable-eev14.10.0-rc42
Diffstat (limited to 'spec/lib/backup')
-rw-r--r-- | spec/lib/backup/artifacts_spec.rb | 24 | ||||
-rw-r--r-- | spec/lib/backup/files_spec.rb | 26 | ||||
-rw-r--r-- | spec/lib/backup/gitaly_backup_spec.rb | 38 | ||||
-rw-r--r-- | spec/lib/backup/gitaly_rpc_backup_spec.rb | 154 | ||||
-rw-r--r-- | spec/lib/backup/lfs_spec.rb | 26 | ||||
-rw-r--r-- | spec/lib/backup/manager_spec.rb | 300 | ||||
-rw-r--r-- | spec/lib/backup/object_backup_spec.rb | 35 | ||||
-rw-r--r-- | spec/lib/backup/pages_spec.rb | 25 | ||||
-rw-r--r-- | spec/lib/backup/repositories_spec.rb | 153 | ||||
-rw-r--r-- | spec/lib/backup/task_spec.rb | 8 | ||||
-rw-r--r-- | spec/lib/backup/uploads_spec.rb | 25 |
11 files changed, 318 insertions, 496 deletions
diff --git a/spec/lib/backup/artifacts_spec.rb b/spec/lib/backup/artifacts_spec.rb deleted file mode 100644 index d830692d96b..00000000000 --- a/spec/lib/backup/artifacts_spec.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Backup::Artifacts do - let(:progress) { StringIO.new } - - subject(:backup) { described_class.new(progress) } - - describe '#dump' do - before do - allow(File).to receive(:realpath).with('/var/gitlab-artifacts').and_return('/var/gitlab-artifacts') - allow(File).to receive(:realpath).with('/var/gitlab-artifacts/..').and_return('/var') - allow(JobArtifactUploader).to receive(:root) { '/var/gitlab-artifacts' } - end - - it 'excludes tmp from backup tar' do - expect(backup).to receive(:tar).and_return('blabla-tar') - expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./tmp -C /var/gitlab-artifacts -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], '']) - expect(backup).to receive(:pipeline_succeeded?).and_return(true) - backup.dump('artifacts.tar.gz') - end - end -end diff --git a/spec/lib/backup/files_spec.rb b/spec/lib/backup/files_spec.rb index bbc465a26c9..f98b5e1414f 100644 --- a/spec/lib/backup/files_spec.rb +++ b/spec/lib/backup/files_spec.rb @@ -39,7 +39,7 @@ RSpec.describe Backup::Files do end describe '#restore' do - subject { described_class.new(progress, 'registry', '/var/gitlab-registry') } + subject { described_class.new(progress, '/var/gitlab-registry') } let(:timestamp) { Time.utc(2017, 3, 22) } @@ -110,7 +110,7 @@ RSpec.describe Backup::Files do end describe '#dump' do - subject { described_class.new(progress, 'pages', '/var/gitlab-pages', excludes: ['@pages.tmp']) } + subject { described_class.new(progress, '/var/gitlab-pages', excludes: ['@pages.tmp']) } before do allow(subject).to receive(:run_pipeline!).and_return([[true, true], '']) @@ -118,14 +118,14 @@ RSpec.describe Backup::Files do end it 'raises no errors' do - expect { subject.dump('registry.tar.gz') }.not_to raise_error + expect { subject.dump('registry.tar.gz', 'backup_id') }.not_to raise_error end it 'excludes tmp dirs from archive' do expect(subject).to receive(:tar).and_return('blabla-tar') expect(subject).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./@pages.tmp -C /var/gitlab-pages -cf - .), 'gzip -c -1'], any_args) - subject.dump('registry.tar.gz') + subject.dump('registry.tar.gz', 'backup_id') end it 'raises an error on failure' do @@ -133,7 +133,7 @@ RSpec.describe Backup::Files do expect(subject).to receive(:pipeline_succeeded?).and_return(false) expect do - subject.dump('registry.tar.gz') + subject.dump('registry.tar.gz', 'backup_id') end.to raise_error(/Failed to create compressed file/) end @@ -149,7 +149,7 @@ RSpec.describe Backup::Files do .with(%w(rsync -a --delete --exclude=lost+found --exclude=/gitlab-pages/@pages.tmp /var/gitlab-pages /var/gitlab-backup)) .and_return(['', 0]) - subject.dump('registry.tar.gz') + subject.dump('registry.tar.gz', 'backup_id') end it 'retries if rsync fails due to vanishing files' do @@ -158,7 +158,7 @@ RSpec.describe Backup::Files do .and_return(['rsync failed', 24], ['', 0]) expect do - subject.dump('registry.tar.gz') + subject.dump('registry.tar.gz', 'backup_id') end.to output(/files vanished during rsync, retrying/).to_stdout end @@ -168,7 +168,7 @@ RSpec.describe Backup::Files do .and_return(['rsync failed', 1]) expect do - subject.dump('registry.tar.gz') + subject.dump('registry.tar.gz', 'backup_id') end.to output(/rsync failed/).to_stdout .and raise_error(/Failed to create compressed file/) end @@ -176,7 +176,7 @@ RSpec.describe Backup::Files do end describe '#exclude_dirs' do - subject { described_class.new(progress, 'pages', '/var/gitlab-pages', excludes: ['@pages.tmp']) } + subject { described_class.new(progress, '/var/gitlab-pages', excludes: ['@pages.tmp']) } it 'prepends a leading dot slash to tar excludes' do expect(subject.exclude_dirs(:tar)).to eq(['--exclude=lost+found', '--exclude=./@pages.tmp']) @@ -188,7 +188,7 @@ RSpec.describe Backup::Files do end describe '#run_pipeline!' do - subject { described_class.new(progress, 'registry', '/var/gitlab-registry') } + subject { described_class.new(progress, '/var/gitlab-registry') } it 'executes an Open3.pipeline for cmd_list' do expect(Open3).to receive(:pipeline).with(%w[whew command], %w[another cmd], any_args) @@ -222,7 +222,7 @@ RSpec.describe Backup::Files do end describe '#pipeline_succeeded?' do - subject { described_class.new(progress, 'registry', '/var/gitlab-registry') } + subject { described_class.new(progress, '/var/gitlab-registry') } it 'returns true if both tar and gzip succeeeded' do expect( @@ -262,7 +262,7 @@ RSpec.describe Backup::Files do end describe '#tar_ignore_non_success?' do - subject { described_class.new(progress, 'registry', '/var/gitlab-registry') } + subject { described_class.new(progress, '/var/gitlab-registry') } context 'if `tar` command exits with 1 exitstatus' do it 'returns true' do @@ -310,7 +310,7 @@ RSpec.describe Backup::Files do end describe '#noncritical_warning?' do - subject { described_class.new(progress, 'registry', '/var/gitlab-registry') } + subject { described_class.new(progress, '/var/gitlab-registry') } it 'returns true if given text matches noncritical warnings list' do expect( diff --git a/spec/lib/backup/gitaly_backup_spec.rb b/spec/lib/backup/gitaly_backup_spec.rb index f5295c2b04c..399e4ffa72b 100644 --- a/spec/lib/backup/gitaly_backup_spec.rb +++ b/spec/lib/backup/gitaly_backup_spec.rb @@ -25,11 +25,11 @@ RSpec.describe Backup::GitalyBackup do progress.close end - subject { described_class.new(progress, max_parallelism: max_parallelism, storage_parallelism: storage_parallelism, backup_id: backup_id) } + subject { described_class.new(progress, max_parallelism: max_parallelism, storage_parallelism: storage_parallelism) } context 'unknown' do it 'fails to start unknown' do - expect { subject.start(:unknown, destination) }.to raise_error(::Backup::Error, 'unknown backup type: unknown') + expect { subject.start(:unknown, destination, backup_id: backup_id) }.to raise_error(::Backup::Error, 'unknown backup type: unknown') end end @@ -44,7 +44,7 @@ RSpec.describe Backup::GitalyBackup do expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything, '-layout', 'pointer', '-id', backup_id).and_call_original - subject.start(:create, destination) + subject.start(:create, destination, backup_id: backup_id) subject.enqueue(project, Gitlab::GlRepository::PROJECT) subject.enqueue(project, Gitlab::GlRepository::WIKI) subject.enqueue(project, Gitlab::GlRepository::DESIGN) @@ -65,7 +65,7 @@ RSpec.describe Backup::GitalyBackup do it 'passes parallel option through' do expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything, '-parallel', '3', '-layout', 'pointer', '-id', backup_id).and_call_original - subject.start(:create, destination) + subject.start(:create, destination, backup_id: backup_id) subject.finish! end end @@ -76,7 +76,7 @@ RSpec.describe Backup::GitalyBackup do it 'passes parallel option through' do expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything, '-parallel-storage', '3', '-layout', 'pointer', '-id', backup_id).and_call_original - subject.start(:create, destination) + subject.start(:create, destination, backup_id: backup_id) subject.finish! end end @@ -84,10 +84,16 @@ RSpec.describe Backup::GitalyBackup do it 'raises when the exit code not zero' do expect(subject).to receive(:bin_path).and_return(Gitlab::Utils.which('false')) - subject.start(:create, destination) + subject.start(:create, destination, backup_id: backup_id) expect { subject.finish! }.to raise_error(::Backup::Error, 'gitaly-backup exit status 1') end + it 'raises when gitaly_backup_path is not set' do + stub_backup_setting(gitaly_backup_path: nil) + + expect { subject.start(:create, destination, backup_id: backup_id) }.to raise_error(::Backup::Error, 'gitaly-backup binary not found and gitaly_backup_path is not configured') + end + context 'feature flag incremental_repository_backup disabled' do before do stub_feature_flags(incremental_repository_backup: false) @@ -102,7 +108,7 @@ RSpec.describe Backup::GitalyBackup do expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything).and_call_original - subject.start(:create, destination) + subject.start(:create, destination, backup_id: backup_id) subject.enqueue(project, Gitlab::GlRepository::PROJECT) subject.enqueue(project, Gitlab::GlRepository::WIKI) subject.enqueue(project, Gitlab::GlRepository::DESIGN) @@ -146,7 +152,7 @@ RSpec.describe Backup::GitalyBackup do it 'passes through SSL envs' do expect(Open3).to receive(:popen2).with(ssl_env, anything, 'create', '-path', anything, '-layout', 'pointer', '-id', backup_id).and_call_original - subject.start(:create, destination) + subject.start(:create, destination, backup_id: backup_id) subject.finish! end end @@ -171,7 +177,7 @@ RSpec.describe Backup::GitalyBackup do expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything, '-layout', 'pointer').and_call_original - subject.start(:restore, destination) + subject.start(:restore, destination, backup_id: backup_id) subject.enqueue(project, Gitlab::GlRepository::PROJECT) subject.enqueue(project, Gitlab::GlRepository::WIKI) subject.enqueue(project, Gitlab::GlRepository::DESIGN) @@ -194,7 +200,7 @@ RSpec.describe Backup::GitalyBackup do it 'passes parallel option through' do expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything, '-parallel', '3', '-layout', 'pointer').and_call_original - subject.start(:restore, destination) + subject.start(:restore, destination, backup_id: backup_id) subject.finish! end end @@ -205,7 +211,7 @@ RSpec.describe Backup::GitalyBackup do it 'passes parallel option through' do expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything, '-parallel-storage', '3', '-layout', 'pointer').and_call_original - subject.start(:restore, destination) + subject.start(:restore, destination, backup_id: backup_id) subject.finish! end end @@ -224,7 +230,7 @@ RSpec.describe Backup::GitalyBackup do expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything).and_call_original - subject.start(:restore, destination) + subject.start(:restore, destination, backup_id: backup_id) subject.enqueue(project, Gitlab::GlRepository::PROJECT) subject.enqueue(project, Gitlab::GlRepository::WIKI) subject.enqueue(project, Gitlab::GlRepository::DESIGN) @@ -245,8 +251,14 @@ RSpec.describe Backup::GitalyBackup do it 'raises when the exit code not zero' do expect(subject).to receive(:bin_path).and_return(Gitlab::Utils.which('false')) - subject.start(:restore, destination) + subject.start(:restore, destination, backup_id: backup_id) expect { subject.finish! }.to raise_error(::Backup::Error, 'gitaly-backup exit status 1') end + + it 'raises when gitaly_backup_path is not set' do + stub_backup_setting(gitaly_backup_path: nil) + + expect { subject.start(:restore, destination, backup_id: backup_id) }.to raise_error(::Backup::Error, 'gitaly-backup binary not found and gitaly_backup_path is not configured') + end end end diff --git a/spec/lib/backup/gitaly_rpc_backup_spec.rb b/spec/lib/backup/gitaly_rpc_backup_spec.rb deleted file mode 100644 index 6cba8c5c9b1..00000000000 --- a/spec/lib/backup/gitaly_rpc_backup_spec.rb +++ /dev/null @@ -1,154 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Backup::GitalyRpcBackup do - let(:progress) { spy(:stdout) } - let(:destination) { File.join(Gitlab.config.backup.path, 'repositories') } - - subject { described_class.new(progress) } - - after do - # make sure we do not leave behind any backup files - FileUtils.rm_rf(File.join(Gitlab.config.backup.path, 'repositories')) - end - - context 'unknown' do - it 'fails to start unknown' do - expect { subject.start(:unknown, destination) }.to raise_error(::Backup::Error, 'unknown backup type: unknown') - end - end - - context 'create' do - RSpec.shared_examples 'creates a repository backup' do - it 'creates repository bundles', :aggregate_failures do - # Add data to the wiki, design repositories, and snippets, so they will be included in the dump. - create(:wiki_page, container: project) - create(:design, :with_file, issue: create(:issue, project: project)) - project_snippet = create(:project_snippet, :repository, project: project) - personal_snippet = create(:personal_snippet, :repository, author: project.first_owner) - - subject.start(:create, destination) - subject.enqueue(project, Gitlab::GlRepository::PROJECT) - subject.enqueue(project, Gitlab::GlRepository::WIKI) - subject.enqueue(project, Gitlab::GlRepository::DESIGN) - subject.enqueue(personal_snippet, Gitlab::GlRepository::SNIPPET) - subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET) - subject.finish! - - expect(File).to exist(File.join(destination, project.disk_path + '.bundle')) - expect(File).to exist(File.join(destination, project.disk_path + '.wiki.bundle')) - expect(File).to exist(File.join(destination, project.disk_path + '.design.bundle')) - expect(File).to exist(File.join(destination, personal_snippet.disk_path + '.bundle')) - expect(File).to exist(File.join(destination, project_snippet.disk_path + '.bundle')) - end - - context 'failure' do - before do - allow_next_instance_of(Repository) do |repository| - allow(repository).to receive(:bundle_to_disk) { raise 'Fail in tests' } - end - end - - it 'logs an appropriate message', :aggregate_failures do - subject.start(:create, destination) - subject.enqueue(project, Gitlab::GlRepository::PROJECT) - subject.finish! - - expect(progress).to have_received(:puts).with("[Failed] backing up #{project.full_path} (#{project.disk_path})") - expect(progress).to have_received(:puts).with("Error Fail in tests") - end - end - end - - context 'hashed storage' do - let_it_be(:project) { create(:project, :repository) } - - it_behaves_like 'creates a repository backup' - end - - context 'legacy storage' do - let_it_be(:project) { create(:project, :repository, :legacy_storage) } - - it_behaves_like 'creates a repository backup' - end - end - - context 'restore' do - let_it_be(:project) { create(:project, :repository) } - let_it_be(:personal_snippet) { create(:personal_snippet, author: project.first_owner) } - let_it_be(:project_snippet) { create(:project_snippet, project: project, author: project.first_owner) } - - def copy_bundle_to_backup_path(bundle_name, destination) - FileUtils.mkdir_p(File.join(Gitlab.config.backup.path, 'repositories', File.dirname(destination))) - FileUtils.cp(Rails.root.join('spec/fixtures/lib/backup', bundle_name), File.join(Gitlab.config.backup.path, 'repositories', destination)) - end - - it 'restores from repository bundles', :aggregate_failures do - copy_bundle_to_backup_path('project_repo.bundle', project.disk_path + '.bundle') - copy_bundle_to_backup_path('wiki_repo.bundle', project.disk_path + '.wiki.bundle') - copy_bundle_to_backup_path('design_repo.bundle', project.disk_path + '.design.bundle') - copy_bundle_to_backup_path('personal_snippet_repo.bundle', personal_snippet.disk_path + '.bundle') - copy_bundle_to_backup_path('project_snippet_repo.bundle', project_snippet.disk_path + '.bundle') - - subject.start(:restore, destination) - subject.enqueue(project, Gitlab::GlRepository::PROJECT) - subject.enqueue(project, Gitlab::GlRepository::WIKI) - subject.enqueue(project, Gitlab::GlRepository::DESIGN) - subject.enqueue(personal_snippet, Gitlab::GlRepository::SNIPPET) - subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET) - subject.finish! - - collect_commit_shas = -> (repo) { repo.commits('master', limit: 10).map(&:sha) } - - expect(collect_commit_shas.call(project.repository)).to eq(['393a7d860a5a4c3cc736d7eb00604e3472bb95ec']) - expect(collect_commit_shas.call(project.wiki.repository)).to eq(['c74b9948d0088d703ee1fafeddd9ed9add2901ea']) - expect(collect_commit_shas.call(project.design_repository)).to eq(['c3cd4d7bd73a51a0f22045c3a4c871c435dc959d']) - expect(collect_commit_shas.call(personal_snippet.repository)).to eq(['3b3c067a3bc1d1b695b51e2be30c0f8cf698a06e']) - expect(collect_commit_shas.call(project_snippet.repository)).to eq(['6e44ba56a4748be361a841e759c20e421a1651a1']) - end - - it 'cleans existing repositories', :aggregate_failures do - expect_next_instance_of(DesignManagement::Repository) do |repository| - expect(repository).to receive(:remove) - end - - # 4 times = project repo + wiki repo + project_snippet repo + personal_snippet repo - expect(Repository).to receive(:new).exactly(4).times.and_wrap_original do |method, *original_args| - full_path, container, kwargs = original_args - - repository = method.call(full_path, container, **kwargs) - - expect(repository).to receive(:remove) - - repository - end - - subject.start(:restore, destination) - subject.enqueue(project, Gitlab::GlRepository::PROJECT) - subject.enqueue(project, Gitlab::GlRepository::WIKI) - subject.enqueue(project, Gitlab::GlRepository::DESIGN) - subject.enqueue(personal_snippet, Gitlab::GlRepository::SNIPPET) - subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET) - subject.finish! - end - - context 'failure' do - before do - allow_next_instance_of(Repository) do |repository| - allow(repository).to receive(:create_repository) { raise 'Fail in tests' } - allow(repository).to receive(:create_from_bundle) { raise 'Fail in tests' } - end - end - - it 'logs an appropriate message', :aggregate_failures do - subject.start(:restore, destination) - subject.enqueue(project, Gitlab::GlRepository::PROJECT) - subject.finish! - - expect(progress).to have_received(:puts).with("[Failed] restoring #{project.full_path} (#{project.disk_path})") - expect(progress).to have_received(:puts).with("Error Fail in tests") - end - end - end -end diff --git a/spec/lib/backup/lfs_spec.rb b/spec/lib/backup/lfs_spec.rb deleted file mode 100644 index a27f60f20d0..00000000000 --- a/spec/lib/backup/lfs_spec.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Backup::Lfs do - let(:progress) { StringIO.new } - - subject(:backup) { described_class.new(progress) } - - describe '#dump' do - before do - allow(File).to receive(:realpath).and_call_original - allow(File).to receive(:realpath).with('/var/lfs-objects').and_return('/var/lfs-objects') - allow(File).to receive(:realpath).with('/var/lfs-objects/..').and_return('/var') - allow(Settings.lfs).to receive(:storage_path).and_return('/var/lfs-objects') - end - - it 'uses the correct lfs dir in tar command', :aggregate_failures do - expect(backup).to receive(:tar).and_return('blabla-tar') - expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found -C /var/lfs-objects -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], '']) - expect(backup).to receive(:pipeline_succeeded?).and_return(true) - - backup.dump('lfs.tar.gz') - end - end -end diff --git a/spec/lib/backup/manager_spec.rb b/spec/lib/backup/manager_spec.rb index 9cf78a11bc7..192739d05a7 100644 --- a/spec/lib/backup/manager_spec.rb +++ b/spec/lib/backup/manager_spec.rb @@ -22,13 +22,13 @@ RSpec.describe Backup::Manager do describe '#run_create_task' do let(:enabled) { true } - let(:task) { instance_double(Backup::Task, human_name: 'my task', enabled: enabled) } - let(:definitions) { { 'my_task' => Backup::Manager::TaskDefinition.new(task: task, destination_path: 'my_task.tar.gz') } } + let(:task) { instance_double(Backup::Task) } + let(:definitions) { { 'my_task' => Backup::Manager::TaskDefinition.new(task: task, enabled: enabled, destination_path: 'my_task.tar.gz', human_name: 'my task') } } it 'calls the named task' do expect(task).to receive(:dump) expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Dumping my task ... ') - expect(Gitlab::BackupLogger).to receive(:info).with(message: 'done') + expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Dumping my task ... done') subject.run_create_task('my_task') end @@ -37,8 +37,7 @@ RSpec.describe Backup::Manager do let(:enabled) { false } it 'informs the user' do - expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Dumping my task ... ') - expect(Gitlab::BackupLogger).to receive(:info).with(message: '[DISABLED]') + expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Dumping my task ... [DISABLED]') subject.run_create_task('my_task') end @@ -48,8 +47,7 @@ RSpec.describe Backup::Manager do it 'informs the user' do stub_env('SKIP', 'my_task') - expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Dumping my task ... ') - expect(Gitlab::BackupLogger).to receive(:info).with(message: '[SKIPPED]') + expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Dumping my task ... [SKIPPED]') subject.run_create_task('my_task') end @@ -60,12 +58,10 @@ RSpec.describe Backup::Manager do let(:enabled) { true } let(:pre_restore_warning) { nil } let(:post_restore_warning) { nil } - let(:definitions) { { 'my_task' => Backup::Manager::TaskDefinition.new(task: task, destination_path: 'my_task.tar.gz') } } + let(:definitions) { { 'my_task' => Backup::Manager::TaskDefinition.new(task: task, enabled: enabled, human_name: 'my task', destination_path: 'my_task.tar.gz') } } let(:backup_information) { {} } let(:task) do instance_double(Backup::Task, - human_name: 'my task', - enabled: enabled, pre_restore_warning: pre_restore_warning, post_restore_warning: post_restore_warning) end @@ -78,7 +74,7 @@ RSpec.describe Backup::Manager do it 'calls the named task' do expect(task).to receive(:restore) expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... ').ordered - expect(Gitlab::BackupLogger).to receive(:info).with(message: 'done').ordered + expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... done').ordered subject.run_restore_task('my_task') end @@ -87,8 +83,7 @@ RSpec.describe Backup::Manager do let(:enabled) { false } it 'informs the user' do - expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... ').ordered - expect(Gitlab::BackupLogger).to receive(:info).with(message: '[DISABLED]').ordered + expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... [DISABLED]').ordered subject.run_restore_task('my_task') end @@ -100,7 +95,7 @@ RSpec.describe Backup::Manager do it 'displays and waits for the user' do expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... ').ordered expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Watch out!').ordered - expect(Gitlab::BackupLogger).to receive(:info).with(message: 'done').ordered + expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... done').ordered expect(Gitlab::TaskHelpers).to receive(:ask_to_continue) expect(task).to receive(:restore) @@ -124,7 +119,7 @@ RSpec.describe Backup::Manager do it 'displays and waits for the user' do expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... ').ordered - expect(Gitlab::BackupLogger).to receive(:info).with(message: 'done').ordered + expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... done').ordered expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Watch out!').ordered expect(Gitlab::TaskHelpers).to receive(:ask_to_continue) expect(task).to receive(:restore) @@ -134,7 +129,7 @@ RSpec.describe Backup::Manager do it 'does not continue when the user quits' do expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... ').ordered - expect(Gitlab::BackupLogger).to receive(:info).with(message: 'done').ordered + expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Restoring my task ... done').ordered expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Watch out!').ordered expect(Gitlab::BackupLogger).to receive(:info).with(message: 'Quitting...').ordered expect(task).to receive(:restore) @@ -148,8 +143,10 @@ RSpec.describe Backup::Manager do end describe '#create' do + let(:incremental_env) { 'false' } let(:expected_backup_contents) { %w{backup_information.yml task1.tar.gz task2.tar.gz} } - let(:tar_file) { '1546300800_2019_01_01_12.3_gitlab_backup.tar' } + let(:backup_id) { '1546300800_2019_01_01_12.3' } + let(:tar_file) { "#{backup_id}_gitlab_backup.tar" } let(:tar_system_options) { { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] } } let(:tar_cmdline) { ['tar', '-cf', '-', *expected_backup_contents, tar_system_options] } let(:backup_information) do @@ -159,24 +156,27 @@ RSpec.describe Backup::Manager do } end - let(:task1) { instance_double(Backup::Task, human_name: 'task 1', enabled: true) } - let(:task2) { instance_double(Backup::Task, human_name: 'task 2', enabled: true) } + let(:task1) { instance_double(Backup::Task) } + let(:task2) { instance_double(Backup::Task) } let(:definitions) do { - 'task1' => Backup::Manager::TaskDefinition.new(task: task1, destination_path: 'task1.tar.gz'), - 'task2' => Backup::Manager::TaskDefinition.new(task: task2, destination_path: 'task2.tar.gz') + 'task1' => Backup::Manager::TaskDefinition.new(task: task1, human_name: 'task 1', destination_path: 'task1.tar.gz'), + 'task2' => Backup::Manager::TaskDefinition.new(task: task2, human_name: 'task 2', destination_path: 'task2.tar.gz') } end before do + stub_env('INCREMENTAL', incremental_env) allow(ActiveRecord::Base.connection).to receive(:reconnect!) + allow(Gitlab::BackupLogger).to receive(:info) allow(Kernel).to receive(:system).and_return(true) + allow(YAML).to receive(:load_file).and_call_original allow(YAML).to receive(:load_file).with(File.join(Gitlab.config.backup.path, 'backup_information.yml')) .and_return(backup_information) allow(subject).to receive(:backup_information).and_return(backup_information) - allow(task1).to receive(:dump).with(File.join(Gitlab.config.backup.path, 'task1.tar.gz')) - allow(task2).to receive(:dump).with(File.join(Gitlab.config.backup.path, 'task2.tar.gz')) + allow(task1).to receive(:dump).with(File.join(Gitlab.config.backup.path, 'task1.tar.gz'), backup_id) + allow(task2).to receive(:dump).with(File.join(Gitlab.config.backup.path, 'task2.tar.gz'), backup_id) end it 'executes tar' do @@ -185,8 +185,22 @@ RSpec.describe Backup::Manager do expect(Kernel).to have_received(:system).with(*tar_cmdline) end + context 'tar fails' do + before do + expect(Kernel).to receive(:system).with(*tar_cmdline).and_return(false) + end + + it 'logs a failure' do + expect do + subject.create # rubocop:disable Rails/SaveBang + end.to raise_error(Backup::Error, 'Backup failed') + + expect(Gitlab::BackupLogger).to have_received(:info).with(message: "Creating archive #{tar_file} failed") + end + end + context 'when BACKUP is set' do - let(:tar_file) { 'custom_gitlab_backup.tar' } + let(:backup_id) { 'custom' } it 'uses the given value as tar file name' do stub_env('BACKUP', '/ignored/path/custom') @@ -213,6 +227,20 @@ RSpec.describe Backup::Manager do end end + context 'when SKIP env is set' do + let(:expected_backup_contents) { %w{backup_information.yml task1.tar.gz} } + + before do + stub_env('SKIP', 'task2') + end + + it 'executes tar' do + subject.create # rubocop:disable Rails/SaveBang + + expect(Kernel).to have_received(:system).with(*tar_cmdline) + end + end + context 'when the destination is optional' do let(:expected_backup_contents) { %w{backup_information.yml task1.tar.gz} } let(:definitions) do @@ -248,6 +276,7 @@ RSpec.describe Backup::Manager do end before do + allow(Gitlab::BackupLogger).to receive(:info) allow(Dir).to receive(:chdir).and_yield allow(Dir).to receive(:glob).and_return(files) allow(FileUtils).to receive(:rm) @@ -266,7 +295,7 @@ RSpec.describe Backup::Manager do end it 'prints a skipped message' do - expect(progress).to have_received(:puts).with('skipping') + expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Deleting old backups ... [SKIPPED]') end end @@ -290,7 +319,7 @@ RSpec.describe Backup::Manager do end it 'prints a done message' do - expect(progress).to have_received(:puts).with('done. (0 removed)') + expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Deleting old backups ... done. (0 removed)') end end @@ -307,7 +336,7 @@ RSpec.describe Backup::Manager do end it 'prints a done message' do - expect(progress).to have_received(:puts).with('done. (0 removed)') + expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Deleting old backups ... done. (0 removed)') end end @@ -348,7 +377,7 @@ RSpec.describe Backup::Manager do end it 'prints a done message' do - expect(progress).to have_received(:puts).with('done. (8 removed)') + expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Deleting old backups ... done. (8 removed)') end end @@ -372,11 +401,11 @@ RSpec.describe Backup::Manager do end it 'sets the correct removed count' do - expect(progress).to have_received(:puts).with('done. (7 removed)') + expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Deleting old backups ... done. (7 removed)') end it 'prints the error from file that could not be removed' do - expect(progress).to have_received(:puts).with(a_string_matching(message)) + expect(Gitlab::BackupLogger).to have_received(:info).with(message: a_string_matching(message)) end end end @@ -386,6 +415,7 @@ RSpec.describe Backup::Manager do let(:backup_filename) { File.basename(backup_file.path) } before do + allow(Gitlab::BackupLogger).to receive(:info) allow(subject).to receive(:tar_file).and_return(backup_filename) stub_backup_setting( @@ -410,6 +440,23 @@ RSpec.describe Backup::Manager do connection.directories.create(key: Gitlab.config.backup.upload.remote_directory) # rubocop:disable Rails/SaveBang end + context 'skipped upload' do + let(:backup_information) do + { + backup_created_at: Time.zone.parse('2019-01-01'), + gitlab_version: '12.3', + skipped: ['remote'] + } + end + + it 'informs the user' do + stub_env('SKIP', 'remote') + subject.create # rubocop:disable Rails/SaveBang + + expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Uploading backup archive to remote storage directory ... [SKIPPED]') + end + end + context 'target path' do it 'uses the tar filename by default' do expect_any_instance_of(Fog::Collection).to receive(:create) @@ -462,7 +509,7 @@ RSpec.describe Backup::Manager do it 'sets encryption attributes' do subject.create # rubocop:disable Rails/SaveBang - expect(progress).to have_received(:puts).with("done (encrypted with AES256)") + expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Uploading backup archive to remote storage directory ... done (encrypted with AES256)') end end @@ -473,7 +520,7 @@ RSpec.describe Backup::Manager do it 'sets encryption attributes' do subject.create # rubocop:disable Rails/SaveBang - expect(progress).to have_received(:puts).with("done (encrypted with AES256)") + expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Uploading backup archive to remote storage directory ... done (encrypted with AES256)') end end @@ -488,7 +535,7 @@ RSpec.describe Backup::Manager do it 'sets encryption attributes' do subject.create # rubocop:disable Rails/SaveBang - expect(progress).to have_received(:puts).with("done (encrypted with aws:kms)") + expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Uploading backup archive to remote storage directory ... done (encrypted with aws:kms)') end end end @@ -546,15 +593,169 @@ RSpec.describe Backup::Manager do end end end + + context 'incremental' do + let(:incremental_env) { 'true' } + let(:gitlab_version) { Gitlab::VERSION } + let(:backup_id) { "1546300800_2019_01_01_#{gitlab_version}" } + let(:tar_file) { "#{backup_id}_gitlab_backup.tar" } + let(:backup_information) do + { + backup_created_at: Time.zone.parse('2019-01-01'), + gitlab_version: gitlab_version + } + end + + context 'when there are no backup files in the directory' do + before do + allow(Dir).to receive(:glob).and_return([]) + end + + it 'fails the operation and prints an error' do + expect { subject.create }.to raise_error SystemExit # rubocop:disable Rails/SaveBang + expect(progress).to have_received(:puts) + .with(a_string_matching('No backups found')) + end + end + + context 'when there are two backup files in the directory and BACKUP variable is not set' do + before do + allow(Dir).to receive(:glob).and_return( + [ + '1451606400_2016_01_01_1.2.3_gitlab_backup.tar', + '1451520000_2015_12_31_gitlab_backup.tar' + ] + ) + end + + it 'prints the list of available backups' do + expect { subject.create }.to raise_error SystemExit # rubocop:disable Rails/SaveBang + expect(progress).to have_received(:puts) + .with(a_string_matching('1451606400_2016_01_01_1.2.3\n 1451520000_2015_12_31')) + end + + it 'fails the operation and prints an error' do + expect { subject.create }.to raise_error SystemExit # rubocop:disable Rails/SaveBang + expect(progress).to have_received(:puts) + .with(a_string_matching('Found more than one backup')) + end + end + + context 'when BACKUP variable is set to a non-existing file' do + before do + allow(Dir).to receive(:glob).and_return( + [ + '1451606400_2016_01_01_gitlab_backup.tar' + ] + ) + allow(File).to receive(:exist?).and_return(false) + + stub_env('BACKUP', 'wrong') + end + + it 'fails the operation and prints an error' do + expect { subject.create }.to raise_error SystemExit # rubocop:disable Rails/SaveBang + expect(File).to have_received(:exist?).with('wrong_gitlab_backup.tar') + expect(progress).to have_received(:puts) + .with(a_string_matching('The backup file wrong_gitlab_backup.tar does not exist')) + end + end + + context 'when BACKUP variable is set to a correct file' do + let(:backup_id) { '1451606400_2016_01_01_1.2.3' } + let(:tar_cmdline) { %w{tar -xf 1451606400_2016_01_01_1.2.3_gitlab_backup.tar} } + + before do + allow(Gitlab::BackupLogger).to receive(:info) + allow(Dir).to receive(:glob).and_return( + [ + '1451606400_2016_01_01_1.2.3_gitlab_backup.tar' + ] + ) + allow(File).to receive(:exist?).and_return(true) + allow(Kernel).to receive(:system).and_return(true) + + stub_env('BACKUP', '/ignored/path/1451606400_2016_01_01_1.2.3') + end + + it 'unpacks the file' do + subject.create # rubocop:disable Rails/SaveBang + + expect(Kernel).to have_received(:system).with(*tar_cmdline) + end + + context 'tar fails' do + before do + expect(Kernel).to receive(:system).with(*tar_cmdline).and_return(false) + end + + it 'logs a failure' do + expect do + subject.create # rubocop:disable Rails/SaveBang + end.to raise_error(SystemExit) + + expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Unpacking backup failed') + end + end + + context 'on version mismatch' do + let(:backup_information) do + { + backup_created_at: Time.zone.parse('2019-01-01'), + gitlab_version: "not #{gitlab_version}" + } + end + + it 'stops the process' do + expect { subject.create }.to raise_error SystemExit # rubocop:disable Rails/SaveBang + expect(progress).to have_received(:puts) + .with(a_string_matching('GitLab version mismatch')) + end + end + end + + context 'when there is a non-tarred backup in the directory' do + before do + allow(Dir).to receive(:glob).and_return( + [ + 'backup_information.yml' + ] + ) + allow(File).to receive(:exist?).and_return(true) + end + + it 'selects the non-tarred backup to restore from' do + subject.create # rubocop:disable Rails/SaveBang + + expect(progress).to have_received(:puts) + .with(a_string_matching('Non tarred backup found ')) + end + + context 'on version mismatch' do + let(:backup_information) do + { + backup_created_at: Time.zone.parse('2019-01-01'), + gitlab_version: "not #{gitlab_version}" + } + end + + it 'stops the process' do + expect { subject.create }.to raise_error SystemExit # rubocop:disable Rails/SaveBang + expect(progress).to have_received(:puts) + .with(a_string_matching('GitLab version mismatch')) + end + end + end + end end describe '#restore' do - let(:task1) { instance_double(Backup::Task, human_name: 'task 1', enabled: true, pre_restore_warning: nil, post_restore_warning: nil) } - let(:task2) { instance_double(Backup::Task, human_name: 'task 2', enabled: true, pre_restore_warning: nil, post_restore_warning: nil) } + let(:task1) { instance_double(Backup::Task, pre_restore_warning: nil, post_restore_warning: nil) } + let(:task2) { instance_double(Backup::Task, pre_restore_warning: nil, post_restore_warning: nil) } let(:definitions) do { - 'task1' => Backup::Manager::TaskDefinition.new(task: task1, destination_path: 'task1.tar.gz'), - 'task2' => Backup::Manager::TaskDefinition.new(task: task2, destination_path: 'task2.tar.gz') + 'task1' => Backup::Manager::TaskDefinition.new(task: task1, human_name: 'task 1', destination_path: 'task1.tar.gz'), + 'task2' => Backup::Manager::TaskDefinition.new(task: task2, human_name: 'task 2', destination_path: 'task2.tar.gz') } end @@ -570,6 +771,7 @@ RSpec.describe Backup::Manager do Rake.application.rake_require 'tasks/gitlab/shell' Rake.application.rake_require 'tasks/cache' + allow(Gitlab::BackupLogger).to receive(:info) allow(task1).to receive(:restore).with(File.join(Gitlab.config.backup.path, 'task1.tar.gz')) allow(task2).to receive(:restore).with(File.join(Gitlab.config.backup.path, 'task2.tar.gz')) allow(YAML).to receive(:load_file).with(File.join(Gitlab.config.backup.path, 'backup_information.yml')) @@ -634,7 +836,10 @@ RSpec.describe Backup::Manager do end context 'when BACKUP variable is set to a correct file' do + let(:tar_cmdline) { %w{tar -xf 1451606400_2016_01_01_1.2.3_gitlab_backup.tar} } + before do + allow(Gitlab::BackupLogger).to receive(:info) allow(Dir).to receive(:glob).and_return( [ '1451606400_2016_01_01_1.2.3_gitlab_backup.tar' @@ -649,8 +854,21 @@ RSpec.describe Backup::Manager do it 'unpacks the file' do subject.restore - expect(Kernel).to have_received(:system) - .with("tar", "-xf", "1451606400_2016_01_01_1.2.3_gitlab_backup.tar") + expect(Kernel).to have_received(:system).with(*tar_cmdline) + end + + context 'tar fails' do + before do + expect(Kernel).to receive(:system).with(*tar_cmdline).and_return(false) + end + + it 'logs a failure' do + expect do + subject.restore + end.to raise_error(SystemExit) + + expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Unpacking backup failed') + end end context 'on version mismatch' do @@ -680,7 +898,7 @@ RSpec.describe Backup::Manager do subject.restore - expect(progress).to have_received(:print).with('Deleting backups/tmp ... ') + expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Deleting backups/tmp ... ') end end end @@ -731,7 +949,7 @@ RSpec.describe Backup::Manager do subject.restore - expect(progress).to have_received(:print).with('Deleting backups/tmp ... ') + expect(Gitlab::BackupLogger).to have_received(:info).with(message: 'Deleting backups/tmp ... ') end end end diff --git a/spec/lib/backup/object_backup_spec.rb b/spec/lib/backup/object_backup_spec.rb deleted file mode 100644 index 85658173b0e..00000000000 --- a/spec/lib/backup/object_backup_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.shared_examples 'backup object' do |setting| - let(:progress) { StringIO.new } - let(:backup_path) { "/var/#{setting}" } - - subject(:backup) { described_class.new(progress) } - - describe '#dump' do - before do - allow(File).to receive(:realpath).and_call_original - allow(File).to receive(:realpath).with(backup_path).and_return(backup_path) - allow(File).to receive(:realpath).with("#{backup_path}/..").and_return('/var') - allow(Settings.send(setting)).to receive(:storage_path).and_return(backup_path) - end - - it 'uses the correct storage dir in tar command and excludes tmp', :aggregate_failures do - expect(backup).to receive(:tar).and_return('blabla-tar') - expect(backup).to receive(:run_pipeline!).with([%W(blabla-tar --exclude=lost+found --exclude=./tmp -C #{backup_path} -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], '']) - expect(backup).to receive(:pipeline_succeeded?).and_return(true) - - backup.dump('backup_object.tar.gz') - end - end -end - -RSpec.describe Backup::Packages do - it_behaves_like 'backup object', 'packages' -end - -RSpec.describe Backup::TerraformState do - it_behaves_like 'backup object', 'terraform_state' -end diff --git a/spec/lib/backup/pages_spec.rb b/spec/lib/backup/pages_spec.rb deleted file mode 100644 index 095dda61cf4..00000000000 --- a/spec/lib/backup/pages_spec.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Backup::Pages do - let(:progress) { StringIO.new } - - subject { described_class.new(progress) } - - before do - allow(File).to receive(:realpath).with("/var/gitlab-pages").and_return("/var/gitlab-pages") - allow(File).to receive(:realpath).with("/var/gitlab-pages/..").and_return("/var") - end - - describe '#dump' do - it 'excludes tmp from backup tar' do - allow(Gitlab.config.pages).to receive(:path) { '/var/gitlab-pages' } - - expect(subject).to receive(:tar).and_return('blabla-tar') - expect(subject).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./@pages.tmp -C /var/gitlab-pages -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], '']) - expect(subject).to receive(:pipeline_succeeded?).and_return(true) - subject.dump('pages.tar.gz') - end - end -end diff --git a/spec/lib/backup/repositories_spec.rb b/spec/lib/backup/repositories_spec.rb index db3e507596f..c6f611e727c 100644 --- a/spec/lib/backup/repositories_spec.rb +++ b/spec/lib/backup/repositories_spec.rb @@ -4,18 +4,14 @@ require 'spec_helper' RSpec.describe Backup::Repositories do let(:progress) { spy(:stdout) } - let(:parallel_enqueue) { true } - let(:strategy) { spy(:strategy, parallel_enqueue?: parallel_enqueue) } - let(:max_concurrency) { 1 } - let(:max_storage_concurrency) { 1 } + let(:strategy) { spy(:strategy) } let(:destination) { 'repositories' } + let(:backup_id) { 'backup_id' } subject do described_class.new( progress, - strategy: strategy, - max_concurrency: max_concurrency, - max_storage_concurrency: max_storage_concurrency + strategy: strategy ) end @@ -27,9 +23,9 @@ RSpec.describe Backup::Repositories do project_snippet = create(:project_snippet, :repository, project: project) personal_snippet = create(:personal_snippet, :repository, author: project.first_owner) - subject.dump(destination) + subject.dump(destination, backup_id) - expect(strategy).to have_received(:start).with(:create, destination) + expect(strategy).to have_received(:start).with(:create, destination, backup_id: backup_id) expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::PROJECT) expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::WIKI) expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::DESIGN) @@ -51,139 +47,30 @@ RSpec.describe Backup::Repositories do it_behaves_like 'creates repository bundles' end - context 'no concurrency' do - it 'creates the expected number of threads' do - expect(Thread).not_to receive(:new) + describe 'command failure' do + it 'enqueue_project raises an error' do + allow(strategy).to receive(:enqueue).with(anything, Gitlab::GlRepository::PROJECT).and_raise(IOError) - expect(strategy).to receive(:start).with(:create, destination) - projects.each do |project| - expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT) - end - expect(strategy).to receive(:finish!) - - subject.dump(destination) - end - - describe 'command failure' do - it 'enqueue_project raises an error' do - allow(strategy).to receive(:enqueue).with(anything, Gitlab::GlRepository::PROJECT).and_raise(IOError) - - expect { subject.dump(destination) }.to raise_error(IOError) - end - - it 'project query raises an error' do - allow(Project).to receive_message_chain(:includes, :find_each).and_raise(ActiveRecord::StatementTimeout) - - expect { subject.dump(destination) }.to raise_error(ActiveRecord::StatementTimeout) - end + expect { subject.dump(destination, backup_id) }.to raise_error(IOError) end - it 'avoids N+1 database queries' do - control_count = ActiveRecord::QueryRecorder.new do - subject.dump(destination) - end.count + it 'project query raises an error' do + allow(Project).to receive_message_chain(:includes, :find_each).and_raise(ActiveRecord::StatementTimeout) - create_list(:project, 2, :repository) - - expect do - subject.dump(destination) - end.not_to exceed_query_limit(control_count) + expect { subject.dump(destination, backup_id) }.to raise_error(ActiveRecord::StatementTimeout) end end - context 'concurrency with a strategy without parallel enqueueing support' do - let(:parallel_enqueue) { false } - let(:max_concurrency) { 2 } - let(:max_storage_concurrency) { 2 } - - it 'enqueues all projects sequentially' do - expect(Thread).not_to receive(:new) - - expect(strategy).to receive(:start).with(:create, destination) - projects.each do |project| - expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT) - end - expect(strategy).to receive(:finish!) - - subject.dump(destination) - end - end - - [4, 10].each do |max_storage_concurrency| - context "max_storage_concurrency #{max_storage_concurrency}", quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/241701' do - let(:storage_keys) { %w[default test_second_storage] } - let(:max_storage_concurrency) { max_storage_concurrency } - - before do - allow(Gitlab.config.repositories.storages).to receive(:keys).and_return(storage_keys) - end - - it 'creates the expected number of threads' do - expect(Thread).to receive(:new) - .exactly(storage_keys.length * (max_storage_concurrency + 1)).times - .and_call_original + it 'avoids N+1 database queries' do + control_count = ActiveRecord::QueryRecorder.new do + subject.dump(destination, backup_id) + end.count - expect(strategy).to receive(:start).with(:create, destination) - projects.each do |project| - expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT) - end - expect(strategy).to receive(:finish!) + create_list(:project, 2, :repository) - subject.dump(destination) - end - - context 'with extra max concurrency' do - let(:max_concurrency) { 3 } - - it 'creates the expected number of threads' do - expect(Thread).to receive(:new) - .exactly(storage_keys.length * (max_storage_concurrency + 1)).times - .and_call_original - - expect(strategy).to receive(:start).with(:create, destination) - projects.each do |project| - expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT) - end - expect(strategy).to receive(:finish!) - - subject.dump(destination) - end - end - - describe 'command failure' do - it 'enqueue_project raises an error' do - allow(strategy).to receive(:enqueue).and_raise(IOError) - - expect { subject.dump(destination) }.to raise_error(IOError) - end - - it 'project query raises an error' do - allow(Project).to receive_message_chain(:for_repository_storage, :includes, :find_each).and_raise(ActiveRecord::StatementTimeout) - - expect { subject.dump(destination) }.to raise_error(ActiveRecord::StatementTimeout) - end - - context 'misconfigured storages' do - let(:storage_keys) { %w[test_second_storage] } - - it 'raises an error' do - expect { subject.dump(destination) }.to raise_error(Backup::Error, 'repositories.storages in gitlab.yml is misconfigured') - end - end - end - - it 'avoids N+1 database queries' do - control_count = ActiveRecord::QueryRecorder.new do - subject.dump(destination) - end.count - - create_list(:project, 2, :repository) - - expect do - subject.dump(destination) - end.not_to exceed_query_limit(control_count) - end - end + expect do + subject.dump(destination, backup_id) + end.not_to exceed_query_limit(control_count) end end diff --git a/spec/lib/backup/task_spec.rb b/spec/lib/backup/task_spec.rb index b0eb885d3f4..80f1fe01b78 100644 --- a/spec/lib/backup/task_spec.rb +++ b/spec/lib/backup/task_spec.rb @@ -7,15 +7,9 @@ RSpec.describe Backup::Task do subject { described_class.new(progress) } - describe '#human_name' do - it 'must be implemented by the subclass' do - expect { subject.human_name }.to raise_error(NotImplementedError) - end - end - describe '#dump' do it 'must be implemented by the subclass' do - expect { subject.dump('some/path') }.to raise_error(NotImplementedError) + expect { subject.dump('some/path', 'backup_id') }.to raise_error(NotImplementedError) end end diff --git a/spec/lib/backup/uploads_spec.rb b/spec/lib/backup/uploads_spec.rb deleted file mode 100644 index 0cfc80a9cb9..00000000000 --- a/spec/lib/backup/uploads_spec.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Backup::Uploads do - let(:progress) { StringIO.new } - - subject(:backup) { described_class.new(progress) } - - describe '#dump' do - before do - allow(File).to receive(:realpath).and_call_original - allow(File).to receive(:realpath).with('/var/uploads').and_return('/var/uploads') - allow(File).to receive(:realpath).with('/var/uploads/..').and_return('/var') - allow(Gitlab.config.uploads).to receive(:storage_path) { '/var' } - end - - it 'excludes tmp from backup tar' do - expect(backup).to receive(:tar).and_return('blabla-tar') - expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./tmp -C /var/uploads -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], '']) - expect(backup).to receive(:pipeline_succeeded?).and_return(true) - backup.dump('uploads.tar.gz') - end - end -end |