diff options
author | Nick Thomas <nick@gitlab.com> | 2018-02-27 22:15:25 +0300 |
---|---|---|
committer | Nick Thomas <nick@gitlab.com> | 2018-03-02 17:16:17 +0300 |
commit | 6f945f20b4c3683bc862ebc476bad9331d72784e (patch) | |
tree | 9504637169d3a410e03e9e75f1d72ee6a323187f /spec | |
parent | 40c61acb6a9ba84928cebcbce8b57630bd439615 (diff) |
Foreground verification of uploads and LFS objects
Diffstat (limited to 'spec')
-rw-r--r-- | spec/factories/lfs_objects.rb | 6 | ||||
-rw-r--r-- | spec/lib/gitlab/verify/lfs_objects_spec.rb | 35 | ||||
-rw-r--r-- | spec/lib/gitlab/verify/uploads_spec.rb | 44 | ||||
-rw-r--r-- | spec/support/gitlab_verify.rb | 45 | ||||
-rw-r--r-- | spec/tasks/gitlab/lfs/check_rake_spec.rb | 28 | ||||
-rw-r--r-- | spec/tasks/gitlab/uploads/check_rake_spec.rb (renamed from spec/tasks/gitlab/uploads_rake_spec.rb) | 13 |
6 files changed, 165 insertions, 6 deletions
diff --git a/spec/factories/lfs_objects.rb b/spec/factories/lfs_objects.rb index 8eb709022ce..caaed4d5246 100644 --- a/spec/factories/lfs_objects.rb +++ b/spec/factories/lfs_objects.rb @@ -9,4 +9,10 @@ FactoryBot.define do trait :with_file do file { fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "`/png") } end + + # The uniqueness constraint means we can't use the correct OID for all LFS + # objects, so the test needs to decide which (if any) object gets it + trait :correct_oid do + oid 'b804383982bb89b00e828e3f44c038cc991d3d1768009fc39ba8e2c081b9fb75' + end end diff --git a/spec/lib/gitlab/verify/lfs_objects_spec.rb b/spec/lib/gitlab/verify/lfs_objects_spec.rb new file mode 100644 index 00000000000..64f3a9660e0 --- /dev/null +++ b/spec/lib/gitlab/verify/lfs_objects_spec.rb @@ -0,0 +1,35 @@ +require 'spec_helper' + +describe Gitlab::Verify::LfsObjects do + include GitlabVerifyHelpers + + it_behaves_like 'Gitlab::Verify::BatchVerifier subclass' do + let!(:objects) { create_list(:lfs_object, 3, :with_file) } + end + + describe '#run_batches' do + let(:failures) { collect_failures } + let(:failure) { failures[lfs_object] } + + let!(:lfs_object) { create(:lfs_object, :with_file, :correct_oid) } + + it 'passes LFS objects with the correct file' do + expect(failures).to eq({}) + end + + it 'fails LFS objects with a missing file' do + FileUtils.rm_f(lfs_object.file.path) + + expect(failures.keys).to contain_exactly(lfs_object) + expect(failure).to be_a(Errno::ENOENT) + expect(failure.to_s).to include(lfs_object.file.path) + end + + it 'fails LFS objects with a mismatched oid' do + File.truncate(lfs_object.file.path, 0) + + expect(failures.keys).to contain_exactly(lfs_object) + expect(failure.to_s).to include('Checksum mismatch') + end + end +end diff --git a/spec/lib/gitlab/verify/uploads_spec.rb b/spec/lib/gitlab/verify/uploads_spec.rb new file mode 100644 index 00000000000..6146ce61226 --- /dev/null +++ b/spec/lib/gitlab/verify/uploads_spec.rb @@ -0,0 +1,44 @@ +require 'spec_helper' + +describe Gitlab::Verify::Uploads do + include GitlabVerifyHelpers + + it_behaves_like 'Gitlab::Verify::BatchVerifier subclass' do + let(:projects) { create_list(:project, 3, :with_avatar) } + let!(:objects) { projects.flat_map(&:uploads) } + end + + describe '#run_batches' do + let(:project) { create(:project, :with_avatar) } + let(:failures) { collect_failures } + let(:failure) { failures[upload] } + + let!(:upload) { project.uploads.first } + + it 'passes uploads with the correct file' do + expect(failures).to eq({}) + end + + it 'fails uploads with a missing file' do + FileUtils.rm_f(upload.absolute_path) + + expect(failures.keys).to contain_exactly(upload) + expect(failure).to be_a(Errno::ENOENT) + expect(failure.to_s).to include(upload.absolute_path) + end + + it 'fails uploads with a mismatched checksum' do + upload.update!(checksum: 'something incorrect') + + expect(failures.keys).to contain_exactly(upload) + expect(failure.to_s).to include('Checksum mismatch') + end + + it 'fails uploads with a missing precalculated checksum' do + upload.update!(checksum: '') + + expect(failures.keys).to contain_exactly(upload) + expect(failure.to_s).to include('Checksum missing') + end + end +end diff --git a/spec/support/gitlab_verify.rb b/spec/support/gitlab_verify.rb new file mode 100644 index 00000000000..13e2e37624d --- /dev/null +++ b/spec/support/gitlab_verify.rb @@ -0,0 +1,45 @@ +RSpec.shared_examples 'Gitlab::Verify::BatchVerifier subclass' do + describe 'batching' do + let(:first_batch) { objects[0].id..objects[0].id } + let(:second_batch) { objects[1].id..objects[1].id } + let(:third_batch) { objects[2].id..objects[2].id } + + it 'iterates through objects in batches' do + expect(collect_ranges).to eq([first_batch, second_batch, third_batch]) + end + + it 'allows the starting ID to be specified' do + expect(collect_ranges(start: second_batch.first)).to eq([second_batch, third_batch]) + end + + it 'allows the finishing ID to be specified' do + expect(collect_ranges(finish: second_batch.last)).to eq([first_batch, second_batch]) + end + end +end + +module GitlabVerifyHelpers + def collect_ranges(args = {}) + verifier = described_class.new(args.merge(batch_size: 1)) + + collect_results(verifier).map { |range, _| range } + end + + def collect_failures + verifier = described_class.new(batch_size: 1) + + out = {} + + collect_results(verifier).map { |_, failures| out.merge!(failures) } + + out + end + + def collect_results(verifier) + out = [] + + verifier.run_batches { |*args| out << args } + + out + end +end diff --git a/spec/tasks/gitlab/lfs/check_rake_spec.rb b/spec/tasks/gitlab/lfs/check_rake_spec.rb new file mode 100644 index 00000000000..2610edf8bac --- /dev/null +++ b/spec/tasks/gitlab/lfs/check_rake_spec.rb @@ -0,0 +1,28 @@ +require 'rake_helper' + +describe 'gitlab:lfs rake tasks' do + describe 'check' do + let!(:lfs_object) { create(:lfs_object, :with_file, :correct_oid) } + + before do + Rake.application.rake_require('tasks/gitlab/lfs/check') + stub_env('VERBOSE' => 'true') + end + + it 'outputs the integrity check for each batch' do + expect { run_rake_task('gitlab:lfs:check') }.to output(/Failures: 0/).to_stdout + end + + it 'errors out about missing files on the file system' do + FileUtils.rm_f(lfs_object.file.path) + + expect { run_rake_task('gitlab:lfs:check') }.to output(/No such file.*#{Regexp.quote(lfs_object.file.path)}/).to_stdout + end + + it 'errors out about invalid checksum' do + File.truncate(lfs_object.file.path, 0) + + expect { run_rake_task('gitlab:lfs:check') }.to output(/Checksum mismatch/).to_stdout + end + end +end diff --git a/spec/tasks/gitlab/uploads_rake_spec.rb b/spec/tasks/gitlab/uploads/check_rake_spec.rb index ac0005e51e0..5d597c66133 100644 --- a/spec/tasks/gitlab/uploads_rake_spec.rb +++ b/spec/tasks/gitlab/uploads/check_rake_spec.rb @@ -5,23 +5,24 @@ describe 'gitlab:uploads rake tasks' do let!(:upload) { create(:upload, path: Rails.root.join('spec/fixtures/banana_sample.gif')) } before do - Rake.application.rake_require 'tasks/gitlab/uploads' + Rake.application.rake_require('tasks/gitlab/uploads/check') + stub_env('VERBOSE' => 'true') end - it 'outputs the integrity check for each uploaded file' do - expect { run_rake_task('gitlab:uploads:check') }.to output(/Checking file \(#{upload.id}\): #{Regexp.quote(upload.absolute_path)}/).to_stdout + it 'outputs the integrity check for each batch' do + expect { run_rake_task('gitlab:uploads:check') }.to output(/Failures: 0/).to_stdout end it 'errors out about missing files on the file system' do - create(:upload) + missing_upload = create(:upload) - expect { run_rake_task('gitlab:uploads:check') }.to output(/File does not exist on the file system/).to_stdout + expect { run_rake_task('gitlab:uploads:check') }.to output(/No such file.*#{Regexp.quote(missing_upload.absolute_path)}/).to_stdout end it 'errors out about invalid checksum' do upload.update_column(:checksum, '01a3156db2cf4f67ec823680b40b7302f89ab39179124ad219f94919b8a1769e') - expect { run_rake_task('gitlab:uploads:check') }.to output(/File checksum \(9e697aa09fe196909813ee36103e34f721fe47a5fdc8aac0e4e4ac47b9b38282\) does not match the one in the database \(#{upload.checksum}\)/).to_stdout + expect { run_rake_task('gitlab:uploads:check') }.to output(/Checksum mismatch/).to_stdout end end end |