[cherry-pick] '4862-verify-file-checksums'

See merge request gitlab-org/gitlab-ee!4753

7 files changed, 223 insertions, 1 deletions

diff --git a/spec/factories/lfs_objects.rb b/spec/factories/lfs_objects.rb
index 8eb709022ce..eaf3a4ed497 100644
--- a/spec/factories/lfs_objects.rb
+++ b/spec/factories/lfs_objects.rb
@@ -9,4 +9,14 @@ FactoryBot.define do
   trait :with_file do
     file { fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "`/png") }
   end
+
+  # The uniqueness constraint means we can't use the correct OID for all LFS
+  # objects, so the test needs to decide which (if any) object gets it
+  trait :correct_oid do
+    oid 'b804383982bb89b00e828e3f44c038cc991d3d1768009fc39ba8e2c081b9fb75'
+  end
+
+  trait :object_storage do
+    file_store { LfsObjectUploader::Store::REMOTE }
+  end
 end
diff --git a/spec/lib/gitlab/verify/lfs_objects_spec.rb b/spec/lib/gitlab/verify/lfs_objects_spec.rb
new file mode 100644
index 00000000000..0f890e2c7ce
--- /dev/null
+++ b/spec/lib/gitlab/verify/lfs_objects_spec.rb
@@ -0,0 +1,51 @@
+require 'spec_helper'
+
+describe Gitlab::Verify::LfsObjects do
+  include GitlabVerifyHelpers
+
+  it_behaves_like 'Gitlab::Verify::BatchVerifier subclass' do
+    let!(:objects) { create_list(:lfs_object, 3, :with_file) }
+  end
+
+  describe '#run_batches' do
+    let(:failures) { collect_failures }
+    let(:failure) { failures[lfs_object] }
+
+    let!(:lfs_object) { create(:lfs_object, :with_file, :correct_oid) }
+
+    it 'passes LFS objects with the correct file' do
+      expect(failures).to eq({})
+    end
+
+    it 'fails LFS objects with a missing file' do
+      FileUtils.rm_f(lfs_object.file.path)
+
+      expect(failures.keys).to contain_exactly(lfs_object)
+      expect(failure).to be_a(Errno::ENOENT)
+      expect(failure.to_s).to include(lfs_object.file.path)
+    end
+
+    it 'fails LFS objects with a mismatched oid' do
+      File.truncate(lfs_object.file.path, 0)
+
+      expect(failures.keys).to contain_exactly(lfs_object)
+      expect(failure.to_s).to include('Checksum mismatch')
+    end
+
+    context 'with remote files' do
+      before do
+        stub_lfs_object_storage
+      end
+
+      it 'skips LFS objects in object storage' do
+        local_failure = create(:lfs_object)
+        create(:lfs_object, :object_storage)
+
+        failures = {}
+        described_class.new(batch_size: 10).run_batches { |_, failed| failures.merge!(failed) }
+
+        expect(failures.keys).to contain_exactly(local_failure)
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/verify/uploads_spec.rb b/spec/lib/gitlab/verify/uploads_spec.rb
new file mode 100644
index 00000000000..85768308edc
--- /dev/null
+++ b/spec/lib/gitlab/verify/uploads_spec.rb
@@ -0,0 +1,60 @@
+require 'spec_helper'
+
+describe Gitlab::Verify::Uploads do
+  include GitlabVerifyHelpers
+
+  it_behaves_like 'Gitlab::Verify::BatchVerifier subclass' do
+    let(:projects) { create_list(:project, 3, :with_avatar) }
+    let!(:objects) { projects.flat_map(&:uploads) }
+  end
+
+  describe '#run_batches' do
+    let(:project) { create(:project, :with_avatar) }
+    let(:failures) { collect_failures }
+    let(:failure) { failures[upload] }
+
+    let!(:upload) { project.uploads.first }
+
+    it 'passes uploads with the correct file' do
+      expect(failures).to eq({})
+    end
+
+    it 'fails uploads with a missing file' do
+      FileUtils.rm_f(upload.absolute_path)
+
+      expect(failures.keys).to contain_exactly(upload)
+      expect(failure).to be_a(Errno::ENOENT)
+      expect(failure.to_s).to include(upload.absolute_path)
+    end
+
+    it 'fails uploads with a mismatched checksum' do
+      upload.update!(checksum: 'something incorrect')
+
+      expect(failures.keys).to contain_exactly(upload)
+      expect(failure.to_s).to include('Checksum mismatch')
+    end
+
+    it 'fails uploads with a missing precalculated checksum' do
+      upload.update!(checksum: '')
+
+      expect(failures.keys).to contain_exactly(upload)
+      expect(failure.to_s).to include('Checksum missing')
+    end
+
+    context 'with remote files' do
+      before do
+        stub_uploads_object_storage(AvatarUploader)
+      end
+
+      it 'skips uploads in object storage' do
+        local_failure = create(:upload)
+        create(:upload, :object_storage)
+
+        failures = {}
+        described_class.new(batch_size: 10).run_batches { |_, failed| failures.merge!(failed) }
+
+        expect(failures.keys).to contain_exactly(local_failure)
+      end
+    end
+  end
+end
diff --git a/spec/support/gitlab_verify.rb b/spec/support/gitlab_verify.rb
new file mode 100644
index 00000000000..13e2e37624d
--- /dev/null
+++ b/spec/support/gitlab_verify.rb
@@ -0,0 +1,45 @@
+RSpec.shared_examples 'Gitlab::Verify::BatchVerifier subclass' do
+  describe 'batching' do
+    let(:first_batch) { objects[0].id..objects[0].id }
+    let(:second_batch) { objects[1].id..objects[1].id }
+    let(:third_batch) { objects[2].id..objects[2].id }
+
+    it 'iterates through objects in batches' do
+      expect(collect_ranges).to eq([first_batch, second_batch, third_batch])
+    end
+
+    it 'allows the starting ID to be specified' do
+      expect(collect_ranges(start: second_batch.first)).to eq([second_batch, third_batch])
+    end
+
+    it 'allows the finishing ID to be specified' do
+      expect(collect_ranges(finish: second_batch.last)).to eq([first_batch, second_batch])
+    end
+  end
+end
+
+module GitlabVerifyHelpers
+  def collect_ranges(args = {})
+    verifier = described_class.new(args.merge(batch_size: 1))
+
+    collect_results(verifier).map { |range, _| range }
+  end
+
+  def collect_failures
+    verifier = described_class.new(batch_size: 1)
+
+    out = {}
+
+    collect_results(verifier).map { |_, failures| out.merge!(failures) }
+
+    out
+  end
+
+  def collect_results(verifier)
+    out = []
+
+    verifier.run_batches { |*args| out << args }
+
+    out
+  end
+end
diff --git a/spec/tasks/gitlab/lfs/check_rake_spec.rb b/spec/tasks/gitlab/lfs/check_rake_spec.rb
new file mode 100644
index 00000000000..2610edf8bac
--- /dev/null
+++ b/spec/tasks/gitlab/lfs/check_rake_spec.rb
@@ -0,0 +1,28 @@
+require 'rake_helper'
+
+describe 'gitlab:lfs rake tasks' do
+  describe 'check' do
+    let!(:lfs_object) { create(:lfs_object, :with_file, :correct_oid) }
+
+    before do
+      Rake.application.rake_require('tasks/gitlab/lfs/check')
+      stub_env('VERBOSE' => 'true')
+    end
+
+    it 'outputs the integrity check for each batch' do
+      expect { run_rake_task('gitlab:lfs:check') }.to output(/Failures: 0/).to_stdout
+    end
+
+    it 'errors out about missing files on the file system' do
+      FileUtils.rm_f(lfs_object.file.path)
+
+      expect { run_rake_task('gitlab:lfs:check') }.to output(/No such file.*#{Regexp.quote(lfs_object.file.path)}/).to_stdout
+    end
+
+    it 'errors out about invalid checksum' do
+      File.truncate(lfs_object.file.path, 0)
+
+      expect { run_rake_task('gitlab:lfs:check') }.to output(/Checksum mismatch/).to_stdout
+    end
+  end
+end
diff --git a/spec/tasks/gitlab/lfs_rake_spec.rb b/spec/tasks/gitlab/lfs/migrate_rake_spec.rb
index f1b677bd6ee..66d1a192a96 100644
--- a/spec/tasks/gitlab/lfs_rake_spec.rb
+++ b/spec/tasks/gitlab/lfs/migrate_rake_spec.rb
@@ -2,7 +2,7 @@ require 'rake_helper'
 
 describe 'gitlab:lfs namespace rake task' do
   before :all do
-    Rake.application.rake_require 'tasks/gitlab/lfs'
+    Rake.application.rake_require 'tasks/gitlab/lfs/migrate'
   end
 
   describe 'migrate' do
diff --git a/spec/tasks/gitlab/uploads/check_rake_spec.rb b/spec/tasks/gitlab/uploads/check_rake_spec.rb
new file mode 100644
index 00000000000..5d597c66133
--- /dev/null
+++ b/spec/tasks/gitlab/uploads/check_rake_spec.rb
@@ -0,0 +1,28 @@
+require 'rake_helper'
+
+describe 'gitlab:uploads rake tasks' do
+  describe 'check' do
+    let!(:upload) { create(:upload, path: Rails.root.join('spec/fixtures/banana_sample.gif')) }
+
+    before do
+      Rake.application.rake_require('tasks/gitlab/uploads/check')
+      stub_env('VERBOSE' => 'true')
+    end
+
+    it 'outputs the integrity check for each batch' do
+      expect { run_rake_task('gitlab:uploads:check') }.to output(/Failures: 0/).to_stdout
+    end
+
+    it 'errors out about missing files on the file system' do
+      missing_upload = create(:upload)
+
+      expect { run_rake_task('gitlab:uploads:check') }.to output(/No such file.*#{Regexp.quote(missing_upload.absolute_path)}/).to_stdout
+    end
+
+    it 'errors out about invalid checksum' do
+      upload.update_column(:checksum, '01a3156db2cf4f67ec823680b40b7302f89ab39179124ad219f94919b8a1769e')
+
+      expect { run_rake_task('gitlab:uploads:check') }.to output(/Checksum mismatch/).to_stdout
+    end
+  end
+end