Detect changes to LFS pointers for pruning and integrity check

Gitlab::Git::Blob.batch_lfs_metadata can be used to check for LFS pointers. It uses a lazy enumorator and filters by blob size

4 files changed, 179 insertions, 29 deletions

diff --git a/spec/lib/gitlab/git/blob_spec.rb b/spec/lib/gitlab/git/blob_spec.rb
index 412a0093d97..c04a9688503 100644
--- a/spec/lib/gitlab/git/blob_spec.rb
+++ b/spec/lib/gitlab/git/blob_spec.rb
@@ -143,6 +143,16 @@ describe Gitlab::Git::Blob, seed_helper: true do
         expect(blob.loaded_size).to eq(blob_size)
       end
     end
+
+    context 'when sha references a tree' do
+      it 'returns nil' do
+        tree = Gitlab::Git::Commit.find(repository, 'master').tree
+
+        blob = Gitlab::Git::Blob.raw(repository, tree.oid)
+
+        expect(blob).to be_nil
+      end
+    end
   end
 
   describe '.raw' do
@@ -226,6 +236,51 @@ describe Gitlab::Git::Blob, seed_helper: true do
     end
   end
 
+  describe '.batch_lfs_pointers' do
+    let(:tree_object) { Gitlab::Git::Commit.find(repository, 'master').tree }
+
+    let(:non_lfs_blob) do
+      Gitlab::Git::Blob.find(
+        repository,
+        'master',
+        'README.md'
+      )
+    end
+
+    let(:lfs_blob) do
+      Gitlab::Git::Blob.find(
+        repository,
+        '33bcff41c232a11727ac6d660bd4b0c2ba86d63d',
+        'files/lfs/image.jpg'
+      )
+    end
+
+    it 'returns a list of Gitlab::Git::Blob' do
+      blobs = described_class.batch_lfs_pointers(repository, [lfs_blob.id])
+
+      expect(blobs.count).to eq(1)
+      expect(blobs).to all( be_a(Gitlab::Git::Blob) )
+    end
+
+    it 'silently ignores tree objects' do
+      blobs = described_class.batch_lfs_pointers(repository, [tree_object.oid])
+
+      expect(blobs).to eq([])
+    end
+
+    it 'silently ignores non lfs objects' do
+      blobs = described_class.batch_lfs_pointers(repository, [non_lfs_blob.id])
+
+      expect(blobs).to eq([])
+    end
+
+    it 'avoids loading large blobs into memory' do
+      expect(repository).not_to receive(:lookup)
+
+      described_class.batch_lfs_pointers(repository, [non_lfs_blob.id])
+    end
+  end
+
   describe 'encoding' do
     context 'file with russian text' do
       let(:blob) { Gitlab::Git::Blob.find(repository, SeedRepo::Commit::ID, "encoding/russian.rb") }
diff --git a/spec/lib/gitlab/git/lfs_changes_spec.rb b/spec/lib/gitlab/git/lfs_changes_spec.rb
new file mode 100644
index 00000000000..c2d2c6e1bc8
--- /dev/null
+++ b/spec/lib/gitlab/git/lfs_changes_spec.rb
@@ -0,0 +1,48 @@
+require 'spec_helper'
+
+describe Gitlab::Git::LfsChanges do
+  let(:project) { create(:project, :repository) }
+  let(:newrev) { '54fcc214b94e78d7a41a9a8fe6d87a5e59500e51' }
+  let(:blob_object_id) { '0c304a93cb8430108629bbbcaa27db3343299bc0' }
+
+  subject { described_class.new(project.repository, newrev) }
+
+  describe 'new_pointers' do
+    before do
+      allow_any_instance_of(Gitlab::Git::RevList).to receive(:new_objects).and_return([blob_object_id])
+    end
+
+    it 'uses rev-list to find new objects' do
+      rev_list = double
+      allow(Gitlab::Git::RevList).to receive(:new).and_return(rev_list)
+
+      expect(rev_list).to receive(:new_objects).and_return([])
+
+      subject.new_pointers
+    end
+
+    it 'filters new objects to find lfs pointers' do
+      expect(Gitlab::Git::Blob).to receive(:batch_lfs_pointers).with(project.repository, [blob_object_id])
+
+      subject.new_pointers(object_limit: 1)
+    end
+
+    it 'limits new_objects using object_limit' do
+      expect(Gitlab::Git::Blob).to receive(:batch_lfs_pointers).with(project.repository, [])
+
+      subject.new_pointers(object_limit: 0)
+    end
+  end
+
+  describe 'all_pointers' do
+    it 'uses rev-list to find all objects' do
+      rev_list = double
+      allow(Gitlab::Git::RevList).to receive(:new).and_return(rev_list)
+      allow(rev_list).to receive(:all_objects).and_return([blob_object_id])
+
+      expect(Gitlab::Git::Blob).to receive(:batch_lfs_pointers).with(project.repository, [blob_object_id])
+
+      subject.all_pointers
+    end
+  end
+end
diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb
index b2d2f770e3d..ad3d81b41e2 100644
--- a/spec/lib/gitlab/git/repository_spec.rb
+++ b/spec/lib/gitlab/git/repository_spec.rb
@@ -1310,6 +1310,24 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
+  describe '#batch_existence' do
+    let(:refs) { ['deadbeef', SeedRepo::RubyBlob::ID, '909e6157199'] }
+
+    it 'returns existing refs back' do
+      result = repository.batch_existence(refs)
+
+      expect(result).to eq([SeedRepo::RubyBlob::ID])
+    end
+
+    context 'existing: true' do
+      it 'inverts meaning and returns non-existing refs' do
+        result = repository.batch_existence(refs, existing: false)
+
+        expect(result).to eq(%w(deadbeef 909e6157199))
+      end
+    end
+  end
+
   describe '#local_branches' do
     before(:all) do
       @repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '')
diff --git a/spec/lib/gitlab/git/rev_list_spec.rb b/spec/lib/gitlab/git/rev_list_spec.rb
index c0eac98d718..643a4b2d03e 100644
--- a/spec/lib/gitlab/git/rev_list_spec.rb
+++ b/spec/lib/gitlab/git/rev_list_spec.rb
@@ -2,53 +2,82 @@ require 'spec_helper'
 
 describe Gitlab::Git::RevList do
   let(:project) { create(:project, :repository) }
+  let(:rev_list) { described_class.new(newrev: 'newrev', path_to_repo: project.repository.path_to_repo) }
 
   before do
-    expect(Gitlab::Git::Env).to receive(:all).and_return({
+    allow(Gitlab::Git::Env).to receive(:all).and_return({
       GIT_OBJECT_DIRECTORY: 'foo',
       GIT_ALTERNATE_OBJECT_DIRECTORIES: 'bar'
     })
   end
 
-  context "#new_refs" do
-    let(:rev_list) { described_class.new(newrev: 'newrev', path_to_repo: project.repository.path_to_repo) }
+  def stub_popen_rev_list(*additional_args, output:)
+    expect(rev_list).to receive(:popen).with([
+      Gitlab.config.git.bin_path,
+      "--git-dir=#{project.repository.path_to_repo}",
+      'rev-list',
+      *additional_args
+    ],
+    nil,
+    {
+      'GIT_OBJECT_DIRECTORY' => 'foo',
+      'GIT_ALTERNATE_OBJECT_DIRECTORIES' => 'bar'
+    }).and_return([output, 0])
+  end
 
+  context "#new_refs" do
     it 'calls out to `popen`' do
-      expect(rev_list).to receive(:popen).with([
-        Gitlab.config.git.bin_path,
-        "--git-dir=#{project.repository.path_to_repo}",
-        'rev-list',
-        'newrev',
-        '--not',
-        '--all'
-      ],
-      nil,
-      {
-        'GIT_OBJECT_DIRECTORY' => 'foo',
-        'GIT_ALTERNATE_OBJECT_DIRECTORIES' => 'bar'
-      }).and_return(["sha1\nsha2", 0])
+      stub_popen_rev_list('newrev', '--not', '--all', output: "sha1\nsha2")
 
       expect(rev_list.new_refs).to eq(%w[sha1 sha2])
     end
   end
 
+  context '#new_objects' do
+    it 'fetches list of newly pushed objects using rev-list' do
+      stub_popen_rev_list('newrev', '--not', '--all', '--objects', output: "sha1\nsha2")
+
+      expect(rev_list.new_objects).to eq(%w[sha1 sha2])
+    end
+
+    it 'can skip pathless objects' do
+      stub_popen_rev_list('newrev', '--not', '--all', '--objects', output: "sha1\nsha2 path/to/file")
+
+      expect(rev_list.new_objects(require_path: true)).to eq(%w[sha2])
+    end
+
+    it 'can return a lazy enumerator' do
+      stub_popen_rev_list('newrev', '--not', '--all', '--objects', output: "sha1\nsha2")
+
+      expect(rev_list.new_objects(lazy: true)).to be_a Enumerator::Lazy
+    end
+
+    it 'can accept list of references to exclude' do
+      stub_popen_rev_list('newrev', '--not', 'master', '--objects', output: "sha1\nsha2")
+
+      expect(rev_list.new_objects(not_in: ['master'])).to eq(%w[sha1 sha2])
+    end
+
+    it 'handles empty list of references to exclude as listing all known objects' do
+      stub_popen_rev_list('newrev', '--objects', output: "sha1\nsha2")
+
+      expect(rev_list.new_objects(not_in: [])).to eq(%w[sha1 sha2])
+    end
+  end
+
+  context '#all_objects' do
+    it 'fetches list of all pushed objects using rev-list' do
+      stub_popen_rev_list('--all', '--objects', output: "sha1\nsha2")
+
+      expect(rev_list.all_objects.force).to eq(%w[sha1 sha2])
+    end
+  end
+
   context "#missed_ref" do
     let(:rev_list) { described_class.new(oldrev: 'oldrev', newrev: 'newrev', path_to_repo: project.repository.path_to_repo) }
 
     it 'calls out to `popen`' do
-      expect(rev_list).to receive(:popen).with([
-        Gitlab.config.git.bin_path,
-        "--git-dir=#{project.repository.path_to_repo}",
-        'rev-list',
-        '--max-count=1',
-        'oldrev',
-        '^newrev'
-      ],
-      nil,
-      {
-        'GIT_OBJECT_DIRECTORY' => 'foo',
-        'GIT_ALTERNATE_OBJECT_DIRECTORIES' => 'bar'
-      }).and_return(["sha1\nsha2", 0])
+      stub_popen_rev_list('--max-count=1', 'oldrev', '^newrev', output: "sha1\nsha2")
 
       expect(rev_list.missed_ref).to eq(%w[sha1 sha2])
     end