Detect changes to LFS pointers for pruning and integrity check

Gitlab::Git::Blob.batch_lfs_metadata can be used to check for LFS pointers. It uses a lazy enumorator and filters by blob size

1 files changed, 44 insertions, 1 deletions

diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb
index 60b2a4ec411..e0c884aceaa 100644
--- a/lib/gitlab/git/rev_list.rb
+++ b/lib/gitlab/git/rev_list.rb
@@ -13,11 +13,31 @@ module Gitlab
         @path_to_repo = path_to_repo
       end
 
-      # This method returns an array of new references
+      # This method returns an array of new commit references
       def new_refs
         execute([*base_args, newrev, '--not', '--all'])
       end
 
+      # Finds newly added objects
+      # Returns an array of shas
+      #
+      # Can skip objects which do not have a path using required_path: true
+      # This skips commit objects and root trees, which might not be needed when
+      # looking for blobs
+      #
+      # Can return a lazy enumerator to limit work done on megabytes of data
+      def new_objects(require_path: nil, lazy: false, not_in: nil)
+        object_output = execute([*base_args, newrev, *not_in_refs(not_in), '--objects'])
+
+        objects_from_output(object_output, require_path: require_path, lazy: lazy)
+      end
+
+      def all_objects(require_path: nil)
+        object_output = execute([*base_args, '--all', '--objects'])
+
+        objects_from_output(object_output, require_path: require_path, lazy: true)
+      end
+
       # This methods returns an array of missed references
       #
       # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348.
@@ -27,6 +47,13 @@ module Gitlab
 
       private
 
+      def not_in_refs(references)
+        return ['--not', '--all'] unless references
+        return [] if references.empty?
+
+        references.prepend('--not')
+      end
+
       def execute(args)
         output, status = popen(args, nil, Gitlab::Git::Env.to_env_hash)
 
@@ -44,6 +71,22 @@ module Gitlab
           'rev-list'
         ]
       end
+
+      def objects_from_output(object_output, require_path: nil, lazy: nil)
+        objects = object_output.lazy.map do |output_line|
+          sha, path = output_line.split(' ', 2)
+
+          next if require_path && path.blank?
+
+          sha
+        end.reject(&:nil?)
+
+        if lazy
+          objects
+        else
+          objects.force
+        end
+      end
     end
   end
 end