Merge branch 'jej/lfs-change-detection' into 'master'

Detect changes to LFS pointers for pruning and integrity check

See merge request gitlab-org/gitlab-ce!14785

4 files changed, 134 insertions, 12 deletions

diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index a4336facee5..cc6c7609ec7 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -12,6 +12,12 @@ module Gitlab
       # blob data should use load_all_data!.
       MAX_DATA_DISPLAY_SIZE = 10.megabytes
 
+      # These limits are used as a heuristic to ignore files which can't be LFS
+      # pointers. The format of these is described in
+      # https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md#the-pointer
+      LFS_POINTER_MIN_SIZE = 120.bytes
+      LFS_POINTER_MAX_SIZE = 200.bytes
+
       attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary
 
       class << self
@@ -30,16 +36,7 @@ module Gitlab
             if is_enabled
               Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE)
             else
-              blob = repository.lookup(sha)
-
-              next unless blob.is_a?(Rugged::Blob)
-
-              new(
-                id: blob.oid,
-                size: blob.size,
-                data: blob.content(MAX_DATA_DISPLAY_SIZE),
-                binary: blob.binary?
-              )
+              rugged_raw(repository, sha, limit: MAX_DATA_DISPLAY_SIZE)
             end
           end
         end
@@ -59,10 +56,25 @@ module Gitlab
           end
         end
 
+        # Find LFS blobs given an array of sha ids
+        # Returns array of Gitlab::Git::Blob
+        # Does not guarantee blob data will be set
+        def batch_lfs_pointers(repository, blob_ids)
+          blob_ids.lazy
+                  .select { |sha| possible_lfs_blob?(repository, sha) }
+                  .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) }
+                  .select(&:lfs_pointer?)
+                  .force
+        end
+
         def binary?(data)
           EncodingHelper.detect_libgit2_binary?(data)
         end
 
+        def size_could_be_lfs?(size)
+          size.between?(LFS_POINTER_MIN_SIZE, LFS_POINTER_MAX_SIZE)
+        end
+
         private
 
         # Recursive search of blob id by path
@@ -167,6 +179,29 @@ module Gitlab
             end
           end
         end
+
+        def rugged_raw(repository, sha, limit:)
+          blob = repository.lookup(sha)
+
+          return unless blob.is_a?(Rugged::Blob)
+
+          new(
+            id: blob.oid,
+            size: blob.size,
+            data: blob.content(limit),
+            binary: blob.binary?
+          )
+        end
+
+        # Efficient lookup to determine if object size
+        # and type make it a possible LFS blob without loading
+        # blob content into memory with repository.lookup(sha)
+        def possible_lfs_blob?(repository, sha)
+          object_header = repository.rugged.read_header(sha)
+
+          object_header[:type] == :blob &&
+            size_could_be_lfs?(object_header[:len])
+        end
       end
 
       def initialize(options)
@@ -226,7 +261,7 @@ module Gitlab
       # size
       # see https://github.com/github/git-lfs/blob/v1.1.0/docs/spec.md#the-pointer
       def lfs_pointer?
-        has_lfs_version_key? && lfs_oid.present? && lfs_size.present?
+        self.class.size_could_be_lfs?(size) && has_lfs_version_key? && lfs_oid.present? && lfs_size.present?
       end
 
       def lfs_oid
diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb
new file mode 100644
index 00000000000..2749e2e69e2
--- /dev/null
+++ b/lib/gitlab/git/lfs_changes.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module Git
+    class LfsChanges
+      def initialize(repository, newrev)
+        @repository = repository
+        @newrev = newrev
+      end
+
+      def new_pointers(object_limit: nil, not_in: nil)
+        @new_pointers ||= begin
+          object_ids = new_objects(not_in: not_in)
+          object_ids = object_ids.take(object_limit) if object_limit
+
+          Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids)
+        end
+      end
+
+      def all_pointers
+        object_ids = rev_list.all_objects(require_path: true)
+
+        Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids)
+      end
+
+      private
+
+      def new_objects(not_in:)
+        rev_list.new_objects(require_path: true, lazy: true, not_in: not_in)
+      end
+
+      def rev_list
+        ::Gitlab::Git::RevList.new(path_to_repo: @repository.path_to_repo,
+                                   newrev: @newrev)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index fc8af38d4d9..a9e4e1130c3 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -290,6 +290,14 @@ module Gitlab
         end
       end
 
+      def batch_existence(object_ids, existing: true)
+        filter_method = existing ? :select : :reject
+
+        object_ids.public_send(filter_method) do |oid| # rubocop:disable GitlabSecurity/PublicSend
+          rugged.exists?(oid)
+        end
+      end
+
       # Returns an Array of branch and tag names
       def ref_names
         branch_names + tag_names
diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb
index 60b2a4ec411..e0c884aceaa 100644
--- a/lib/gitlab/git/rev_list.rb
+++ b/lib/gitlab/git/rev_list.rb
@@ -13,11 +13,31 @@ module Gitlab
         @path_to_repo = path_to_repo
       end
 
-      # This method returns an array of new references
+      # This method returns an array of new commit references
       def new_refs
         execute([*base_args, newrev, '--not', '--all'])
       end
 
+      # Finds newly added objects
+      # Returns an array of shas
+      #
+      # Can skip objects which do not have a path using required_path: true
+      # This skips commit objects and root trees, which might not be needed when
+      # looking for blobs
+      #
+      # Can return a lazy enumerator to limit work done on megabytes of data
+      def new_objects(require_path: nil, lazy: false, not_in: nil)
+        object_output = execute([*base_args, newrev, *not_in_refs(not_in), '--objects'])
+
+        objects_from_output(object_output, require_path: require_path, lazy: lazy)
+      end
+
+      def all_objects(require_path: nil)
+        object_output = execute([*base_args, '--all', '--objects'])
+
+        objects_from_output(object_output, require_path: require_path, lazy: true)
+      end
+
       # This methods returns an array of missed references
       #
       # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348.
@@ -27,6 +47,13 @@ module Gitlab
 
       private
 
+      def not_in_refs(references)
+        return ['--not', '--all'] unless references
+        return [] if references.empty?
+
+        references.prepend('--not')
+      end
+
       def execute(args)
         output, status = popen(args, nil, Gitlab::Git::Env.to_env_hash)
 
@@ -44,6 +71,22 @@ module Gitlab
           'rev-list'
         ]
       end
+
+      def objects_from_output(object_output, require_path: nil, lazy: nil)
+        objects = object_output.lazy.map do |output_line|
+          sha, path = output_line.split(' ', 2)
+
+          next if require_path && path.blank?
+
+          sha
+        end.reject(&:nil?)
+
+        if lazy
+          objects
+        else
+          objects.force
+        end
+      end
     end
   end
 end