Merge branch 'rd-upgrade-vendor' into 'master'

Upgrade vendor to d2aa3e3d5fae1017373cc047a9403cfa111b2031

See merge request gitlab-org/gitaly!755

10 files changed, 154 insertions, 10 deletions

diff --git a/_support/vendor-gitlab-git b/_support/vendor-gitlab-git
index 705de3239..389766557 100755
--- a/_support/vendor-gitlab-git
+++ b/_support/vendor-gitlab-git
@@ -5,7 +5,9 @@ FILE_LIST = %w[
   lib/gitlab/git.rb
   lib/gitlab/git
   lib/gitlab/encoding_helper.rb
-  lib/gitlab/utils/strong_memoize.rb  
+  lib/gitlab/utils/strong_memoize.rb
+  lib/gitlab/version_info.rb
+  lib/gitlab/popen.rb
 ].freeze
 
 REMOTE = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze
diff --git a/changelogs/unreleased/rd-upgrade-vendor.yml b/changelogs/unreleased/rd-upgrade-vendor.yml
new file mode 100644
index 000000000..1c1483413
--- /dev/null
+++ b/changelogs/unreleased/rd-upgrade-vendor.yml
@@ -0,0 +1,5 @@
+---
+title: Upgrade vendor to d2aa3e3d5fae1017373cc047a9403cfa111b2031
+merge_request:
+author:
+type: other
diff --git a/ruby/lib/gitlab/git.rb b/ruby/lib/gitlab/git.rb
index fd984d3db..5dbd174e2 100644
--- a/ruby/lib/gitlab/git.rb
+++ b/ruby/lib/gitlab/git.rb
@@ -28,6 +28,8 @@ vendor_gitlab_git = '../../vendor/gitlab_git/'
 # Some later requires are order-sensitive. Manually require whatever we need.
 require_relative File.join(vendor_gitlab_git, 'lib/gitlab/encoding_helper.rb')
 require_relative File.join(vendor_gitlab_git, 'lib/gitlab/utils/strong_memoize.rb')
+require_relative File.join(vendor_gitlab_git, 'lib/gitlab/version_info.rb')
+require_relative File.join(vendor_gitlab_git, 'lib/gitlab/popen.rb')
 require_relative File.join(vendor_gitlab_git, 'lib/gitlab/git.rb')
 require_relative File.join(vendor_gitlab_git, 'lib/gitlab/git/popen.rb')
 require_relative File.join(vendor_gitlab_git, 'lib/gitlab/git/ref.rb')
diff --git a/ruby/vendor/gitlab_git/REVISION b/ruby/vendor/gitlab_git/REVISION
index a5f63575a..ddd1a8d54 100644
--- a/ruby/vendor/gitlab_git/REVISION
+++ b/ruby/vendor/gitlab_git/REVISION
@@ -1 +1 @@
-930ad88a87b0814173989aaccc6dc8af00d1bf65
+d2aa3e3d5fae1017373cc047a9403cfa111b2031
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/commit.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/commit.rb
index 89f761dd5..c9806cdb8 100644
--- a/ruby/vendor/gitlab_git/lib/gitlab/git/commit.rb
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git/commit.rb
@@ -60,6 +60,9 @@ module Gitlab
           # Some weird thing?
           return nil unless commit_id.is_a?(String)
 
+          # This saves us an RPC round trip.
+          return nil if commit_id.include?(':')
+
           commit = repo.gitaly_migrate(:find_commit) do |is_enabled|
             if is_enabled
               repo.gitaly_commit_client.find_commit(commit_id)
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/lfs_changes.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/lfs_changes.rb
index b9e5cf258..f3cc388ea 100644
--- a/ruby/vendor/gitlab_git/lib/gitlab/git/lfs_changes.rb
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git/lfs_changes.rb
@@ -30,7 +30,7 @@ module Gitlab
 
       def git_new_pointers(object_limit, not_in)
         @new_pointers ||= begin
-          rev_list.new_objects(not_in: not_in, require_path: true) do |object_ids|
+          rev_list.new_objects(rev_list_params(not_in: not_in)) do |object_ids|
             object_ids = object_ids.take(object_limit) if object_limit
 
             Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids)
@@ -39,7 +39,12 @@ module Gitlab
       end
 
       def git_all_pointers
-        rev_list.all_objects(require_path: true) do |object_ids|
+        params = {}
+        if rev_list_supports_new_options?
+          params[:options] = ["--filter=blob:limit=#{Gitlab::Git::Blob::LFS_POINTER_MAX_SIZE}"]
+        end
+
+        rev_list.all_objects(rev_list_params(params)) do |object_ids|
           Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids)
         end
       end
@@ -47,6 +52,23 @@ module Gitlab
       def rev_list
         Gitlab::Git::RevList.new(@repository, newrev: @newrev)
       end
+
+      # We're passing the `--in-commit-order` arg to ensure we don't wait
+      # for git to traverse all commits before returning pointers.
+      # This is required in order to improve the performance of LFS integrity check
+      def rev_list_params(params = {})
+        params[:options] ||= []
+        params[:options] << "--in-commit-order" if rev_list_supports_new_options?
+        params[:require_path] = true
+
+        params
+      end
+
+      def rev_list_supports_new_options?
+        return @option_supported if defined?(@option_supported)
+
+        @option_supported = Gitlab::Git.version >= Gitlab::VersionInfo.parse('2.16.0')
+      end
     end
   end
 end
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb
index d1b13ca23..93f9adaf1 100644
--- a/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb
@@ -1543,7 +1543,7 @@ module Gitlab
         end
       end
 
-      def rev_list(including: [], excluding: [], objects: false, &block)
+      def rev_list(including: [], excluding: [], options: [], objects: false, &block)
         args = ['rev-list']
 
         args.push(*rev_list_param(including))
@@ -1556,6 +1556,10 @@ module Gitlab
 
         args.push('--objects') if objects
 
+        if options.any?
+          args.push(*options)
+        end
+
         run_git!(args, lazy_block: block)
       end
 
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/rev_list.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/rev_list.rb
index 38c3a55f9..4e661ecef 100644
--- a/ruby/vendor/gitlab_git/lib/gitlab/git/rev_list.rb
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git/rev_list.rb
@@ -27,9 +27,10 @@ module Gitlab
       #
       # When given a block it will yield objects as a lazy enumerator so
       # the caller can limit work done instead of processing megabytes of data
-      def new_objects(require_path: nil, not_in: nil, &lazy_block)
+      def new_objects(options: [], require_path: nil, not_in: nil, &lazy_block)
         opts = {
           including: newrev,
+          options: options,
           excluding: not_in.nil? ? :all : not_in,
           require_path: require_path
         }
@@ -37,8 +38,11 @@ module Gitlab
         get_objects(opts, &lazy_block)
       end
 
-      def all_objects(require_path: nil, &lazy_block)
-        get_objects(including: :all, require_path: require_path, &lazy_block)
+      def all_objects(options: [], require_path: nil, &lazy_block)
+        get_objects(including: :all,
+                    options: options,
+                    require_path: require_path,
+                    &lazy_block)
       end
 
       # This methods returns an array of missed references
@@ -54,8 +58,8 @@ module Gitlab
         repository.rev_list(args).split("\n")
       end
 
-      def get_objects(including: [], excluding: [], require_path: nil)
-        opts = { including: including, excluding: excluding, objects: true }
+      def get_objects(including: [], excluding: [], options: [], require_path: nil)
+        opts = { including: including, excluding: excluding, options: options, objects: true }
 
         repository.rev_list(opts) do |lazy_output|
           objects = objects_from_output(lazy_output, require_path: require_path)
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/popen.rb b/ruby/vendor/gitlab_git/lib/gitlab/popen.rb
new file mode 100644
index 000000000..b9832a724
--- /dev/null
+++ b/ruby/vendor/gitlab_git/lib/gitlab/popen.rb
@@ -0,0 +1,48 @@
+require 'fileutils'
+require 'open3'
+
+module Gitlab
+  module Popen
+    extend self
+
+    Result = Struct.new(:cmd, :stdout, :stderr, :status, :duration)
+
+    # Returns [stdout + stderr, status]
+    def popen(cmd, path = nil, vars = {}, &block)
+      result = popen_with_detail(cmd, path, vars, &block)
+
+      [result.stdout << result.stderr, result.status&.exitstatus]
+    end
+
+    # Returns Result
+    def popen_with_detail(cmd, path = nil, vars = {})
+      unless cmd.is_a?(Array)
+        raise "System commands must be given as an array of strings"
+      end
+
+      path ||= Dir.pwd
+      vars['PWD'] = path
+      options = { chdir: path }
+
+      unless File.directory?(path)
+        FileUtils.mkdir_p(path)
+      end
+
+      cmd_stdout = ''
+      cmd_stderr = ''
+      cmd_status = nil
+      start = Time.now
+
+      Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr|
+        yield(stdin) if block_given?
+        stdin.close
+
+        cmd_stdout = stdout.read
+        cmd_stderr = stderr.read
+        cmd_status = wait_thr.value
+      end
+
+      Result.new(cmd, cmd_stdout, cmd_stderr, cmd_status, Time.now - start)
+    end
+  end
+end
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/version_info.rb b/ruby/vendor/gitlab_git/lib/gitlab/version_info.rb
new file mode 100644
index 000000000..6ee41e85c
--- /dev/null
+++ b/ruby/vendor/gitlab_git/lib/gitlab/version_info.rb
@@ -0,0 +1,54 @@
+module Gitlab
+  class VersionInfo
+    include Comparable
+
+    attr_reader :major, :minor, :patch
+
+    def self.parse(str)
+      if str && m = str.match(/(\d+)\.(\d+)\.(\d+)/)
+        VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i)
+      else
+        VersionInfo.new
+      end
+    end
+
+    def initialize(major = 0, minor = 0, patch = 0)
+      @major = major
+      @minor = minor
+      @patch = patch
+    end
+
+    def <=>(other)
+      return unless other.is_a? VersionInfo
+      return unless valid? && other.valid?
+
+      if other.major < @major
+        1
+      elsif @major < other.major
+        -1
+      elsif other.minor < @minor
+        1
+      elsif @minor < other.minor
+        -1
+      elsif other.patch < @patch
+        1
+      elsif @patch < other.patch
+        -1
+      else
+        0
+      end
+    end
+
+    def to_s
+      if valid?
+        "%d.%d.%d" % [@major, @minor, @patch]
+      else
+        "Unknown"
+      end
+    end
+
+    def valid?
+      @major >= 0 && @minor >= 0 && @patch >= 0 && @major + @minor + @patch > 0
+    end
+  end
+end