Merge branch 'vendor-gg-update' into 'master'

Update vendored gitlab_git to 31fa9313991881258b4697cb507cfc8ab205b7dc

See merge request gitlab-org/gitaly!486

15 files changed, 467 insertions, 210 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b1d43997d..a3707670a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,10 @@
 # Gitaly changelog
 
+UNRELEASED
+
+- Update vendored gitlab_git to 31fa9313991881258b4697cb507cfc8ab205b7dc
+  https://gitlab.com/gitlab-org/gitaly/merge_requests/486
+
 v0.60.0
 
 - Implement FindMergeBase RPC
diff --git a/ruby/lib/gitlab/git.rb b/ruby/lib/gitlab/git.rb
index 7be27bef6..ace759f49 100644
--- a/ruby/lib/gitlab/git.rb
+++ b/ruby/lib/gitlab/git.rb
@@ -24,6 +24,7 @@ require_relative File.join(vendor_gitlab_git, 'lib/gitlab/git.rb')
 require_relative File.join(vendor_gitlab_git, 'lib/gitlab/git/popen.rb')
 require_relative File.join(vendor_gitlab_git, 'lib/gitlab/git/ref.rb')
 require_relative File.join(vendor_gitlab_git, 'lib/gitlab/git/repository_mirroring.rb')
+require_relative File.join(vendor_gitlab_git, 'lib/gitlab/git/storage/circuit_breaker_settings.rb')
 
 # Require all .rb files we can find in the vendored gitlab/git directory
 dir = File.expand_path(File.join('..', vendor_gitlab_git, 'lib/gitlab/'), __FILE__)
diff --git a/ruby/lib/gitlab/git/gitaly_remote_repository.rb b/ruby/lib/gitlab/git/gitaly_remote_repository.rb
index 8e9b3035c..be15f5df7 100644
--- a/ruby/lib/gitlab/git/gitaly_remote_repository.rb
+++ b/ruby/lib/gitlab/git/gitaly_remote_repository.rb
@@ -15,7 +15,7 @@ module Gitlab
         raise 'gitaly-ruby cannot access remote repositories by path'
       end
 
-      def empty_repo?
+      def empty?
         !exists? || !has_visible_content?
       end
 
diff --git a/ruby/vendor/gitlab_git/REVISION b/ruby/vendor/gitlab_git/REVISION
index d55645df4..7e9eb81ed 100644
--- a/ruby/vendor/gitlab_git/REVISION
+++ b/ruby/vendor/gitlab_git/REVISION
@@ -1 +1 @@
-359b65beac43e009b715c2db048e06b6f96b0ee8
+31fa9313991881258b4697cb507cfc8ab205b7dc
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git.rb b/ruby/vendor/gitlab_git/lib/gitlab/git.rb
index 1f31cdbc9..1f7c35caf 100644
--- a/ruby/vendor/gitlab_git/lib/gitlab/git.rb
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git.rb
@@ -70,6 +70,18 @@ module Gitlab
       def diff_line_code(file_path, new_line_position, old_line_position)
         "#{Digest::SHA1.hexdigest(file_path)}_#{old_line_position}_#{new_line_position}"
       end
+
+      def shas_eql?(sha1, sha2)
+        return false if sha1.nil? || sha2.nil?
+        return false unless sha1.class == sha2.class
+
+        # If either of the shas is below the minimum length, we cannot be sure
+        # that they actually refer to the same commit because of hash collision.
+        length = [sha1.length, sha2.length].min
+        return false if length < Gitlab::Git::Commit::MIN_SHA_LENGTH
+
+        sha1[0, length] == sha2[0, length]
+      end
     end
   end
 end
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/commit.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/commit.rb
index c85dcfa04..e90b158fb 100644
--- a/ruby/vendor/gitlab_git/lib/gitlab/git/commit.rb
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git/commit.rb
@@ -6,6 +6,7 @@ module Gitlab
 
       attr_accessor :raw_commit, :head
 
+      MIN_SHA_LENGTH = 7
       SERIALIZE_KEYS = [
         :id, :message, :parent_ids,
         :authored_date, :author_name, :author_email,
@@ -213,11 +214,17 @@ module Gitlab
         end
 
         def shas_with_signatures(repository, shas)
-          shas.select do |sha|
-            begin
-              Rugged::Commit.extract_signature(repository.rugged, sha)
-            rescue Rugged::OdbError
-              false
+          GitalyClient.migrate(:filter_shas_with_signatures) do |is_enabled|
+            if is_enabled
+              Gitlab::GitalyClient::CommitService.new(repository).filter_shas_with_signatures(shas)
+            else
+              shas.select do |sha|
+                begin
+                  Rugged::Commit.extract_signature(repository.rugged, sha)
+                rescue Rugged::OdbError
+                  false
+                end
+              end
             end
           end
         end
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/operation_service.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/operation_service.rb
index e36d54104..7e8fe1730 100644
--- a/ruby/vendor/gitlab_git/lib/gitlab/git/operation_service.rb
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git/operation_service.rb
@@ -83,7 +83,7 @@ module Gitlab
         Gitlab::Git.check_namespace!(start_repository)
         start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository)
 
-        start_branch_name = nil if start_repository.empty_repo?
+        start_branch_name = nil if start_repository.empty?
 
         if start_branch_name && !start_repository.branch_exists?(start_branch_name)
           raise ArgumentError, "Cannot find branch #{start_branch_name} in #{start_repository.relative_path}"
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/remote_repository.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/remote_repository.rb
index 3685aa206..6bd6e58fe 100644
--- a/ruby/vendor/gitlab_git/lib/gitlab/git/remote_repository.rb
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git/remote_repository.rb
@@ -24,10 +24,12 @@ module Gitlab
         @path = repository.path
       end
 
-      def empty_repo?
+      def empty?
         # We will override this implementation in gitaly-ruby because we cannot
         # use '@repository' there.
-        @repository.empty_repo?
+        #
+        # Caches and memoization used on the Rails side
+        !@repository.exists? || @repository.empty?
       end
 
       def commit_id(revision)
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb
index f2bbaed64..0e0a1987c 100644
--- a/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb
@@ -18,6 +18,9 @@ module Gitlab
         GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE
       ].freeze
       SEARCH_CONTEXT_LINES = 3
+      REBASE_WORKTREE_PREFIX = 'rebase'.freeze
+      SQUASH_WORKTREE_PREFIX = 'squash'.freeze
+      GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze
 
       NoRepository = Class.new(StandardError)
       InvalidBlobName = Class.new(StandardError)
@@ -73,9 +76,6 @@ module Gitlab
         @attributes = Gitlab::Git::Attributes.new(path)
       end
 
-      delegate  :empty?,
-                to: :rugged
-
       def ==(other)
         path == other.path
       end
@@ -204,6 +204,13 @@ module Gitlab
         end
       end
 
+      # Git repository can contains some hidden refs like:
+      #   /refs/notes/*
+      #   /refs/git-as-svn/*
+      #   /refs/pulls/*
+      # This refs by default not visible in project page and not cloned to client side.
+      alias_method :has_visible_content?, :has_local_branches?
+
       def has_local_branches_rugged?
         rugged.branches.each(:local).any? do |ref|
           begin
@@ -774,24 +781,21 @@ module Gitlab
       end
 
       def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
-        OperationService.new(user, self).with_branch(
-          branch_name,
-          start_branch_name: start_branch_name,
-          start_repository: start_repository
-        ) do |start_commit|
-
-          Gitlab::Git.check_namespace!(commit, start_repository)
-
-          revert_tree_id = check_revert_content(commit, start_commit.sha)
-          raise CreateTreeError unless revert_tree_id
-
-          committer = user_to_committer(user)
+        gitaly_migrate(:revert) do |is_enabled|
+          args = {
+            user: user,
+            commit: commit,
+            branch_name: branch_name,
+            message: message,
+            start_branch_name: start_branch_name,
+            start_repository: start_repository
+          }
 
-          create_commit(message: message,
-                        author: committer,
-                        committer: committer,
-                        tree: revert_tree_id,
-                        parents: [start_commit.sha])
+          if is_enabled
+            gitaly_operations_client.user_revert(args)
+          else
+            rugged_revert(args)
+          end
         end
       end
 
@@ -884,8 +888,11 @@ module Gitlab
         end
       end
 
-      def add_remote(remote_name, url)
+      # If `mirror_refmap` is present the remote is set as mirror with that mapping
+      def add_remote(remote_name, url, mirror_refmap: nil)
         rugged.remotes.create(remote_name, url)
+
+        set_remote_as_mirror(remote_name, refmap: mirror_refmap) if mirror_refmap
       rescue Rugged::ConfigError
         remote_update(remote_name, url: url)
       end
@@ -1005,7 +1012,7 @@ module Gitlab
         Gitlab::Git.check_namespace!(start_repository)
         start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository)
 
-        return yield nil if start_repository.empty_repo?
+        return yield nil if start_repository.empty?
 
         if start_repository.same_repository?(self)
           yield commit(start_branch_name)
@@ -1066,17 +1073,17 @@ module Gitlab
         end
       end
 
-      def write_ref(ref_path, ref)
+      def write_ref(ref_path, ref, force: false)
         raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ')
         raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00")
 
-        command = [Gitlab.config.git.bin_path] + %w[update-ref --stdin -z]
-        input = "update #{ref_path}\x00#{ref}\x00\x00"
-        output, status = circuit_breaker.perform do
-          popen(command, path) { |stdin| stdin.write(input) }
-        end
+        ref = "refs/heads/#{ref}" unless ref.start_with?("refs") || ref =~ /\A[a-f0-9]+\z/i
 
-        raise GitError, output unless status.zero?
+        rugged.references.create(ref_path, ref, force: force)
+      rescue Rugged::ReferenceError => ex
+        raise GitError, "could not create ref #{ref_path}: #{ex}"
+      rescue Rugged::OSError => ex
+        raise GitError, "could not create ref #{ref_path}: #{ex}"
       end
 
       def fetch_ref(source_repository, source_ref:, target_ref:)
@@ -1098,14 +1105,22 @@ module Gitlab
       end
 
       # Refactoring aid; allows us to copy code from app/models/repository.rb
-      def run_git(args, env: {}, nice: false)
+      def run_git(args, chdir: path, env: {}, nice: false, &block)
         cmd = [Gitlab.config.git.bin_path, *args]
         cmd.unshift("nice") if nice
         circuit_breaker.perform do
-          popen(cmd, path, env)
+          popen(cmd, chdir, env, &block)
         end
       end
 
+      def run_git!(args, chdir: path, env: {}, nice: false, &block)
+        output, status = run_git(args, chdir: chdir, env: env, nice: nice, &block)
+
+        raise GitError, output unless status.zero?
+
+        output
+      end
+
       # Refactoring aid; allows us to copy code from app/models/repository.rb
       def run_git_with_timeout(args, timeout, env: {})
         circuit_breaker.perform do
@@ -1118,32 +1133,28 @@ module Gitlab
         Gitlab::Git::Commit.find(self, ref)
       end
 
-      # Refactoring aid; allows us to copy code from app/models/repository.rb
-      def empty_repo?
-        !exists? || !has_visible_content?
+      def empty?
+        !has_visible_content?
       end
 
-      #
-      # Git repository can contains some hidden refs like:
-      #   /refs/notes/*
-      #   /refs/git-as-svn/*
-      #   /refs/pulls/*
-      # This refs by default not visible in project page and not cloned to client side.
-      #
-      # This method return true if repository contains some content visible in project page.
-      #
-      def has_visible_content?
-        return @has_visible_content if defined?(@has_visible_content)
+      def fetch_repository_as_mirror(repository)
+        remote_name = "tmp-#{SecureRandom.hex}"
 
-        @has_visible_content = has_local_branches?
-      end
+        # Notice that this feature flag is not for `fetch_repository_as_mirror`
+        # as a whole but for the fetching mechanism (file path or gitaly-ssh).
+        url, env = gitaly_migrate(:fetch_internal) do |is_enabled|
+          if is_enabled
+            repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository)
+            [GITALY_INTERNAL_URL, repository.fetch_env]
+          else
+            [repository.path, nil]
+          end
+        end
 
-      # Like all public `Gitlab::Git::Repository` methods, this method is part
-      # of `Repository`'s interface through `method_missing`.
-      # `Repository` has its own `fetch_remote` which uses `gitlab-shell` and
-      # takes some extra attributes, so we qualify this method name to prevent confusion.
-      def fetch_remote_without_shell(remote = 'origin')
-        run_git(['fetch', remote]).last.zero?
+        add_remote(remote_name, url, mirror_refmap: :all_refs)
+        fetch_remote(remote_name, env: env)
+      ensure
+        remove_remote(remote_name)
       end
 
       def blob_at(sha, path)
@@ -1170,9 +1181,73 @@ module Gitlab
       end
 
       def fsck
-        output, status = run_git(%W[--git-dir=#{path} fsck], nice: true)
+        gitaly_migrate(:git_fsck) do |is_enabled|
+          msg, status = if is_enabled
+                          gitaly_fsck
+                        else
+                          shell_fsck
+                        end
+
+          raise GitError.new("Could not fsck repository: #{msg}") unless status.zero?
+        end
+      end
+
+      def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:)
+        rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)
+        env = git_env_for_user(user)
+
+        with_worktree(rebase_path, branch, env: env) do
+          run_git!(
+            %W(pull --rebase #{remote_repository.path} #{remote_branch}),
+            chdir: rebase_path, env: env
+          )
+
+          rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip
 
-        raise GitError.new("Could not fsck repository:\n#{output}") unless status.zero?
+          Gitlab::Git::OperationService.new(user, self)
+            .update_branch(branch, rebase_sha, branch_sha)
+
+          rebase_sha
+        end
+      end
+
+      def rebase_in_progress?(rebase_id)
+        fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id))
+      end
+
+      def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:)
+        squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)
+        env = git_env_for_user(user).merge(
+          'GIT_AUTHOR_NAME' => author.name,
+          'GIT_AUTHOR_EMAIL' => author.email
+        )
+        diff_range = "#{start_sha}...#{end_sha}"
+        diff_files = run_git!(
+          %W(diff --name-only --diff-filter=a --binary #{diff_range})
+        ).chomp
+
+        with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do
+          # Apply diff of the `diff_range` to the worktree
+          diff = run_git!(%W(diff --binary #{diff_range}))
+          run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin|
+            stdin.write(diff)
+          end
+
+          # Commit the `diff_range` diff
+          run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env)
+
+          # Return the squash sha. May print a warning for ambiguous refs, but
+          # we can ignore that with `--quiet` and just take the SHA, if present.
+          # HEAD here always refers to the current HEAD commit, even if there is
+          # another ref called HEAD.
+          run_git!(
+            %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env
+          ).chomp
+        end
+      end
+
+      def squash_in_progress?(squash_id)
+        fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id))
       end
 
       def gitaly_repository
@@ -1211,6 +1286,65 @@ module Gitlab
 
       private
 
+      def fresh_worktree?(path)
+        File.exist?(path) && !clean_stuck_worktree(path)
+      end
+
+      def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:)
+        base_args = %w(worktree add --detach)
+
+        # Note that we _don't_ want to test for `.present?` here: If the caller
+        # passes an non nil empty value it means it still wants sparse checkout
+        # but just isn't interested in any file, perhaps because it wants to
+        # checkout files in by a changeset but that changeset only adds files.
+        if sparse_checkout_files
+          # Create worktree without checking out
+          run_git!(base_args + ['--no-checkout', worktree_path], env: env)
+          worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path)
+
+          configure_sparse_checkout(worktree_git_path, sparse_checkout_files)
+
+          # After sparse checkout configuration, checkout `branch` in worktree
+          run_git!(%W(checkout --detach #{branch}), chdir: worktree_path, env: env)
+        else
+          # Create worktree and checkout `branch` in it
+          run_git!(base_args + [worktree_path, branch], env: env)
+        end
+
+        yield
+      ensure
+        FileUtils.rm_rf(worktree_path) if File.exist?(worktree_path)
+        FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path)
+      end
+
+      def clean_stuck_worktree(path)
+        return false unless File.mtime(path) < 15.minutes.ago
+
+        FileUtils.rm_rf(path)
+        true
+      end
+
+      # Adding a worktree means checking out the repository. For large repos,
+      # this can be very expensive, so set up sparse checkout for the worktree
+      # to only check out the files we're interested in.
+      def configure_sparse_checkout(worktree_git_path, files)
+        run_git!(%w(config core.sparseCheckout true))
+
+        return if files.empty?
+
+        worktree_info_path = File.join(worktree_git_path, 'info')
+        FileUtils.mkdir_p(worktree_info_path)
+        File.write(File.join(worktree_info_path, 'sparse-checkout'), files)
+      end
+
+      def gitaly_fsck
+        gitaly_repository_client.fsck
+      end
+
+      def shell_fsck
+        run_git(%W[--git-dir=#{path} fsck], nice: true)
+      end
+
       def rugged_fetch_source_branch(source_repository, source_branch, local_ref)
         with_repo_branch_commit(source_repository, source_branch) do |commit|
           if commit
@@ -1222,6 +1356,24 @@ module Gitlab
         end
       end
 
+      def worktree_path(prefix, id)
+        id = id.to_s
+        raise ArgumentError, "worktree id can't be empty" unless id.present?
+        raise ArgumentError, "worktree id can't contain slashes " if id.include?("/")
+
+        File.join(path, 'gitlab-worktree', "#{prefix}-#{id}")
+      end
+
+      def git_env_for_user(user)
+        {
+          'GIT_COMMITTER_NAME' => user.name,
+          'GIT_COMMITTER_EMAIL' => user.email,
+          'GL_ID' => Gitlab::GlId.gl_id(user),
+          'GL_PROTOCOL' => Gitlab::Git::Hook::GL_PROTOCOL,
+          'GL_REPOSITORY' => gl_repository
+        }
+      end
+
       # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'.
       def branches_filter(filter: nil, sort_by: nil)
         # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37464
@@ -1637,6 +1789,28 @@ module Gitlab
         end
       end
 
+      def rugged_revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
+        OperationService.new(user, self).with_branch(
+          branch_name,
+          start_branch_name: start_branch_name,
+          start_repository: start_repository
+        ) do |start_commit|
+
+          Gitlab::Git.check_namespace!(commit, start_repository)
+
+          revert_tree_id = check_revert_content(commit, start_commit.sha)
+          raise CreateTreeError unless revert_tree_id
+
+          committer = user_to_committer(user)
+
+          create_commit(message: message,
+                        author: committer,
+                        committer: committer,
+                        tree: revert_tree_id,
+                        parents: [start_commit.sha])
+        end
+      end
+
       def gitaly_add_branch(branch_name, user, target)
         gitaly_operation_client.user_create_branch(branch_name, user, target)
       rescue GRPC::FailedPrecondition => ex
@@ -1698,7 +1872,7 @@ module Gitlab
       end
 
       def gitaly_fetch_ref(source_repository, source_ref:, target_ref:)
-        args = %W(fetch --no-tags -f ssh://gitaly/internal.git #{source_ref}:#{target_ref})
+        args = %W(fetch --no-tags -f #{GITALY_INTERNAL_URL} #{source_ref}:#{target_ref})
 
         run_git(args, env: source_repository.fetch_env)
       end
@@ -1718,6 +1892,10 @@ module Gitlab
       rescue Rugged::ReferenceError
         raise ArgumentError, 'Invalid merge source'
       end
+
+      def fetch_remote(remote_name = 'origin', env: nil)
+        run_git(['fetch', remote_name], env: env).last.zero?
+      end
     end
   end
 end
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb
index 392bef69e..effb1f0ca 100644
--- a/ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb
@@ -17,33 +17,6 @@ module Gitlab
         rugged.config["remote.#{remote_name}.prune"] = true
       end
 
-      def set_remote_refmap(remote_name, refmap)
-        Array(refmap).each_with_index do |refspec, i|
-          refspec = REFMAPS[refspec] || refspec
-
-          # We need multiple `fetch` entries, but Rugged only allows replacing a config, not adding to it.
-          # To make sure we start from scratch, we set the first using rugged, and use `git` for any others
-          if i == 0
-            rugged.config["remote.#{remote_name}.fetch"] = refspec
-          else
-            run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
-          end
-        end
-      end
-
-      # Like all_refs public `Gitlab::Git::Repository` methods, this method is part
-      # of `Repository`'s interface through `method_missing`.
-      # `Repository` has its own `fetch_as_mirror` which uses `gitlab-shell` and
-      # takes some extra attributes, so we qualify this method name to prevent confusion.
-      def fetch_as_mirror_without_shell(url)
-        remote_name = "tmp-#{SecureRandom.hex}"
-        add_remote(remote_name, url)
-        set_remote_as_mirror(remote_name)
-        fetch_remote_without_shell(remote_name)
-      ensure
-        remove_remote(remote_name) if remote_name
-      end
-
       def remote_tags(remote)
         # Each line has this format: "dc872e9fa6963f8f03da6c8f6f264d0845d6b092\trefs/tags/v1.10.0\n"
         # We want to convert it to: [{ 'v1.10.0' => 'dc872e9fa6963f8f03da6c8f6f264d0845d6b092' }, ...]
@@ -85,6 +58,20 @@ module Gitlab
 
       private
 
+      def set_remote_refmap(remote_name, refmap)
+        Array(refmap).each_with_index do |refspec, i|
+          refspec = REFMAPS[refspec] || refspec
+
+          # We need multiple `fetch` entries, but Rugged only allows replacing a config, not adding to it.
+          # To make sure we start from scratch, we set the first using rugged, and use `git` for any others
+          if i == 0
+            rugged.config["remote.#{remote_name}.fetch"] = refspec
+          else
+            run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
+          end
+        end
+      end
+
       def list_remote_tags(remote)
         tag_list, exit_code, error = nil
         cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-remote --tags #{remote})
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/storage/checker.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/storage/checker.rb
new file mode 100644
index 000000000..d3c37f821
--- /dev/null
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git/storage/checker.rb
@@ -0,0 +1,120 @@
+module Gitlab
+  module Git
+    module Storage
+      class Checker
+        include CircuitBreakerSettings
+
+        attr_reader :storage_path, :storage, :hostname, :logger
+        METRICS_MUTEX = Mutex.new
+        STORAGE_TIMING_BUCKETS = [0.1, 0.15, 0.25, 0.33, 0.5, 1, 1.5, 2.5, 5, 10, 15].freeze
+
+        def self.check_all(logger = Rails.logger)
+          threads = Gitlab.config.repositories.storages.keys.map do |storage_name|
+            Thread.new do
+              Thread.current[:result] = new(storage_name, logger).check_with_lease
+            end
+          end
+
+          threads.map do |thread|
+            thread.join
+            thread[:result]
+          end
+        end
+
+        def self.check_histogram
+          @check_histogram ||=
+            METRICS_MUTEX.synchronize do
+              @check_histogram || Gitlab::Metrics.histogram(:circuitbreaker_storage_check_duration_seconds,
+                                                            'Storage check time in seconds',
+                                                            {},
+                                                            STORAGE_TIMING_BUCKETS
+                                                           )
+            end
+        end
+
+        def initialize(storage, logger = Rails.logger)
+          @storage = storage
+          config = Gitlab.config.repositories.storages[@storage]
+          @storage_path = config['path']
+          @logger = logger
+
+          @hostname = Gitlab::Environment.hostname
+        end
+
+        def check_with_lease
+          lease_key = "storage_check:#{cache_key}"
+          lease = Gitlab::ExclusiveLease.new(lease_key, timeout: storage_timeout)
+          result = { storage: storage, success: nil }
+
+          if uuid = lease.try_obtain
+            result[:success] = check
+
+            Gitlab::ExclusiveLease.cancel(lease_key, uuid)
+          else
+            logger.warn("#{hostname}: #{storage}: Skipping check, previous check still running")
+          end
+
+          result
+        end
+
+        def check
+          if perform_access_check
+            track_storage_accessible
+            true
+          else
+            track_storage_inaccessible
+            logger.error("#{hostname}: #{storage}: Not accessible.")
+            false
+          end
+        end
+
+        private
+
+        def perform_access_check
+          start_time = Gitlab::Metrics::System.monotonic_time
+
+          Gitlab::Git::Storage::ForkedStorageCheck.storage_available?(storage_path, storage_timeout, access_retries)
+        ensure
+          execution_time = Gitlab::Metrics::System.monotonic_time - start_time
+          self.class.check_histogram.observe({ storage: storage }, execution_time)
+        end
+
+        def track_storage_inaccessible
+          first_failure = current_failure_info.first_failure || Time.now
+          last_failure = Time.now
+
+          Gitlab::Git::Storage.redis.with do |redis|
+            redis.pipelined do
+              redis.hset(cache_key, :first_failure, first_failure.to_i)
+              redis.hset(cache_key, :last_failure, last_failure.to_i)
+              redis.hincrby(cache_key, :failure_count, 1)
+              redis.expire(cache_key, failure_reset_time)
+              maintain_known_keys(redis)
+            end
+          end
+        end
+
+        def track_storage_accessible
+          Gitlab::Git::Storage.redis.with do |redis|
+            redis.pipelined do
+              redis.hset(cache_key, :first_failure, nil)
+              redis.hset(cache_key, :last_failure, nil)
+              redis.hset(cache_key, :failure_count, 0)
+              maintain_known_keys(redis)
+            end
+          end
+        end
+
+        def maintain_known_keys(redis)
+          expire_time = Time.now.to_i + failure_reset_time
+          redis.zadd(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, expire_time, cache_key)
+          redis.zremrangebyscore(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, '-inf', Time.now.to_i)
+        end
+
+        def current_failure_info
+          FailureInfo.load(cache_key)
+        end
+      end
+    end
+  end
+end
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/storage/circuit_breaker.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/storage/circuit_breaker.rb
index 4328c0ea2..898bb1b65 100644
--- a/ruby/vendor/gitlab_git/lib/gitlab/git/storage/circuit_breaker.rb
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git/storage/circuit_breaker.rb
@@ -4,22 +4,11 @@ module Gitlab
       class CircuitBreaker
         include CircuitBreakerSettings
 
-        FailureInfo = Struct.new(:last_failure, :failure_count)
-
         attr_reader :storage,
-                    :hostname,
-                    :storage_path
-
-        delegate :last_failure, :failure_count, to: :failure_info
-
-        def self.reset_all!
-          Gitlab::Git::Storage.redis.with do |redis|
-            all_storage_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1)
-            redis.del(*all_storage_keys) unless all_storage_keys.empty?
-          end
+                    :hostname
 
-          RequestStore.delete(:circuitbreaker_cache)
-        end
+        delegate :last_failure, :failure_count, :no_failures?,
+                 to: :failure_info
 
         def self.for_storage(storage)
           cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do
@@ -46,9 +35,6 @@ module Gitlab
         def initialize(storage, hostname)
           @storage = storage
           @hostname = hostname
-
-          config = Gitlab.config.repositories.storages[@storage]
-          @storage_path = config['path']
         end
 
         def perform
@@ -65,15 +51,6 @@ module Gitlab
           failure_count > failure_count_threshold
         end
 
-        def backing_off?
-          return false if no_failures?
-
-          recent_failure = last_failure > failure_wait_time.seconds.ago
-          too_many_failures = failure_count > backoff_threshold
-
-          recent_failure && too_many_failures
-        end
-
         private
 
         # The circuitbreaker can be enabled for the entire fleet using a Feature
@@ -86,88 +63,13 @@ module Gitlab
         end
 
         def failure_info
-          @failure_info ||= get_failure_info
-        end
-
-        # Memoizing the `storage_available` call means we only do it once per
-        # request when the storage is available.
-        #
-        # When the storage appears not available, and the memoized value is `false`
-        # we might want to try again.
-        def storage_available?
-          return @storage_available if @storage_available
-
-          if @storage_available = Gitlab::Git::Storage::ForkedStorageCheck
-                                    .storage_available?(storage_path, storage_timeout, access_retries)
-            track_storage_accessible
-          else
-            track_storage_inaccessible
-          end
-
-          @storage_available
+          @failure_info ||= FailureInfo.load(cache_key)
         end
 
         def check_storage_accessible!
           if circuit_broken?
             raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_reset_time)
           end
-
-          if backing_off?
-            raise Gitlab::Git::Storage::Failing.new("Backing off access to #{storage}", failure_wait_time)
-          end
-
-          unless storage_available?
-            raise Gitlab::Git::Storage::Inaccessible.new("#{storage} not accessible", failure_wait_time)
-          end
-        end
-
-        def no_failures?
-          last_failure.blank? && failure_count == 0
-        end
-
-        def track_storage_inaccessible
-          @failure_info = FailureInfo.new(Time.now, failure_count + 1)
-
-          Gitlab::Git::Storage.redis.with do |redis|
-            redis.pipelined do
-              redis.hset(cache_key, :last_failure, last_failure.to_i)
-              redis.hincrby(cache_key, :failure_count, 1)
-              redis.expire(cache_key, failure_reset_time)
-              maintain_known_keys(redis)
-            end
-          end
-        end
-
-        def track_storage_accessible
-          @failure_info = FailureInfo.new(nil, 0)
-
-          Gitlab::Git::Storage.redis.with do |redis|
-            redis.pipelined do
-              redis.hset(cache_key, :last_failure, nil)
-              redis.hset(cache_key, :failure_count, 0)
-              maintain_known_keys(redis)
-            end
-          end
-        end
-
-        def maintain_known_keys(redis)
-          expire_time = Time.now.to_i + failure_reset_time
-          redis.zadd(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, expire_time, cache_key)
-          redis.zremrangebyscore(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, '-inf', Time.now.to_i)
-        end
-
-        def get_failure_info
-          last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis|
-            redis.hmget(cache_key, :last_failure, :failure_count)
-          end
-
-          last_failure = Time.at(last_failure.to_i) if last_failure.present?
-
-          FailureInfo.new(last_failure, failure_count.to_i)
-        end
-
-        def cache_key
-          @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}"
         end
       end
     end
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/storage/circuit_breaker_settings.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/storage/circuit_breaker_settings.rb
index 257fe8cd8..c9e225f18 100644
--- a/ruby/vendor/gitlab_git/lib/gitlab/git/storage/circuit_breaker_settings.rb
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git/storage/circuit_breaker_settings.rb
@@ -6,10 +6,6 @@ module Gitlab
           application_settings.circuitbreaker_failure_count_threshold
         end
 
-        def failure_wait_time
-          application_settings.circuitbreaker_failure_wait_time
-        end
-
         def failure_reset_time
           application_settings.circuitbreaker_failure_reset_time
         end
@@ -22,8 +18,12 @@ module Gitlab
           application_settings.circuitbreaker_access_retries
         end
 
-        def backoff_threshold
-          application_settings.circuitbreaker_backoff_threshold
+        def check_interval
+          application_settings.circuitbreaker_check_interval
+        end
+
+        def cache_key
+          @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}"
         end
 
         private
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/storage/failure_info.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/storage/failure_info.rb
new file mode 100644
index 000000000..387279c11
--- /dev/null
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git/storage/failure_info.rb
@@ -0,0 +1,39 @@
+module Gitlab
+  module Git
+    module Storage
+      class FailureInfo
+        attr_accessor :first_failure, :last_failure, :failure_count
+
+        def self.reset_all!
+          Gitlab::Git::Storage.redis.with do |redis|
+            all_storage_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1)
+            redis.del(*all_storage_keys) unless all_storage_keys.empty?
+          end
+
+          RequestStore.delete(:circuitbreaker_cache)
+        end
+
+        def self.load(cache_key)
+          first_failure, last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis|
+            redis.hmget(cache_key, :first_failure, :last_failure, :failure_count)
+          end
+
+          last_failure = Time.at(last_failure.to_i) if last_failure.present?
+          first_failure = Time.at(first_failure.to_i) if first_failure.present?
+
+          new(first_failure, last_failure, failure_count.to_i)
+        end
+
+        def initialize(first_failure, last_failure, failure_count)
+          @first_failure = first_failure
+          @last_failure = last_failure
+          @failure_count = failure_count
+        end
+
+        def no_failures?
+          first_failure.blank? && last_failure.blank? && failure_count == 0
+        end
+      end
+    end
+  end
+end
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/storage/null_circuit_breaker.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/storage/null_circuit_breaker.rb
index a12d52d29..261c936c6 100644
--- a/ruby/vendor/gitlab_git/lib/gitlab/git/storage/null_circuit_breaker.rb
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git/storage/null_circuit_breaker.rb
@@ -11,6 +11,9 @@ module Gitlab
         # These will always have nil values
         attr_reader :storage_path
 
+        delegate :last_failure, :failure_count, :no_failures?,
+                 to: :failure_info
+
         def initialize(storage, hostname, error: nil)
           @storage = storage
           @hostname = hostname
@@ -29,16 +32,17 @@ module Gitlab
           false
         end
 
-        def last_failure
-          circuit_broken? ? Time.now : nil
-        end
-
-        def failure_count
-          circuit_broken? ? failure_count_threshold : 0
-        end
-
         def failure_info
-          Gitlab::Git::Storage::CircuitBreaker::FailureInfo.new(last_failure, failure_count)
+          @failure_info ||=
+            if circuit_broken?
+              Gitlab::Git::Storage::FailureInfo.new(Time.now,
+                                                    Time.now,
+                                                    failure_count_threshold)
+            else
+              Gitlab::Git::Storage::FailureInfo.new(nil,
+                                                    nil,
+                                                    0)
+            end
         end
       end
     end