Merge branch 'gitlab-git-c594659fea15c6dd17b' into 'master'

Update vendored gitlab_git to c594659fea15c6dd17b

Closes #792

See merge request gitlab-org/gitaly!473

14 files changed, 213 insertions, 57 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7f40669ea..2cf4f102a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,8 @@
 
 UNRELEASED
 
+- Update vendored gitlab_git to c594659fea15c6dd17b
+  https://gitlab.com/gitlab-org/gitaly/merge_requests/473
 - More logging in housekeeping
   https://gitlab.com/gitlab-org/gitaly/merge_requests/435
 
diff --git a/ruby/lib/gitaly_server/commit_service.rb b/ruby/lib/gitaly_server/commit_service.rb
index 20e49e177..38637e12d 100644
--- a/ruby/lib/gitaly_server/commit_service.rb
+++ b/ruby/lib/gitaly_server/commit_service.rb
@@ -37,7 +37,7 @@ module GitalyServer
               commit.is_a?(Rugged::Commit) ? gitaly_commit_from_rugged(commit) : nil
             end.compact
 
-            y.yield  Gitaly::ListCommitsByOidResponse.new(commits: commits)
+            y.yield Gitaly::ListCommitsByOidResponse.new(commits: commits)
           end
         end
       end
diff --git a/ruby/lib/gitaly_server/operations_service.rb b/ruby/lib/gitaly_server/operations_service.rb
index e5780d9a6..e1db64ce1 100644
--- a/ruby/lib/gitaly_server/operations_service.rb
+++ b/ruby/lib/gitaly_server/operations_service.rb
@@ -118,7 +118,7 @@ module GitalyServer
 
             second_request = session.next
             unless second_request.apply
-              raise GRPC::FailedPrecondition, 'merge aborted by client'
+              raise GRPC::FailedPrecondition.new('merge aborted by client')
             end
           end
 
diff --git a/ruby/lib/gitaly_server/repository_service.rb b/ruby/lib/gitaly_server/repository_service.rb
index 725714350..cee25cba7 100644
--- a/ruby/lib/gitaly_server/repository_service.rb
+++ b/ruby/lib/gitaly_server/repository_service.rb
@@ -2,9 +2,9 @@ module GitalyServer
   class RepositoryService < Gitaly::RepositoryService::Service
     include Utils
 
-    def create_repository(request, _call)
+    def create_repository(request, call)
       bridge_exceptions do
-        repo_path = GitalyServer.repo_path(_call)
+        repo_path = GitalyServer.repo_path(call)
 
         Gitlab::Git::Repository.create(repo_path, bare: true, symlink_hooks_to: Gitlab.config.gitlab_shell.hooks_path)
 
@@ -12,7 +12,7 @@ module GitalyServer
       end
     end
 
-    def has_local_branches(request, call)
+    def has_local_branches(request, call) # rubocop:disable Naming/PredicateName
       repo = Gitlab::Git::Repository.from_gitaly(request.repository, call)
 
       Gitaly::HasLocalBranchesResponse.new(value: repo.has_local_branches?)
diff --git a/ruby/lib/gitlab/git.rb b/ruby/lib/gitlab/git.rb
index d80532e35..0423c9571 100644
--- a/ruby/lib/gitlab/git.rb
+++ b/ruby/lib/gitlab/git.rb
@@ -137,3 +137,9 @@ class FakeCircuitBreaker
     yield
   end
 end
+
+class RequestStore
+  def self.active?
+    false
+  end
+end
diff --git a/ruby/lib/gitlab/gitaly_client.rb b/ruby/lib/gitlab/gitaly_client.rb
index 5e7b71559..49d6a3120 100644
--- a/ruby/lib/gitlab/gitaly_client.rb
+++ b/ruby/lib/gitlab/gitaly_client.rb
@@ -1,7 +1,9 @@
 module Gitlab
   module GitalyClient
     module MigrationStatus
-      OPT_IN = :fake_constant
+      DISABLED = :fake_disabled
+      OPT_IN = :fake_opt_in
+      OPT_OUT = :fake_opt_out
     end
 
     class << self
diff --git a/ruby/lib/gitlab/gollum.rb b/ruby/lib/gitlab/gollum.rb
index 7374925f9..4c8b236fd 100644
--- a/ruby/lib/gitlab/gollum.rb
+++ b/ruby/lib/gitlab/gollum.rb
@@ -10,4 +10,28 @@ module Gollum
       index.send(name, *args) # rubocop:disable GitlabSecurity/PublicSend
     end
   end
+
+  class Wiki
+    def pages(treeish=nil, limit: nil)
+      tree_list((treeish || @ref), limit: limit)
+    end
+
+    def tree_list(ref, limit: nil)
+      if (sha = @access.ref_to_sha(ref))
+        commit = @access.commit(sha)
+        tree_map_for(sha).each_with_object([]) do |entry, list|
+          next list unless @page_class.valid_page_name?(entry.name)
+
+          list << entry.page(self, commit)
+          break list if limit && list.size >= limit
+
+          list
+        end
+      else
+        []
+      end
+    end
+  end
 end
+
+Gollum::Page.per_page = 20 # Magic number from Kaminari.config.default_per_page
diff --git a/ruby/vendor/gitlab_git/REVISION b/ruby/vendor/gitlab_git/REVISION
index 32820efd3..9f671e5ad 100644
--- a/ruby/vendor/gitlab_git/REVISION
+++ b/ruby/vendor/gitlab_git/REVISION
@@ -1 +1 @@
-f7537ce03a29b1af85461b2883c33bb41b2382d5
+c594659fea15c6dd17b9ea4c6b88c5a418f10ab9
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/encoding_helper.rb b/ruby/vendor/gitlab_git/lib/gitlab/encoding_helper.rb
index 99dfee3dd..582028493 100644
--- a/ruby/vendor/gitlab_git/lib/gitlab/encoding_helper.rb
+++ b/ruby/vendor/gitlab_git/lib/gitlab/encoding_helper.rb
@@ -17,6 +17,10 @@ module Gitlab
       return nil unless message.respond_to?(:force_encoding)
       return message if message.encoding == Encoding::UTF_8 && message.valid_encoding?
 
+      if message.respond_to?(:frozen?) && message.frozen?
+        message = message.dup
+      end
+
       message.force_encoding("UTF-8")
       return message if message.valid_encoding?
 
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/blob.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/blob.rb
index cc6c7609e..228d97a87 100644
--- a/ruby/vendor/gitlab_git/lib/gitlab/git/blob.rb
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git/blob.rb
@@ -49,6 +49,7 @@ module Gitlab
         # Keep in mind that this method may allocate a lot of memory. It is up
         # to the caller to limit the number of blobs and blob_size_limit.
         #
+        # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/798
         def batch(repository, blob_references, blob_size_limit: nil)
           blob_size_limit ||= MAX_DATA_DISPLAY_SIZE
           blob_references.map do |sha, path|
@@ -102,6 +103,7 @@ module Gitlab
 
           if path_arr.size > 1
             return nil unless entry[:type] == :tree
+
             path_arr.shift
             find_entry_by_path(repository, entry[:oid], path_arr.join('/'))
           else
@@ -178,6 +180,8 @@ module Gitlab
               )
             end
           end
+        rescue Rugged::ReferenceError
+          nil
         end
 
         def rugged_raw(repository, sha, limit:)
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb
index 14a8d5515..dbc087472 100644
--- a/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb
@@ -104,7 +104,7 @@ module Gitlab
       end
 
       def exists?
-        Gitlab::GitalyClient.migrate(:repository_exists) do |enabled|
+        Gitlab::GitalyClient.migrate(:repository_exists, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
           if enabled
             gitaly_repository_client.exists?
           else
@@ -304,7 +304,13 @@ module Gitlab
       end
 
       def delete_all_refs_except(prefixes)
-        delete_refs(*all_ref_names_except(prefixes))
+        gitaly_migrate(:ref_delete_refs) do |is_enabled|
+          if is_enabled
+            gitaly_ref_client.delete_refs(except_with_prefixes: prefixes)
+          else
+            delete_refs(*all_ref_names_except(prefixes))
+          end
+        end
       end
 
       # Returns an Array of all ref names, except when it's matching pattern
@@ -499,7 +505,7 @@ module Gitlab
 
       # Counts the amount of commits between `from` and `to`.
       def count_commits_between(from, to)
-        Commit.between(self, from, to).size
+        count_commits(ref: "#{from}..#{to}")
       end
 
       # Returns the SHA of the most recent common ancestor of +from+ and +to+
@@ -984,6 +990,10 @@ module Gitlab
         @attributes.attributes(path)
       end
 
+      def gitattribute(path, name)
+        attributes(path)[name]
+      end
+
       def languages(ref = nil)
         Gitlab::GitalyClient.migrate(:commit_languages) do |is_enabled|
           if is_enabled
@@ -1036,9 +1046,15 @@ module Gitlab
       end
 
       def with_repo_tmp_commit(start_repository, start_branch_name, sha)
+        source_ref = start_branch_name
+
+        unless Gitlab::Git.branch_ref?(source_ref)
+          source_ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{source_ref}"
+        end
+
         tmp_ref = fetch_ref(
           start_repository,
-          source_ref: "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}",
+          source_ref: source_ref,
           target_ref: "refs/tmp/#{SecureRandom.hex}"
         )
 
@@ -1048,12 +1064,11 @@ module Gitlab
       end
 
       def fetch_source_branch!(source_repository, source_branch, local_ref)
-        with_repo_branch_commit(source_repository, source_branch) do |commit|
-          if commit
-            write_ref(local_ref, commit.sha)
-            true
+        Gitlab::GitalyClient.migrate(:fetch_source_branch) do |is_enabled|
+          if is_enabled
+            gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref)
           else
-            false
+            rugged_fetch_source_branch(source_repository, source_branch, local_ref)
           end
         end
       end
@@ -1141,16 +1156,23 @@ module Gitlab
         @has_visible_content = has_local_branches?
       end
 
-      def fetch(remote = 'origin')
-        args = %W(#{Gitlab.config.git.bin_path} fetch #{remote})
-
-        popen(args, @path).last.zero?
+      # Like all public `Gitlab::Git::Repository` methods, this method is part
+      # of `Repository`'s interface through `method_missing`.
+      # `Repository` has its own `fetch_remote` which uses `gitlab-shell` and
+      # takes some extra attributes, so we qualify this method name to prevent confusion.
+      def fetch_remote_without_shell(remote = 'origin')
+        run_git(['fetch', remote]).last.zero?
       end
 
       def blob_at(sha, path)
         Gitlab::Git::Blob.find(self, sha, path) unless Gitlab::Git.blank_ref?(sha)
       end
 
+      # Items should be of format [[commit_id, path], [commit_id1, path1]]
+      def batch_blobs(items, blob_size_limit: nil)
+        Gitlab::Git::Blob.batch(self, items, blob_size_limit: blob_size_limit)
+      end
+
       def commit_index(user, branch_name, index, options)
         committer = user_to_committer(user)
 
@@ -1201,6 +1223,17 @@ module Gitlab
 
       private
 
+      def rugged_fetch_source_branch(source_repository, source_branch, local_ref)
+        with_repo_branch_commit(source_repository, source_branch) do |commit|
+          if commit
+            write_ref(local_ref, commit.sha)
+            true
+          else
+            false
+          end
+        end
+      end
+
       # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'.
       def branches_filter(filter: nil, sort_by: nil)
         # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37464
@@ -1218,11 +1251,25 @@ module Gitlab
         sort_branches(branches, sort_by)
       end
 
+      # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/695
       def git_merged_branch_names(branch_names = [])
-        lines = run_git(['branch', '--merged', root_ref] + branch_names)
-          .first.lines
+        return [] unless root_ref
 
-        lines.map(&:strip)
+        root_sha = find_branch(root_ref)&.target
+
+        return [] unless root_sha
+
+        git_arguments =
+          %W[branch --merged #{root_sha}
+             --format=%(refname:short)\ %(objectname)] + branch_names
+
+        lines = run_git(git_arguments).first.lines
+
+        lines.each_with_object([]) do |line, branches|
+          name, sha = line.strip.split(' ', 2)
+
+          branches << name if sha != root_sha
+        end
       end
 
       def log_using_shell?(options)
@@ -1376,6 +1423,7 @@ module Gitlab
             end
 
             return nil unless tmp_entry.type == :tree
+
             tmp_entry = tmp_entry[dir]
           end
         end
@@ -1496,6 +1544,7 @@ module Gitlab
       # Ref names must start with `refs/`.
       def rugged_ref_exists?(ref_name)
         raise ArgumentError, 'invalid refname' unless ref_name.start_with?('refs/')
+
         rugged.references.exist?(ref_name)
       rescue Rugged::ReferenceError
         false
@@ -1562,6 +1611,7 @@ module Gitlab
         Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
       rescue Rugged::ReferenceError => e
         raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/
+
         raise InvalidRef.new("Invalid reference #{start_point}")
       end
 
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb
index 637e7a065..392bef69e 100644
--- a/ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb
@@ -1,38 +1,47 @@
 module Gitlab
   module Git
     module RepositoryMirroring
-      IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze
-      IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze
-      MIRROR_REMOTE = 'mirror'.freeze
+      REFMAPS = {
+        # With `:all_refs`, the repository is equivalent to the result of `git clone --mirror`
+        all_refs: '+refs/*:refs/*',
+        heads: '+refs/heads/*:refs/heads/*',
+        tags: '+refs/tags/*:refs/tags/*'
+      }.freeze
 
       RemoteError = Class.new(StandardError)
 
-      def set_remote_as_mirror(remote_name)
-        # This is used to define repository as equivalent as "git clone --mirror"
-        rugged.config["remote.#{remote_name}.fetch"] = 'refs/*:refs/*'
-        rugged.config["remote.#{remote_name}.mirror"] = true
-        rugged.config["remote.#{remote_name}.prune"] = true
-      end
-
-      def set_import_remote_as_mirror(remote_name)
-        # Add first fetch with Rugged so it does not create its own.
-        rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS
-
-        add_remote_fetch_config(remote_name, IMPORT_TAG_REFS)
+      def set_remote_as_mirror(remote_name, refmap: :all_refs)
+        set_remote_refmap(remote_name, refmap)
 
         rugged.config["remote.#{remote_name}.mirror"] = true
         rugged.config["remote.#{remote_name}.prune"] = true
       end
 
-      def add_remote_fetch_config(remote_name, refspec)
-        run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
+      def set_remote_refmap(remote_name, refmap)
+        Array(refmap).each_with_index do |refspec, i|
+          refspec = REFMAPS[refspec] || refspec
+
+          # We need multiple `fetch` entries, but Rugged only allows replacing a config, not adding to it.
+          # To make sure we start from scratch, we set the first using rugged, and use `git` for any others
+          if i == 0
+            rugged.config["remote.#{remote_name}.fetch"] = refspec
+          else
+            run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
+          end
+        end
       end
 
-      def fetch_mirror(url)
-        add_remote(MIRROR_REMOTE, url)
-        set_remote_as_mirror(MIRROR_REMOTE)
-        fetch(MIRROR_REMOTE)
-        remove_remote(MIRROR_REMOTE)
+      # Like all_refs public `Gitlab::Git::Repository` methods, this method is part
+      # of `Repository`'s interface through `method_missing`.
+      # `Repository` has its own `fetch_as_mirror` which uses `gitlab-shell` and
+      # takes some extra attributes, so we qualify this method name to prevent confusion.
+      def fetch_as_mirror_without_shell(url)
+        remote_name = "tmp-#{SecureRandom.hex}"
+        add_remote(remote_name, url)
+        set_remote_as_mirror(remote_name)
+        fetch_remote_without_shell(remote_name)
+      ensure
+        remove_remote(remote_name) if remote_name
       end
 
       def remote_tags(remote)
@@ -78,7 +87,7 @@ module Gitlab
 
       def list_remote_tags(remote)
         tag_list, exit_code, error = nil
-        cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{full_path} ls-remote --tags #{remote})
+        cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-remote --tags #{remote})
 
         Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr|
           tag_list  = stdout.read
@@ -88,7 +97,7 @@ module Gitlab
 
         raise RemoteError, error unless exit_code.zero?
 
-        tag_list.split('\n')
+        tag_list.split("\n")
       end
     end
   end
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/user.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/user.rb
index e6b61417d..e573cd0e1 100644
--- a/ruby/vendor/gitlab_git/lib/gitlab/git/user.rb
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git/user.rb
@@ -8,7 +8,12 @@ module Gitlab
       end
 
       def self.from_gitaly(gitaly_user)
-        new(gitaly_user.gl_username, gitaly_user.name, gitaly_user.email, gitaly_user.gl_id)
+        new(
+          gitaly_user.gl_username,
+          Gitlab::EncodingHelper.encode!(gitaly_user.name),
+          Gitlab::EncodingHelper.encode!(gitaly_user.email),
+          gitaly_user.gl_id
+        )
       end
 
       def initialize(username, name, email, gl_id)
@@ -23,7 +28,7 @@ module Gitlab
       end
 
       def to_gitaly
-        Gitaly::User.new(gl_username: username, gl_id: gl_id, name: name, email: email)
+        Gitaly::User.new(gl_username: username, gl_id: gl_id, name: name.b, email: email.b)
       end
     end
   end
diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/wiki.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/wiki.rb
index 5d36f16ab..d4a53d32c 100644
--- a/ruby/vendor/gitlab_git/lib/gitlab/git/wiki.rb
+++ b/ruby/vendor/gitlab_git/lib/gitlab/git/wiki.rb
@@ -58,8 +58,14 @@ module Gitlab
         end
       end
 
-      def pages
-        gollum_wiki.pages.map { |gollum_page| new_page(gollum_page) }
+      def pages(limit: nil)
+        @repository.gitaly_migrate(:wiki_get_all_pages, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled|
+          if is_enabled
+            gitaly_get_all_pages
+          else
+            gollum_get_all_pages(limit: limit)
+          end
+        end
       end
 
       def page(title:, version: nil, dir: nil)
@@ -82,14 +88,23 @@ module Gitlab
         end
       end
 
-      def page_versions(page_path)
+      # options:
+      #  :page     - The Integer page number.
+      #  :per_page - The number of items per page.
+      #  :limit    - Total number of items to return.
+      def page_versions(page_path, options = {})
         current_page = gollum_page_by_path(page_path)
-        current_page.versions.map do |gollum_git_commit|
-          gollum_page = gollum_wiki.page(current_page.title, gollum_git_commit.id)
-          new_version(gollum_page, gollum_git_commit.id)
+
+        commits_from_page(current_page, options).map do |gitlab_git_commit|
+          gollum_page = gollum_wiki.page(current_page.title, gitlab_git_commit.id)
+          Gitlab::Git::WikiPageVersion.new(gitlab_git_commit, gollum_page&.format)
         end
       end
 
+      def count_page_versions(page_path)
+        @repository.count_commits(ref: 'HEAD', path: page_path)
+      end
+
       def preview_slug(title, format)
         # Adapted from gollum gem (Gollum::Wiki#preview_page) to avoid
         # using Rugged through a Gollum::Wiki instance
@@ -104,6 +119,22 @@ module Gitlab
 
       private
 
+      # options:
+      #  :page     - The Integer page number.
+      #  :per_page - The number of items per page.
+      #  :limit    - Total number of items to return.
+      def commits_from_page(gollum_page, options = {})
+        unless options[:limit]
+          options[:offset] = ([1, options.delete(:page).to_i].max - 1) * Gollum::Page.per_page
+          options[:limit] = (options.delete(:per_page) || Gollum::Page.per_page).to_i
+        end
+
+        @repository.log(ref: gollum_page.last_version.id,
+                        path: gollum_page.path,
+                        limit: options[:limit],
+                        offset: options[:offset])
+      end
+
       def gollum_wiki
         @gollum_wiki ||= Gollum::Wiki.new(@repository.path)
       end
@@ -120,8 +151,17 @@ module Gitlab
       end
 
       def new_version(gollum_page, commit_id)
-        commit = Gitlab::Git::Commit.find(@repository, commit_id)
-        Gitlab::Git::WikiPageVersion.new(commit, gollum_page&.format)
+        Gitlab::Git::WikiPageVersion.new(version(commit_id), gollum_page&.format)
+      end
+
+      def version(commit_id)
+        commit_find_proc = -> { Gitlab::Git::Commit.find(@repository, commit_id) }
+
+        if RequestStore.active?
+          RequestStore.fetch([:wiki_version_commit, commit_id]) { commit_find_proc.call }
+        else
+          commit_find_proc.call
+        end
       end
 
       def assert_type!(object, klass)
@@ -179,6 +219,10 @@ module Gitlab
         Gitlab::Git::WikiFile.new(gollum_file)
       end
 
+      def gollum_get_all_pages(limit: nil)
+        gollum_wiki.pages(limit: limit).map { |gollum_page| new_page(gollum_page) }
+      end
+
       def gitaly_write_page(name, format, content, commit_details)
         gitaly_wiki_client.write_page(name, format, content, commit_details)
       end
@@ -204,6 +248,12 @@ module Gitlab
 
         Gitlab::Git::WikiFile.new(wiki_file)
       end
+
+      def gitaly_get_all_pages
+        gitaly_wiki_client.get_all_pages.map do |wiki_page, version|
+          Gitlab::Git::WikiPage.new(wiki_page, version)
+        end
+      end
     end
   end
 end