Add latest changes from gitlab-org/gitlab@13-2-stable-ee

9 files changed, 165 insertions, 82 deletions

diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb
index 8c704c2ceea..c66b3335d89 100644
--- a/lib/gitlab/gitaly_client/blob_service.rb
+++ b/lib/gitlab/gitaly_client/blob_service.rb
@@ -16,27 +16,7 @@ module Gitlab
           limit: limit
         )
         response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_blob, request, timeout: GitalyClient.fast_timeout)
-
-        data = []
-        blob = nil
-        response.each do |msg|
-          if blob.nil?
-            blob = msg
-          end
-
-          data << msg.data
-        end
-
-        return if blob.oid.blank?
-
-        data = data.join
-
-        Gitlab::Git::Blob.new(
-          id: blob.oid,
-          size: blob.size,
-          data: data,
-          binary: Gitlab::Git::Blob.binary?(data)
-        )
+        consume_blob_response(response)
       end
 
       def batch_lfs_pointers(blob_ids)
@@ -48,7 +28,6 @@ module Gitlab
         )
 
         response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request, timeout: GitalyClient.medium_timeout)
-
         map_lfs_pointers(response)
       end
 
@@ -70,8 +49,7 @@ module Gitlab
           :blob_service,
           :get_blobs,
           request,
-          timeout: GitalyClient.fast_timeout
-        )
+          timeout: GitalyClient.fast_timeout)
 
         GitalyClient::BlobsStitcher.new(response)
       end
@@ -96,7 +74,6 @@ module Gitlab
           request,
           timeout: GitalyClient.fast_timeout
         )
-
         map_blob_types(response)
       end
 
@@ -127,7 +104,6 @@ module Gitlab
           request,
           timeout: timeout
         )
-
         map_lfs_pointers(response)
       end
 
@@ -137,12 +113,34 @@ module Gitlab
         )
 
         response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_all_lfs_pointers, request, timeout: GitalyClient.medium_timeout)
-
         map_lfs_pointers(response)
       end
 
       private
 
+      def consume_blob_response(response)
+        data = []
+        blob = nil
+        response.each do |msg|
+          if blob.nil?
+            blob = msg
+          end
+
+          data << msg.data
+        end
+
+        return if blob.oid.blank?
+
+        data = data.join
+
+        Gitlab::Git::Blob.new(
+          id: blob.oid,
+          size: blob.size,
+          data: data,
+          binary: Gitlab::Git::Blob.binary?(data)
+        )
+      end
+
       def map_lfs_pointers(response)
         response.flat_map do |message|
           message.lfs_pointers.map do |lfs_pointer|
diff --git a/lib/gitlab/gitaly_client/call.rb b/lib/gitlab/gitaly_client/call.rb
new file mode 100644
index 00000000000..9d4d86997ad
--- /dev/null
+++ b/lib/gitlab/gitaly_client/call.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GitalyClient
+    class Call
+      def initialize(storage, service, rpc, request, remote_storage, timeout)
+        @storage = storage
+        @service = service
+        @rpc = rpc
+        @request = request
+        @remote_storage = remote_storage
+        @timeout = timeout
+        @duration = 0
+      end
+
+      def call(&block)
+        response = recording_request do
+          GitalyClient.execute(@storage, @service, @rpc, @request, remote_storage: @remote_storage, timeout: @timeout, &block)
+        end
+
+        if response.is_a?(Enumerator)
+          # When the given response is an enumerator (coming from streamed
+          # responses), we wrap it in order to properly measure the stream
+          # consumption as it happens.
+          #
+          # store_timings is not called in that scenario as needs to be
+          # handled lazily in the custom Enumerator context.
+          instrument_stream(response)
+        else
+          store_timings
+          response
+        end
+      rescue => err
+        store_timings
+        raise err
+      end
+
+      private
+
+      def instrument_stream(response)
+        Enumerator.new do |yielder|
+          loop do
+            value = recording_request { response.next }
+
+            yielder.yield(value)
+          end
+        ensure
+          store_timings
+        end
+      end
+
+      def recording_request
+        start = Gitlab::Metrics::System.monotonic_time
+
+        yield
+      ensure
+        @duration += Gitlab::Metrics::System.monotonic_time - start
+      end
+
+      def store_timings
+        GitalyClient.add_query_time(@duration)
+
+        return unless Gitlab::PerformanceBar.enabled_for_request?
+
+        request_hash = @request.is_a?(Google::Protobuf::MessageExts) ? @request.to_h : {}
+
+        GitalyClient.add_call_details(feature: "#{@service}##{@rpc}", duration: @duration, request: request_hash, rpc: @rpc,
+                                      backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gitaly_client/cleanup_service.rb b/lib/gitlab/gitaly_client/cleanup_service.rb
index e2293d3121a..649aaa46362 100644
--- a/lib/gitlab/gitaly_client/cleanup_service.rb
+++ b/lib/gitlab/gitaly_client/cleanup_service.rb
@@ -13,15 +13,14 @@ module Gitlab
       end
 
       def apply_bfg_object_map_stream(io, &blk)
-        responses = GitalyClient.call(
+        response = GitalyClient.call(
           storage,
           :cleanup_service,
           :apply_bfg_object_map_stream,
           build_object_map_enum(io),
           timeout: GitalyClient.long_timeout
         )
-
-        responses.each(&blk)
+        response.each(&blk)
       end
 
       private
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index aed132aaca0..464d2519b27 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -73,7 +73,6 @@ module Gitlab
       def commit_deltas(commit)
         request = Gitaly::CommitDeltaRequest.new(diff_from_parent_request_params(commit))
         response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request, timeout: GitalyClient.fast_timeout)
-
         response.flat_map { |msg| msg.deltas }
       end
 
@@ -162,13 +161,14 @@ module Gitlab
         [response.left_count, response.right_count]
       end
 
-      def list_last_commits_for_tree(revision, path, offset: 0, limit: 25)
+      def list_last_commits_for_tree(revision, path, offset: 0, limit: 25, literal_pathspec: false)
         request = Gitaly::ListLastCommitsForTreeRequest.new(
           repository: @gitaly_repo,
           revision: encode_binary(revision),
           path: encode_binary(path.to_s),
           offset: offset,
-          limit: limit
+          limit: limit,
+          global_options: parse_global_options!(literal_pathspec: literal_pathspec)
         )
 
         response = GitalyClient.call(@repository.storage, :commit_service, :list_last_commits_for_tree, request, timeout: GitalyClient.medium_timeout)
@@ -180,11 +180,12 @@ module Gitlab
         end
       end
 
-      def last_commit_for_path(revision, path)
+      def last_commit_for_path(revision, path, literal_pathspec: false)
         request = Gitaly::LastCommitForPathRequest.new(
           repository: @gitaly_repo,
           revision: encode_binary(revision),
-          path: encode_binary(path.to_s)
+          path: encode_binary(path.to_s),
+          global_options: parse_global_options!(literal_pathspec: literal_pathspec)
         )
 
         gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request, timeout: GitalyClient.fast_timeout).commit
@@ -200,9 +201,8 @@ module Gitlab
           to: to
         )
 
-        GitalyClient.streaming_call(@repository.storage, :commit_service, :commits_between, request, timeout: GitalyClient.medium_timeout) do |response|
-          consume_commits_response(response)
-        end
+        response = GitalyClient.call(@repository.storage, :commit_service, :commits_between, request, timeout: GitalyClient.medium_timeout)
+        consume_commits_response(response)
       end
 
       def diff_stats(left_commit_sha, right_commit_sha)
@@ -212,9 +212,8 @@ module Gitlab
           right_commit_id: right_commit_sha
         )
 
-        GitalyClient.streaming_call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout) do |response|
-          response.flat_map(&:stats)
-        end
+        response = GitalyClient.call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout)
+        response.flat_map(&:stats)
       end
 
       def find_all_commits(opts = {})
@@ -226,9 +225,8 @@ module Gitlab
         )
         request.order = opts[:order].upcase if opts[:order].present?
 
-        GitalyClient.streaming_call(@repository.storage, :commit_service, :find_all_commits, request, timeout: GitalyClient.medium_timeout) do |response|
-          consume_commits_response(response)
-        end
+        response = GitalyClient.call(@repository.storage, :commit_service, :find_all_commits, request, timeout: GitalyClient.medium_timeout)
+        consume_commits_response(response)
       end
 
       def list_commits_by_oid(oids)
@@ -236,26 +234,25 @@ module Gitlab
 
         request = Gitaly::ListCommitsByOidRequest.new(repository: @gitaly_repo, oid: oids)
 
-        GitalyClient.streaming_call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout) do |response|
-          consume_commits_response(response)
-        end
+        response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout)
+        consume_commits_response(response)
       rescue GRPC::NotFound # If no repository is found, happens mainly during testing
         []
       end
 
-      def commits_by_message(query, revision: '', path: '', limit: 1000, offset: 0)
+      def commits_by_message(query, revision: '', path: '', limit: 1000, offset: 0, literal_pathspec: true)
         request = Gitaly::CommitsByMessageRequest.new(
           repository: @gitaly_repo,
           query: query,
           revision: encode_binary(revision),
           path: encode_binary(path),
           limit: limit.to_i,
-          offset: offset.to_i
+          offset: offset.to_i,
+          global_options: parse_global_options!(literal_pathspec: literal_pathspec)
         )
 
-        GitalyClient.streaming_call(@repository.storage, :commit_service, :commits_by_message, request, timeout: GitalyClient.medium_timeout) do |response|
-          consume_commits_response(response)
-        end
+        response = GitalyClient.call(@repository.storage, :commit_service, :commits_by_message, request, timeout: GitalyClient.medium_timeout)
+        consume_commits_response(response)
       end
 
       def languages(ref = nil)
@@ -320,6 +317,7 @@ module Gitlab
           skip_merges:  options[:skip_merges],
           all:          !!options[:all],
           first_parent: !!options[:first_parent],
+          global_options: parse_global_options!(options),
           disable_walk: true # This option is deprecated. The 'walk' implementation is being removed.
         )
         request.after    = GitalyClient.timestamp(options[:after]) if options[:after]
@@ -330,9 +328,8 @@ module Gitlab
 
         request.paths = encode_repeated(Array(options[:path])) if options[:path].present?
 
-        GitalyClient.streaming_call(@repository.storage, :commit_service, :find_commits, request, timeout: GitalyClient.medium_timeout) do |response|
-          consume_commits_response(response)
-        end
+        response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request, timeout: GitalyClient.medium_timeout)
+        consume_commits_response(response)
       end
 
       def filter_shas_with_signatures(shas)
@@ -349,7 +346,6 @@ module Gitlab
         end
 
         response = GitalyClient.call(@repository.storage, :commit_service, :filter_shas_with_signatures, enum, timeout: GitalyClient.fast_timeout)
-
         response.flat_map do |msg|
           msg.shas.map { |sha| EncodingHelper.encode!(sha) }
         end
@@ -390,8 +386,28 @@ module Gitlab
         messages
       end
 
+      def list_commits_by_ref_name(refs)
+        request = Gitaly::ListCommitsByRefNameRequest
+          .new(repository: @gitaly_repo, ref_names: refs.map { |ref| encode_binary(ref) })
+
+        response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_ref_name, request, timeout: GitalyClient.medium_timeout)
+
+        commit_refs = response.flat_map do |message|
+          message.commit_refs.map do |commit_ref|
+            [encode_utf8(commit_ref.ref_name), Gitlab::Git::Commit.new(@repository, commit_ref.commit)]
+          end
+        end
+
+        Hash[commit_refs]
+      end
+
       private
 
+      def parse_global_options!(options)
+        literal_pathspec = options.delete(:literal_pathspec)
+        Gitaly::GlobalOptions.new(literal_pathspecs: literal_pathspec)
+      end
+
       def call_commit_diff(request_params, options = {})
         request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false)
         request_params[:enforce_limits] = options.fetch(:limits, true)
diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb
index f7eb4b45197..6f08dcc69b6 100644
--- a/lib/gitlab/gitaly_client/conflicts_service.rb
+++ b/lib/gitlab/gitaly_client/conflicts_service.rb
@@ -21,7 +21,6 @@ module Gitlab
           their_commit_oid: @their_commit_oid
         )
         response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request, timeout: GitalyClient.long_timeout)
-
         GitalyClient::ConflictFilesStitcher.new(response, @gitaly_repo)
       end
 
diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb
index 9ed4b2da09a..87505418ae9 100644
--- a/lib/gitlab/gitaly_client/operation_service.rb
+++ b/lib/gitlab/gitaly_client/operation_service.rb
@@ -178,6 +178,10 @@ module Gitlab
           timeout: GitalyClient.long_timeout
         )
 
+        if response.pre_receive_error.present?
+          raise Gitlab::Git::PreReceiveError.new("GL-HOOK-ERR: pre-receive hook failed.")
+        end
+
         Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update)
       rescue GRPC::FailedPrecondition => e
         raise Gitlab::Git::CommitError, e
diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb
index 63def4e29c9..97b6813c080 100644
--- a/lib/gitlab/gitaly_client/ref_service.rb
+++ b/lib/gitlab/gitaly_client/ref_service.rb
@@ -15,14 +15,12 @@ module Gitlab
       def branches
         request = Gitaly::FindAllBranchesRequest.new(repository: @gitaly_repo)
         response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request, timeout: GitalyClient.fast_timeout)
-
         consume_find_all_branches_response(response)
       end
 
       def remote_branches(remote_name)
         request = Gitaly::FindAllRemoteBranchesRequest.new(repository: @gitaly_repo, remote_name: remote_name)
-        response = GitalyClient.call(@repository.storage, :ref_service, :find_all_remote_branches, request, timeout: GitalyClient.medium_timeout)
-
+        response = GitalyClient.call(@storage, :ref_service, :find_all_remote_branches, request, timeout: GitalyClient.medium_timeout)
         consume_find_all_remote_branches_response(remote_name, response)
       end
 
@@ -33,7 +31,6 @@ module Gitlab
           merged_branches: branch_names.map { |s| encode_binary(s) }
         )
         response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request, timeout: GitalyClient.fast_timeout)
-
         consume_find_all_branches_response(response)
       end
 
@@ -71,10 +68,9 @@ module Gitlab
           commit_id: newrev
         )
 
-        response = GitalyClient
-          .call(@storage, :ref_service, :list_new_commits, request, timeout: GitalyClient.medium_timeout)
-
         commits = []
+
+        response = GitalyClient.call(@storage, :ref_service, :list_new_commits, request, timeout: GitalyClient.medium_timeout)
         response.each do |msg|
           msg.commits.each do |c|
             commits << Gitlab::Git::Commit.new(@repository, c)
@@ -98,9 +94,7 @@ module Gitlab
             GitalyClient.medium_timeout
           end
 
-        response = GitalyClient
-          .call(@storage, :ref_service, :list_new_blobs, request, timeout: timeout)
-
+        response = GitalyClient.call(@storage, :ref_service, :list_new_blobs, request, timeout: timeout)
         response.flat_map do |msg|
           # Returns an Array of Gitaly::NewBlobObject objects
           # Available methods are: #size, #oid and #path
@@ -116,8 +110,8 @@ module Gitlab
         branch_names.count
       end
 
-      def local_branches(sort_by: nil)
-        request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo)
+      def local_branches(sort_by: nil, pagination_params: nil)
+        request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo, pagination_params: pagination_params)
         request.sort_by = sort_by_param(sort_by) if sort_by
         response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request, timeout: GitalyClient.fast_timeout)
         consume_find_local_branches_response(response)
@@ -171,9 +165,8 @@ module Gitlab
           limit: limit
         )
 
-        stream = GitalyClient.call(@repository.storage, :ref_service, :list_tag_names_containing_commit, request, timeout: GitalyClient.medium_timeout)
-
-        consume_ref_contains_sha_response(stream, :tag_names)
+        response = GitalyClient.call(@storage, :ref_service, :list_tag_names_containing_commit, request, timeout: GitalyClient.medium_timeout)
+        consume_ref_contains_sha_response(response, :tag_names)
       end
 
       # Limit: 0 implies no limit, thus all tag names will be returned
@@ -184,18 +177,16 @@ module Gitlab
           limit: limit
         )
 
-        stream = GitalyClient.call(@repository.storage, :ref_service, :list_branch_names_containing_commit, request, timeout: GitalyClient.medium_timeout)
-
-        consume_ref_contains_sha_response(stream, :branch_names)
+        response = GitalyClient.call(@storage, :ref_service, :list_branch_names_containing_commit, request, timeout: GitalyClient.medium_timeout)
+        consume_ref_contains_sha_response(response, :branch_names)
       end
 
       def get_tag_messages(tag_ids)
         request = Gitaly::GetTagMessagesRequest.new(repository: @gitaly_repo, tag_ids: tag_ids)
-        response = GitalyClient.call(@repository.storage, :ref_service, :get_tag_messages, request, timeout: GitalyClient.fast_timeout)
-
         messages = Hash.new { |h, k| h[k] = +''.b }
         current_tag_id = nil
 
+        response = GitalyClient.call(@storage, :ref_service, :get_tag_messages, request, timeout: GitalyClient.fast_timeout)
         response.each do |rpc_message|
           current_tag_id = rpc_message.tag_id if rpc_message.tag_id.present?
 
diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb
index 4566c59bbe0..06aaf460751 100644
--- a/lib/gitlab/gitaly_client/remote_service.rb
+++ b/lib/gitlab/gitaly_client/remote_service.rb
@@ -8,9 +8,11 @@ module Gitlab
       MAX_MSG_SIZE = 128.kilobytes.freeze
 
       def self.exists?(remote_url)
-        request = Gitaly::FindRemoteRepositoryRequest.new(remote: remote_url)
+        storage = GitalyClient.random_storage
 
-        response = GitalyClient.call(GitalyClient.random_storage,
+        request = Gitaly::FindRemoteRepositoryRequest.new(remote: remote_url, storage_name: storage)
+
+        response = GitalyClient.call(storage,
                                      :remote_service,
                                      :find_remote_repository, request,
                                      timeout: GitalyClient.medium_timeout)
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index f74c9ea4192..20ad6d0184b 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -201,9 +201,9 @@ module Gitlab
         response = GitalyClient.call(@storage, :repository_service, :fsck, request, timeout: GitalyClient.long_timeout)
 
         if response.error.empty?
-          return "", 0
+          ["", 0]
         else
-          return response.error.b, 1
+          [response.error.b, 1]
         end
       end
 
@@ -335,7 +335,6 @@ module Gitlab
       def search_files_by_content(ref, query, options = {})
         request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query)
         response = GitalyClient.call(@storage, :repository_service, :search_files_by_content, request, timeout: GitalyClient.default_timeout)
-
         search_results_from_response(response, options)
       end
 
@@ -410,7 +409,10 @@ module Gitlab
           request,
           timeout: timeout
         )
+        write_stream_to_file(response, save_path)
+      end
 
+      def write_stream_to_file(response, save_path)
         File.open(save_path, 'wb') do |f|
           response.each do |message|
             f.write(message.data)