diff options
Diffstat (limited to 'lib/gitlab/git')
| -rw-r--r-- | lib/gitlab/git/blob.rb | 20 | ||||
| -rw-r--r-- | lib/gitlab/git/commit.rb | 64 | ||||
| -rw-r--r-- | lib/gitlab/git/path_helper.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/git/repository.rb | 83 | ||||
| -rw-r--r-- | lib/gitlab/git/repository_mirroring.rb | 6 | ||||
| -rw-r--r-- | lib/gitlab/git/tag.rb | 88 |
6 files changed, 194 insertions, 69 deletions
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index d78d29b7ac6..156d077a69c 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -104,25 +104,22 @@ module Gitlab # file.rb # oid: 4a # # - # Blob.find_entry_by_path(repo, '1a', 'app/file.rb') # => '4a' + # Blob.find_entry_by_path(repo, '1a', 'blog', 'app', 'file.rb') # => '4a' # - def find_entry_by_path(repository, root_id, path) + def find_entry_by_path(repository, root_id, *path_parts) root_tree = repository.lookup(root_id) - # Strip leading slashes - path[%r{^/*}] = '' - path_arr = path.split('/') entry = root_tree.find do |entry| - entry[:name] == path_arr[0] + entry[:name] == path_parts[0] end return nil unless entry - if path_arr.size > 1 + if path_parts.size > 1 return nil unless entry[:type] == :tree - path_arr.shift - find_entry_by_path(repository, entry[:oid], path_arr.join('/')) + path_parts.shift + find_entry_by_path(repository, entry[:oid], *path_parts) else [:blob, :commit].include?(entry[:type]) ? entry : nil end @@ -185,10 +182,13 @@ module Gitlab def find_by_rugged(repository, sha, path, limit:) return unless path + # Strip any leading / characters from the path + path = path.sub(%r{\A/*}, '') + rugged_commit = repository.lookup(sha) root_tree = rugged_commit.tree - blob_entry = find_entry_by_path(repository, root_tree.oid, path) + blob_entry = find_entry_by_path(repository, root_tree.oid, *path.split('/')) return nil unless blob_entry diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index d79a4dbeee4..89f761dd515 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -6,6 +6,7 @@ module Gitlab attr_accessor :raw_commit, :head + MAX_COMMIT_MESSAGE_DISPLAY_SIZE = 10.megabytes MIN_SHA_LENGTH = 7 SERIALIZE_KEYS = [ :id, :message, :parent_ids, @@ -63,9 +64,7 @@ module Gitlab if is_enabled repo.gitaly_commit_client.find_commit(commit_id) else - obj = repo.rev_parse_target(commit_id) - - obj.is_a?(Rugged::Commit) ? obj : nil + rugged_find(repo, commit_id) end end @@ -76,6 +75,12 @@ module Gitlab nil end + def rugged_find(repo, commit_id) + obj = repo.rev_parse_target(commit_id) + + obj.is_a?(Rugged::Commit) ? obj : nil + end + # Get last commit for HEAD # # Ex. @@ -297,11 +302,40 @@ module Gitlab nil end end + + def get_message(repository, commit_id) + BatchLoader.for({ repository: repository, commit_id: commit_id }).batch do |items, loader| + items_by_repo = items.group_by { |i| i[:repository] } + + items_by_repo.each do |repo, items| + commit_ids = items.map { |i| i[:commit_id] } + + messages = get_messages(repository, commit_ids) + + messages.each do |commit_sha, message| + loader.call({ repository: repository, commit_id: commit_sha }, message) + end + end + end + end + + def get_messages(repository, commit_ids) + repository.gitaly_migrate(:commit_messages) do |is_enabled| + if is_enabled + repository.gitaly_commit_client.get_commit_messages(commit_ids) + else + commit_ids.map { |id| [id, rugged_find(repository, id).message] }.to_h + end + end + end end def initialize(repository, raw_commit, head = nil) raise "Nil as raw commit passed" unless raw_commit + @repository = repository + @head = head + case raw_commit when Hash init_from_hash(raw_commit) @@ -312,9 +346,6 @@ module Gitlab else raise "Invalid raw commit type: #{raw_commit.class}" end - - @repository = repository - @head = head end def sha @@ -518,7 +549,7 @@ module Gitlab # TODO: Once gitaly "takes over" Rugged consider separating the # subject from the message to make it clearer when there's one # available but not the other. - @message = (commit.body.presence || commit.subject).dup + @message = message_from_gitaly_body @authored_date = Time.at(commit.author.date.seconds).utc @author_name = commit.author.name.dup @author_email = commit.author.email.dup @@ -570,6 +601,25 @@ module Gitlab def refs(repo) repo.refs_hash[id] end + + def message_from_gitaly_body + return @raw_commit.subject.dup if @raw_commit.body_size.zero? + return @raw_commit.body.dup if full_body_fetched_from_gitaly? + + if @raw_commit.body_size > MAX_COMMIT_MESSAGE_DISPLAY_SIZE + "#{@raw_commit.subject}\n\n--commit message is too big".strip + else + fetch_body_from_gitaly + end + end + + def full_body_fetched_from_gitaly? + @raw_commit.body.bytesize == @raw_commit.body_size + end + + def fetch_body_from_gitaly + self.class.get_message(@repository, id) + end end end end diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb index 155cf52f050..57b82a37d6c 100644 --- a/lib/gitlab/git/path_helper.rb +++ b/lib/gitlab/git/path_helper.rb @@ -6,7 +6,7 @@ module Gitlab class << self def normalize_path(filename) # Strip all leading slashes so that //foo -> foo - filename[%r{^/*}] = '' + filename = filename.sub(%r{\A/*}, '') # Expand relative paths (e.g. foo/../bar) filename = Pathname.new(filename) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 061865a7acf..2be604a5b13 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -403,10 +403,10 @@ module Gitlab prefix = archive_prefix(ref, commit.id, append_sha: append_sha) { - 'RepoPath' => path, 'ArchivePrefix' => prefix, 'ArchivePath' => archive_file_path(storage_path, commit.id, prefix, format), - 'CommitId' => commit.id + 'CommitId' => commit.id, + 'GitalyRepository' => gitaly_repository.to_h } end @@ -1473,10 +1473,19 @@ module Gitlab def search_files_by_content(query, ref) return [] if empty? || query.blank? - offset = 2 - args = %W(grep -i -I -n -z --before-context #{offset} --after-context #{offset} -E -e #{Regexp.escape(query)} #{ref || root_ref}) + safe_query = Regexp.escape(query) + ref ||= root_ref + + gitaly_migrate(:search_files_by_content) do |is_enabled| + if is_enabled + gitaly_repository_client.search_files_by_content(ref, safe_query) + else + offset = 2 + args = %W(grep -i -I -n -z --before-context #{offset} --after-context #{offset} -E -e #{safe_query} #{ref}) - run_git(args).first.scrub.split(/^--\n/) + run_git(args).first.scrub.split(/^--\n/) + end + end end def can_be_merged?(source_sha, target_branch) @@ -1491,12 +1500,19 @@ module Gitlab def search_files_by_name(query, ref) safe_query = Regexp.escape(query.sub(%r{^/*}, "")) + ref ||= root_ref return [] if empty? || safe_query.blank? - args = %W(ls-tree -r --name-status --full-tree #{ref || root_ref} -- #{safe_query}) + gitaly_migrate(:search_files_by_name) do |is_enabled| + if is_enabled + gitaly_repository_client.search_files_by_name(ref, safe_query) + else + args = %W(ls-tree -r --name-status --full-tree #{ref} -- #{safe_query}) - run_git(args).first.lines.map(&:strip) + run_git(args).first.lines.map(&:strip) + end + end end def find_commits_by_message(query, ref, path, limit, offset) @@ -1572,14 +1588,12 @@ module Gitlab end def checksum - gitaly_migrate(:calculate_checksum, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_repository_client.calculate_checksum - else - calculate_checksum_by_shelling_out - end - end + # The exists? RPC is much cheaper, so we perform this request first + raise NoRepository, "Repository does not exists" unless exists? + + gitaly_repository_client.calculate_checksum + rescue GRPC::NotFound + raise NoRepository # Guard against data races. end private @@ -1948,7 +1962,12 @@ module Gitlab end target_commit = Gitlab::Git::Commit.find(self, ref.target) - Gitlab::Git::Tag.new(self, ref.name, ref.target, target_commit, message) + Gitlab::Git::Tag.new(self, { + name: ref.name, + target: ref.target, + target_commit: target_commit, + message: message + }) end.sort_by(&:name) end @@ -2348,7 +2367,7 @@ module Gitlab end def gitaly_delete_refs(*ref_names) - gitaly_ref_client.delete_refs(refs: ref_names) + gitaly_ref_client.delete_refs(refs: ref_names) if ref_names.any? end def rugged_remove_remote(remote_name) @@ -2478,36 +2497,6 @@ module Gitlab rev_parse_target(ref).oid end - def calculate_checksum_by_shelling_out - raise NoRepository unless exists? - - args = %W(--git-dir=#{path} show-ref --heads --tags) - output, status = run_git(args) - - if status.nil? || !status.zero? - # Non-valid git repositories return 128 as the status code and an error output - raise InvalidRepository if status == 128 && output.to_s.downcase =~ /not a git repository/ - # Empty repositories returns with a non-zero status and an empty output. - raise ChecksumError, output unless output.blank? - - return EMPTY_REPOSITORY_CHECKSUM - end - - refs = output.split("\n") - - result = refs.inject(nil) do |checksum, ref| - value = Digest::SHA1.hexdigest(ref).hex - - if checksum.nil? - value - else - checksum ^ value - end - end - - result.to_s(16) - end - def build_git_cmd(*args) object_directories = alternate_object_directories.join(File::PATH_SEPARATOR) diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb index 8a01f92e2af..e35ea5762eb 100644 --- a/lib/gitlab/git/repository_mirroring.rb +++ b/lib/gitlab/git/repository_mirroring.rb @@ -35,7 +35,11 @@ module Gitlab next if name =~ /\^\{\}\Z/ target_commit = Gitlab::Git::Commit.find(self, target) - Gitlab::Git::Tag.new(self, name, target, target_commit) + Gitlab::Git::Tag.new(self, { + name: name, + target: target, + target_commit: target_commit + }) end.compact end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index 8a8f7b051ed..e44284572fd 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -1,17 +1,99 @@ module Gitlab module Git class Tag < Ref - attr_reader :object_sha + extend Gitlab::EncodingHelper + + attr_reader :object_sha, :repository + + MAX_TAG_MESSAGE_DISPLAY_SIZE = 10.megabytes + SERIALIZE_KEYS = %i[name target target_commit message].freeze + + attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator + + class << self + def get_message(repository, tag_id) + BatchLoader.for({ repository: repository, tag_id: tag_id }).batch do |items, loader| + items_by_repo = items.group_by { |i| i[:repository] } + + items_by_repo.each do |repo, items| + tag_ids = items.map { |i| i[:tag_id] } + + messages = get_messages(repository, tag_ids) + + messages.each do |id, message| + loader.call({ repository: repository, tag_id: id }, message) + end + end + end + end + + def get_messages(repository, tag_ids) + repository.gitaly_migrate(:tag_messages) do |is_enabled| + if is_enabled + repository.gitaly_ref_client.get_tag_messages(tag_ids) + else + tag_ids.map do |id| + tag = repository.rugged.lookup(id) + message = tag.is_a?(Rugged::Commit) ? "" : tag.message + + [id, message] + end.to_h + end + end + end + end + + def initialize(repository, raw_tag) + @repository = repository + @raw_tag = raw_tag + + case raw_tag + when Hash + init_from_hash + when Gitaly::Tag + init_from_gitaly + end - def initialize(repository, name, target, target_commit, message = nil) super(repository, name, target, target_commit) + end + + def init_from_hash + raw_tag = @raw_tag.symbolize_keys + + SERIALIZE_KEYS.each do |key| + send("#{key}=", raw_tag[key]) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def init_from_gitaly + @name = encode!(@raw_tag.name.dup) + @target = @raw_tag.id + @message = message_from_gitaly_tag - @message = message + if @raw_tag.target_commit.present? + @target_commit = Gitlab::Git::Commit.decorate(repository, @raw_tag.target_commit) + end end def message encode! @message end + + private + + def message_from_gitaly_tag + return @raw_tag.message.dup if full_message_fetched_from_gitaly? + + if @raw_tag.message_size > MAX_TAG_MESSAGE_DISPLAY_SIZE + '--tag message is too big' + else + self.class.get_message(@repository, target) + end + end + + def full_message_fetched_from_gitaly? + @raw_tag.message.bytesize == @raw_tag.message_size + end end end end |