diff options
author | Zeger-Jan van de Weg <zegerjan@gitlab.com> | 2017-12-04 14:24:19 +0300 |
---|---|---|
committer | Zeger-Jan van de Weg <zegerjan@gitlab.com> | 2017-12-04 14:24:19 +0300 |
commit | eb1511c83aba99852f573abddf67ca86e67fa8f1 (patch) | |
tree | 41100a6ad21d93c695fc28b551b17137c4e71af2 | |
parent | 143c86e482617e934ae3ad522560564730b0de46 (diff) | |
parent | 8dfabdae7d1688035721423adc7eb1032955b6af (diff) |
Merge branch 'gitlab-git-c594659fea15c6dd17b' into 'master'
Update vendored gitlab_git to c594659fea15c6dd17b
Closes #792
See merge request gitlab-org/gitaly!473
-rw-r--r-- | CHANGELOG.md | 2 | ||||
-rw-r--r-- | ruby/lib/gitaly_server/commit_service.rb | 2 | ||||
-rw-r--r-- | ruby/lib/gitaly_server/operations_service.rb | 2 | ||||
-rw-r--r-- | ruby/lib/gitaly_server/repository_service.rb | 6 | ||||
-rw-r--r-- | ruby/lib/gitlab/git.rb | 6 | ||||
-rw-r--r-- | ruby/lib/gitlab/gitaly_client.rb | 4 | ||||
-rw-r--r-- | ruby/lib/gitlab/gollum.rb | 24 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/REVISION | 2 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/lib/gitlab/encoding_helper.rb | 4 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/lib/gitlab/git/blob.rb | 4 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb | 82 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb | 57 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/lib/gitlab/git/user.rb | 9 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/lib/gitlab/git/wiki.rb | 66 |
14 files changed, 213 insertions, 57 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f40669ea..2cf4f102a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ UNRELEASED +- Update vendored gitlab_git to c594659fea15c6dd17b + https://gitlab.com/gitlab-org/gitaly/merge_requests/473 - More logging in housekeeping https://gitlab.com/gitlab-org/gitaly/merge_requests/435 diff --git a/ruby/lib/gitaly_server/commit_service.rb b/ruby/lib/gitaly_server/commit_service.rb index 20e49e177..38637e12d 100644 --- a/ruby/lib/gitaly_server/commit_service.rb +++ b/ruby/lib/gitaly_server/commit_service.rb @@ -37,7 +37,7 @@ module GitalyServer commit.is_a?(Rugged::Commit) ? gitaly_commit_from_rugged(commit) : nil end.compact - y.yield Gitaly::ListCommitsByOidResponse.new(commits: commits) + y.yield Gitaly::ListCommitsByOidResponse.new(commits: commits) end end end diff --git a/ruby/lib/gitaly_server/operations_service.rb b/ruby/lib/gitaly_server/operations_service.rb index e5780d9a6..e1db64ce1 100644 --- a/ruby/lib/gitaly_server/operations_service.rb +++ b/ruby/lib/gitaly_server/operations_service.rb @@ -118,7 +118,7 @@ module GitalyServer second_request = session.next unless second_request.apply - raise GRPC::FailedPrecondition, 'merge aborted by client' + raise GRPC::FailedPrecondition.new('merge aborted by client') end end diff --git a/ruby/lib/gitaly_server/repository_service.rb b/ruby/lib/gitaly_server/repository_service.rb index 725714350..cee25cba7 100644 --- a/ruby/lib/gitaly_server/repository_service.rb +++ b/ruby/lib/gitaly_server/repository_service.rb @@ -2,9 +2,9 @@ module GitalyServer class RepositoryService < Gitaly::RepositoryService::Service include Utils - def create_repository(request, _call) + def create_repository(request, call) bridge_exceptions do - repo_path = GitalyServer.repo_path(_call) + repo_path = GitalyServer.repo_path(call) Gitlab::Git::Repository.create(repo_path, bare: true, symlink_hooks_to: Gitlab.config.gitlab_shell.hooks_path) @@ -12,7 +12,7 @@ module GitalyServer end end - def has_local_branches(request, call) + def has_local_branches(request, call) # rubocop:disable Naming/PredicateName repo = Gitlab::Git::Repository.from_gitaly(request.repository, call) Gitaly::HasLocalBranchesResponse.new(value: repo.has_local_branches?) diff --git a/ruby/lib/gitlab/git.rb b/ruby/lib/gitlab/git.rb index d80532e35..0423c9571 100644 --- a/ruby/lib/gitlab/git.rb +++ b/ruby/lib/gitlab/git.rb @@ -137,3 +137,9 @@ class FakeCircuitBreaker yield end end + +class RequestStore + def self.active? + false + end +end diff --git a/ruby/lib/gitlab/gitaly_client.rb b/ruby/lib/gitlab/gitaly_client.rb index 5e7b71559..49d6a3120 100644 --- a/ruby/lib/gitlab/gitaly_client.rb +++ b/ruby/lib/gitlab/gitaly_client.rb @@ -1,7 +1,9 @@ module Gitlab module GitalyClient module MigrationStatus - OPT_IN = :fake_constant + DISABLED = :fake_disabled + OPT_IN = :fake_opt_in + OPT_OUT = :fake_opt_out end class << self diff --git a/ruby/lib/gitlab/gollum.rb b/ruby/lib/gitlab/gollum.rb index 7374925f9..4c8b236fd 100644 --- a/ruby/lib/gitlab/gollum.rb +++ b/ruby/lib/gitlab/gollum.rb @@ -10,4 +10,28 @@ module Gollum index.send(name, *args) # rubocop:disable GitlabSecurity/PublicSend end end + + class Wiki + def pages(treeish=nil, limit: nil) + tree_list((treeish || @ref), limit: limit) + end + + def tree_list(ref, limit: nil) + if (sha = @access.ref_to_sha(ref)) + commit = @access.commit(sha) + tree_map_for(sha).each_with_object([]) do |entry, list| + next list unless @page_class.valid_page_name?(entry.name) + + list << entry.page(self, commit) + break list if limit && list.size >= limit + + list + end + else + [] + end + end + end end + +Gollum::Page.per_page = 20 # Magic number from Kaminari.config.default_per_page diff --git a/ruby/vendor/gitlab_git/REVISION b/ruby/vendor/gitlab_git/REVISION index 32820efd3..9f671e5ad 100644 --- a/ruby/vendor/gitlab_git/REVISION +++ b/ruby/vendor/gitlab_git/REVISION @@ -1 +1 @@ -f7537ce03a29b1af85461b2883c33bb41b2382d5 +c594659fea15c6dd17b9ea4c6b88c5a418f10ab9 diff --git a/ruby/vendor/gitlab_git/lib/gitlab/encoding_helper.rb b/ruby/vendor/gitlab_git/lib/gitlab/encoding_helper.rb index 99dfee3dd..582028493 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/encoding_helper.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/encoding_helper.rb @@ -17,6 +17,10 @@ module Gitlab return nil unless message.respond_to?(:force_encoding) return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? + if message.respond_to?(:frozen?) && message.frozen? + message = message.dup + end + message.force_encoding("UTF-8") return message if message.valid_encoding? diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/blob.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/blob.rb index cc6c7609e..228d97a87 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/blob.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/blob.rb @@ -49,6 +49,7 @@ module Gitlab # Keep in mind that this method may allocate a lot of memory. It is up # to the caller to limit the number of blobs and blob_size_limit. # + # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/798 def batch(repository, blob_references, blob_size_limit: nil) blob_size_limit ||= MAX_DATA_DISPLAY_SIZE blob_references.map do |sha, path| @@ -102,6 +103,7 @@ module Gitlab if path_arr.size > 1 return nil unless entry[:type] == :tree + path_arr.shift find_entry_by_path(repository, entry[:oid], path_arr.join('/')) else @@ -178,6 +180,8 @@ module Gitlab ) end end + rescue Rugged::ReferenceError + nil end def rugged_raw(repository, sha, limit:) diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb index 14a8d5515..dbc087472 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb @@ -104,7 +104,7 @@ module Gitlab end def exists? - Gitlab::GitalyClient.migrate(:repository_exists) do |enabled| + Gitlab::GitalyClient.migrate(:repository_exists, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_repository_client.exists? else @@ -304,7 +304,13 @@ module Gitlab end def delete_all_refs_except(prefixes) - delete_refs(*all_ref_names_except(prefixes)) + gitaly_migrate(:ref_delete_refs) do |is_enabled| + if is_enabled + gitaly_ref_client.delete_refs(except_with_prefixes: prefixes) + else + delete_refs(*all_ref_names_except(prefixes)) + end + end end # Returns an Array of all ref names, except when it's matching pattern @@ -499,7 +505,7 @@ module Gitlab # Counts the amount of commits between `from` and `to`. def count_commits_between(from, to) - Commit.between(self, from, to).size + count_commits(ref: "#{from}..#{to}") end # Returns the SHA of the most recent common ancestor of +from+ and +to+ @@ -984,6 +990,10 @@ module Gitlab @attributes.attributes(path) end + def gitattribute(path, name) + attributes(path)[name] + end + def languages(ref = nil) Gitlab::GitalyClient.migrate(:commit_languages) do |is_enabled| if is_enabled @@ -1036,9 +1046,15 @@ module Gitlab end def with_repo_tmp_commit(start_repository, start_branch_name, sha) + source_ref = start_branch_name + + unless Gitlab::Git.branch_ref?(source_ref) + source_ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{source_ref}" + end + tmp_ref = fetch_ref( start_repository, - source_ref: "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", + source_ref: source_ref, target_ref: "refs/tmp/#{SecureRandom.hex}" ) @@ -1048,12 +1064,11 @@ module Gitlab end def fetch_source_branch!(source_repository, source_branch, local_ref) - with_repo_branch_commit(source_repository, source_branch) do |commit| - if commit - write_ref(local_ref, commit.sha) - true + Gitlab::GitalyClient.migrate(:fetch_source_branch) do |is_enabled| + if is_enabled + gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref) else - false + rugged_fetch_source_branch(source_repository, source_branch, local_ref) end end end @@ -1141,16 +1156,23 @@ module Gitlab @has_visible_content = has_local_branches? end - def fetch(remote = 'origin') - args = %W(#{Gitlab.config.git.bin_path} fetch #{remote}) - - popen(args, @path).last.zero? + # Like all public `Gitlab::Git::Repository` methods, this method is part + # of `Repository`'s interface through `method_missing`. + # `Repository` has its own `fetch_remote` which uses `gitlab-shell` and + # takes some extra attributes, so we qualify this method name to prevent confusion. + def fetch_remote_without_shell(remote = 'origin') + run_git(['fetch', remote]).last.zero? end def blob_at(sha, path) Gitlab::Git::Blob.find(self, sha, path) unless Gitlab::Git.blank_ref?(sha) end + # Items should be of format [[commit_id, path], [commit_id1, path1]] + def batch_blobs(items, blob_size_limit: nil) + Gitlab::Git::Blob.batch(self, items, blob_size_limit: blob_size_limit) + end + def commit_index(user, branch_name, index, options) committer = user_to_committer(user) @@ -1201,6 +1223,17 @@ module Gitlab private + def rugged_fetch_source_branch(source_repository, source_branch, local_ref) + with_repo_branch_commit(source_repository, source_branch) do |commit| + if commit + write_ref(local_ref, commit.sha) + true + else + false + end + end + end + # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. def branches_filter(filter: nil, sort_by: nil) # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37464 @@ -1218,11 +1251,25 @@ module Gitlab sort_branches(branches, sort_by) end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/695 def git_merged_branch_names(branch_names = []) - lines = run_git(['branch', '--merged', root_ref] + branch_names) - .first.lines + return [] unless root_ref - lines.map(&:strip) + root_sha = find_branch(root_ref)&.target + + return [] unless root_sha + + git_arguments = + %W[branch --merged #{root_sha} + --format=%(refname:short)\ %(objectname)] + branch_names + + lines = run_git(git_arguments).first.lines + + lines.each_with_object([]) do |line, branches| + name, sha = line.strip.split(' ', 2) + + branches << name if sha != root_sha + end end def log_using_shell?(options) @@ -1376,6 +1423,7 @@ module Gitlab end return nil unless tmp_entry.type == :tree + tmp_entry = tmp_entry[dir] end end @@ -1496,6 +1544,7 @@ module Gitlab # Ref names must start with `refs/`. def rugged_ref_exists?(ref_name) raise ArgumentError, 'invalid refname' unless ref_name.start_with?('refs/') + rugged.references.exist?(ref_name) rescue Rugged::ReferenceError false @@ -1562,6 +1611,7 @@ module Gitlab Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) rescue Rugged::ReferenceError => e raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ + raise InvalidRef.new("Invalid reference #{start_point}") end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb index 637e7a065..392bef69e 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb @@ -1,38 +1,47 @@ module Gitlab module Git module RepositoryMirroring - IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze - IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze - MIRROR_REMOTE = 'mirror'.freeze + REFMAPS = { + # With `:all_refs`, the repository is equivalent to the result of `git clone --mirror` + all_refs: '+refs/*:refs/*', + heads: '+refs/heads/*:refs/heads/*', + tags: '+refs/tags/*:refs/tags/*' + }.freeze RemoteError = Class.new(StandardError) - def set_remote_as_mirror(remote_name) - # This is used to define repository as equivalent as "git clone --mirror" - rugged.config["remote.#{remote_name}.fetch"] = 'refs/*:refs/*' - rugged.config["remote.#{remote_name}.mirror"] = true - rugged.config["remote.#{remote_name}.prune"] = true - end - - def set_import_remote_as_mirror(remote_name) - # Add first fetch with Rugged so it does not create its own. - rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS - - add_remote_fetch_config(remote_name, IMPORT_TAG_REFS) + def set_remote_as_mirror(remote_name, refmap: :all_refs) + set_remote_refmap(remote_name, refmap) rugged.config["remote.#{remote_name}.mirror"] = true rugged.config["remote.#{remote_name}.prune"] = true end - def add_remote_fetch_config(remote_name, refspec) - run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}]) + def set_remote_refmap(remote_name, refmap) + Array(refmap).each_with_index do |refspec, i| + refspec = REFMAPS[refspec] || refspec + + # We need multiple `fetch` entries, but Rugged only allows replacing a config, not adding to it. + # To make sure we start from scratch, we set the first using rugged, and use `git` for any others + if i == 0 + rugged.config["remote.#{remote_name}.fetch"] = refspec + else + run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}]) + end + end end - def fetch_mirror(url) - add_remote(MIRROR_REMOTE, url) - set_remote_as_mirror(MIRROR_REMOTE) - fetch(MIRROR_REMOTE) - remove_remote(MIRROR_REMOTE) + # Like all_refs public `Gitlab::Git::Repository` methods, this method is part + # of `Repository`'s interface through `method_missing`. + # `Repository` has its own `fetch_as_mirror` which uses `gitlab-shell` and + # takes some extra attributes, so we qualify this method name to prevent confusion. + def fetch_as_mirror_without_shell(url) + remote_name = "tmp-#{SecureRandom.hex}" + add_remote(remote_name, url) + set_remote_as_mirror(remote_name) + fetch_remote_without_shell(remote_name) + ensure + remove_remote(remote_name) if remote_name end def remote_tags(remote) @@ -78,7 +87,7 @@ module Gitlab def list_remote_tags(remote) tag_list, exit_code, error = nil - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{full_path} ls-remote --tags #{remote}) + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-remote --tags #{remote}) Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr| tag_list = stdout.read @@ -88,7 +97,7 @@ module Gitlab raise RemoteError, error unless exit_code.zero? - tag_list.split('\n') + tag_list.split("\n") end end end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/user.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/user.rb index e6b61417d..e573cd0e1 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/user.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/user.rb @@ -8,7 +8,12 @@ module Gitlab end def self.from_gitaly(gitaly_user) - new(gitaly_user.gl_username, gitaly_user.name, gitaly_user.email, gitaly_user.gl_id) + new( + gitaly_user.gl_username, + Gitlab::EncodingHelper.encode!(gitaly_user.name), + Gitlab::EncodingHelper.encode!(gitaly_user.email), + gitaly_user.gl_id + ) end def initialize(username, name, email, gl_id) @@ -23,7 +28,7 @@ module Gitlab end def to_gitaly - Gitaly::User.new(gl_username: username, gl_id: gl_id, name: name, email: email) + Gitaly::User.new(gl_username: username, gl_id: gl_id, name: name.b, email: email.b) end end end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/wiki.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/wiki.rb index 5d36f16ab..d4a53d32c 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/wiki.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/wiki.rb @@ -58,8 +58,14 @@ module Gitlab end end - def pages - gollum_wiki.pages.map { |gollum_page| new_page(gollum_page) } + def pages(limit: nil) + @repository.gitaly_migrate(:wiki_get_all_pages, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| + if is_enabled + gitaly_get_all_pages + else + gollum_get_all_pages(limit: limit) + end + end end def page(title:, version: nil, dir: nil) @@ -82,14 +88,23 @@ module Gitlab end end - def page_versions(page_path) + # options: + # :page - The Integer page number. + # :per_page - The number of items per page. + # :limit - Total number of items to return. + def page_versions(page_path, options = {}) current_page = gollum_page_by_path(page_path) - current_page.versions.map do |gollum_git_commit| - gollum_page = gollum_wiki.page(current_page.title, gollum_git_commit.id) - new_version(gollum_page, gollum_git_commit.id) + + commits_from_page(current_page, options).map do |gitlab_git_commit| + gollum_page = gollum_wiki.page(current_page.title, gitlab_git_commit.id) + Gitlab::Git::WikiPageVersion.new(gitlab_git_commit, gollum_page&.format) end end + def count_page_versions(page_path) + @repository.count_commits(ref: 'HEAD', path: page_path) + end + def preview_slug(title, format) # Adapted from gollum gem (Gollum::Wiki#preview_page) to avoid # using Rugged through a Gollum::Wiki instance @@ -104,6 +119,22 @@ module Gitlab private + # options: + # :page - The Integer page number. + # :per_page - The number of items per page. + # :limit - Total number of items to return. + def commits_from_page(gollum_page, options = {}) + unless options[:limit] + options[:offset] = ([1, options.delete(:page).to_i].max - 1) * Gollum::Page.per_page + options[:limit] = (options.delete(:per_page) || Gollum::Page.per_page).to_i + end + + @repository.log(ref: gollum_page.last_version.id, + path: gollum_page.path, + limit: options[:limit], + offset: options[:offset]) + end + def gollum_wiki @gollum_wiki ||= Gollum::Wiki.new(@repository.path) end @@ -120,8 +151,17 @@ module Gitlab end def new_version(gollum_page, commit_id) - commit = Gitlab::Git::Commit.find(@repository, commit_id) - Gitlab::Git::WikiPageVersion.new(commit, gollum_page&.format) + Gitlab::Git::WikiPageVersion.new(version(commit_id), gollum_page&.format) + end + + def version(commit_id) + commit_find_proc = -> { Gitlab::Git::Commit.find(@repository, commit_id) } + + if RequestStore.active? + RequestStore.fetch([:wiki_version_commit, commit_id]) { commit_find_proc.call } + else + commit_find_proc.call + end end def assert_type!(object, klass) @@ -179,6 +219,10 @@ module Gitlab Gitlab::Git::WikiFile.new(gollum_file) end + def gollum_get_all_pages(limit: nil) + gollum_wiki.pages(limit: limit).map { |gollum_page| new_page(gollum_page) } + end + def gitaly_write_page(name, format, content, commit_details) gitaly_wiki_client.write_page(name, format, content, commit_details) end @@ -204,6 +248,12 @@ module Gitlab Gitlab::Git::WikiFile.new(wiki_file) end + + def gitaly_get_all_pages + gitaly_wiki_client.get_all_pages.map do |wiki_page, version| + Gitlab::Git::WikiPage.new(wiki_page, version) + end + end end end end |