diff options
author | Jacob Vosmaer (GitLab) <jacob@gitlab.com> | 2018-07-24 11:56:56 +0300 |
---|---|---|
committer | Jacob Vosmaer (GitLab) <jacob@gitlab.com> | 2018-07-24 11:56:56 +0300 |
commit | 4437abad38bd83fcd8f0ed88b2c2208219a37cc5 (patch) | |
tree | aa56bf330990fe4c5fb2fdff2d4a4c3666bec6b0 | |
parent | 9d7920fb8cc133bea541375a23de789bf70f375f (diff) | |
parent | 4f35817c2d6008d720b45d8dbdc231b888680320 (diff) |
Merge branch 'vendor-mirroring' into 'master'
Vendor gitlab-git at 92802e51
Closes #554 and #343
See merge request gitlab-org/gitaly!825
-rw-r--r-- | changelogs/unreleased/vendor-mirroring.yml | 5 | ||||
-rw-r--r-- | ruby/lib/gitlab/git/repository.rb | 47 | ||||
-rw-r--r-- | ruby/lib/gitlab/git/repository_mirroring.rb | 76 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/REVISION | 2 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/lib/gitlab/git/operation_service.rb | 2 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb | 87 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb | 81 |
7 files changed, 162 insertions, 138 deletions
diff --git a/changelogs/unreleased/vendor-mirroring.yml b/changelogs/unreleased/vendor-mirroring.yml new file mode 100644 index 000000000..a6b9aa293 --- /dev/null +++ b/changelogs/unreleased/vendor-mirroring.yml @@ -0,0 +1,5 @@ +--- +title: Vendor gitlab-git at 92802e51 +merge_request: 825 +author: +type: other diff --git a/ruby/lib/gitlab/git/repository.rb b/ruby/lib/gitlab/git/repository.rb index 069904a66..4cb6e9044 100644 --- a/ruby/lib/gitlab/git/repository.rb +++ b/ruby/lib/gitlab/git/repository.rb @@ -543,6 +543,53 @@ module Gitlab Ref.dereference_object(obj) end + def add_remote(remote_name, url, mirror_refmap: nil) + rugged.remotes.create(remote_name, url) + + set_remote_as_mirror(remote_name, refmap: mirror_refmap) if mirror_refmap + rescue Rugged::ConfigError + remote_update(remote_name, url: url) + end + + def remove_remote(remote_name) + # When a remote is deleted all its remote refs are deleted too, but in + # the case of mirrors we map its refs (that would usualy go under + # [remote_name]/) to the top level namespace. We clean the mapping so + # those don't get deleted. + if rugged.config["remote.#{remote_name}.mirror"] + rugged.config.delete("remote.#{remote_name}.fetch") + end + + rugged.remotes.delete(remote_name) + true + rescue Rugged::ConfigError + false + end + + # Update the specified remote using the values in the +options+ hash + # + # Example + # repo.update_remote("origin", url: "path/to/repo") + def remote_update(remote_name, url:) + # TODO: Implement other remote options + rugged.remotes.set_url(remote_name, url) + nil + end + + def fetch_repository_as_mirror(repository) + remote_name = "tmp-#{SecureRandom.hex}" + repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository) + + add_remote(remote_name, GITALY_INTERNAL_URL, mirror_refmap: :all_refs) + fetch_remote(remote_name, env: repository.fetch_env) + ensure + remove_remote(remote_name) + end + + def fetch_remote(remote_name = 'origin', env: nil) + run_git(['fetch', remote_name], env: env).last.zero? + end + private def uncached_has_local_branches? diff --git a/ruby/lib/gitlab/git/repository_mirroring.rb b/ruby/lib/gitlab/git/repository_mirroring.rb new file mode 100644 index 000000000..6050289f8 --- /dev/null +++ b/ruby/lib/gitlab/git/repository_mirroring.rb @@ -0,0 +1,76 @@ +module Gitlab + module Git + module RepositoryMirroring + REFMAPS = { + # With `:all_refs`, the repository is equivalent to the result of `git clone --mirror` + all_refs: '+refs/*:refs/*', + heads: '+refs/heads/*:refs/heads/*', + tags: '+refs/tags/*:refs/tags/*' + }.freeze + + def set_remote_as_mirror(remote_name, refmap: :all_refs) + set_remote_refmap(remote_name, refmap) + + rugged.config["remote.#{remote_name}.mirror"] = true + rugged.config["remote.#{remote_name}.prune"] = true + end + + def remote_tags(remote) + # Each line has this format: "dc872e9fa6963f8f03da6c8f6f264d0845d6b092\trefs/tags/v1.10.0\n" + # We want to convert it to: [{ 'v1.10.0' => 'dc872e9fa6963f8f03da6c8f6f264d0845d6b092' }, ...] + list_remote_tags(remote).map do |line| + target, path = line.strip.split("\t") + + # When the remote repo does not have tags. + if target.nil? || path.nil? + Rails.logger.info "Empty or invalid list of tags for remote: #{remote}. Output: #{output}" + break [] + end + + name = path.split('/', 3).last + # We're only interested in tag references + # See: http://stackoverflow.com/questions/15472107/when-listing-git-ls-remote-why-theres-after-the-tag-name + next if name =~ /\^\{\}\Z/ + + target_commit = Gitlab::Git::Commit.find(self, target) + Gitlab::Git::Tag.new(self, { + name: name, + target: target, + target_commit: target_commit + }) + end.compact + end + + private + + def set_remote_refmap(remote_name, refmap) + Array(refmap).each_with_index do |refspec, i| + refspec = REFMAPS[refspec] || refspec + + # We need multiple `fetch` entries, but Rugged only allows replacing a config, not adding to it. + # To make sure we start from scratch, we set the first using rugged, and use `git` for any others + if i == 0 + rugged.config["remote.#{remote_name}.fetch"] = refspec + else + run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}]) + end + end + end + + def list_remote_tags(remote) + tag_list, exit_code, error = nil + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-remote --tags #{remote}) + + Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr| + tag_list = stdout.read + error = stderr.read + exit_code = wait_thr.value.exitstatus + end + + raise RemoteError, error unless exit_code.zero? + + tag_list.split("\n") + end + end + end +end diff --git a/ruby/vendor/gitlab_git/REVISION b/ruby/vendor/gitlab_git/REVISION index 0bcc27ec7..cccb4ae43 100644 --- a/ruby/vendor/gitlab_git/REVISION +++ b/ruby/vendor/gitlab_git/REVISION @@ -1 +1 @@ -740ae2d194f3833e224c326cc909d833c5807484 +92802e51075f485ce2aa80d19f40688f1dd95fb7 diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/operation_service.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/operation_service.rb index 280def182..57d748343 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/operation_service.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/operation_service.rb @@ -8,6 +8,8 @@ module Gitlab alias_method :branch_created?, :branch_created def self.from_gitaly(branch_update) + return if branch_update.nil? + new( branch_update.commit_id, branch_update.repo_created, diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb index 19c79b6f7..21ac43f80 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb @@ -1,4 +1,3 @@ -# Gitlab::Git::Repository is a wrapper around native Rugged::Repository object require 'tempfile' require 'forwardable' require "rubygems/package" @@ -381,6 +380,21 @@ module Gitlab end end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/1233 + def new_commits(newrev) + gitaly_migrate(:new_commits) do |is_enabled| + if is_enabled + gitaly_ref_client.list_new_commits(newrev) + else + refs = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + rev_list(including: newrev, excluding: :all).split("\n").map(&:strip) + end + + Gitlab::Git::Commit.batch_by_oid(self, refs) + end + end + end + def count_commits(options) options = process_count_commits_options(options.dup) @@ -658,35 +672,17 @@ module Gitlab # If `mirror_refmap` is present the remote is set as mirror with that mapping def add_remote(remote_name, url, mirror_refmap: nil) - gitaly_migrate(:remote_add_remote) do |is_enabled| - if is_enabled - gitaly_remote_client.add_remote(remote_name, url, mirror_refmap) - else - rugged_add_remote(remote_name, url, mirror_refmap) - end + wrapped_gitaly_errors do + gitaly_remote_client.add_remote(remote_name, url, mirror_refmap) end end def remove_remote(remote_name) - gitaly_migrate(:remote_remove_remote) do |is_enabled| - if is_enabled - gitaly_remote_client.remove_remote(remote_name) - else - rugged_remove_remote(remote_name) - end + wrapped_gitaly_errors do + gitaly_remote_client.remove_remote(remote_name) end end - # Update the specified remote using the values in the +options+ hash - # - # Example - # repo.update_remote("origin", url: "path/to/repo") - def remote_update(remote_name, url:) - # TODO: Implement other remote options - rugged.remotes.set_url(remote_name, url) - nil - end - AUTOCRLF_VALUES = { "true" => true, "false" => false, @@ -860,12 +856,8 @@ module Gitlab end def fetch_repository_as_mirror(repository) - gitaly_migrate(:remote_fetch_internal_remote) do |is_enabled| - if is_enabled - gitaly_remote_client.fetch_internal_remote(repository) - else - rugged_fetch_repository_as_mirror(repository) - end + wrapped_gitaly_errors do + gitaly_remote_client.fetch_internal_remote(repository) end end @@ -1261,47 +1253,10 @@ module Gitlab run_git(args, env: source_repository.fetch_env) end - def rugged_add_remote(remote_name, url, mirror_refmap) - rugged.remotes.create(remote_name, url) - - set_remote_as_mirror(remote_name, refmap: mirror_refmap) if mirror_refmap - rescue Rugged::ConfigError - remote_update(remote_name, url: url) - end - def gitaly_delete_refs(*ref_names) gitaly_ref_client.delete_refs(refs: ref_names) if ref_names.any? end - def rugged_remove_remote(remote_name) - # When a remote is deleted all its remote refs are deleted too, but in - # the case of mirrors we map its refs (that would usualy go under - # [remote_name]/) to the top level namespace. We clean the mapping so - # those don't get deleted. - if rugged.config["remote.#{remote_name}.mirror"] - rugged.config.delete("remote.#{remote_name}.fetch") - end - - rugged.remotes.delete(remote_name) - true - rescue Rugged::ConfigError - false - end - - def rugged_fetch_repository_as_mirror(repository) - remote_name = "tmp-#{SecureRandom.hex}" - repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository) - - add_remote(remote_name, GITALY_INTERNAL_URL, mirror_refmap: :all_refs) - fetch_remote(remote_name, env: repository.fetch_env) - ensure - remove_remote(remote_name) - end - - def fetch_remote(remote_name = 'origin', env: nil) - run_git(['fetch', remote_name], env: env).last.zero? - end - def gitlab_projects_error raise CommandError, @gitlab_projects.output end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb index 9faa62be2..8835bfb24 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb @@ -1,49 +1,19 @@ module Gitlab module Git module RepositoryMirroring - REFMAPS = { - # With `:all_refs`, the repository is equivalent to the result of `git clone --mirror` - all_refs: '+refs/*:refs/*', - heads: '+refs/heads/*:refs/heads/*', - tags: '+refs/tags/*:refs/tags/*' - }.freeze - - RemoteError = Class.new(StandardError) - - def set_remote_as_mirror(remote_name, refmap: :all_refs) - set_remote_refmap(remote_name, refmap) - - rugged.config["remote.#{remote_name}.mirror"] = true - rugged.config["remote.#{remote_name}.prune"] = true - end - - def remote_tags(remote) - # Each line has this format: "dc872e9fa6963f8f03da6c8f6f264d0845d6b092\trefs/tags/v1.10.0\n" - # We want to convert it to: [{ 'v1.10.0' => 'dc872e9fa6963f8f03da6c8f6f264d0845d6b092' }, ...] - list_remote_tags(remote).map do |line| - target, path = line.strip.split("\t") - - # When the remote repo does not have tags. - if target.nil? || path.nil? - Rails.logger.info "Empty or invalid list of tags for remote: #{remote}. Output: #{output}" - break [] + def remote_branches(remote_name) + gitaly_migrate(:ref_find_all_remote_branches) do |is_enabled| + if is_enabled + gitaly_ref_client.remote_branches(remote_name) + else + rugged_remote_branches(remote_name) end - - name = path.split('/', 3).last - # We're only interested in tag references - # See: http://stackoverflow.com/questions/15472107/when-listing-git-ls-remote-why-theres-after-the-tag-name - next if name =~ /\^\{\}\Z/ - - target_commit = Gitlab::Git::Commit.find(self, target) - Gitlab::Git::Tag.new(self, { - name: name, - target: target, - target_commit: target_commit - }) - end.compact + end end - def remote_branches(remote_name) + private + + def rugged_remote_branches(remote_name) branches = [] rugged.references.each("refs/remotes/#{remote_name}/*").map do |ref| @@ -59,37 +29,6 @@ module Gitlab branches end - - private - - def set_remote_refmap(remote_name, refmap) - Array(refmap).each_with_index do |refspec, i| - refspec = REFMAPS[refspec] || refspec - - # We need multiple `fetch` entries, but Rugged only allows replacing a config, not adding to it. - # To make sure we start from scratch, we set the first using rugged, and use `git` for any others - if i == 0 - rugged.config["remote.#{remote_name}.fetch"] = refspec - else - run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}]) - end - end - end - - def list_remote_tags(remote) - tag_list, exit_code, error = nil - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-remote --tags #{remote}) - - Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr| - tag_list = stdout.read - error = stderr.read - exit_code = wait_thr.value.exitstatus - end - - raise RemoteError, error unless exit_code.zero? - - tag_list.split("\n") - end end end end |