diff options
author | Sean McGivern <sean@mcgivern.me.uk> | 2017-11-06 13:39:36 +0300 |
---|---|---|
committer | Sean McGivern <sean@mcgivern.me.uk> | 2017-11-06 13:39:36 +0300 |
commit | d4ceec9d47a7da5fa17cb6e161ac491e13fcb8bd (patch) | |
tree | 2e7170aed7ea79da51f7693eeda32e8cc29fcc43 /lib | |
parent | 46dc343f605f0c98d2b9ec44617d313a1548b4e2 (diff) | |
parent | fe4874c43d3387aed52d95bc2892b27e551ac7a9 (diff) |
Merge branch 'ce-gitlab-git-backport' into 'master'
CE repository mirroring changes backport
See merge request gitlab-org/gitlab-ce!15184
Diffstat (limited to 'lib')
-rw-r--r-- | lib/github/import.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/git/repository.rb | 26 | ||||
-rw-r--r-- | lib/gitlab/git/repository_mirroring.rb | 95 | ||||
-rw-r--r-- | lib/gitlab/git/wiki.rb | 2 |
4 files changed, 118 insertions, 9 deletions
diff --git a/lib/github/import.rb b/lib/github/import.rb index 76612799412..8cabbdec940 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -60,7 +60,9 @@ module Github project.repository.set_import_remote_as_mirror('github') project.repository.add_remote_fetch_config('github', '+refs/pull/*/head:refs/merge-requests/*/head') fetch_remote(forced: true) - rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e + rescue Gitlab::Git::Repository::NoRepository, + Gitlab::Git::RepositoryMirroring::RemoteError, + Gitlab::Shell::Error => e error(:project, repo_url, e.message) raise Github::RepositoryFetchError end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 4f9eac92d9a..182ffc96ef9 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -6,6 +6,7 @@ require "rubygems/package" module Gitlab module Git class Repository + include Gitlab::Git::RepositoryMirroring include Gitlab::Git::Popen ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[ @@ -898,16 +899,25 @@ module Gitlab end end - # Delete the specified remote from this repository. - def remote_delete(remote_name) - rugged.remotes.delete(remote_name) - nil + def add_remote(remote_name, url) + rugged.remotes.create(remote_name, url) + rescue Rugged::ConfigError + remote_update(remote_name, url: url) end - # Add a new remote to this repository. - def remote_add(remote_name, url) - rugged.remotes.create(remote_name, url) - nil + def remove_remote(remote_name) + # When a remote is deleted all its remote refs are deleted too, but in + # the case of mirrors we map its refs (that would usualy go under + # [remote_name]/) to the top level namespace. We clean the mapping so + # those don't get deleted. + if rugged.config["remote.#{remote_name}.mirror"] + rugged.config.delete("remote.#{remote_name}.fetch") + end + + rugged.remotes.delete(remote_name) + true + rescue Rugged::ConfigError + false end # Update the specified remote using the values in the +options+ hash diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb new file mode 100644 index 00000000000..637e7a0659c --- /dev/null +++ b/lib/gitlab/git/repository_mirroring.rb @@ -0,0 +1,95 @@ +module Gitlab + module Git + module RepositoryMirroring + IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze + IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze + MIRROR_REMOTE = 'mirror'.freeze + + RemoteError = Class.new(StandardError) + + def set_remote_as_mirror(remote_name) + # This is used to define repository as equivalent as "git clone --mirror" + rugged.config["remote.#{remote_name}.fetch"] = 'refs/*:refs/*' + rugged.config["remote.#{remote_name}.mirror"] = true + rugged.config["remote.#{remote_name}.prune"] = true + end + + def set_import_remote_as_mirror(remote_name) + # Add first fetch with Rugged so it does not create its own. + rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS + + add_remote_fetch_config(remote_name, IMPORT_TAG_REFS) + + rugged.config["remote.#{remote_name}.mirror"] = true + rugged.config["remote.#{remote_name}.prune"] = true + end + + def add_remote_fetch_config(remote_name, refspec) + run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}]) + end + + def fetch_mirror(url) + add_remote(MIRROR_REMOTE, url) + set_remote_as_mirror(MIRROR_REMOTE) + fetch(MIRROR_REMOTE) + remove_remote(MIRROR_REMOTE) + end + + def remote_tags(remote) + # Each line has this format: "dc872e9fa6963f8f03da6c8f6f264d0845d6b092\trefs/tags/v1.10.0\n" + # We want to convert it to: [{ 'v1.10.0' => 'dc872e9fa6963f8f03da6c8f6f264d0845d6b092' }, ...] + list_remote_tags(remote).map do |line| + target, path = line.strip.split("\t") + + # When the remote repo does not have tags. + if target.nil? || path.nil? + Rails.logger.info "Empty or invalid list of tags for remote: #{remote}. Output: #{output}" + return [] + end + + name = path.split('/', 3).last + # We're only interested in tag references + # See: http://stackoverflow.com/questions/15472107/when-listing-git-ls-remote-why-theres-after-the-tag-name + next if name =~ /\^\{\}\Z/ + + target_commit = Gitlab::Git::Commit.find(self, target) + Gitlab::Git::Tag.new(self, name, target, target_commit) + end.compact + end + + def remote_branches(remote_name) + branches = [] + + rugged.references.each("refs/remotes/#{remote_name}/*").map do |ref| + name = ref.name.sub(/\Arefs\/remotes\/#{remote_name}\//, '') + + begin + target_commit = Gitlab::Git::Commit.find(self, ref.target) + branches << Gitlab::Git::Branch.new(self, name, ref.target, target_commit) + rescue Rugged::ReferenceError + # Omit invalid branch + end + end + + branches + end + + private + + def list_remote_tags(remote) + tag_list, exit_code, error = nil + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{full_path} ls-remote --tags #{remote}) + + Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr| + tag_list = stdout.read + error = stderr.read + exit_code = wait_thr.value.exitstatus + end + + raise RemoteError, error unless exit_code.zero? + + tag_list.split('\n') + end + end + end +end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 45362ac438b..fe901d049d4 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -10,6 +10,8 @@ module Gitlab end PageBlob = Struct.new(:name) + attr_reader :repository + def self.default_ref 'master' end |