diff options
author | Zeger-Jan van de Weg <zegerjan@gitlab.com> | 2018-09-13 18:35:15 +0300 |
---|---|---|
committer | Zeger-Jan van de Weg <zegerjan@gitlab.com> | 2018-09-13 18:35:15 +0300 |
commit | 8f853e4f6770dd02a5997bc0389e6ada966114f0 (patch) | |
tree | 21b9af74f21588ed7853d9fd77088c147f418d09 | |
parent | 244549f23b61d0966b5b0b3b858191ec4a5c922f (diff) | |
parent | b515c74b780778d5ee5262d3e0687fad69f5a8c3 (diff) |
Merge branch 'stop-gg-vendor' into 'master'
Stop vendoring Gitlab::Git
Closes #1279
See merge request gitlab-org/gitaly!883
-rw-r--r-- | .gitlab-ci.yml | 12 | ||||
-rwxr-xr-x | _support/vendor-gitlab-git | 80 | ||||
-rw-r--r-- | changelogs/unreleased/stop-gg-vendor.yml | 5 | ||||
-rw-r--r-- | ruby/lib/gitlab/encoding_helper.rb (renamed from ruby/vendor/gitlab_git/lib/gitlab/encoding_helper.rb) | 2 | ||||
-rw-r--r-- | ruby/lib/gitlab/git.rb | 131 | ||||
-rw-r--r-- | ruby/lib/gitlab/git/committer_with_hooks.rb (renamed from ruby/vendor/gitlab_git/lib/gitlab/git/committer_with_hooks.rb) | 0 | ||||
-rw-r--r-- | ruby/lib/gitlab/git/conflict/file.rb (renamed from ruby/vendor/gitlab_git/lib/gitlab/git/conflict/file.rb) | 0 | ||||
-rw-r--r-- | ruby/lib/gitlab/git/conflict/parser.rb (renamed from ruby/vendor/gitlab_git/lib/gitlab/git/conflict/parser.rb) | 0 | ||||
-rw-r--r-- | ruby/lib/gitlab/git/conflict/resolution.rb (renamed from ruby/vendor/gitlab_git/lib/gitlab/git/conflict/resolution.rb) | 0 | ||||
-rw-r--r-- | ruby/lib/gitlab/git/gitlab_projects.rb | 240 | ||||
-rw-r--r-- | ruby/lib/gitlab/git/hook.rb (renamed from ruby/vendor/gitlab_git/lib/gitlab/git/hook.rb) | 0 | ||||
-rw-r--r-- | ruby/lib/gitlab/git/hooks_service.rb (renamed from ruby/vendor/gitlab_git/lib/gitlab/git/hooks_service.rb) | 0 | ||||
-rw-r--r-- | ruby/lib/gitlab/git/operation_service.rb (renamed from ruby/vendor/gitlab_git/lib/gitlab/git/operation_service.rb) | 1 | ||||
-rw-r--r-- | ruby/lib/gitlab/git/path_helper.rb (renamed from ruby/vendor/gitlab_git/lib/gitlab/git/path_helper.rb) | 0 | ||||
-rw-r--r-- | ruby/lib/gitlab/git/popen.rb (renamed from ruby/vendor/gitlab_git/lib/gitlab/git/popen.rb) | 0 | ||||
-rw-r--r-- | ruby/lib/gitlab/git/pre_receive_error.rb | 10 | ||||
-rw-r--r-- | ruby/lib/gitlab/git/remote_repository.rb (renamed from ruby/vendor/gitlab_git/lib/gitlab/git/remote_repository.rb) | 20 | ||||
-rw-r--r-- | ruby/lib/gitlab/git/repository.rb | 285 | ||||
-rw-r--r-- | ruby/lib/gitlab/git/repository_mirroring.rb | 17 | ||||
-rw-r--r-- | ruby/lib/gitlab/git/user.rb (renamed from ruby/vendor/gitlab_git/lib/gitlab/git/user.rb) | 0 | ||||
-rw-r--r-- | ruby/lib/gitlab/gitaly_client.rb | 28 | ||||
-rw-r--r-- | ruby/lib/gitlab/utils/strong_memoize.rb (renamed from ruby/vendor/gitlab_git/lib/gitlab/utils/strong_memoize.rb) | 0 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/ORIGIN | 1 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/REVISION | 1 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/lib/gitlab/git.rb | 96 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/lib/gitlab/git/gitlab_projects.rb | 253 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/lib/gitlab/git/pre_receive_error.rb | 21 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/lib/gitlab/git/remote_mirror.rb | 16 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb | 1242 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb | 36 | ||||
-rw-r--r-- | ruby/vendor/gitlab_git/lib/gitlab/git/util.rb | 20 |
31 files changed, 626 insertions, 1891 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 21496d3f3..4c9ee6980 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -192,15 +192,3 @@ update-downstream-server-version: script: - ruby _support/update-downstream-server-version allow_failure: true - -# Ensure that gitlab-git vendoring from gitlab-ce is functioning -test-vendor-gitaly-ruby: - stage: test - only: - - schedules - before_script: - - DEBIAN_FRONTEND=noninteractive apt-get update -qq && apt-get -q -y install rsync - script: - - _support/vendor-gitlab-git master - - git diff - - make test diff --git a/_support/vendor-gitlab-git b/_support/vendor-gitlab-git deleted file mode 100755 index b66baa763..000000000 --- a/_support/vendor-gitlab-git +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env ruby - -# These files and directories of gitlab-ce will be vendored -FILE_LIST = %w[ - lib/gitlab/git.rb - lib/gitlab/git - lib/gitlab/encoding_helper.rb - lib/gitlab/utils/strong_memoize.rb -].freeze - -# We have (already) stopped vendoring these files. -EXCLUDE = %w[ - lib/gitlab/git/attributes_at_ref_parser.rb - lib/gitlab/git/attributes_parser.rb - lib/gitlab/git/blame.rb - lib/gitlab/git/blob.rb - lib/gitlab/git/branch.rb - lib/gitlab/git/blob_snippet.rb - lib/gitlab/git/commit.rb - lib/gitlab/git/commit_stats.rb - lib/gitlab/git/compare.rb - lib/gitlab/git/conflict/resolver.rb - lib/gitlab/git/diff.rb - lib/gitlab/git/diff_collection.rb - lib/gitlab/git/gitmodules_parser.rb - lib/gitlab/git/hook_env.rb - lib/gitlab/git/index.rb - lib/gitlab/git/lfs_changes.rb - lib/gitlab/git/lfs_pointer_file.rb - lib/gitlab/git/merge_base.rb - lib/gitlab/git/ref.rb - lib/gitlab/git/rev_list.rb - lib/gitlab/git/raw_diff_change.rb - lib/gitlab/git/storage/ - lib/gitlab/git/storage.rb - lib/gitlab/git/tag.rb - lib/gitlab/git/tree.rb - lib/gitlab/git/version.rb - lib/gitlab/git/wiki.rb - lib/gitlab/git/wiki_file.rb - lib/gitlab/git/wiki_page.rb - lib/gitlab/git/wiki_page_version.rb - lib/gitlab/version_info.rb -].freeze - -REMOTE = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze - -# This directory in Gitaly will be the 'root' of gitlab-ce -VENDOR_DIR = 'ruby/vendor/gitlab_git'.freeze - -require_relative 'run.rb' -require 'tempfile' - -def main - if ARGV.count != 1 - abort "usage: #{$0} BRANCH_OR_TAG" - end - - revision = ARGV.first - revision_sha = nil - - Dir.mktmpdir do |dir| - gitlab_dir = File.join(dir, 'gitlab') - run!(%W[git clone --quiet --depth=1 -b #{revision} #{REMOTE}.git #{gitlab_dir}]) - revision_sha = capture!(%w[git rev-parse HEAD], gitlab_dir).chomp - - FileUtils.rm_rf(VENDOR_DIR) - FileUtils.mkdir_p(VENDOR_DIR) - args = %w[rsync -avR] - args += EXCLUDE.map { |e| "--exclude=#{e}" } - args += FILE_LIST - args += %W[#{File.join(Dir.pwd, VENDOR_DIR)}/] - run!(args, gitlab_dir) - end - - File.write(File.join(VENDOR_DIR, 'REVISION'), "#{revision_sha}\n") - File.write(File.join(VENDOR_DIR, 'ORIGIN'), "Cloned from #{REMOTE}.\n") -end - -main diff --git a/changelogs/unreleased/stop-gg-vendor.yml b/changelogs/unreleased/stop-gg-vendor.yml new file mode 100644 index 000000000..1b29d9dad --- /dev/null +++ b/changelogs/unreleased/stop-gg-vendor.yml @@ -0,0 +1,5 @@ +--- +title: Stop vendoring Gitlab::Git +merge_request: 883 +author: +type: other diff --git a/ruby/vendor/gitlab_git/lib/gitlab/encoding_helper.rb b/ruby/lib/gitlab/encoding_helper.rb index d1fd5dfe0..0f336fbaa 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/encoding_helper.rb +++ b/ruby/lib/gitlab/encoding_helper.rb @@ -75,7 +75,7 @@ module Gitlab end def binary_stringio(str) - StringIO.new(str || '').tap { |io| io.set_encoding(Encoding::ASCII_8BIT) } + StringIO.new(str.freeze || '').tap { |io| io.set_encoding(Encoding::ASCII_8BIT) } end private diff --git a/ruby/lib/gitlab/git.rb b/ruby/lib/gitlab/git.rb index d60554deb..bf3ea44ec 100644 --- a/ruby/lib/gitlab/git.rb +++ b/ruby/lib/gitlab/git.rb @@ -12,38 +12,23 @@ require 'active_support/core_ext/module/delegation' require 'active_support/core_ext/hash/transform_values' require 'active_support/core_ext/enumerable' -# We split our mock implementation of Gitlab::GitalyClient into a separate file -require_relative 'gitaly_client.rb' require_relative 'git_logger.rb' require_relative 'rails_logger.rb' require_relative 'gollum.rb' require_relative 'config.rb' require_relative 'version_info' -def require_dependency(_arg) - # no-op -end - -vendor_gitlab_git = '../../vendor/gitlab_git/' +dir = __dir__ # Some later requires are order-sensitive. Manually require whatever we need. -require_relative File.join(vendor_gitlab_git, 'lib/gitlab/encoding_helper.rb') -require_relative File.join(vendor_gitlab_git, 'lib/gitlab/utils/strong_memoize.rb') -require_relative File.join(vendor_gitlab_git, 'lib/gitlab/git.rb') -require_relative File.join(vendor_gitlab_git, 'lib/gitlab/git/popen.rb') -require_relative File.join(vendor_gitlab_git, 'lib/gitlab/git/repository_mirroring.rb') - -# Require all .rb files we can find in the vendored gitlab/git directory -dir = File.expand_path(File.join('..', vendor_gitlab_git, 'lib/gitlab/'), __FILE__) -Dir["#{dir}/git/**/*.rb"].sort.each do |ruby_file| - next if ruby_file.include?('circuit_breaker') - - require_relative ruby_file.sub(dir, File.join(vendor_gitlab_git, 'lib/gitlab/')).sub(%r{^/*}, '') -end - -# Require all .rb files we can find in the local gitlab/git directory -dir = __dir__ -Dir["#{dir}/git/**/*.rb"].sort.each do |ruby_file| +require_relative "#{dir}/encoding_helper.rb" +require_relative "#{dir}/utils/strong_memoize.rb" +require_relative "#{dir}/git/remote_repository.rb" +require_relative "#{dir}/git/popen.rb" +require_relative "#{dir}/git/repository_mirroring.rb" + +# Require all .rb files we can find in the gitlab lib directory +Dir["#{dir}/**/*.rb"].sort.each do |ruby_file| require_relative ruby_file.sub(dir, '').sub(%r{^/*}, '') end @@ -54,12 +39,6 @@ class String end end -class FakeCircuitBreaker - def self.perform - yield - end -end - class RequestStore def self.active? false @@ -68,11 +47,95 @@ end module Gitlab module Git - class Env - NotAvailableInGitalyRuby = Class.new(StandardError) + # The ID of empty tree. + # See http://stackoverflow.com/a/40884093/1856239 and + # https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 + EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze + BLANK_SHA = ('0' * 40).freeze + TAG_REF_PREFIX = "refs/tags/".freeze + BRANCH_REF_PREFIX = "refs/heads/".freeze + + BaseError = Class.new(StandardError) + CommandError = Class.new(BaseError) + CommitError = Class.new(BaseError) + OSError = Class.new(BaseError) + UnknownRef = Class.new(BaseError) + PreReceiveError = Class.new(BaseError) + + class << self + include Gitlab::EncodingHelper + + def ref_name(ref) + encode!(ref).sub(%r{\Arefs/(tags|heads|remotes)/}, '') + end + + def branch_name(ref) + ref = ref.to_s + if self.branch_ref?(ref) + self.ref_name(ref) + else + nil + end + end + + def committer_hash(email:, name:) + return if email.nil? || name.nil? + + { + email: email, + name: name, + time: Time.now + } + end + + def tag_name(ref) + ref = ref.to_s + if self.tag_ref?(ref) + self.ref_name(ref) + else + nil + end + end + + def tag_ref?(ref) + ref.start_with?(TAG_REF_PREFIX) + end + + def branch_ref?(ref) + ref.start_with?(BRANCH_REF_PREFIX) + end + + def blank_ref?(ref) + ref == BLANK_SHA + end + + def version + Gitlab::Git::Version.git_version + end + + def check_namespace!(*objects) + expected_namespace = self.name + '::' + objects.each do |object| + unless object.class.name.start_with?(expected_namespace) + raise ArgumentError, "expected object in #{expected_namespace}, got #{object}" + end + end + end + + def diff_line_code(file_path, new_line_position, old_line_position) + "#{Digest::SHA1.hexdigest(file_path)}_#{old_line_position}_#{new_line_position}" + end + + def shas_eql?(sha1, sha2) + return false if sha1.nil? || sha2.nil? + return false unless sha1.class == sha2.class + + # If either of the shas is below the minimum length, we cannot be sure + # that they actually refer to the same commit because of hash collision. + length = [sha1.length, sha2.length].min + return false if length < Gitlab::Git::Commit::MIN_SHA_LENGTH - def self.all - raise NotAvailableInGitalyRuby + sha1[0, length] == sha2[0, length] end end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/committer_with_hooks.rb b/ruby/lib/gitlab/git/committer_with_hooks.rb index 4198be7c9..4198be7c9 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/committer_with_hooks.rb +++ b/ruby/lib/gitlab/git/committer_with_hooks.rb diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/conflict/file.rb b/ruby/lib/gitlab/git/conflict/file.rb index f08dab59c..f08dab59c 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/conflict/file.rb +++ b/ruby/lib/gitlab/git/conflict/file.rb diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/conflict/parser.rb b/ruby/lib/gitlab/git/conflict/parser.rb index fb5717dd5..fb5717dd5 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/conflict/parser.rb +++ b/ruby/lib/gitlab/git/conflict/parser.rb diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/conflict/resolution.rb b/ruby/lib/gitlab/git/conflict/resolution.rb index ab9be683e..ab9be683e 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/conflict/resolution.rb +++ b/ruby/lib/gitlab/git/conflict/resolution.rb diff --git a/ruby/lib/gitlab/git/gitlab_projects.rb b/ruby/lib/gitlab/git/gitlab_projects.rb index f2d0004de..d78c7db1b 100644 --- a/ruby/lib/gitlab/git/gitlab_projects.rb +++ b/ruby/lib/gitlab/git/gitlab_projects.rb @@ -1,7 +1,24 @@ module Gitlab module Git - # These are monkey patches on top of the vendored version of GitlabProjects. class GitlabProjects + include Gitlab::Git::Popen + include Gitlab::Utils::StrongMemoize + + # Name of shard where repositories are stored. + # Example: nfs-file06 + attr_reader :shard_name + + # Relative path is a directory name for repository with .git at the end. + # Example: gitlab-org/gitlab-test.git + attr_reader :repository_relative_path + + # This is the path at which the gitlab-shell hooks directory can be found. + # It's essential for integration between git and GitLab proper. All new + # repositories should have their hooks directory symlinked here. + attr_reader :global_hooks_path + + attr_reader :logger + def self.from_gitaly(gitaly_repository, call) storage_path = GitalyServer.storage_path(call) @@ -30,6 +47,227 @@ module Gitlab def shard_path @shard_path end + + def output + io = @output.dup + io.rewind + io.read + end + + # Absolute path to the repository. + # Example: /home/git/repositorities/gitlab-org/gitlab-test.git + # Probably will be removed when we fully migrate to Gitaly, part of + # https://gitlab.com/gitlab-org/gitaly/issues/1124. + def repository_absolute_path + strong_memoize(:repository_absolute_path) do + File.join(shard_path, repository_relative_path) + end + end + + def shard_path + strong_memoize(:shard_path) do + Gitlab.config.repositories.storages.fetch(shard_name).legacy_disk_path + end + end + + # Import project via git clone --bare + # URL must be publicly cloneable + def import_project(source, timeout) + git_import_repository(source, timeout) + end + + def fork_repository(new_shard_name, new_repository_relative_path) + git_fork_repository(new_shard_name, new_repository_relative_path) + end + + def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil, prune: true) + logger.info "Fetching remote #{name} for repository #{repository_absolute_path}." + cmd = fetch_remote_command(name, tags, prune, force) + + setup_ssh_auth(ssh_key, known_hosts) do |env| + run_with_timeout(cmd, timeout, repository_absolute_path, env).tap do |success| + unless success + logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed." + end + end + end + end + + def push_branches(remote_name, timeout, force, branch_names) + logger.info "Pushing branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}" + cmd = %W(#{Gitlab.config.git.bin_path} push) + cmd << '--force' if force + cmd += %W(-- #{remote_name}).concat(branch_names) + + success = run_with_timeout(cmd, timeout, repository_absolute_path) + + unless success + logger.error("Pushing branches to remote #{remote_name} failed.") + end + + success + end + + def delete_remote_branches(remote_name, branch_names) + branches = branch_names.map { |branch_name| ":#{branch_name}" } + + logger.info "Pushing deleted branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}" + cmd = %W(#{Gitlab.config.git.bin_path} push -- #{remote_name}).concat(branches) + + success = run(cmd, repository_absolute_path) + + unless success + logger.error("Pushing deleted branches to remote #{remote_name} failed.") + end + + success + end + + protected + + def run(*args) + output, exitstatus = popen(*args) + @output << output + + exitstatus&.zero? + end + + def run_with_timeout(*args) + output, exitstatus = popen_with_timeout(*args) + @output << output + + exitstatus&.zero? + rescue Timeout::Error + @output.puts('Timed out') + + false + end + + def mask_password_in_url(url) + result = URI(url) + result.password = "*****" unless result.password.nil? + result.user = "*****" unless result.user.nil? # it's needed for oauth access_token + result + rescue + url + end + + def remove_origin_in_repo + cmd = %W(#{Gitlab.config.git.bin_path} remote rm origin) + run(cmd, repository_absolute_path) + end + + # Builds a small shell script that can be used to execute SSH with a set of + # custom options. + # + # Options are expanded as `'-oKey="Value"'`, so SSH will correctly interpret + # paths with spaces in them. We trust the user not to embed single or double + # quotes in the key or value. + def custom_ssh_script(options = {}) + args = options.map { |k, v| %Q{'-o#{k}="#{v}"'} }.join(' ') + + [ + "#!/bin/sh", + "exec ssh #{args} \"$@\"" + ].join("\n") + end + + # Known hosts data and private keys can be passed to gitlab-shell in the + # environment. If present, this method puts them into temporary files, writes + # a script that can substitute as `ssh`, setting the options to respect those + # files, and yields: { "GIT_SSH" => "/tmp/myScript" } + def setup_ssh_auth(key, known_hosts) + options = {} + + if key + key_file = Tempfile.new('gitlab-shell-key-file') + key_file.chmod(0o400) + key_file.write(key) + key_file.close + + options['IdentityFile'] = key_file.path + options['IdentitiesOnly'] = 'yes' + end + + if known_hosts + known_hosts_file = Tempfile.new('gitlab-shell-known-hosts') + known_hosts_file.chmod(0o400) + known_hosts_file.write(known_hosts) + known_hosts_file.close + + options['StrictHostKeyChecking'] = 'yes' + options['UserKnownHostsFile'] = known_hosts_file.path + end + + return yield({}) if options.empty? + + script = Tempfile.new('gitlab-shell-ssh-wrapper') + script.chmod(0o755) + script.write(custom_ssh_script(options)) + script.close + + yield('GIT_SSH' => script.path) + ensure + key_file&.close! + known_hosts_file&.close! + script&.close! + end + + private + + def fetch_remote_command(name, tags, prune, force) + %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet).tap do |cmd| + cmd << '--prune' if prune + cmd << '--force' if force + cmd << (tags ? '--tags' : '--no-tags') + end + end + + def git_import_repository(source, timeout) + # Skip import if repo already exists + return false if File.exist?(repository_absolute_path) + + masked_source = mask_password_in_url(source) + + logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>." + cmd = %W(#{Gitlab.config.git.bin_path} clone --bare -- #{source} #{repository_absolute_path}) + + success = run_with_timeout(cmd, timeout, nil) + + unless success + logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.") + FileUtils.rm_rf(repository_absolute_path) + return false + end + + Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path) + + # The project was imported successfully. + # Remove the origin URL since it may contain password. + remove_origin_in_repo + + true + end + + def git_fork_repository(new_shard_name, new_repository_relative_path) + from_path = repository_absolute_path + new_shard_path = Gitlab.config.repositories.storages.fetch(new_shard_name).legacy_disk_path + to_path = File.join(new_shard_path, new_repository_relative_path) + + # The repository cannot already exist + if File.exist?(to_path) + logger.error "fork-repository failed: destination repository <#{to_path}> already exists." + return false + end + + # Ensure the namepsace / hashed storage directory exists + FileUtils.mkdir_p(File.dirname(to_path), mode: 0770) + + logger.info "Forking repository from <#{from_path}> to <#{to_path}>." + cmd = %W(#{Gitlab.config.git.bin_path} clone --bare --no-local -- #{from_path} #{to_path}) + + run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) + end end end end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/hook.rb b/ruby/lib/gitlab/git/hook.rb index 94ff5b498..94ff5b498 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/hook.rb +++ b/ruby/lib/gitlab/git/hook.rb diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/hooks_service.rb b/ruby/lib/gitlab/git/hooks_service.rb index e67cacdb9..e67cacdb9 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/hooks_service.rb +++ b/ruby/lib/gitlab/git/hooks_service.rb diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/operation_service.rb b/ruby/lib/gitlab/git/operation_service.rb index 57d748343..99ee18166 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/operation_service.rb +++ b/ruby/lib/gitlab/git/operation_service.rb @@ -161,7 +161,6 @@ module Gitlab end end - # Gitaly note: JV: wait with migrating #update_ref until we know how to migrate its call sites. def update_ref(ref, newrev, oldrev) # We use 'git update-ref' because libgit2/rugged currently does not # offer 'compare and swap' ref updates. Without compare-and-swap we can diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/path_helper.rb b/ruby/lib/gitlab/git/path_helper.rb index 57b82a37d..57b82a37d 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/path_helper.rb +++ b/ruby/lib/gitlab/git/path_helper.rb diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/popen.rb b/ruby/lib/gitlab/git/popen.rb index 7426688fc..7426688fc 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/popen.rb +++ b/ruby/lib/gitlab/git/popen.rb diff --git a/ruby/lib/gitlab/git/pre_receive_error.rb b/ruby/lib/gitlab/git/pre_receive_error.rb deleted file mode 100644 index 775b00830..000000000 --- a/ruby/lib/gitlab/git/pre_receive_error.rb +++ /dev/null @@ -1,10 +0,0 @@ -module Gitlab - module Git - class PreReceiveError - # In gitlab-rails this method applies HTML sanitization. - def nlbr(str) - str - end - end - end -end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/remote_repository.rb b/ruby/lib/gitlab/git/remote_repository.rb index f40e59a8d..494da0c55 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/remote_repository.rb +++ b/ruby/lib/gitlab/git/remote_repository.rb @@ -7,10 +7,6 @@ module Gitlab # call is being made on (a Repository instance), and the "other" # repository (a RemoteRepository instance). This is the reason why we # have the RemoteRepository class in Gitlab::Git. - # - # When you make changes, be aware that gitaly-ruby sub-classes this - # class. - # class RemoteRepository attr_reader :relative_path, :gitaly_repository @@ -18,34 +14,22 @@ module Gitlab @relative_path = repository.relative_path @gitaly_repository = repository.gitaly_repository - # These instance variables will not be available in gitaly-ruby, where - # we have no disk access to this repository. @repository = repository end def empty? - # We will override this implementation in gitaly-ruby because we cannot - # use '@repository' there. - # - # Caches and memoization used on the Rails side !@repository.exists? || @repository.empty? end def commit_id(revision) - # We will override this implementation in gitaly-ruby because we cannot - # use '@repository' there. @repository.commit(revision)&.sha end def branch_exists?(name) - # We will override this implementation in gitaly-ruby because we cannot - # use '@repository' there. @repository.branch_exists?(name) end - # Compares self to a Gitlab::Git::Repository. This implementation uses - # 'self.gitaly_repository' so that it will also work in the - # GitalyRemoteRepository subclass defined in gitaly-ruby. + # Compares self to a Gitlab::Git::Repository def same_repository?(other_repository) gitaly_repository.storage_name == other_repository.storage && gitaly_repository.relative_path == other_repository.relative_path @@ -76,7 +60,7 @@ module Gitlab # Must return an object that responds to 'address' and 'storage'. def gitaly_client - Gitlab::GitalyClient + raise NotImplementedError.new("Can't perform remote operations on superclass") end def storage diff --git a/ruby/lib/gitlab/git/repository.rb b/ruby/lib/gitlab/git/repository.rb index 7995c98bc..d6ea03041 100644 --- a/ruby/lib/gitlab/git/repository.rb +++ b/ruby/lib/gitlab/git/repository.rb @@ -2,6 +2,41 @@ module Gitlab module Git # These are monkey patches on top of the vendored version of Repository. class Repository + include Gitlab::Git::RepositoryMirroring + include Gitlab::Git::Popen + include Gitlab::EncodingHelper + include Gitlab::Utils::StrongMemoize + + ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[ + GIT_OBJECT_DIRECTORY + GIT_ALTERNATE_OBJECT_DIRECTORIES + ].freeze + ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES = %w[ + GIT_OBJECT_DIRECTORY_RELATIVE + GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE + ].freeze + SEARCH_CONTEXT_LINES = 3 + REV_LIST_COMMIT_LIMIT = 2_000 + # In https://gitlab.com/gitlab-org/gitaly/merge_requests/698 + # We copied these two prefixes into gitaly-go, so don't change these + # or things will break! (REBASE_WORKTREE_PREFIX and SQUASH_WORKTREE_PREFIX) + REBASE_WORKTREE_PREFIX = 'rebase'.freeze + SQUASH_WORKTREE_PREFIX = 'squash'.freeze + GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze + GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout + EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000'.freeze + AUTOCRLF_VALUES = { 'true' => true, 'false' => false, 'input' => :input }.freeze + + NoRepository = Class.new(StandardError) + InvalidRepository = Class.new(StandardError) + InvalidBlobName = Class.new(StandardError) + InvalidRef = Class.new(StandardError) + GitError = Class.new(StandardError) + DeleteBranchError = Class.new(StandardError) + CreateTreeError = Class.new(StandardError) + TagExistsError = Class.new(StandardError) + ChecksumError = Class.new(StandardError) + class << self def from_gitaly(gitaly_repository, call) new( @@ -53,6 +88,14 @@ module Gitlab attr_reader :path + # Directory name of repo + attr_reader :name + + # Relative path of repo + attr_reader :relative_path + + attr_reader :gitlab_projects, :storage, :gl_repository, :relative_path + def initialize(gitaly_repository, path, gl_repository, gitlab_projects, combined_alt_dirs="") @gitaly_repository = gitaly_repository @@ -67,6 +110,10 @@ module Gitlab @gitlab_projects = gitlab_projects end + def ==(other) + [storage, relative_path] == [other.storage, other.relative_path] + end + def add_branch(branch_name, user:, target:) target_object = Ref.dereference_object(lookup(target)) raise InvalidRef.new("target not found: #{target}") unless target_object @@ -77,15 +124,6 @@ module Gitlab raise InvalidRef, ex end - # Fake implementation, so we wrap correctly on the client side - def wrapped_gitaly_errors - yield - end - - def circuit_breaker - FakeCircuitBreaker - end - def gitaly_repository @gitaly_repository end @@ -94,8 +132,21 @@ module Gitlab @alternate_object_directories end - def relative_object_directories - raise "don't use relative object directories in gitaly-ruby" + def sort_branches(branches, sort_by) + case sort_by + when 'name' + branches.sort_by(&:name) + when 'updated_desc' + branches.sort do |a, b| + b.dereferenced_target.committed_date <=> a.dereferenced_target.committed_date + end + when 'updated_asc' + branches.sort do |a, b| + a.dereferenced_target.committed_date <=> b.dereferenced_target.committed_date + end + else + branches + end end # TODO: Can be removed once https://gitlab.com/gitlab-org/gitaly/merge_requests/738 @@ -113,6 +164,12 @@ module Gitlab @root_ref ||= discover_default_branch end + def rugged + Rugged::Repository.new(path, alternates: alternate_object_directories) + rescue Rugged::RepositoryError, Rugged::OSError + raise NoRepository.new('no repository for such path') + end + def branch_names branches.map(&:name) end @@ -125,14 +182,21 @@ module Gitlab branches_filter(filter: :local, sort_by: sort_by) end - def has_local_branches_rugged? - branches_filter(filter: :local).any? do |ref| - begin - ref.name && ref.target # ensures the branch is valid + # Git repository can contains some hidden refs like: + # /refs/notes/* + # /refs/git-as-svn/* + # /refs/pulls/* + # This refs by default not visible in project page and not cloned to client side. + def has_visible_content? + strong_memoize(:has_visible_content) do + branches_filter(filter: :local).any? do |ref| + begin + ref.name && ref.target # ensures the branch is valid - true - rescue Rugged::ReferenceError - false + true + rescue Rugged::ReferenceError + false + end end end end @@ -278,6 +342,10 @@ module Gitlab raise TagExistsError end + def update_branch(branch_name, user:, newrev:, oldrev:) + OperationService.new(user, self).update_branch(branch_name, newrev, oldrev) + end + def rm_branch(branch_name, user:) branch = find_branch(branch_name) @@ -294,6 +362,10 @@ module Gitlab Gitlab::Git::OperationService.new(user, self).rm_tag(tag) end + def find_tag(name) + tags.find { |tag| tag.name == name } + end + def merge(user, source_sha, target_branch, message, &block) committer = Gitlab::Git.committer_hash(email: user.email, name: user.name) @@ -370,6 +442,10 @@ module Gitlab rugged_cherry_pick(args) end + def diff_exists?(sha1, sha2) + rugged.diff(sha1, sha2).size > 0 + end + def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) env = git_env_for_user(user) @@ -427,6 +503,18 @@ module Gitlab end end + def push_remote_branches(remote_name, branch_names, forced: true) + success = @gitlab_projects.push_branches(remote_name, GITLAB_PROJECTS_TIMEOUT, forced, branch_names) + + success || gitlab_projects_error + end + + def delete_remote_branches(remote_name, branch_names) + success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) + + success || gitlab_projects_error + end + def multi_action( user, branch_name:, message:, actions:, author_email: nil, author_name: nil, @@ -477,6 +565,48 @@ module Gitlab log_by_shell(sha, options) end + def with_repo_branch_commit(start_repository, start_branch_name) + Gitlab::Git.check_namespace!(start_repository) + start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) + + return yield nil if start_repository.empty? + + if start_repository.same_repository?(self) + yield commit(start_branch_name) + else + start_commit_id = start_repository.commit_id(start_branch_name) + + return yield nil unless start_commit_id + + if branch_commit = commit(start_commit_id) + yield branch_commit + else + with_repo_tmp_commit( + start_repository, start_branch_name, start_commit_id) do |tmp_commit| + yield tmp_commit + end + end + end + end + + def with_repo_tmp_commit(start_repository, start_branch_name, sha) + source_ref = start_branch_name + + unless Gitlab::Git.branch_ref?(source_ref) + source_ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{source_ref}" + end + + tmp_ref = fetch_ref( + start_repository, + source_ref: source_ref, + target_ref: "refs/tmp/#{SecureRandom.hex}" + ) + + yield commit(sha) + ensure + delete_refs(tmp_ref) if tmp_ref + end + def fetch_source_branch!(source_repository, source_branch, local_ref) rugged_fetch_source_branch(source_repository, source_branch, local_ref) end @@ -513,6 +643,15 @@ module Gitlab delete_refs(*all_ref_names_except(prefixes)) end + # Returns an Array of all ref names, except when it's matching pattern + # + # regexp - The pattern for ref names we don't want + def all_ref_names_except(prefixes) + rugged.references.reject do |ref| + prefixes.any? { |p| ref.name.start_with?(p) } + end.map(&:name) + end + # Returns true if the given branch exists # # name - The name of the branch as a String. @@ -533,6 +672,10 @@ module Gitlab nil end + def user_to_committer(user) + Gitlab::Git.committer_hash(email: user.email, name: user.name) + end + def write_ref(ref_path, ref, old_ref: nil, shell: true) if shell shell_write_ref(ref_path, ref, old_ref) @@ -541,6 +684,17 @@ module Gitlab end end + def fetch_ref(source_repository, source_ref:, target_ref:) + Gitlab::Git.check_namespace!(source_repository) + source_repository = RemoteRepository.new(source_repository) unless source_repository.is_a?(RemoteRepository) + + args = %W(fetch --no-tags -f #{GITALY_INTERNAL_URL} #{source_ref}:#{target_ref}) + message, status = run_git(args, env: source_repository.fetch_env) + raise Gitlab::Git::CommandError, message if status != 0 + + target_ref + end + # Lookup for rugged object by oid or ref name def lookup(oid_or_ref_name) rugged.rev_parse(oid_or_ref_name) @@ -600,6 +754,26 @@ module Gitlab nil end + def commit(ref = 'HEAD') + Gitlab::Git::Commit.find(self, ref) + end + + def empty? + !has_visible_content? + end + + def autocrlf + AUTOCRLF_VALUES[rugged.config['core.autocrlf']] + end + + def autocrlf=(value) + rugged.config['core.autocrlf'] = AUTOCRLF_VALUES.invert[value] + end + + def blob_at(sha, path) + Gitlab::Git::Blob.find(self, sha, path) unless Gitlab::Git.blank_ref?(sha) + end + def fetch_repository_as_mirror(repository) remote_name = "tmp-#{SecureRandom.hex}" repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository) @@ -614,10 +788,73 @@ module Gitlab run_git(['fetch', remote_name], env: env).last.zero? end + def rev_list(including: [], excluding: [], options: [], objects: false, &block) + args = ['rev-list'] + + args.push(*rev_list_param(including)) + + exclude_param = *rev_list_param(excluding) + if exclude_param.any? + args.push('--not') + args.push(*exclude_param) + end + + args.push('--objects') if objects + + if options.any? + args.push(*options) + end + + run_git!(args, lazy_block: block) + end + private - def uncached_has_local_branches? - has_local_branches_rugged? + def run_git(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) + cmd = [Gitlab.config.git.bin_path, *args] + cmd.unshift("nice") if nice + + object_directories = alternate_object_directories + if object_directories.any? + env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = object_directories.join(File::PATH_SEPARATOR) + end + + popen(cmd, chdir, env, lazy_block: lazy_block, &block) + end + + def run_git!(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) + output, status = run_git(args, chdir: chdir, env: env, nice: nice, lazy_block: lazy_block, &block) + + raise GitError, output unless status.zero? + + output + end + + def run_git_with_timeout(args, timeout, env: {}) + popen_with_timeout([Gitlab.config.git.bin_path, *args], timeout, path, env) + end + + def git_env_for_user(user) + { + 'GIT_COMMITTER_NAME' => user.name, + 'GIT_COMMITTER_EMAIL' => user.email, + 'GL_ID' => Gitlab::GlId.gl_id(user), + 'GL_PROTOCOL' => Gitlab::Git::Hook::GL_PROTOCOL, + 'GL_REPOSITORY' => gl_repository + } + end + + def check_revert_content(target_commit, source_sha) + args = [target_commit.sha, source_sha] + args << { mainline: 1 } if target_commit.merge_commit? + + revert_index = rugged.revert_commit(*args) + return false if revert_index.conflicts? + + tree_id = revert_index.write_tree(rugged) + return false unless diff_exists?(source_sha, tree_id) + + tree_id end def branches_filter(filter: nil, sort_by: nil) @@ -841,6 +1078,14 @@ module Gitlab def sha_from_ref(ref) rev_parse_target(ref).oid end + + def gitlab_projects_error + raise CommandError, @gitlab_projects.output + end + + def rev_list_param(spec) + spec == :all ? ['--all'] : spec + end end end end diff --git a/ruby/lib/gitlab/git/repository_mirroring.rb b/ruby/lib/gitlab/git/repository_mirroring.rb index 6050289f8..08c44af9a 100644 --- a/ruby/lib/gitlab/git/repository_mirroring.rb +++ b/ruby/lib/gitlab/git/repository_mirroring.rb @@ -8,6 +8,23 @@ module Gitlab tags: '+refs/tags/*:refs/tags/*' }.freeze + def remote_branches(remote_name) + branches = [] + + rugged.references.each("refs/remotes/#{remote_name}/*").map do |ref| + name = ref.name.sub(%r{\Arefs/remotes/#{remote_name}/}, '') + + begin + target_commit = Gitlab::Git::Commit.find(self, ref.target.oid) + branches << Gitlab::Git::Branch.new(self, name, ref.target, target_commit) + rescue Rugged::ReferenceError + # Omit invalid branch + end + end + + branches + end + def set_remote_as_mirror(remote_name, refmap: :all_refs) set_remote_refmap(remote_name, refmap) diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/user.rb b/ruby/lib/gitlab/git/user.rb index e573cd0e1..e573cd0e1 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/user.rb +++ b/ruby/lib/gitlab/git/user.rb diff --git a/ruby/lib/gitlab/gitaly_client.rb b/ruby/lib/gitlab/gitaly_client.rb deleted file mode 100644 index fd1f7cf23..000000000 --- a/ruby/lib/gitlab/gitaly_client.rb +++ /dev/null @@ -1,28 +0,0 @@ -module Gitlab - module GitalyClient - module MigrationStatus - DISABLED = :fake_disabled - OPT_IN = :fake_opt_in - OPT_OUT = :fake_opt_out - end - - class StorageSettings - def self.allow_disk_access - yield - end - end - - class << self - # In case we hit a method that tries to do a Gitaly RPC, we want to - # prevent this most of the time. - def migrate(*args) - whitelist = [:fetch_ref, :fetch_internal] - yield whitelist.include?(args.first) - end - - def allow_n_plus_1_calls - yield - end - end - end -end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/utils/strong_memoize.rb b/ruby/lib/gitlab/utils/strong_memoize.rb index fe091f461..fe091f461 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/utils/strong_memoize.rb +++ b/ruby/lib/gitlab/utils/strong_memoize.rb diff --git a/ruby/vendor/gitlab_git/ORIGIN b/ruby/vendor/gitlab_git/ORIGIN deleted file mode 100644 index 14c24eb7c..000000000 --- a/ruby/vendor/gitlab_git/ORIGIN +++ /dev/null @@ -1 +0,0 @@ -Cloned from https://gitlab.com/gitlab-org/gitlab-ce. diff --git a/ruby/vendor/gitlab_git/REVISION b/ruby/vendor/gitlab_git/REVISION deleted file mode 100644 index 85c7be410..000000000 --- a/ruby/vendor/gitlab_git/REVISION +++ /dev/null @@ -1 +0,0 @@ -c87ca8322636db69b6f097336f163d0373bb415e diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git.rb b/ruby/vendor/gitlab_git/lib/gitlab/git.rb deleted file mode 100644 index 2913a3e41..000000000 --- a/ruby/vendor/gitlab_git/lib/gitlab/git.rb +++ /dev/null @@ -1,96 +0,0 @@ -require_dependency 'gitlab/encoding_helper' - -module Gitlab - module Git - # The ID of empty tree. - # See http://stackoverflow.com/a/40884093/1856239 and - # https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 - EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze - BLANK_SHA = ('0' * 40).freeze - TAG_REF_PREFIX = "refs/tags/".freeze - BRANCH_REF_PREFIX = "refs/heads/".freeze - - BaseError = Class.new(StandardError) - CommandError = Class.new(BaseError) - CommitError = Class.new(BaseError) - OSError = Class.new(BaseError) - UnknownRef = Class.new(BaseError) - - class << self - include Gitlab::EncodingHelper - - def ref_name(ref) - encode!(ref).sub(%r{\Arefs/(tags|heads|remotes)/}, '') - end - - def branch_name(ref) - ref = ref.to_s - if self.branch_ref?(ref) - self.ref_name(ref) - else - nil - end - end - - def committer_hash(email:, name:) - return if email.nil? || name.nil? - - { - email: email, - name: name, - time: Time.now - } - end - - def tag_name(ref) - ref = ref.to_s - if self.tag_ref?(ref) - self.ref_name(ref) - else - nil - end - end - - def tag_ref?(ref) - ref.start_with?(TAG_REF_PREFIX) - end - - def branch_ref?(ref) - ref.start_with?(BRANCH_REF_PREFIX) - end - - def blank_ref?(ref) - ref == BLANK_SHA - end - - def version - Gitlab::Git::Version.git_version - end - - def check_namespace!(*objects) - expected_namespace = self.name + '::' - objects.each do |object| - unless object.class.name.start_with?(expected_namespace) - raise ArgumentError, "expected object in #{expected_namespace}, got #{object}" - end - end - end - - def diff_line_code(file_path, new_line_position, old_line_position) - "#{Digest::SHA1.hexdigest(file_path)}_#{old_line_position}_#{new_line_position}" - end - - def shas_eql?(sha1, sha2) - return false if sha1.nil? || sha2.nil? - return false unless sha1.class == sha2.class - - # If either of the shas is below the minimum length, we cannot be sure - # that they actually refer to the same commit because of hash collision. - length = [sha1.length, sha2.length].min - return false if length < Gitlab::Git::Commit::MIN_SHA_LENGTH - - sha1[0, length] == sha2[0, length] - end - end - end -end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/gitlab_projects.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/gitlab_projects.rb deleted file mode 100644 index 5ff15a787..000000000 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/gitlab_projects.rb +++ /dev/null @@ -1,253 +0,0 @@ -module Gitlab - module Git - class GitlabProjects - include Gitlab::Git::Popen - include Gitlab::Utils::StrongMemoize - - # Name of shard where repositories are stored. - # Example: nfs-file06 - attr_reader :shard_name - - # Relative path is a directory name for repository with .git at the end. - # Example: gitlab-org/gitlab-test.git - attr_reader :repository_relative_path - - # This is the path at which the gitlab-shell hooks directory can be found. - # It's essential for integration between git and GitLab proper. All new - # repositories should have their hooks directory symlinked here. - attr_reader :global_hooks_path - - attr_reader :logger - - def initialize(shard_name, repository_relative_path, global_hooks_path:, logger:) - @shard_name = shard_name - @repository_relative_path = repository_relative_path - - @logger = logger - @global_hooks_path = global_hooks_path - @output = StringIO.new - end - - def output - io = @output.dup - io.rewind - io.read - end - - # Absolute path to the repository. - # Example: /home/git/repositorities/gitlab-org/gitlab-test.git - # Probably will be removed when we fully migrate to Gitaly, part of - # https://gitlab.com/gitlab-org/gitaly/issues/1124. - def repository_absolute_path - strong_memoize(:repository_absolute_path) do - File.join(shard_path, repository_relative_path) - end - end - - def shard_path - strong_memoize(:shard_path) do - Gitlab.config.repositories.storages.fetch(shard_name).legacy_disk_path - end - end - - # Import project via git clone --bare - # URL must be publicly cloneable - def import_project(source, timeout) - git_import_repository(source, timeout) - end - - def fork_repository(new_shard_name, new_repository_relative_path) - git_fork_repository(new_shard_name, new_repository_relative_path) - end - - def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil, prune: true) - logger.info "Fetching remote #{name} for repository #{repository_absolute_path}." - cmd = fetch_remote_command(name, tags, prune, force) - - setup_ssh_auth(ssh_key, known_hosts) do |env| - run_with_timeout(cmd, timeout, repository_absolute_path, env).tap do |success| - unless success - logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed." - end - end - end - end - - def push_branches(remote_name, timeout, force, branch_names) - logger.info "Pushing branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}" - cmd = %W(#{Gitlab.config.git.bin_path} push) - cmd << '--force' if force - cmd += %W(-- #{remote_name}).concat(branch_names) - - success = run_with_timeout(cmd, timeout, repository_absolute_path) - - unless success - logger.error("Pushing branches to remote #{remote_name} failed.") - end - - success - end - - def delete_remote_branches(remote_name, branch_names) - branches = branch_names.map { |branch_name| ":#{branch_name}" } - - logger.info "Pushing deleted branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}" - cmd = %W(#{Gitlab.config.git.bin_path} push -- #{remote_name}).concat(branches) - - success = run(cmd, repository_absolute_path) - - unless success - logger.error("Pushing deleted branches to remote #{remote_name} failed.") - end - - success - end - - protected - - def run(*args) - output, exitstatus = popen(*args) - @output << output - - exitstatus&.zero? - end - - def run_with_timeout(*args) - output, exitstatus = popen_with_timeout(*args) - @output << output - - exitstatus&.zero? - rescue Timeout::Error - @output.puts('Timed out') - - false - end - - def mask_password_in_url(url) - result = URI(url) - result.password = "*****" unless result.password.nil? - result.user = "*****" unless result.user.nil? # it's needed for oauth access_token - result - rescue - url - end - - def remove_origin_in_repo - cmd = %W(#{Gitlab.config.git.bin_path} remote rm origin) - run(cmd, repository_absolute_path) - end - - # Builds a small shell script that can be used to execute SSH with a set of - # custom options. - # - # Options are expanded as `'-oKey="Value"'`, so SSH will correctly interpret - # paths with spaces in them. We trust the user not to embed single or double - # quotes in the key or value. - def custom_ssh_script(options = {}) - args = options.map { |k, v| %Q{'-o#{k}="#{v}"'} }.join(' ') - - [ - "#!/bin/sh", - "exec ssh #{args} \"$@\"" - ].join("\n") - end - - # Known hosts data and private keys can be passed to gitlab-shell in the - # environment. If present, this method puts them into temporary files, writes - # a script that can substitute as `ssh`, setting the options to respect those - # files, and yields: { "GIT_SSH" => "/tmp/myScript" } - def setup_ssh_auth(key, known_hosts) - options = {} - - if key - key_file = Tempfile.new('gitlab-shell-key-file') - key_file.chmod(0o400) - key_file.write(key) - key_file.close - - options['IdentityFile'] = key_file.path - options['IdentitiesOnly'] = 'yes' - end - - if known_hosts - known_hosts_file = Tempfile.new('gitlab-shell-known-hosts') - known_hosts_file.chmod(0o400) - known_hosts_file.write(known_hosts) - known_hosts_file.close - - options['StrictHostKeyChecking'] = 'yes' - options['UserKnownHostsFile'] = known_hosts_file.path - end - - return yield({}) if options.empty? - - script = Tempfile.new('gitlab-shell-ssh-wrapper') - script.chmod(0o755) - script.write(custom_ssh_script(options)) - script.close - - yield('GIT_SSH' => script.path) - ensure - key_file&.close! - known_hosts_file&.close! - script&.close! - end - - private - - def fetch_remote_command(name, tags, prune, force) - %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet).tap do |cmd| - cmd << '--prune' if prune - cmd << '--force' if force - cmd << (tags ? '--tags' : '--no-tags') - end - end - - def git_import_repository(source, timeout) - # Skip import if repo already exists - return false if File.exist?(repository_absolute_path) - - masked_source = mask_password_in_url(source) - - logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>." - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare -- #{source} #{repository_absolute_path}) - - success = run_with_timeout(cmd, timeout, nil) - - unless success - logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.") - FileUtils.rm_rf(repository_absolute_path) - return false - end - - Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path) - - # The project was imported successfully. - # Remove the origin URL since it may contain password. - remove_origin_in_repo - - true - end - - def git_fork_repository(new_shard_name, new_repository_relative_path) - from_path = repository_absolute_path - new_shard_path = Gitlab.config.repositories.storages.fetch(new_shard_name).legacy_disk_path - to_path = File.join(new_shard_path, new_repository_relative_path) - - # The repository cannot already exist - if File.exist?(to_path) - logger.error "fork-repository failed: destination repository <#{to_path}> already exists." - return false - end - - # Ensure the namepsace / hashed storage directory exists - FileUtils.mkdir_p(File.dirname(to_path), mode: 0770) - - logger.info "Forking repository from <#{from_path}> to <#{to_path}>." - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare --no-local -- #{from_path} #{to_path}) - - run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) - end - end - end -end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/pre_receive_error.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/pre_receive_error.rb deleted file mode 100644 index ac1ab7c39..000000000 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/pre_receive_error.rb +++ /dev/null @@ -1,21 +0,0 @@ -module Gitlab - module Git - # - # PreReceiveError is special because its message gets displayed to users - # in the web UI. To prevent XSS we sanitize the message on - # initialization. - class PreReceiveError < StandardError - def initialize(msg = '') - super(nlbr(msg)) - end - - private - - # In gitaly-ruby we override this method to do nothing, so that - # sanitization happens in gitlab-rails only. - def nlbr(str) - Gitlab::Utils.nlbr(str) - end - end - end -end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/remote_mirror.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/remote_mirror.rb deleted file mode 100644 index e4743b4db..000000000 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/remote_mirror.rb +++ /dev/null @@ -1,16 +0,0 @@ -module Gitlab - module Git - class RemoteMirror - def initialize(repository, ref_name) - @repository = repository - @ref_name = ref_name - end - - def update(only_branches_matching: []) - @repository.wrapped_gitaly_errors do - @repository.gitaly_remote_client.update_remote_mirror(@ref_name, only_branches_matching) - end - end - end - end -end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb deleted file mode 100644 index e9c901f85..000000000 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb +++ /dev/null @@ -1,1242 +0,0 @@ -require 'tempfile' -require 'forwardable' -require "rubygems/package" - -module Gitlab - module Git - class Repository - include Gitlab::Git::RepositoryMirroring - include Gitlab::Git::Popen - include Gitlab::EncodingHelper - include Gitlab::Utils::StrongMemoize - - ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[ - GIT_OBJECT_DIRECTORY - GIT_ALTERNATE_OBJECT_DIRECTORIES - ].freeze - ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES = %w[ - GIT_OBJECT_DIRECTORY_RELATIVE - GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE - ].freeze - SEARCH_CONTEXT_LINES = 3 - REV_LIST_COMMIT_LIMIT = 2_000 - # In https://gitlab.com/gitlab-org/gitaly/merge_requests/698 - # We copied these two prefixes into gitaly-go, so don't change these - # or things will break! (REBASE_WORKTREE_PREFIX and SQUASH_WORKTREE_PREFIX) - REBASE_WORKTREE_PREFIX = 'rebase'.freeze - SQUASH_WORKTREE_PREFIX = 'squash'.freeze - GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze - GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout - EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000'.freeze - - NoRepository = Class.new(StandardError) - InvalidRepository = Class.new(StandardError) - InvalidBlobName = Class.new(StandardError) - InvalidRef = Class.new(StandardError) - GitError = Class.new(StandardError) - DeleteBranchError = Class.new(StandardError) - CreateTreeError = Class.new(StandardError) - TagExistsError = Class.new(StandardError) - ChecksumError = Class.new(StandardError) - - class << self - def create_hooks(repo_path, global_hooks_path) - local_hooks_path = File.join(repo_path, 'hooks') - real_local_hooks_path = :not_found - - begin - real_local_hooks_path = File.realpath(local_hooks_path) - rescue Errno::ENOENT - # real_local_hooks_path == :not_found - end - - # Do nothing if hooks already exist - unless real_local_hooks_path == File.realpath(global_hooks_path) - if File.exist?(local_hooks_path) - # Move the existing hooks somewhere safe - FileUtils.mv( - local_hooks_path, - "#{local_hooks_path}.old.#{Time.now.to_i}") - end - - # Create the hooks symlink - FileUtils.ln_sf(global_hooks_path, local_hooks_path) - end - - true - end - end - - # Directory name of repo - attr_reader :name - - # Relative path of repo - attr_reader :relative_path - - attr_reader :gitlab_projects, :storage, :gl_repository, :relative_path - - # This initializer method is only used on the client side (gitlab-ce). - # Gitaly-ruby uses a different initializer. - def initialize(storage, relative_path, gl_repository) - @storage = storage - @relative_path = relative_path - @gl_repository = gl_repository - - @gitlab_projects = Gitlab::Git::GitlabProjects.new( - storage, - relative_path, - global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, - logger: Rails.logger - ) - - @name = @relative_path.split("/").last - end - - def ==(other) - [storage, relative_path] == [other.storage, other.relative_path] - end - - # This method will be removed when Gitaly reaches v1.1. - def path - File.join( - Gitlab.config.repositories.storages[@storage].legacy_disk_path, @relative_path - ) - end - - # Default branch in the repository - def root_ref - gitaly_ref_client.default_branch_name - rescue GRPC::NotFound => e - raise NoRepository.new(e.message) - rescue GRPC::Unknown => e - raise Gitlab::Git::CommandError.new(e.message) - end - - # This method will be removed when Gitaly reaches v1.1. - def rugged - circuit_breaker.perform do - Rugged::Repository.new(path, alternates: alternate_object_directories) - end - rescue Rugged::RepositoryError, Rugged::OSError - raise NoRepository.new('no repository for such path') - end - - def cleanup - @rugged&.close - end - - def circuit_breaker - @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) - end - - def exists? - gitaly_repository_client.exists? - end - - # Returns an Array of branch names - # sorted by name ASC - def branch_names - wrapped_gitaly_errors do - gitaly_ref_client.branch_names - end - end - - # Returns an Array of Branches - def branches - wrapped_gitaly_errors do - gitaly_ref_client.branches - end - end - - def reload_rugged - @rugged = nil - end - - # Directly find a branch with a simple name (e.g. master) - # - def find_branch(name) - wrapped_gitaly_errors do - gitaly_ref_client.find_branch(name) - end - end - - def local_branches(sort_by: nil) - wrapped_gitaly_errors do - gitaly_ref_client.local_branches(sort_by: sort_by) - end - end - - # Returns the number of valid branches - def branch_count - wrapped_gitaly_errors do - gitaly_ref_client.count_branch_names - end - end - - def expire_has_local_branches_cache - clear_memoization(:has_local_branches) - end - - def has_local_branches? - strong_memoize(:has_local_branches) do - uncached_has_local_branches? - end - end - - # Git repository can contains some hidden refs like: - # /refs/notes/* - # /refs/git-as-svn/* - # /refs/pulls/* - # This refs by default not visible in project page and not cloned to client side. - alias_method :has_visible_content?, :has_local_branches? - - # Returns the number of valid tags - def tag_count - wrapped_gitaly_errors do - gitaly_ref_client.count_tag_names - end - end - - # Returns an Array of tag names - def tag_names - wrapped_gitaly_errors do - gitaly_ref_client.tag_names - end - end - - # Returns an Array of Tags - # - def tags - wrapped_gitaly_errors do - gitaly_ref_client.tags - end - end - - # Returns true if the given ref name exists - # - # Ref names must start with `refs/`. - def ref_exists?(ref_name) - wrapped_gitaly_errors do - gitaly_ref_exists?(ref_name) - end - end - - # Returns true if the given tag exists - # - # name - The name of the tag as a String. - def tag_exists?(name) - wrapped_gitaly_errors do - gitaly_ref_exists?("refs/tags/#{name}") - end - end - - # Returns true if the given branch exists - # - # name - The name of the branch as a String. - def branch_exists?(name) - wrapped_gitaly_errors do - gitaly_ref_exists?("refs/heads/#{name}") - end - end - - # Returns an Array of branch and tag names - def ref_names - branch_names + tag_names - end - - def delete_all_refs_except(prefixes) - wrapped_gitaly_errors do - gitaly_ref_client.delete_refs(except_with_prefixes: prefixes) - end - end - - # Returns an Array of all ref names, except when it's matching pattern - # - # regexp - The pattern for ref names we don't want - def all_ref_names_except(prefixes) - rugged.references.reject do |ref| - prefixes.any? { |p| ref.name.start_with?(p) } - end.map(&:name) - end - - def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:) - ref ||= root_ref - commit = Gitlab::Git::Commit.find(self, ref) - return {} if commit.nil? - - prefix = archive_prefix(ref, commit.id, project_path, append_sha: append_sha) - - { - 'ArchivePrefix' => prefix, - 'ArchivePath' => archive_file_path(storage_path, commit.id, prefix, format), - 'CommitId' => commit.id, - 'GitalyRepository' => gitaly_repository.to_h - } - end - - # This is both the filename of the archive (missing the extension) and the - # name of the top-level member of the archive under which all files go - def archive_prefix(ref, sha, project_path, append_sha:) - append_sha = (ref != sha) if append_sha.nil? - - formatted_ref = ref.tr('/', '-') - - prefix_segments = [project_path, formatted_ref] - prefix_segments << sha if append_sha - - prefix_segments.join('-') - end - private :archive_prefix - - # The full path on disk where the archive should be stored. This is used - # to cache the archive between requests. - # - # The path is a global namespace, so needs to be globally unique. This is - # achieved by including `gl_repository` in the path. - # - # Archives relating to a particular ref when the SHA is not present in the - # filename must be invalidated when the ref is updated to point to a new - # SHA. This is achieved by including the SHA in the path. - # - # As this is a full path on disk, it is not "cloud native". This should - # be resolved by either removing the cache, or moving the implementation - # into Gitaly and removing the ArchivePath parameter from the git-archive - # senddata response. - def archive_file_path(storage_path, sha, name, format = "tar.gz") - # Build file path - return nil unless name - - extension = - case format - when "tar.bz2", "tbz", "tbz2", "tb2", "bz2" - "tar.bz2" - when "tar" - "tar" - when "zip" - "zip" - else - # everything else should fall back to tar.gz - "tar.gz" - end - - file_name = "#{name}.#{extension}" - File.join(storage_path, self.gl_repository, sha, file_name) - end - private :archive_file_path - - # Return repo size in megabytes - def size - size = gitaly_repository_client.repository_size - - (size.to_f / 1024).round(2) - end - - # Use the Rugged Walker API to build an array of commits. - # - # Usage. - # repo.log( - # ref: 'master', - # path: 'app/models', - # limit: 10, - # offset: 5, - # after: Time.new(2016, 4, 21, 14, 32, 10) - # ) - def log(options) - default_options = { - limit: 10, - offset: 0, - path: nil, - follow: false, - skip_merges: false, - after: nil, - before: nil, - all: false - } - - options = default_options.merge(options) - options[:offset] ||= 0 - - limit = options[:limit] - if limit == 0 || !limit.is_a?(Integer) - raise ArgumentError.new("invalid Repository#log limit: #{limit.inspect}") - end - - wrapped_gitaly_errors do - gitaly_commit_client.find_commits(options) - end - end - - def new_commits(newrev) - wrapped_gitaly_errors do - gitaly_ref_client.list_new_commits(newrev) - end - end - - def new_blobs(newrev) - return [] if newrev.blank? || newrev == ::Gitlab::Git::BLANK_SHA - - strong_memoize("new_blobs_#{newrev}") do - wrapped_gitaly_errors do - gitaly_ref_client.list_new_blobs(newrev, REV_LIST_COMMIT_LIMIT) - end - end - end - - def count_commits(options) - options = process_count_commits_options(options.dup) - - wrapped_gitaly_errors do - if options[:left_right] - from = options[:from] - to = options[:to] - - right_count = gitaly_commit_client - .commit_count("#{from}..#{to}", options) - left_count = gitaly_commit_client - .commit_count("#{to}..#{from}", options) - - [left_count, right_count] - else - gitaly_commit_client.commit_count(options[:ref], options) - end - end - end - - # Counts the amount of commits between `from` and `to`. - def count_commits_between(from, to, options = {}) - count_commits(from: from, to: to, **options) - end - - # old_rev and new_rev are commit ID's - # the result of this method is an array of Gitlab::Git::RawDiffChange - def raw_changes_between(old_rev, new_rev) - @raw_changes_between ||= {} - - @raw_changes_between[[old_rev, new_rev]] ||= - begin - return [] if new_rev.blank? || new_rev == Gitlab::Git::BLANK_SHA - - wrapped_gitaly_errors do - gitaly_repository_client.raw_changes_between(old_rev, new_rev) - .each_with_object([]) do |msg, arr| - msg.raw_changes.each { |change| arr << ::Gitlab::Git::RawDiffChange.new(change) } - end - end - end - rescue ArgumentError => e - raise Gitlab::Git::Repository::GitError.new(e) - end - - # Returns the SHA of the most recent common ancestor of +from+ and +to+ - def merge_base(from, to) - wrapped_gitaly_errors do - gitaly_repository_client.find_merge_base(from, to) - end - end - - # Returns true is +from+ is direct ancestor to +to+, otherwise false - def ancestor?(from, to) - gitaly_commit_client.ancestor?(from, to) - end - - def merged_branch_names(branch_names = []) - return [] unless root_ref - - root_sha = find_branch(root_ref)&.target - - return [] unless root_sha - - branches = wrapped_gitaly_errors do - gitaly_merged_branch_names(branch_names, root_sha) - end - - Set.new(branches) - end - - # Return an array of Diff objects that represent the diff - # between +from+ and +to+. See Diff::filter_diff_options for the allowed - # diff options. The +options+ hash can also include :break_rewrites to - # split larger rewrites into delete/add pairs. - def diff(from, to, options = {}, *paths) - iterator = gitaly_commit_client.diff(from, to, options.merge(paths: paths)) - - Gitlab::Git::DiffCollection.new(iterator, options) - end - - # Returns a RefName for a given SHA - def ref_name_for_sha(ref_path, sha) - raise ArgumentError, "sha can't be empty" unless sha.present? - - gitaly_ref_client.find_ref_name(sha, ref_path) - end - - # Get refs hash which key is is the commit id - # and value is a Gitlab::Git::Tag or Gitlab::Git::Branch - # Note that both inherit from Gitlab::Git::Ref - def refs_hash - return @refs_hash if @refs_hash - - @refs_hash = Hash.new { |h, k| h[k] = [] } - - (tags + branches).each do |ref| - next unless ref.target && ref.name - - @refs_hash[ref.dereferenced_target.id] << ref.name - end - - @refs_hash - end - - # Returns url for submodule - # - # Ex. - # @repository.submodule_url_for('master', 'rack') - # # => git@localhost:rack.git - # - def submodule_url_for(ref, path) - wrapped_gitaly_errors do - gitaly_submodule_url_for(ref, path) - end - end - - # Return total commits count accessible from passed ref - def commit_count(ref) - wrapped_gitaly_errors do - gitaly_commit_client.commit_count(ref) - end - end - - # Mimic the `git clean` command and recursively delete untracked files. - # Valid keys that can be passed in the +options+ hash are: - # - # :d - Remove untracked directories - # :f - Remove untracked directories that are managed by a different - # repository - # :x - Remove ignored files - # - # The value in +options+ must evaluate to true for an option to take - # effect. - # - # Examples: - # - # repo.clean(d: true, f: true) # Enable the -d and -f options - # - # repo.clean(d: false, x: true) # -x is enabled, -d is not - def clean(options = {}) - strategies = [:remove_untracked] - strategies.push(:force) if options[:f] - strategies.push(:remove_ignored) if options[:x] - - # TODO: implement this method - end - - def add_branch(branch_name, user:, target:) - wrapped_gitaly_errors do - gitaly_operation_client.user_create_branch(branch_name, user, target) - end - end - - def add_tag(tag_name, user:, target:, message: nil) - wrapped_gitaly_errors do - gitaly_operation_client.add_tag(tag_name, user, target, message) - end - end - - def update_branch(branch_name, user:, newrev:, oldrev:) - gitaly_migrate(:operation_user_update_branch) do |is_enabled| - if is_enabled - gitaly_operation_client.user_update_branch(branch_name, user, newrev, oldrev) - else - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - OperationService.new(user, self).update_branch(branch_name, newrev, oldrev) - end - end - end - end - - def rm_branch(branch_name, user:) - wrapped_gitaly_errors do - gitaly_operation_client.user_delete_branch(branch_name, user) - end - end - - def rm_tag(tag_name, user:) - wrapped_gitaly_errors do - gitaly_operation_client.rm_tag(tag_name, user) - end - end - - def find_tag(name) - tags.find { |tag| tag.name == name } - end - - def merge(user, source_sha, target_branch, message, &block) - wrapped_gitaly_errors do - gitaly_operation_client.user_merge_branch(user, source_sha, target_branch, message, &block) - end - end - - def ff_merge(user, source_sha, target_branch) - wrapped_gitaly_errors do - gitaly_operation_client.user_ff_branch(user, source_sha, target_branch) - end - end - - def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - args = { - user: user, - commit: commit, - branch_name: branch_name, - message: message, - start_branch_name: start_branch_name, - start_repository: start_repository - } - - wrapped_gitaly_errors do - gitaly_operation_client.user_revert(args) - end - end - - def check_revert_content(target_commit, source_sha) - args = [target_commit.sha, source_sha] - args << { mainline: 1 } if target_commit.merge_commit? - - revert_index = rugged.revert_commit(*args) - return false if revert_index.conflicts? - - tree_id = revert_index.write_tree(rugged) - return false unless diff_exists?(source_sha, tree_id) - - tree_id - end - - def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - args = { - user: user, - commit: commit, - branch_name: branch_name, - message: message, - start_branch_name: start_branch_name, - start_repository: start_repository - } - - wrapped_gitaly_errors do - gitaly_operation_client.user_cherry_pick(args) - end - end - - def diff_exists?(sha1, sha2) - rugged.diff(sha1, sha2).size > 0 - end - - def user_to_committer(user) - Gitlab::Git.committer_hash(email: user.email, name: user.name) - end - - # Delete the specified branch from the repository - def delete_branch(branch_name) - wrapped_gitaly_errors do - gitaly_ref_client.delete_branch(branch_name) - end - rescue CommandError => e - raise DeleteBranchError, e - end - - def delete_refs(*ref_names) - wrapped_gitaly_errors do - gitaly_delete_refs(*ref_names) - end - end - - # Create a new branch named **ref+ based on **stat_point+, HEAD by default - # - # Examples: - # create_branch("feature") - # create_branch("other-feature", "master") - def create_branch(ref, start_point = "HEAD") - wrapped_gitaly_errors do - gitaly_ref_client.create_branch(ref, start_point) - end - end - - # If `mirror_refmap` is present the remote is set as mirror with that mapping - def add_remote(remote_name, url, mirror_refmap: nil) - wrapped_gitaly_errors do - gitaly_remote_client.add_remote(remote_name, url, mirror_refmap) - end - end - - def remove_remote(remote_name) - wrapped_gitaly_errors do - gitaly_remote_client.remove_remote(remote_name) - end - end - - AUTOCRLF_VALUES = { - "true" => true, - "false" => false, - "input" => :input - }.freeze - - def autocrlf - AUTOCRLF_VALUES[rugged.config['core.autocrlf']] - end - - def autocrlf=(value) - rugged.config['core.autocrlf'] = AUTOCRLF_VALUES.invert[value] - end - - # Returns result like "git ls-files" , recursive and full file path - # - # Ex. - # repo.ls_files('master') - # - def ls_files(ref) - gitaly_commit_client.ls_files(ref) - end - - def copy_gitattributes(ref) - wrapped_gitaly_errors do - gitaly_repository_client.apply_gitattributes(ref) - end - end - - def info_attributes - return @info_attributes if @info_attributes - - content = gitaly_repository_client.info_attributes - @info_attributes = AttributesParser.new(content) - end - - # Returns the Git attributes for the given file path. - # - # See `Gitlab::Git::Attributes` for more information. - def attributes(path) - info_attributes.attributes(path) - end - - def gitattribute(path, name) - attributes(path)[name] - end - - # Check .gitattributes for a given ref - # - # This only checks the root .gitattributes file, - # it does not traverse subfolders to find additional .gitattributes files - # - # This method is around 30 times slower than `attributes`, which uses - # `$GIT_DIR/info/attributes`. Consider caching AttributesAtRefParser - # and reusing that for multiple calls instead of this method. - def attributes_at(ref, file_path) - parser = AttributesAtRefParser.new(self, ref) - parser.attributes(file_path) - end - - def languages(ref = nil) - wrapped_gitaly_errors do - gitaly_commit_client.languages(ref) - end - end - - def license_short_name - wrapped_gitaly_errors do - gitaly_repository_client.license_short_name - end - end - - def with_repo_branch_commit(start_repository, start_branch_name) - Gitlab::Git.check_namespace!(start_repository) - start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) - - return yield nil if start_repository.empty? - - if start_repository.same_repository?(self) - yield commit(start_branch_name) - else - start_commit_id = start_repository.commit_id(start_branch_name) - - return yield nil unless start_commit_id - - if branch_commit = commit(start_commit_id) - yield branch_commit - else - with_repo_tmp_commit( - start_repository, start_branch_name, start_commit_id) do |tmp_commit| - yield tmp_commit - end - end - end - end - - def with_repo_tmp_commit(start_repository, start_branch_name, sha) - source_ref = start_branch_name - - unless Gitlab::Git.branch_ref?(source_ref) - source_ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{source_ref}" - end - - tmp_ref = fetch_ref( - start_repository, - source_ref: source_ref, - target_ref: "refs/tmp/#{SecureRandom.hex}" - ) - - yield commit(sha) - ensure - delete_refs(tmp_ref) if tmp_ref - end - - def fetch_source_branch!(source_repository, source_branch, local_ref) - wrapped_gitaly_errors do - gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref) - end - end - - def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:) - tmp_ref = "refs/tmp/#{SecureRandom.hex}" - - return unless fetch_source_branch!(source_repository, source_branch_name, tmp_ref) - - Gitlab::Git::Compare.new( - self, - target_branch_name, - tmp_ref, - straight: straight - ) - ensure - delete_refs(tmp_ref) - end - - def write_ref(ref_path, ref, old_ref: nil, shell: true) - ref_path = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{ref_path}" unless ref_path.start_with?("refs/") || ref_path == "HEAD" - - wrapped_gitaly_errors do - gitaly_repository_client.write_ref(ref_path, ref, old_ref, shell) - end - end - - # This method, fetch_ref, is used from within - # Gitlab::Git::OperationService. OperationService will eventually only - # exist in gitaly-ruby. When we delete OperationService from gitlab-ce - # we can also remove fetch_ref. - def fetch_ref(source_repository, source_ref:, target_ref:) - Gitlab::Git.check_namespace!(source_repository) - source_repository = RemoteRepository.new(source_repository) unless source_repository.is_a?(RemoteRepository) - - args = %W(fetch --no-tags -f #{GITALY_INTERNAL_URL} #{source_ref}:#{target_ref}) - message, status = run_git(args, env: source_repository.fetch_env) - raise Gitlab::Git::CommandError, message if status != 0 - - target_ref - end - - # Refactoring aid; allows us to copy code from app/models/repository.rb - def commit(ref = 'HEAD') - Gitlab::Git::Commit.find(self, ref) - end - - def empty? - !has_visible_content? - end - - def fetch_repository_as_mirror(repository) - wrapped_gitaly_errors do - gitaly_remote_client.fetch_internal_remote(repository) - end - end - - def blob_at(sha, path) - Gitlab::Git::Blob.find(self, sha, path) unless Gitlab::Git.blank_ref?(sha) - end - - # Items should be of format [[commit_id, path], [commit_id1, path1]] - def batch_blobs(items, blob_size_limit: Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) - Gitlab::Git::Blob.batch(self, items, blob_size_limit: blob_size_limit) - end - - def fsck - msg, status = gitaly_repository_client.fsck - - raise GitError.new("Could not fsck repository: #{msg}") unless status.zero? - end - - def create_from_bundle(bundle_path) - gitaly_repository_client.create_from_bundle(bundle_path) - end - - def create_from_snapshot(url, auth) - gitaly_repository_client.create_from_snapshot(url, auth) - end - - def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) - wrapped_gitaly_errors do - gitaly_operation_client.user_rebase(user, rebase_id, - branch: branch, - branch_sha: branch_sha, - remote_repository: remote_repository, - remote_branch: remote_branch) - end - end - - def rebase_in_progress?(rebase_id) - wrapped_gitaly_errors do - gitaly_repository_client.rebase_in_progress?(rebase_id) - end - end - - def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:) - wrapped_gitaly_errors do - gitaly_operation_client.user_squash(user, squash_id, branch, - start_sha, end_sha, author, message) - end - end - - def squash_in_progress?(squash_id) - wrapped_gitaly_errors do - gitaly_repository_client.squash_in_progress?(squash_id) - end - end - - def push_remote_branches(remote_name, branch_names, forced: true) - success = @gitlab_projects.push_branches(remote_name, GITLAB_PROJECTS_TIMEOUT, forced, branch_names) - - success || gitlab_projects_error - end - - def delete_remote_branches(remote_name, branch_names) - success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) - - success || gitlab_projects_error - end - - def delete_remote_branches(remote_name, branch_names) - success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) - - success || gitlab_projects_error - end - - def bundle_to_disk(save_path) - wrapped_gitaly_errors do - gitaly_repository_client.create_bundle(save_path) - end - - true - end - - def multi_action( - user, branch_name:, message:, actions:, - author_email: nil, author_name: nil, - start_branch_name: nil, start_repository: self) - - wrapped_gitaly_errors do - gitaly_operation_client.user_commit_files(user, branch_name, - message, actions, author_email, author_name, - start_branch_name, start_repository) - end - end - - def write_config(full_path:) - return unless full_path.present? - - # This guard avoids Gitaly log/error spam - raise NoRepository, 'repository does not exist' unless exists? - - set_config('gitlab.fullpath' => full_path) - end - - def set_config(entries) - wrapped_gitaly_errors do - gitaly_repository_client.set_config(entries) - end - end - - def delete_config(*keys) - wrapped_gitaly_errors do - gitaly_repository_client.delete_config(keys) - end - end - - def gitaly_repository - Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) - end - - def gitaly_ref_client - @gitaly_ref_client ||= Gitlab::GitalyClient::RefService.new(self) - end - - def gitaly_commit_client - @gitaly_commit_client ||= Gitlab::GitalyClient::CommitService.new(self) - end - - def gitaly_repository_client - @gitaly_repository_client ||= Gitlab::GitalyClient::RepositoryService.new(self) - end - - def gitaly_operation_client - @gitaly_operation_client ||= Gitlab::GitalyClient::OperationService.new(self) - end - - def gitaly_remote_client - @gitaly_remote_client ||= Gitlab::GitalyClient::RemoteService.new(self) - end - - def gitaly_blob_client - @gitaly_blob_client ||= Gitlab::GitalyClient::BlobService.new(self) - end - - def gitaly_conflicts_client(our_commit_oid, their_commit_oid) - Gitlab::GitalyClient::ConflictsService.new(self, our_commit_oid, their_commit_oid) - end - - def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) - Gitlab::GitalyClient.migrate(method, status: status, &block) - rescue GRPC::NotFound => e - raise NoRepository.new(e) - rescue GRPC::InvalidArgument => e - raise ArgumentError.new(e) - rescue GRPC::BadStatus => e - raise CommandError.new(e) - end - - def wrapped_gitaly_errors(&block) - yield block - rescue GRPC::NotFound => e - raise NoRepository.new(e) - rescue GRPC::InvalidArgument => e - raise ArgumentError.new(e) - rescue GRPC::BadStatus => e - raise CommandError.new(e) - end - - def clean_stale_repository_files - wrapped_gitaly_errors do - gitaly_repository_client.cleanup if exists? - end - rescue Gitlab::Git::CommandError => e # Don't fail if we can't cleanup - Rails.logger.error("Unable to clean repository on storage #{storage} with relative path #{relative_path}: #{e.message}") - Gitlab::Metrics.counter( - :failed_repository_cleanup_total, - 'Number of failed repository cleanup events' - ).increment - end - - def branch_names_contains_sha(sha) - gitaly_ref_client.branch_names_contains_sha(sha) - end - - def tag_names_contains_sha(sha) - gitaly_ref_client.tag_names_contains_sha(sha) - end - - def search_files_by_content(query, ref) - return [] if empty? || query.blank? - - safe_query = Regexp.escape(query) - ref ||= root_ref - - gitaly_repository_client.search_files_by_content(ref, safe_query) - end - - def can_be_merged?(source_sha, target_branch) - if target_sha = find_branch(target_branch)&.target - !gitaly_conflicts_client(source_sha, target_sha).conflicts? - else - false - end - end - - def search_files_by_name(query, ref) - safe_query = Regexp.escape(query.sub(%r{^/*}, "")) - ref ||= root_ref - - return [] if empty? || safe_query.blank? - - gitaly_repository_client.search_files_by_name(ref, safe_query) - end - - def find_commits_by_message(query, ref, path, limit, offset) - wrapped_gitaly_errors do - gitaly_commit_client - .commits_by_message(query, revision: ref, path: path, limit: limit, offset: offset) - .map { |c| commit(c) } - end - end - - def shell_blame(sha, path) - output, _status = run_git(%W(blame -p #{sha} -- #{path})) - output - end - - def last_commit_for_path(sha, path) - wrapped_gitaly_errors do - gitaly_commit_client.last_commit_for_path(sha, path) - end - end - - def rev_list(including: [], excluding: [], options: [], objects: false, &block) - args = ['rev-list'] - - args.push(*rev_list_param(including)) - - exclude_param = *rev_list_param(excluding) - if exclude_param.any? - args.push('--not') - args.push(*exclude_param) - end - - args.push('--objects') if objects - - if options.any? - args.push(*options) - end - - run_git!(args, lazy_block: block) - end - - def checksum - # The exists? RPC is much cheaper, so we perform this request first - raise NoRepository, "Repository does not exists" unless exists? - - gitaly_repository_client.calculate_checksum - rescue GRPC::NotFound - raise NoRepository # Guard against data races. - end - - private - - def uncached_has_local_branches? - wrapped_gitaly_errors do - gitaly_repository_client.has_local_branches? - end - end - - def run_git(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) - cmd = [Gitlab.config.git.bin_path, *args] - cmd.unshift("nice") if nice - - object_directories = alternate_object_directories - if object_directories.any? - env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = object_directories.join(File::PATH_SEPARATOR) - end - - circuit_breaker.perform do - popen(cmd, chdir, env, lazy_block: lazy_block, &block) - end - end - - def run_git!(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) - output, status = run_git(args, chdir: chdir, env: env, nice: nice, lazy_block: lazy_block, &block) - - raise GitError, output unless status.zero? - - output - end - - def run_git_with_timeout(args, timeout, env: {}) - circuit_breaker.perform do - popen_with_timeout([Gitlab.config.git.bin_path, *args], timeout, path, env) - end - end - - def git_env_for_user(user) - { - 'GIT_COMMITTER_NAME' => user.name, - 'GIT_COMMITTER_EMAIL' => user.email, - 'GL_ID' => Gitlab::GlId.gl_id(user), - 'GL_PROTOCOL' => Gitlab::Git::Hook::GL_PROTOCOL, - 'GL_REPOSITORY' => gl_repository - } - end - - def gitaly_merged_branch_names(branch_names, root_sha) - qualified_branch_names = branch_names.map { |b| "refs/heads/#{b}" } - - gitaly_ref_client.merged_branches(qualified_branch_names) - .reject { |b| b.target == root_sha } - .map(&:name) - end - - def process_count_commits_options(options) - if options[:from] || options[:to] - ref = - if options[:left_right] # Compare with merge-base for left-right - "#{options[:from]}...#{options[:to]}" - else - "#{options[:from]}..#{options[:to]}" - end - - options.merge(ref: ref) - - elsif options[:ref] && options[:left_right] - from, to = options[:ref].match(/\A([^\.]*)\.{2,3}([^\.]*)\z/)[1..2] - - options.merge(from: from, to: to) - else - options - end - end - - def gitaly_submodule_url_for(ref, path) - # We don't care about the contents so 1 byte is enough. Can't request 0 bytes, 0 means unlimited. - commit_object = gitaly_commit_client.tree_entry(ref, path, 1) - - return unless commit_object && commit_object.type == :COMMIT - - gitmodules = gitaly_commit_client.tree_entry(ref, '.gitmodules', Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) - return unless gitmodules - - found_module = GitmodulesParser.new(gitmodules.data).parse[path] - - found_module && found_module['url'] - end - - def alternate_object_directories - relative_object_directories.map { |d| File.join(path, d) } - end - - def relative_object_directories - Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact - end - - def sort_branches(branches, sort_by) - case sort_by - when 'name' - branches.sort_by(&:name) - when 'updated_desc' - branches.sort do |a, b| - b.dereferenced_target.committed_date <=> a.dereferenced_target.committed_date - end - when 'updated_asc' - branches.sort do |a, b| - a.dereferenced_target.committed_date <=> b.dereferenced_target.committed_date - end - else - branches - end - end - - # Returns true if the given ref name exists - # - # Ref names must start with `refs/`. - def gitaly_ref_exists?(ref_name) - gitaly_ref_client.ref_exists?(ref_name) - end - - def gitaly_copy_gitattributes(revision) - gitaly_repository_client.apply_gitattributes(revision) - end - - def gitaly_delete_refs(*ref_names) - gitaly_ref_client.delete_refs(refs: ref_names) if ref_names.any? - end - - def gitlab_projects_error - raise CommandError, @gitlab_projects.output - end - - def rev_list_param(spec) - spec == :all ? ['--all'] : spec - end - end - end -end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb deleted file mode 100644 index 65eb5cc18..000000000 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/repository_mirroring.rb +++ /dev/null @@ -1,36 +0,0 @@ -module Gitlab - module Git - module RepositoryMirroring - def remote_branches(remote_name) - gitaly_migrate(:ref_find_all_remote_branches) do |is_enabled| - if is_enabled - gitaly_ref_client.remote_branches(remote_name) - else - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - rugged_remote_branches(remote_name) - end - end - end - end - - private - - def rugged_remote_branches(remote_name) - branches = [] - - rugged.references.each("refs/remotes/#{remote_name}/*").map do |ref| - name = ref.name.sub(%r{\Arefs/remotes/#{remote_name}/}, '') - - begin - target_commit = Gitlab::Git::Commit.find(self, ref.target.oid) - branches << Gitlab::Git::Branch.new(self, name, ref.target, target_commit) - rescue Rugged::ReferenceError - # Omit invalid branch - end - end - - branches - end - end - end -end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/util.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/util.rb deleted file mode 100644 index 4708f22dc..000000000 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/util.rb +++ /dev/null @@ -1,20 +0,0 @@ -# Gitaly note: JV: no RPC's here. - -module Gitlab - module Git - module Util - LINE_SEP = "\n".freeze - - def self.count_lines(string) - case string[-1] - when nil - 0 - when LINE_SEP - string.count(LINE_SEP) - else - string.count(LINE_SEP) + 1 - end - end - end - end -end |