diff options
author | Jacob Vosmaer (GitLab) <jacob@gitlab.com> | 2017-10-04 17:33:33 +0300 |
---|---|---|
committer | Ahmad Sherif <ahmad.m.sherif@gmail.com> | 2017-10-04 17:33:33 +0300 |
commit | 97a7930d68d4e65f264d6784441f91d65762918b (patch) | |
tree | e96efca1adc9500bf2f57fd849ce76ae96c04515 | |
parent | 0298bb076f46967a174c69582e4794241a392b53 (diff) |
Update gitlab_git to 4a0f720a502ac02423
22 files changed, 526 insertions, 125 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index c580e7acb..711ec0880 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ UNRELEASED +- Update gitlab_git to 4a0f720a502ac02423 + https://gitlab.com/gitlab-org/gitaly/merge_requests/389 - Fix incorrect parsing of diff chunks starting with ++ or -- https://gitlab.com/gitlab-org/gitaly/merge_requests/385 - Implement Raw{Diff,Patch} RPCs diff --git a/ruby/lib/gitaly_server/operations_service.rb b/ruby/lib/gitaly_server/operations_service.rb index 923317cda..36bfb650e 100644 --- a/ruby/lib/gitaly_server/operations_service.rb +++ b/ruby/lib/gitaly_server/operations_service.rb @@ -32,7 +32,7 @@ module GitalyServer Gitaly::UserCreateTagResponse.new(tag: tag) rescue Gitlab::Git::Repository::InvalidRef => e raise GRPC::FailedPrecondition.new(e.message) - rescue Rugged::TagError + rescue Gitlab::Git::Repository::TagExistsError return Gitaly::UserCreateTagResponse.new(exists: true) rescue Gitlab::Git::HooksService::PreReceiveError => e return Gitaly::UserCreateTagResponse.new(pre_receive_error: e.message) diff --git a/ruby/lib/gitaly_server/repository_service.rb b/ruby/lib/gitaly_server/repository_service.rb index d27d6356e..c26c763e3 100644 --- a/ruby/lib/gitaly_server/repository_service.rb +++ b/ruby/lib/gitaly_server/repository_service.rb @@ -6,8 +6,7 @@ module GitalyServer bridge_exceptions do repo_path = GitalyServer.repo_path(_call) - # TODO refactor Repository.create to eliminate bogus '/' argument - Gitlab::Git::Repository.create('/', repo_path, bare: true, symlink_hooks_to: Gitlab.config.gitlab_shell.hooks_path) + Gitlab::Git::Repository.create(repo_path, bare: true, symlink_hooks_to: Gitlab.config.gitlab_shell.hooks_path) Gitaly::CreateRepositoryResponse.new end diff --git a/ruby/lib/gitlab/git.rb b/ruby/lib/gitlab/git.rb index 8ade7f9a5..f92f334f1 100644 --- a/ruby/lib/gitlab/git.rb +++ b/ruby/lib/gitlab/git.rb @@ -10,6 +10,7 @@ require 'active_support/core_ext/enumerable' # We split our mock implementation of Gitlab::GitalyClient into a separate file require_relative 'gitaly_client.rb' +require_relative 'git_logger.rb' vendor_gitlab_git = '../../vendor/gitlab_git/' @@ -83,3 +84,10 @@ module Gitlab end end end + +class String + # Because we are not rendering HTML, this is a no-op in gitaly-ruby. + def html_safe + self + end +end diff --git a/ruby/lib/gitlab/git_logger.rb b/ruby/lib/gitlab/git_logger.rb new file mode 100644 index 000000000..23a163ba6 --- /dev/null +++ b/ruby/lib/gitlab/git_logger.rb @@ -0,0 +1,7 @@ +require 'logger' + +module Gitlab + GitLogger = Logger.new(STDOUT) + GitLogger.progname = 'githost.log' + GitLogger.level = 'info' +end diff --git a/ruby/vendor/gitlab_git/REVISION b/ruby/vendor/gitlab_git/REVISION index a6939283e..7c3aa3cb2 100644 --- a/ruby/vendor/gitlab_git/REVISION +++ b/ruby/vendor/gitlab_git/REVISION @@ -1 +1 @@ -8612b0889482a184c366e0493ab8f17b6e324418 +4a0f720a502ac02423cb9db20727ba386de3e1f1 diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/blob.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/blob.rb index 8d96826f6..a4336face 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/blob.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/blob.rb @@ -32,6 +32,8 @@ module Gitlab else blob = repository.lookup(sha) + next unless blob.is_a?(Rugged::Blob) + new( id: blob.oid, size: blob.size, diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/diff.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/diff.rb index a23c8cf0d..ca94b4baa 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/diff.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/diff.rb @@ -24,41 +24,13 @@ module Gitlab SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze - class << self - # The maximum size of a diff to display. - def size_limit - if RequestStore.active? - RequestStore['gitlab_git_diff_size_limit'] ||= find_size_limit - else - find_size_limit - end - end - - # The maximum size before a diff is collapsed. - def collapse_limit - if RequestStore.active? - RequestStore['gitlab_git_diff_collapse_limit'] ||= find_collapse_limit - else - find_collapse_limit - end - end - - def find_size_limit - if Feature.enabled?('gitlab_git_diff_size_limit_increase') - 200.kilobytes - else - 100.kilobytes - end - end + # The maximum size of a diff to display. + SIZE_LIMIT = 100.kilobytes - def find_collapse_limit - if Feature.enabled?('gitlab_git_diff_size_limit_increase') - 100.kilobytes - else - 10.kilobytes - end - end + # The maximum size before a diff is collapsed. + COLLAPSE_LIMIT = 10.kilobytes + class << self def between(repo, head, base, options = {}, *paths) straight = options.delete(:straight) || false @@ -172,7 +144,7 @@ module Gitlab def too_large? if @too_large.nil? - @too_large = @diff.bytesize >= self.class.size_limit + @too_large = @diff.bytesize >= SIZE_LIMIT else @too_large end @@ -190,7 +162,7 @@ module Gitlab def collapsed? return @collapsed if defined?(@collapsed) - @collapsed = !expanded && @diff.bytesize >= self.class.collapse_limit + @collapsed = !expanded && @diff.bytesize >= COLLAPSE_LIMIT end def collapse! @@ -206,6 +178,10 @@ module Gitlab Diff.binary_message(@old_path, @new_path) end + def has_binary_notice? + @diff.start_with?('Binary') + end + private def init_from_rugged(rugged) @@ -271,14 +247,14 @@ module Gitlab hunk.each_line do |line| size += line.content.bytesize - if size >= self.class.size_limit + if size >= SIZE_LIMIT too_large! return true end end end - if !expanded && size >= self.class.collapse_limit + if !expanded && size >= COLLAPSE_LIMIT collapse! return true end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/hook.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/hook.rb index cc35d77c6..e29a1f7af 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/hook.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/hook.rb @@ -22,22 +22,22 @@ module Gitlab File.exist?(path) end - def trigger(gl_id, oldrev, newrev, ref) + def trigger(gl_id, gl_username, oldrev, newrev, ref) return [true, nil] unless exists? Bundler.with_clean_env do case name when "pre-receive", "post-receive" - call_receive_hook(gl_id, oldrev, newrev, ref) + call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) when "update" - call_update_hook(gl_id, oldrev, newrev, ref) + call_update_hook(gl_id, gl_username, oldrev, newrev, ref) end end end private - def call_receive_hook(gl_id, oldrev, newrev, ref) + def call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) changes = [oldrev, newrev, ref].join(" ") exit_status = false @@ -45,6 +45,7 @@ module Gitlab vars = { 'GL_ID' => gl_id, + 'GL_USERNAME' => gl_username, 'PWD' => repo_path, 'GL_PROTOCOL' => GL_PROTOCOL, 'GL_REPOSITORY' => repository.gl_repository @@ -80,16 +81,21 @@ module Gitlab [exit_status, exit_message] end - def call_update_hook(gl_id, oldrev, newrev, ref) + def call_update_hook(gl_id, gl_username, oldrev, newrev, ref) Dir.chdir(repo_path) do - stdout, stderr, status = Open3.capture3({ 'GL_ID' => gl_id }, path, ref, oldrev, newrev) - [status.success?, stderr.presence || stdout] + env = { + 'GL_ID' => gl_id, + 'GL_USERNAME' => gl_username + } + stdout, stderr, status = Open3.capture3(env, path, ref, oldrev, newrev) + [status.success?, (stderr.presence || stdout).gsub(/\R/, "<br>").html_safe] end end def retrieve_error_message(stderr, stdout) - err_message = stderr.gets - err_message.blank? ? stdout.gets : err_message + err_message = stderr.read + err_message = err_message.blank? ? stdout.read : err_message + err_message.gsub(/\R/, "<br>").html_safe end end end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/hooks_service.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/hooks_service.rb index ea8a87a12..c327e9b16 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/hooks_service.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/hooks_service.rb @@ -5,12 +5,13 @@ module Gitlab attr_accessor :oldrev, :newrev, :ref - def execute(committer, repository, oldrev, newrev, ref) - @repository = repository - @gl_id = committer.gl_id - @oldrev = oldrev - @newrev = newrev - @ref = ref + def execute(pusher, repository, oldrev, newrev, ref) + @repository = repository + @gl_id = pusher.gl_id + @gl_username = pusher.name + @oldrev = oldrev + @newrev = newrev + @ref = ref %w(pre-receive update).each do |hook_name| status, message = run_hook(hook_name) @@ -29,7 +30,7 @@ module Gitlab def run_hook(name) hook = Gitlab::Git::Hook.new(name, @repository) - hook.trigger(@gl_id, oldrev, newrev, ref) + hook.trigger(@gl_id, @gl_username, oldrev, newrev, ref) end end end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/operation_service.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/operation_service.rb index 786e2e7e8..d835dcca8 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/operation_service.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/operation_service.rb @@ -152,13 +152,15 @@ module Gitlab # (and have!) accidentally reset the ref to an earlier state, clobbering # commits. See also https://github.com/libgit2/libgit2/issues/1534. command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] - _, status = popen( + + output, status = popen( command, repository.path) do |stdin| stdin.write("update #{ref}\x00#{newrev}\x00#{oldrev}\x00") end unless status.zero? + Gitlab::GitLogger.error("'git update-ref' in #{repository.path}: #{output}") raise Gitlab::Git::CommitError.new( "Could not update branch #{Gitlab::Git.branch_name(ref)}." \ " Please refresh and try again.") diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb index 0be35034d..89b654253 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb @@ -20,13 +20,11 @@ module Gitlab GitError = Class.new(StandardError) DeleteBranchError = Class.new(StandardError) CreateTreeError = Class.new(StandardError) + TagExistsError = Class.new(StandardError) class << self - # Unlike `new`, `create` takes the storage path, not the storage name - def create(storage_path, name, bare: true, symlink_hooks_to: nil) - repo_path = File.join(storage_path, name) - repo_path += '.git' unless repo_path.end_with?('.git') - + # Unlike `new`, `create` takes the repository path + def create(repo_path, bare: true, symlink_hooks_to: nil) FileUtils.mkdir_p(repo_path, mode: 0770) # Equivalent to `git --git-path=#{repo_path} init [--bare]` @@ -55,14 +53,15 @@ module Gitlab # Rugged repo object attr_reader :rugged - attr_reader :storage, :gl_repository, :relative_path + attr_reader :storage, :gl_repository, :relative_path, :gitaly_resolver - # 'path' must be the path to a _bare_ git repository, e.g. - # /path/to/my-repo.git + # This initializer method is only used on the client side (gitlab-ce). + # Gitaly-ruby uses a different initializer. def initialize(storage, relative_path, gl_repository) @storage = storage @relative_path = relative_path @gl_repository = gl_repository + @gitaly_resolver = Gitlab::GitalyClient storage_path = Gitlab.config.repositories.storages[@storage]['path'] @path = File.join(storage_path, @relative_path) @@ -73,8 +72,6 @@ module Gitlab delegate :empty?, to: :rugged - delegate :exists?, to: :gitaly_repository_client - def ==(other) path == other.path end @@ -102,6 +99,18 @@ module Gitlab @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) end + def exists? + Gitlab::GitalyClient.migrate(:repository_exists) do |enabled| + if enabled + gitaly_repository_client.exists? + else + circuit_breaker.perform do + File.exist?(File.join(@path, 'refs')) + end + end + end + end + # Returns an Array of branch names # sorted by name ASC def branch_names @@ -181,6 +190,28 @@ module Gitlab end end + def has_local_branches? + gitaly_migrate(:has_local_branches) do |is_enabled| + if is_enabled + gitaly_ref_client.has_local_branches? + else + has_local_branches_rugged? + end + end + end + + def has_local_branches_rugged? + rugged.branches.each(:local).any? do |ref| + begin + ref.name && ref.target # ensures the branch is valid + + true + rescue Rugged::ReferenceError + false + end + end + end + # Returns the number of valid tags def tag_count gitaly_migrate(:tag_names) do |is_enabled| @@ -386,7 +417,13 @@ module Gitlab options[:limit] ||= 0 options[:offset] ||= 0 - raw_log(options).map { |c| Commit.decorate(self, c) } + gitaly_migrate(:find_commits) do |is_enabled| + if is_enabled + gitaly_commit_client.find_commits(options) + else + raw_log(options).map { |c| Commit.decorate(self, c) } + end + end end # Used in gitaly-ruby @@ -475,7 +512,15 @@ module Gitlab # diff options. The +options+ hash can also include :break_rewrites to # split larger rewrites into delete/add pairs. def diff(from, to, options = {}, *paths) - Gitlab::Git::DiffCollection.new(diff_patches(from, to, options, *paths), options) + iterator = gitaly_migrate(:diff_between) do |is_enabled| + if is_enabled + gitaly_commit_client.diff(from, to, options.merge(paths: paths)) + else + diff_patches(from, to, options, *paths) + end + end + + Gitlab::Git::DiffCollection.new(iterator, options) end # Returns a RefName for a given SHA @@ -612,42 +657,43 @@ module Gitlab end def add_branch(branch_name, user:, target:) - target_object = Ref.dereference_object(lookup(target)) - raise InvalidRef.new("target not found: #{target}") unless target_object - - OperationService.new(user, self).add_branch(branch_name, target_object.oid) - find_branch(branch_name) - rescue Rugged::ReferenceError => ex - raise InvalidRef, ex + gitaly_migrate(:operation_user_create_branch) do |is_enabled| + if is_enabled + gitaly_add_branch(branch_name, user, target) + else + rugged_add_branch(branch_name, user, target) + end + end end def add_tag(tag_name, user:, target:, message: nil) - target_object = Ref.dereference_object(lookup(target)) - raise InvalidRef.new("target not found: #{target}") unless target_object - - user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) - - options = nil # Use nil, not the empty hash. Rugged cares about this. - if message - options = { - message: message, - tagger: Gitlab::Git.committer_hash(email: user.email, name: user.name) - } + gitaly_migrate(:operation_user_add_tag) do |is_enabled| + if is_enabled + gitaly_add_tag(tag_name, user: user, target: target, message: message) + else + rugged_add_tag(tag_name, user: user, target: target, message: message) + end end - - OperationService.new(user, self).add_tag(tag_name, target_object.oid, options) - - find_tag(tag_name) - rescue Rugged::ReferenceError => ex - raise InvalidRef, ex end def rm_branch(branch_name, user:) - OperationService.new(user, self).rm_branch(find_branch(branch_name)) + gitaly_migrate(:operation_user_delete_branch) do |is_enabled| + if is_enabled + gitaly_operations_client.user_delete_branch(branch_name, user) + else + OperationService.new(user, self).rm_branch(find_branch(branch_name)) + end + end end def rm_tag(tag_name, user:) - OperationService.new(user, self).rm_tag(find_tag(tag_name)) + gitaly_migrate(:operation_user_delete_tag) do |is_enabled| + if is_enabled + gitaly_operations_client.rm_tag(tag_name, user) + else + Gitlab::Git::OperationService.new(user, self).rm_tag(find_tag(tag_name)) + end + end end def find_tag(name) @@ -923,7 +969,11 @@ module Gitlab if start_repository == self yield commit(start_branch_name) else - sha = start_repository.commit(start_branch_name).sha + start_commit = start_repository.commit(start_branch_name) + + return yield nil unless start_commit + + sha = start_commit.sha if branch_commit = commit(sha) yield branch_commit @@ -938,9 +988,9 @@ module Gitlab def with_repo_tmp_commit(start_repository, start_branch_name, sha) tmp_ref = fetch_ref( - start_repository.path, - "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", - "refs/tmp/#{SecureRandom.hex}/head" + start_repository, + source_ref: "#{Gitlab::Git::BRANCH_REF_PREFIX}#{start_branch_name}", + target_ref: "refs/tmp/#{SecureRandom.hex}/head" ) yield commit(sha) @@ -952,8 +1002,9 @@ module Gitlab with_repo_branch_commit(source_repository, source_branch) do |commit| if commit write_ref(local_ref, commit.sha) + true else - raise Rugged::ReferenceError, 'source repository is empty' + false end end end @@ -971,13 +1022,27 @@ module Gitlab end end - def write_ref(ref_path, sha) - rugged.references.create(ref_path, sha, force: true) + def write_ref(ref_path, ref) + raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') + raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") + + command = [Gitlab.config.git.bin_path] + %w[update-ref --stdin -z] + input = "update #{ref_path}\x00#{ref}\x00\x00" + output, status = circuit_breaker.perform do + popen(command, path) { |stdin| stdin.write(input) } + end + + raise GitError, output unless status.zero? end - def fetch_ref(source_path, source_ref, target_ref) - args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) - message, status = run_git(args) + def fetch_ref(source_repository, source_ref:, target_ref:) + message, status = GitalyClient.migrate(:fetch_ref) do |is_enabled| + if is_enabled + gitaly_fetch_ref(source_repository, source_ref: source_ref, target_ref: target_ref) + else + local_fetch_ref(source_repository.path, source_ref: source_ref, target_ref: target_ref) + end + end # Make sure ref was created, and raise Rugged::ReferenceError when not raise Rugged::ReferenceError, message if status != 0 @@ -986,9 +1051,9 @@ module Gitlab end # Refactoring aid; allows us to copy code from app/models/repository.rb - def run_git(args) + def run_git(args, env: {}) circuit_breaker.perform do - popen([Gitlab.config.git.bin_path, *args], path) + popen([Gitlab.config.git.bin_path, *args], path, env) end end @@ -1012,11 +1077,17 @@ module Gitlab # This method return true if repository contains some content visible in project page. # def has_visible_content? - branch_count > 0 + return @has_visible_content if defined?(@has_visible_content) + + @has_visible_content = has_local_branches? end def gitaly_repository - Gitlab::GitalyClient::Util.repository(@storage, @relative_path) + Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) + end + + def gitaly_operations_client + @gitaly_operations_client ||= Gitlab::GitalyClient::OperationService.new(self) end def gitaly_ref_client @@ -1031,6 +1102,10 @@ module Gitlab @gitaly_repository_client ||= Gitlab::GitalyClient::RepositoryService.new(self) end + def gitaly_operation_client + @gitaly_operation_client ||= Gitlab::GitalyClient::OperationService.new(self) + end + def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) Gitlab::GitalyClient.migrate(method, status: status, &block) rescue GRPC::NotFound => e @@ -1353,6 +1428,33 @@ module Gitlab false end + def gitaly_add_tag(tag_name, user:, target:, message: nil) + gitaly_operations_client.add_tag(tag_name, user, target, message) + end + + def rugged_add_tag(tag_name, user:, target:, message: nil) + target_object = Ref.dereference_object(lookup(target)) + raise InvalidRef.new("target not found: #{target}") unless target_object + + user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) + + options = nil # Use nil, not the empty hash. Rugged cares about this. + if message + options = { + message: message, + tagger: Gitlab::Git.committer_hash(email: user.email, name: user.name) + } + end + + Gitlab::Git::OperationService.new(user, self).add_tag(tag_name, target_object.oid, options) + + find_tag(tag_name) + rescue Rugged::ReferenceError => ex + raise InvalidRef, ex + rescue Rugged::TagError + raise TagExistsError + end + def rugged_create_branch(ref, start_point) rugged_ref = rugged.branches.create(ref, start_point) target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) @@ -1395,6 +1497,46 @@ module Gitlab file.write(gitattributes_content) end end + + def gitaly_add_branch(branch_name, user, target) + gitaly_operation_client.user_create_branch(branch_name, user, target) + rescue GRPC::FailedPrecondition => ex + raise InvalidRef, ex + end + + def rugged_add_branch(branch_name, user, target) + target_object = Ref.dereference_object(lookup(target)) + raise InvalidRef.new("target not found: #{target}") unless target_object + + OperationService.new(user, self).add_branch(branch_name, target_object.oid) + find_branch(branch_name) + rescue Rugged::ReferenceError => ex + raise InvalidRef, ex + end + + def local_fetch_ref(source_path, source_ref:, target_ref:) + args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) + run_git(args) + end + + def gitaly_fetch_ref(source_repository, source_ref:, target_ref:) + gitaly_ssh = File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-ssh')) + gitaly_address = gitaly_resolver.address(source_repository.storage) + gitaly_token = gitaly_resolver.token(source_repository.storage) + + request = Gitaly::SSHUploadPackRequest.new(repository: source_repository.gitaly_repository) + env = { + 'GITALY_ADDRESS' => gitaly_address, + 'GITALY_PAYLOAD' => request.to_json, + 'GITALY_WD' => Dir.pwd, + 'GIT_SSH_COMMAND' => "#{gitaly_ssh} upload-pack" + } + env['GITALY_TOKEN'] = gitaly_token if gitaly_token.present? + + args = %W(fetch --no-tags -f ssh://gitaly/internal.git #{source_ref}:#{target_ref}) + + run_git(args, env: env) + end end end end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/rev_list.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/rev_list.rb index e0943d3a3..92a6a6725 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/rev_list.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/rev_list.rb @@ -31,7 +31,7 @@ module Gitlab output, status = popen(args, nil, Gitlab::Git::Env.all.stringify_keys) unless status.zero? - raise "Got a non-zero exit code while calling out `#{args.join(' ')}`." + raise "Got a non-zero exit code while calling out `#{args.join(' ')}`: #{output}" end output.split("\n") diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/storage.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/storage.rb index e28be4b8a..08e6c29ab 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/storage.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/storage.rb @@ -11,6 +11,7 @@ module Gitlab end CircuitOpen = Class.new(Inaccessible) + Misconfiguration = Class.new(Inaccessible) REDIS_KEY_PREFIX = 'storage_accessible:'.freeze diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/storage/circuit_breaker.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/storage/circuit_breaker.rb index 9ea9367d4..1eaa2d83f 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/storage/circuit_breaker.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/storage/circuit_breaker.rb @@ -28,14 +28,26 @@ module Gitlab def self.for_storage(storage) cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do Hash.new do |hash, storage_name| - hash[storage_name] = new(storage_name) + hash[storage_name] = build(storage_name) end end cached_circuitbreakers[storage] end - def initialize(storage, hostname = Gitlab::Environment.hostname) + def self.build(storage, hostname = Gitlab::Environment.hostname) + config = Gitlab.config.repositories.storages[storage] + + if !config.present? + NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Storage '#{storage}' is not configured")) + elsif !config['path'].present? + NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Path for storage '#{storage}' is not configured")) + else + new(storage, hostname) + end + end + + def initialize(storage, hostname) @storage = storage @hostname = hostname @@ -64,6 +76,10 @@ module Gitlab recent_failure || too_many_failures end + def failure_info + @failure_info ||= get_failure_info + end + # Memoizing the `storage_available` call means we only do it once per # request when the storage is available. # @@ -121,10 +137,12 @@ module Gitlab end end - def failure_info - @failure_info ||= get_failure_info + def cache_key + @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" end + private + def get_failure_info last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis| redis.hmget(cache_key, :last_failure, :failure_count) @@ -134,10 +152,6 @@ module Gitlab FailureInfo.new(last_failure, failure_count.to_i) end - - def cache_key - @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" - end end end end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/storage/health.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/storage/health.rb index 2d723147f..1564e94b7 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/storage/health.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/storage/health.rb @@ -78,7 +78,7 @@ module Gitlab def failing_circuit_breakers @failing_circuit_breakers ||= failing_on_hosts.map do |hostname| - CircuitBreaker.new(storage_name, hostname) + CircuitBreaker.build(storage_name, hostname) end end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/storage/null_circuit_breaker.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/storage/null_circuit_breaker.rb new file mode 100644 index 000000000..297c043d0 --- /dev/null +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/storage/null_circuit_breaker.rb @@ -0,0 +1,47 @@ +module Gitlab + module Git + module Storage + class NullCircuitBreaker + # These will have actual values + attr_reader :storage, + :hostname + + # These will always have nil values + attr_reader :storage_path, + :failure_wait_time, + :failure_reset_time, + :storage_timeout + + def initialize(storage, hostname, error: nil) + @storage = storage + @hostname = hostname + @error = error + end + + def perform + @error ? raise(@error) : yield + end + + def circuit_broken? + !!@error + end + + def failure_count_threshold + 1 + end + + def last_failure + circuit_broken? ? Time.now : nil + end + + def failure_count + circuit_broken? ? 1 : 0 + end + + def failure_info + Gitlab::Git::Storage::CircuitBreaker::FailureInfo.new(last_failure, failure_count) + end + end + end + end +end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/user.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/user.rb index ea634d396..da74719ae 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/user.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/user.rb @@ -1,24 +1,26 @@ module Gitlab module Git class User - attr_reader :name, :email, :gl_id + attr_reader :username, :name, :email, :gl_id def self.from_gitlab(gitlab_user) - new(gitlab_user.name, gitlab_user.email, Gitlab::GlId.gl_id(gitlab_user)) + new(gitlab_user.username, gitlab_user.name, gitlab_user.email, Gitlab::GlId.gl_id(gitlab_user)) end + # TODO support the username field in Gitaly https://gitlab.com/gitlab-org/gitaly/issues/628 def self.from_gitaly(gitaly_user) - new(gitaly_user.name, gitaly_user.email, gitaly_user.gl_id) + new('', gitaly_user.name, gitaly_user.email, gitaly_user.gl_id) end - def initialize(name, email, gl_id) + def initialize(username, name, email, gl_id) + @username = username @name = name @email = email @gl_id = gl_id end def ==(other) - [name, email, gl_id] == [other.name, other.email, other.gl_id] + [username, name, email, gl_id] == [other.username, other.name, other.email, other.gl_id] end end end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/wiki.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/wiki.rb new file mode 100644 index 000000000..d651c931a --- /dev/null +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/wiki.rb @@ -0,0 +1,115 @@ +module Gitlab + module Git + class Wiki + DuplicatePageError = Class.new(StandardError) + + CommitDetails = Struct.new(:name, :email, :message) do + def to_h + { name: name, email: email, message: message } + end + end + + def self.default_ref + 'master' + end + + # Initialize with a Gitlab::Git::Repository instance + def initialize(repository) + @repository = repository + end + + def repository_exists? + @repository.exists? + end + + def write_page(name, format, content, commit_details) + assert_type!(format, Symbol) + assert_type!(commit_details, CommitDetails) + + gollum_wiki.write_page(name, format, content, commit_details.to_h) + + nil + rescue Gollum::DuplicatePageError => e + raise Gitlab::Git::Wiki::DuplicatePageError, e.message + end + + def delete_page(page_path, commit_details) + assert_type!(commit_details, CommitDetails) + + gollum_wiki.delete_page(gollum_page_by_path(page_path), commit_details.to_h) + nil + end + + def update_page(page_path, title, format, content, commit_details) + assert_type!(format, Symbol) + assert_type!(commit_details, CommitDetails) + + gollum_wiki.update_page(gollum_page_by_path(page_path), title, format, content, commit_details.to_h) + nil + end + + def pages + gollum_wiki.pages.map { |gollum_page| new_page(gollum_page) } + end + + def page(title:, version: nil, dir: nil) + if version + version = Gitlab::Git::Commit.find(@repository, version).id + end + + gollum_page = gollum_wiki.page(title, version, dir) + return unless gollum_page + + new_page(gollum_page) + end + + def file(name, version) + version ||= self.class.default_ref + gollum_file = gollum_wiki.file(name, version) + return unless gollum_file + + Gitlab::Git::WikiFile.new(gollum_file) + end + + def page_versions(page_path) + current_page = gollum_page_by_path(page_path) + current_page.versions.map do |gollum_git_commit| + gollum_page = gollum_wiki.page(current_page.title, gollum_git_commit.id) + new_version(gollum_page, gollum_git_commit.id) + end + end + + def preview_slug(title, format) + gollum_wiki.preview_page(title, '', format).url_path + end + + private + + def gollum_wiki + @gollum_wiki ||= Gollum::Wiki.new(@repository.path) + end + + def gollum_page_by_path(page_path) + page_name = Gollum::Page.canonicalize_filename(page_path) + page_dir = File.split(page_path).first + + gollum_wiki.paged(page_name, page_dir) + end + + def new_page(gollum_page) + Gitlab::Git::WikiPage.new(gollum_page, new_version(gollum_page, gollum_page.version.id)) + end + + def new_version(gollum_page, commit_id) + commit = Gitlab::Git::Commit.find(@repository, commit_id) + Gitlab::Git::WikiPageVersion.new(commit, gollum_page&.format) + end + + def assert_type!(object, klass) + unless object.is_a?(klass) + raise ArgumentError, "expected a #{klass}, got #{object.inspect}" + end + end + end + end +end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/wiki_file.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/wiki_file.rb new file mode 100644 index 000000000..527f2a44d --- /dev/null +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/wiki_file.rb @@ -0,0 +1,19 @@ +module Gitlab + module Git + class WikiFile + attr_reader :mime_type, :raw_data, :name + + # This class is meant to be serializable so that it can be constructed + # by Gitaly and sent over the network to GitLab. + # + # Because Gollum::File is not serializable we must get all the data from + # 'gollum_file' during initialization, and NOT store it in an instance + # variable. + def initialize(gollum_file) + @mime_type = gollum_file.mime_type + @raw_data = gollum_file.raw_data + @name = gollum_file.name + end + end + end +end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/wiki_page.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/wiki_page.rb new file mode 100644 index 000000000..a06bac441 --- /dev/null +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/wiki_page.rb @@ -0,0 +1,39 @@ +module Gitlab + module Git + class WikiPage + attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical + + # This class is meant to be serializable so that it can be constructed + # by Gitaly and sent over the network to GitLab. + # + # Because Gollum::Page is not serializable we must get all the data from + # 'gollum_page' during initialization, and NOT store it in an instance + # variable. + # + # Note that 'version' is a WikiPageVersion instance which it itself + # serializable. That means it's OK to store 'version' in an instance + # variable. + def initialize(gollum_page, version) + @url_path = gollum_page.url_path + @title = gollum_page.title + @format = gollum_page.format + @path = gollum_page.path + @raw_data = gollum_page.raw_data + @name = gollum_page.name + @historical = gollum_page.historical? + + @version = version + end + + def historical? + @historical + end + + def text_data + return @text_data if defined?(@text_data) + + @text_data = @raw_data && Gitlab::EncodingHelper.encode!(@raw_data.dup) + end + end + end +end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/wiki_page_version.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/wiki_page_version.rb new file mode 100644 index 000000000..55f1afedc --- /dev/null +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/wiki_page_version.rb @@ -0,0 +1,19 @@ +module Gitlab + module Git + class WikiPageVersion + attr_reader :commit, :format + + # This class is meant to be serializable so that it can be constructed + # by Gitaly and sent over the network to GitLab. + # + # Both 'commit' (a Gitlab::Git::Commit) and 'format' (a string) are + # serializable. + def initialize(commit, format) + @commit = commit + @format = format + end + + delegate :message, :sha, :id, :author_name, :authored_date, to: :commit + end + end +end |