diff options
Diffstat (limited to 'lib/gitlab/git/repository.rb')
| -rw-r--r-- | lib/gitlab/git/repository.rb | 398 |
1 files changed, 307 insertions, 91 deletions
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index d399636bb28..369bb16f719 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -18,6 +18,9 @@ module Gitlab GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE ].freeze SEARCH_CONTEXT_LINES = 3 + REBASE_WORKTREE_PREFIX = 'rebase'.freeze + SQUASH_WORKTREE_PREFIX = 'squash'.freeze + GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze NoRepository = Class.new(StandardError) InvalidBlobName = Class.new(StandardError) @@ -73,9 +76,6 @@ module Gitlab @attributes = Gitlab::Git::Attributes.new(path) end - delegate :empty?, - to: :rugged - def ==(other) path == other.path end @@ -204,6 +204,13 @@ module Gitlab end end + # Git repository can contains some hidden refs like: + # /refs/notes/* + # /refs/git-as-svn/* + # /refs/pulls/* + # This refs by default not visible in project page and not cloned to client side. + alias_method :has_visible_content?, :has_local_branches? + def has_local_branches_rugged? rugged.branches.each(:local).any? do |ref| begin @@ -505,13 +512,20 @@ module Gitlab # Counts the amount of commits between `from` and `to`. def count_commits_between(from, to) - Commit.between(self, from, to).size + count_commits(ref: "#{from}..#{to}") end # Returns the SHA of the most recent common ancestor of +from+ and +to+ def merge_base_commit(from, to) - rugged.merge_base(from, to) + gitaly_migrate(:merge_base) do |is_enabled| + if is_enabled + gitaly_repository_client.find_merge_base(from, to) + else + rugged.merge_base(from, to) + end + end end + alias_method :merge_base, :merge_base_commit # Gitaly note: JV: check gitlab-ee before removing this method. def rugged_is_ancestor?(ancestor_id, descendant_id) @@ -774,24 +788,21 @@ module Gitlab end def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - OperationService.new(user, self).with_branch( - branch_name, - start_branch_name: start_branch_name, - start_repository: start_repository - ) do |start_commit| - - Gitlab::Git.check_namespace!(commit, start_repository) - - revert_tree_id = check_revert_content(commit, start_commit.sha) - raise CreateTreeError unless revert_tree_id - - committer = user_to_committer(user) + gitaly_migrate(:revert) do |is_enabled| + args = { + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository + } - create_commit(message: message, - author: committer, - committer: committer, - tree: revert_tree_id, - parents: [start_commit.sha]) + if is_enabled + gitaly_operations_client.user_revert(args) + else + rugged_revert(args) + end end end @@ -809,44 +820,24 @@ module Gitlab end def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - OperationService.new(user, self).with_branch( - branch_name, - start_branch_name: start_branch_name, - start_repository: start_repository - ) do |start_commit| - - Gitlab::Git.check_namespace!(commit, start_repository) - - cherry_pick_tree_id = check_cherry_pick_content(commit, start_commit.sha) - raise CreateTreeError unless cherry_pick_tree_id - - committer = user_to_committer(user) + gitaly_migrate(:cherry_pick) do |is_enabled| + args = { + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository + } - create_commit(message: message, - author: { - email: commit.author_email, - name: commit.author_name, - time: commit.authored_date - }, - committer: committer, - tree: cherry_pick_tree_id, - parents: [start_commit.sha]) + if is_enabled + gitaly_operations_client.user_cherry_pick(args) + else + rugged_cherry_pick(args) + end end end - def check_cherry_pick_content(target_commit, source_sha) - args = [target_commit.sha, source_sha] - args << 1 if target_commit.merge_commit? - - cherry_pick_index = rugged.cherrypick_commit(*args) - return false if cherry_pick_index.conflicts? - - tree_id = cherry_pick_index.write_tree(rugged) - return false unless diff_exists?(source_sha, tree_id) - - tree_id - end - def diff_exists?(sha1, sha2) rugged.diff(sha1, sha2).size > 0 end @@ -904,8 +895,11 @@ module Gitlab end end - def add_remote(remote_name, url) + # If `mirror_refmap` is present the remote is set as mirror with that mapping + def add_remote(remote_name, url, mirror_refmap: nil) rugged.remotes.create(remote_name, url) + + set_remote_as_mirror(remote_name, refmap: mirror_refmap) if mirror_refmap rescue Rugged::ConfigError remote_update(remote_name, url: url) end @@ -1025,7 +1019,7 @@ module Gitlab Gitlab::Git.check_namespace!(start_repository) start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) - return yield nil if start_repository.empty_repo? + return yield nil if start_repository.empty? if start_repository.same_repository?(self) yield commit(start_branch_name) @@ -1086,17 +1080,17 @@ module Gitlab end end - def write_ref(ref_path, ref) + def write_ref(ref_path, ref, force: false) raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") - command = [Gitlab.config.git.bin_path] + %w[update-ref --stdin -z] - input = "update #{ref_path}\x00#{ref}\x00\x00" - output, status = circuit_breaker.perform do - popen(command, path) { |stdin| stdin.write(input) } - end + ref = "refs/heads/#{ref}" unless ref.start_with?("refs") || ref =~ /\A[a-f0-9]+\z/i - raise GitError, output unless status.zero? + rugged.references.create(ref_path, ref, force: force) + rescue Rugged::ReferenceError => ex + raise GitError, "could not create ref #{ref_path}: #{ex}" + rescue Rugged::OSError => ex + raise GitError, "could not create ref #{ref_path}: #{ex}" end def fetch_ref(source_repository, source_ref:, target_ref:) @@ -1118,12 +1112,22 @@ module Gitlab end # Refactoring aid; allows us to copy code from app/models/repository.rb - def run_git(args, env: {}) + def run_git(args, chdir: path, env: {}, nice: false, &block) + cmd = [Gitlab.config.git.bin_path, *args] + cmd.unshift("nice") if nice circuit_breaker.perform do - popen([Gitlab.config.git.bin_path, *args], path, env) + popen(cmd, chdir, env, &block) end end + def run_git!(args, chdir: path, env: {}, nice: false, &block) + output, status = run_git(args, chdir: chdir, env: env, nice: nice, &block) + + raise GitError, output unless status.zero? + + output + end + # Refactoring aid; allows us to copy code from app/models/repository.rb def run_git_with_timeout(args, timeout, env: {}) circuit_breaker.perform do @@ -1136,32 +1140,28 @@ module Gitlab Gitlab::Git::Commit.find(self, ref) end - # Refactoring aid; allows us to copy code from app/models/repository.rb - def empty_repo? - !exists? || !has_visible_content? + def empty? + !has_visible_content? end - # - # Git repository can contains some hidden refs like: - # /refs/notes/* - # /refs/git-as-svn/* - # /refs/pulls/* - # This refs by default not visible in project page and not cloned to client side. - # - # This method return true if repository contains some content visible in project page. - # - def has_visible_content? - return @has_visible_content if defined?(@has_visible_content) + def fetch_repository_as_mirror(repository) + remote_name = "tmp-#{SecureRandom.hex}" - @has_visible_content = has_local_branches? - end + # Notice that this feature flag is not for `fetch_repository_as_mirror` + # as a whole but for the fetching mechanism (file path or gitaly-ssh). + url, env = gitaly_migrate(:fetch_internal) do |is_enabled| + if is_enabled + repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository) + [GITALY_INTERNAL_URL, repository.fetch_env] + else + [repository.path, nil] + end + end - # Like all public `Gitlab::Git::Repository` methods, this method is part - # of `Repository`'s interface through `method_missing`. - # `Repository` has its own `fetch_remote` which uses `gitlab-shell` and - # takes some extra attributes, so we qualify this method name to prevent confusion. - def fetch_remote_without_shell(remote = 'origin') - run_git(['fetch', remote]).last.zero? + add_remote(remote_name, url, mirror_refmap: :all_refs) + fetch_remote(remote_name, env: env) + ensure + remove_remote(remote_name) end def blob_at(sha, path) @@ -1187,6 +1187,76 @@ module Gitlab end end + def fsck + gitaly_migrate(:git_fsck) do |is_enabled| + msg, status = if is_enabled + gitaly_fsck + else + shell_fsck + end + + raise GitError.new("Could not fsck repository: #{msg}") unless status.zero? + end + end + + def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) + env = git_env_for_user(user) + + with_worktree(rebase_path, branch, env: env) do + run_git!( + %W(pull --rebase #{remote_repository.path} #{remote_branch}), + chdir: rebase_path, env: env + ) + + rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip + + Gitlab::Git::OperationService.new(user, self) + .update_branch(branch, rebase_sha, branch_sha) + + rebase_sha + end + end + + def rebase_in_progress?(rebase_id) + fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)) + end + + def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:) + squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id) + env = git_env_for_user(user).merge( + 'GIT_AUTHOR_NAME' => author.name, + 'GIT_AUTHOR_EMAIL' => author.email + ) + diff_range = "#{start_sha}...#{end_sha}" + diff_files = run_git!( + %W(diff --name-only --diff-filter=a --binary #{diff_range}) + ).chomp + + with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do + # Apply diff of the `diff_range` to the worktree + diff = run_git!(%W(diff --binary #{diff_range})) + run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin| + stdin.write(diff) + end + + # Commit the `diff_range` diff + run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env) + + # Return the squash sha. May print a warning for ambiguous refs, but + # we can ignore that with `--quiet` and just take the SHA, if present. + # HEAD here always refers to the current HEAD commit, even if there is + # another ref called HEAD. + run_git!( + %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env + ).chomp + end + end + + def squash_in_progress?(squash_id) + fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)) + end + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) end @@ -1223,6 +1293,65 @@ module Gitlab private + def fresh_worktree?(path) + File.exist?(path) && !clean_stuck_worktree(path) + end + + def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:) + base_args = %w(worktree add --detach) + + # Note that we _don't_ want to test for `.present?` here: If the caller + # passes an non nil empty value it means it still wants sparse checkout + # but just isn't interested in any file, perhaps because it wants to + # checkout files in by a changeset but that changeset only adds files. + if sparse_checkout_files + # Create worktree without checking out + run_git!(base_args + ['--no-checkout', worktree_path], env: env) + worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path) + + configure_sparse_checkout(worktree_git_path, sparse_checkout_files) + + # After sparse checkout configuration, checkout `branch` in worktree + run_git!(%W(checkout --detach #{branch}), chdir: worktree_path, env: env) + else + # Create worktree and checkout `branch` in it + run_git!(base_args + [worktree_path, branch], env: env) + end + + yield + ensure + FileUtils.rm_rf(worktree_path) if File.exist?(worktree_path) + FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path) + end + + def clean_stuck_worktree(path) + return false unless File.mtime(path) < 15.minutes.ago + + FileUtils.rm_rf(path) + true + end + + # Adding a worktree means checking out the repository. For large repos, + # this can be very expensive, so set up sparse checkout for the worktree + # to only check out the files we're interested in. + def configure_sparse_checkout(worktree_git_path, files) + run_git!(%w(config core.sparseCheckout true)) + + return if files.empty? + + worktree_info_path = File.join(worktree_git_path, 'info') + FileUtils.mkdir_p(worktree_info_path) + File.write(File.join(worktree_info_path, 'sparse-checkout'), files) + end + + def gitaly_fsck + gitaly_repository_client.fsck + end + + def shell_fsck + run_git(%W[--git-dir=#{path} fsck], nice: true) + end + def rugged_fetch_source_branch(source_repository, source_branch, local_ref) with_repo_branch_commit(source_repository, source_branch) do |commit| if commit @@ -1234,6 +1363,24 @@ module Gitlab end end + def worktree_path(prefix, id) + id = id.to_s + raise ArgumentError, "worktree id can't be empty" unless id.present? + raise ArgumentError, "worktree id can't contain slashes " if id.include?("/") + + File.join(path, 'gitlab-worktree', "#{prefix}-#{id}") + end + + def git_env_for_user(user) + { + 'GIT_COMMITTER_NAME' => user.name, + 'GIT_COMMITTER_EMAIL' => user.email, + 'GL_ID' => Gitlab::GlId.gl_id(user), + 'GL_PROTOCOL' => Gitlab::Git::Hook::GL_PROTOCOL, + 'GL_REPOSITORY' => gl_repository + } + end + # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. def branches_filter(filter: nil, sort_by: nil) # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37464 @@ -1253,7 +1400,11 @@ module Gitlab # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/695 def git_merged_branch_names(branch_names = []) - root_sha = find_branch(root_ref).target + return [] unless root_ref + + root_sha = find_branch(root_ref)&.target + + return [] unless root_sha git_arguments = %W[branch --merged #{root_sha} @@ -1645,6 +1796,28 @@ module Gitlab end end + def rugged_revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + OperationService.new(user, self).with_branch( + branch_name, + start_branch_name: start_branch_name, + start_repository: start_repository + ) do |start_commit| + + Gitlab::Git.check_namespace!(commit, start_repository) + + revert_tree_id = check_revert_content(commit, start_commit.sha) + raise CreateTreeError unless revert_tree_id + + committer = user_to_committer(user) + + create_commit(message: message, + author: committer, + committer: committer, + tree: revert_tree_id, + parents: [start_commit.sha]) + end + end + def gitaly_add_branch(branch_name, user, target) gitaly_operation_client.user_create_branch(branch_name, user, target) rescue GRPC::FailedPrecondition => ex @@ -1661,13 +1834,52 @@ module Gitlab raise InvalidRef, ex end + def rugged_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + OperationService.new(user, self).with_branch( + branch_name, + start_branch_name: start_branch_name, + start_repository: start_repository + ) do |start_commit| + + Gitlab::Git.check_namespace!(commit, start_repository) + + cherry_pick_tree_id = check_cherry_pick_content(commit, start_commit.sha) + raise CreateTreeError unless cherry_pick_tree_id + + committer = user_to_committer(user) + + create_commit(message: message, + author: { + email: commit.author_email, + name: commit.author_name, + time: commit.authored_date + }, + committer: committer, + tree: cherry_pick_tree_id, + parents: [start_commit.sha]) + end + end + + def check_cherry_pick_content(target_commit, source_sha) + args = [target_commit.sha, source_sha] + args << 1 if target_commit.merge_commit? + + cherry_pick_index = rugged.cherrypick_commit(*args) + return false if cherry_pick_index.conflicts? + + tree_id = cherry_pick_index.write_tree(rugged) + return false unless diff_exists?(source_sha, tree_id) + + tree_id + end + def local_fetch_ref(source_path, source_ref:, target_ref:) args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) run_git(args) end def gitaly_fetch_ref(source_repository, source_ref:, target_ref:) - args = %W(fetch --no-tags -f ssh://gitaly/internal.git #{source_ref}:#{target_ref}) + args = %W(fetch --no-tags -f #{GITALY_INTERNAL_URL} #{source_ref}:#{target_ref}) run_git(args, env: source_repository.fetch_env) end @@ -1687,6 +1899,10 @@ module Gitlab rescue Rugged::ReferenceError raise ArgumentError, 'Invalid merge source' end + + def fetch_remote(remote_name = 'origin', env: nil) + run_git(['fetch', remote_name], env: env).last.zero? + end end end end |