diff options
author | Kim "BKC" Carlbäcker <kim.carlbacker@gmail.com> | 2018-01-25 14:44:04 +0300 |
---|---|---|
committer | Kim "BKC" Carlbäcker <kim.carlbacker@gmail.com> | 2018-01-25 15:06:56 +0300 |
commit | 3b38d340512030df9eb0b43a78af8b84f14e6b46 (patch) | |
tree | 90ec8a1c8e1c6e2645672b779686bd9c661ac30e | |
parent | 0470aecd6801482daf774d2faff3de879d4f2279 (diff) |
Vendor Gitlab::Git at f9b946c1c9756533fd95c8735803d7b54d6dd204
- Fix Attributes => InfoAttributes
16 files changed, 396 insertions, 185 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 855f0fd29..b612e055c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Gitaly changelog +UNRELEASED + +- Vendor Gitlab::Git @ f9b946c1c9756533fd95c8735803d7b54d6dd204 + https://gitlab.com/gitlab-org/gitaly/merge_requests/563 + v0.74.0 - Implement CreateRepositoryFromBundle RPC diff --git a/ruby/lib/gitlab/git.rb b/ruby/lib/gitlab/git.rb index 094464e03..5fc3dcdcd 100644 --- a/ruby/lib/gitlab/git.rb +++ b/ruby/lib/gitlab/git.rb @@ -65,7 +65,7 @@ module Gitlab @gl_repository = gl_repository @gitlab_projects = gitlab_projects @rugged = Rugged::Repository.new(path, alternates: alt_dirs) - @attributes = Gitlab::Git::Attributes.new(path) + @attributes = Gitlab::Git::InfoAttributes.new(path) end # Bypass the CircuitBreaker class which needs Redis diff --git a/ruby/vendor/gitlab_git/REVISION b/ruby/vendor/gitlab_git/REVISION index 95995173d..540fc0f36 100644 --- a/ruby/vendor/gitlab_git/REVISION +++ b/ruby/vendor/gitlab_git/REVISION @@ -1 +1 @@ -b10ea6e386a025759aca5e9ef0d23931e77d1012 +f9b946c1c9756533fd95c8735803d7b54d6dd204 diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/attributes_at_ref_parser.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/attributes_at_ref_parser.rb new file mode 100644 index 000000000..26b5bd520 --- /dev/null +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/attributes_at_ref_parser.rb @@ -0,0 +1,14 @@ +module Gitlab + module Git + # Parses root .gitattributes file at a given ref + class AttributesAtRefParser + delegate :attributes, to: :@parser + + def initialize(repository, ref) + blob = repository.blob_at(ref, '.gitattributes') + + @parser = AttributesParser.new(blob&.data) + end + end + end +end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/attributes.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/attributes_parser.rb index 2d20cd473..d8aeabb6c 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/attributes.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/attributes_parser.rb @@ -1,42 +1,26 @@ -# Gitaly note: JV: not sure what to make of this class. Why does it use -# the full disk path of the repository to look up attributes This is -# problematic in Gitaly, because Gitaly hides the full disk path to the -# repository from gitlab-ce. - module Gitlab module Git # Class for parsing Git attribute files and extracting the attributes for # file patterns. - # - # Unlike Rugged this parser only needs a single IO call (a call to `open`), - # vastly reducing the time spent in extracting attributes. - # - # This class _only_ supports parsing the attributes file located at - # `$GIT_DIR/info/attributes` as GitLab doesn't use any other files - # (`.gitattributes` is copied to this particular path). - # - # Basic usage: - # - # attributes = Gitlab::Git::Attributes.new(some_repo.path) - # - # attributes.attributes('README.md') # => { "eol" => "lf } - class Attributes - # path - The path to the Git repository. - def initialize(path) - @path = File.expand_path(path) - @patterns = nil + class AttributesParser + def initialize(attributes_data) + @data = attributes_data || "" + + if @data.is_a?(File) + @patterns = parse_file + end end # Returns all the Git attributes for the given path. # - # path - A path to a file for which to get the attributes. + # file_path - A path to a file for which to get the attributes. # # Returns a Hash. - def attributes(path) - full_path = File.join(@path, path) + def attributes(file_path) + absolute_path = File.join('/', file_path) patterns.each do |pattern, attrs| - return attrs if File.fnmatch?(pattern, full_path) + return attrs if File.fnmatch?(pattern, absolute_path) end {} @@ -98,16 +82,10 @@ module Gitlab # Iterates over every line in the attributes file. def each_line - full_path = File.join(@path, 'info/attributes') + @data.each_line do |line| + break unless line.valid_encoding? - return unless File.exist?(full_path) - - File.open(full_path, 'r') do |handle| - handle.each_line do |line| - break unless line.valid_encoding? - - yield line.strip - end + yield line.strip end end @@ -125,7 +103,8 @@ module Gitlab parsed = attrs ? parse_attributes(attrs) : {} - pairs << [File.join(@path, pattern), parsed] + absolute_pattern = File.join('/', pattern) + pairs << [absolute_pattern, parsed] end # Newer entries take precedence over older entries. diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/blob.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/blob.rb index 031fccba9..81e460287 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/blob.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/blob.rb @@ -34,7 +34,7 @@ module Gitlab def raw(repository, sha) Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled| if is_enabled - Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) + repository.gitaly_blob_client.get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) else rugged_raw(repository, sha, limit: MAX_DATA_DISPLAY_SIZE) end @@ -70,11 +70,19 @@ module Gitlab # Returns array of Gitlab::Git::Blob # Does not guarantee blob data will be set def batch_lfs_pointers(repository, blob_ids) - blob_ids.lazy - .select { |sha| possible_lfs_blob?(repository, sha) } - .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) } - .select(&:lfs_pointer?) - .force + return [] if blob_ids.empty? + + repository.gitaly_migrate(:batch_lfs_pointers) do |is_enabled| + if is_enabled + repository.gitaly_blob_client.batch_lfs_pointers(blob_ids) + else + blob_ids.lazy + .select { |sha| possible_lfs_blob?(repository, sha) } + .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) } + .select(&:lfs_pointer?) + .force + end + end end def binary?(data) @@ -132,6 +140,8 @@ module Gitlab end def find_by_gitaly(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) + return unless path + path = path.sub(/\A\/*/, '') path = '/' if path.empty? name = File.basename(path) @@ -173,6 +183,8 @@ module Gitlab end def find_by_rugged(repository, sha, path, limit:) + return unless path + rugged_commit = repository.lookup(sha) root_tree = rugged_commit.tree @@ -254,7 +266,7 @@ module Gitlab Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| @data = begin if is_enabled - Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: id, limit: -1).data + repository.gitaly_blob_client.get_blob(oid: id, limit: -1).data else repository.lookup(id).content end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/commit.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/commit.rb index 016437b24..768617e2c 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/commit.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/commit.rb @@ -239,6 +239,24 @@ module Gitlab end end end + + def extract_signature(repository, commit_id) + repository.gitaly_migrate(:extract_commit_signature) do |is_enabled| + if is_enabled + repository.gitaly_commit_client.extract_signature(commit_id) + else + rugged_extract_signature(repository, commit_id) + end + end + end + + def rugged_extract_signature(repository, commit_id) + begin + Rugged::Commit.extract_signature(repository.rugged, commit_id) + rescue Rugged::OdbError + nil + end + end end def initialize(repository, raw_commit, head = nil) @@ -436,6 +454,16 @@ module Gitlab parent_ids.size > 1 end + def tree_entry(path) + @repository.gitaly_migrate(:commit_tree_entry) do |is_migrated| + if is_migrated + gitaly_tree_entry(path) + else + rugged_tree_entry(path) + end + end + end + def to_gitaly_commit return raw_commit if raw_commit.is_a?(Gitaly::GitCommit) @@ -450,11 +478,6 @@ module Gitlab ) end - # Is this the same as Blob.find_entry_by_path ? - def rugged_tree_entry(path) - rugged_commit.tree.path(path) - end - private def init_from_hash(hash) @@ -501,6 +524,28 @@ module Gitlab SERIALIZE_KEYS end + def gitaly_tree_entry(path) + # We're only interested in metadata, so limit actual data to 1 byte + # since Gitaly doesn't support "send no data" option. + entry = @repository.gitaly_commit_client.tree_entry(id, path, 1) + return unless entry + + # To be compatible with the rugged format + entry = entry.to_h + entry.delete(:data) + entry[:name] = File.basename(path) + entry[:type] = entry[:type].downcase + + entry + end + + # Is this the same as Blob.find_entry_by_path ? + def rugged_tree_entry(path) + rugged_commit.tree.path(path) + rescue Rugged::TreeError + nil + end + def gitaly_commit_author_from_rugged(author_or_committer) Gitaly::CommitAuthor.new( name: author_or_committer[:name].b, diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/conflict/resolver.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/conflict/resolver.rb index 74c9874d5..07b7e811a 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/conflict/resolver.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/conflict/resolver.rb @@ -15,7 +15,7 @@ module Gitlab @conflicts ||= begin @target_repository.gitaly_migrate(:conflicts_list_conflict_files) do |is_enabled| if is_enabled - gitaly_conflicts_client(@target_repository).list_conflict_files + gitaly_conflicts_client(@target_repository).list_conflict_files.to_a else rugged_list_conflict_files end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/gitlab_projects.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/gitlab_projects.rb index 976fa1ddf..e5a747cb9 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/gitlab_projects.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/gitlab_projects.rb @@ -44,29 +44,13 @@ module Gitlab # Import project via git clone --bare # URL must be publicly cloneable def import_project(source, timeout) - # Skip import if repo already exists - return false if File.exist?(repository_absolute_path) - - masked_source = mask_password_in_url(source) - - logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>." - cmd = %W(git clone --bare -- #{source} #{repository_absolute_path}) - - success = run_with_timeout(cmd, timeout, nil) - - unless success - logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.") - FileUtils.rm_rf(repository_absolute_path) - return false + Gitlab::GitalyClient.migrate(:import_repository) do |is_enabled| + if is_enabled + gitaly_import_repository(source) + else + git_import_repository(source, timeout) + end end - - Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path) - - # The project was imported successfully. - # Remove the origin URL since it may contain password. - remove_origin_in_repo - - true end def fork_repository(new_shard_path, new_repository_relative_path) @@ -231,6 +215,42 @@ module Gitlab raise(ShardNameNotFoundError, "no shard found for path '#{shard_path}'") end + def git_import_repository(source, timeout) + # Skip import if repo already exists + return false if File.exist?(repository_absolute_path) + + masked_source = mask_password_in_url(source) + + logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>." + cmd = %W(git clone --bare -- #{source} #{repository_absolute_path}) + + success = run_with_timeout(cmd, timeout, nil) + + unless success + logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.") + FileUtils.rm_rf(repository_absolute_path) + return false + end + + Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path) + + # The project was imported successfully. + # Remove the origin URL since it may contain password. + remove_origin_in_repo + + true + end + + def gitaly_import_repository(source) + raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) + + Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source) + true + rescue GRPC::BadStatus => e + @output << e.message + false + end + def git_fork_repository(new_shard_path, new_repository_relative_path) from_path = repository_absolute_path to_path = File.join(new_shard_path, new_repository_relative_path) diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/info_attributes.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/info_attributes.rb new file mode 100644 index 000000000..e79a44095 --- /dev/null +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/info_attributes.rb @@ -0,0 +1,49 @@ +# Gitaly note: JV: not sure what to make of this class. Why does it use +# the full disk path of the repository to look up attributes This is +# problematic in Gitaly, because Gitaly hides the full disk path to the +# repository from gitlab-ce. + +module Gitlab + module Git + # Parses gitattributes at `$GIT_DIR/info/attributes` + # + # Unlike Rugged this parser only needs a single IO call (a call to `open`), + # vastly reducing the time spent in extracting attributes. + # + # This class _only_ supports parsing the attributes file located at + # `$GIT_DIR/info/attributes` as GitLab doesn't use any other files + # (`.gitattributes` is copied to this particular path). + # + # Basic usage: + # + # attributes = Gitlab::Git::InfoAttributes.new(some_repo.path) + # + # attributes.attributes('README.md') # => { "eol" => "lf } + class InfoAttributes + delegate :attributes, :patterns, to: :parser + + # path - The path to the Git repository. + def initialize(path) + @repo_path = File.expand_path(path) + end + + def parser + @parser ||= begin + if File.exist?(attributes_path) + File.open(attributes_path, 'r') do |file_handle| + AttributesParser.new(file_handle) + end + else + AttributesParser.new("") + end + end + end + + private + + def attributes_path + @attributes_path ||= File.join(@repo_path, 'info/attributes') + end + end + end +end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/ref.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/ref.rb index 372ce005b..a3ba9475a 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/ref.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/ref.rb @@ -33,9 +33,9 @@ module Gitlab object end - def initialize(repository, name, target, derefenced_target) + def initialize(repository, name, target, dereferenced_target) @name = Gitlab::Git.ref_name(name) - @dereferenced_target = derefenced_target + @dereferenced_target = dereferenced_target @target = if target.respond_to?(:oid) target.oid elsif target.respond_to?(:name) diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/remote_mirror.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/remote_mirror.rb index 38e9d2a85..ebe467228 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/remote_mirror.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/remote_mirror.rb @@ -6,7 +6,23 @@ module Gitlab @ref_name = ref_name end - def update(only_branches_matching: [], only_tags_matching: []) + def update(only_branches_matching: []) + @repository.gitaly_migrate(:remote_update_remote_mirror) do |is_enabled| + if is_enabled + gitaly_update(only_branches_matching) + else + rugged_update(only_branches_matching) + end + end + end + + private + + def gitaly_update(only_branches_matching) + @repository.gitaly_remote_client.update_remote_mirror(@ref_name, only_branches_matching) + end + + def rugged_update(only_branches_matching) local_branches = refs_obj(@repository.local_branches, only_refs_matching: only_branches_matching) remote_branches = refs_obj(@repository.remote_branches(@ref_name), only_refs_matching: only_branches_matching) @@ -15,8 +31,8 @@ module Gitlab delete_refs(local_branches, remote_branches) - local_tags = refs_obj(@repository.tags, only_refs_matching: only_tags_matching) - remote_tags = refs_obj(@repository.remote_tags(@ref_name), only_refs_matching: only_tags_matching) + local_tags = refs_obj(@repository.tags) + remote_tags = refs_obj(@repository.remote_tags(@ref_name)) updated_tags = changed_refs(local_tags, remote_tags) @repository.push_remote_branches(@ref_name, updated_tags.keys) if updated_tags.present? @@ -24,8 +40,6 @@ module Gitlab delete_refs(local_tags, remote_tags) end - private - def refs_obj(refs, only_refs_matching: []) refs.each_with_object({}) do |ref, refs| next if only_refs_matching.present? && !only_refs_matching.include?(ref.name) diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb index fa9bc57dd..d6c098025 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/repository.rb @@ -102,7 +102,7 @@ module Gitlab ) @path = File.join(storage_path, @relative_path) @name = @relative_path.split("/").last - @attributes = Gitlab::Git::Attributes.new(path) + @attributes = Gitlab::Git::InfoAttributes.new(path) end def ==(other) @@ -571,7 +571,21 @@ module Gitlab end def merged_branch_names(branch_names = []) - Set.new(git_merged_branch_names(branch_names)) + return [] unless root_ref + + root_sha = find_branch(root_ref)&.target + + return [] unless root_sha + + branches = gitaly_migrate(:merged_branch_names) do |is_enabled| + if is_enabled + gitaly_merged_branch_names(branch_names, root_sha) + else + git_merged_branch_names(branch_names, root_sha) + end + end + + Set.new(branches) end # Return an array of Diff objects that represent the diff @@ -607,37 +621,6 @@ module Gitlab end end - # Returns branch names collection that contains the special commit(SHA1 - # or name) - # - # Ex. - # repo.branch_names_contains('master') - # - def branch_names_contains(commit) - branches_contains(commit).map { |c| c.name } - end - - # Returns branch collection that contains the special commit(SHA1 or name) - # - # Ex. - # repo.branch_names_contains('master') - # - def branches_contains(commit) - commit_obj = rugged.rev_parse(commit) - parent = commit_obj.parents.first unless commit_obj.parents.empty? - - walker = Rugged::Walker.new(rugged) - - rugged.branches.select do |branch| - walker.push(branch.target_id) - walker.hide(parent) if parent - result = walker.any? { |c| c.oid == commit_obj.oid } - walker.reset - - result - end - end - # Get refs hash which key is SHA1 # and value is a Rugged::Reference def refs_hash @@ -654,6 +637,7 @@ module Gitlab end end end + @refs_hash end @@ -1011,6 +995,18 @@ module Gitlab attributes(path)[name] end + # Check .gitattributes for a given ref + # + # This only checks the root .gitattributes file, + # it does not traverse subfolders to find additional .gitattributes files + # + # This method is around 30 times slower than `attributes`, + # which uses `$GIT_DIR/info/attributes` + def attributes_at(ref, file_path) + parser = AttributesAtRefParser.new(self, ref) + parser.attributes(file_path) + end + def languages(ref = nil) Gitlab::GitalyClient.migrate(:commit_languages) do |is_enabled| if is_enabled @@ -1111,19 +1107,6 @@ module Gitlab end end - def shell_write_ref(ref_path, ref, old_ref) - raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') - raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") - raise ArgumentError, "invalid old_ref #{old_ref.inspect}" if !old_ref.nil? && old_ref.include?("\x00") - - input = "update #{ref_path}\x00#{ref}\x00#{old_ref}\x00" - run_git!(%w[update-ref --stdin -z]) { |stdin| stdin.write(input) } - end - - def rugged_write_ref(ref_path, ref) - rugged.references.create(ref_path, ref, force: true) - end - def fetch_ref(source_repository, source_ref:, target_ref:) Gitlab::Git.check_namespace!(source_repository) source_repository = RemoteRepository.new(source_repository) unless source_repository.is_a?(RemoteRepository) @@ -1221,33 +1204,31 @@ module Gitlab end def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) - rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) - env = git_env_for_user(user) - - if remote_repository.is_a?(RemoteRepository) - env.merge!(remote_repository.fetch_env) - remote_repo_path = GITALY_INTERNAL_URL - else - remote_repo_path = remote_repository.path - end - - with_worktree(rebase_path, branch, env: env) do - run_git!( - %W(pull --rebase #{remote_repo_path} #{remote_branch}), - chdir: rebase_path, env: env - ) - - rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip - - Gitlab::Git::OperationService.new(user, self) - .update_branch(branch, rebase_sha, branch_sha) - - rebase_sha + gitaly_migrate(:rebase) do |is_enabled| + if is_enabled + gitaly_rebase(user, rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch) + else + git_rebase(user, rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch) + end end end def rebase_in_progress?(rebase_id) - fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)) + gitaly_migrate(:rebase_in_progress) do |is_enabled| + if is_enabled + gitaly_repository_client.rebase_in_progress?(rebase_id) + else + fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)) + end + end end def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:) @@ -1303,42 +1284,41 @@ module Gitlab success || gitlab_projects_error end + def bundle_to_disk(save_path) + gitaly_migrate(:bundle_to_disk) do |is_enabled| + if is_enabled + gitaly_repository_client.create_bundle(save_path) + else + run_git!(%W(bundle create #{save_path} --all)) + end + end + + true + end + # rubocop:disable Metrics/ParameterLists def multi_action( user, branch_name:, message:, actions:, author_email: nil, author_name: nil, start_branch_name: nil, start_repository: self) - OperationService.new(user, self).with_branch( - branch_name, - start_branch_name: start_branch_name, - start_repository: start_repository - ) do |start_commit| - index = Gitlab::Git::Index.new(self) - parents = [] - - if start_commit - index.read_tree(start_commit.rugged_commit.tree) - parents = [start_commit.sha] + gitaly_migrate(:operation_user_commit_files) do |is_enabled| + if is_enabled + gitaly_operation_client.user_commit_files(user, branch_name, + message, actions, author_email, author_name, + start_branch_name, start_repository) + else + rugged_multi_action(user, branch_name, message, actions, + author_email, author_name, start_branch_name, start_repository) end - - actions.each { |opts| index.apply(opts.delete(:action), opts) } - - committer = user_to_committer(user) - author = Gitlab::Git.committer_hash(email: author_email, name: author_name) || committer - options = { - tree: index.write_tree, - message: message, - parents: parents, - author: author, - committer: committer - } - - create_commit(options) end end # rubocop:enable Metrics/ParameterLists + def write_config(full_path:) + rugged.config['gitlab.fullpath'] = full_path if full_path.present? + end + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) end @@ -1367,6 +1347,10 @@ module Gitlab @gitaly_remote_client ||= Gitlab::GitalyClient::RemoteService.new(self) end + def gitaly_blob_client + @gitaly_blob_client ||= Gitlab::GitalyClient::BlobService.new(self) + end + def gitaly_conflicts_client(our_commit_oid, their_commit_oid) Gitlab::GitalyClient::ConflictsService.new(self, our_commit_oid, their_commit_oid) end @@ -1383,6 +1367,25 @@ module Gitlab private + def shell_write_ref(ref_path, ref, old_ref) + raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') + raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") + raise ArgumentError, "invalid old_ref #{old_ref.inspect}" if !old_ref.nil? && old_ref.include?("\x00") + + input = "update #{ref_path}\x00#{ref}\x00#{old_ref}\x00" + run_git!(%w[update-ref --stdin -z]) { |stdin| stdin.write(input) } + end + + def rugged_write_ref(ref_path, ref) + rugged.references.create(ref_path, ref, force: true) + rescue Rugged::ReferenceError => ex + Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}" + rescue Rugged::OSError => ex + raise unless ex.message =~ /Failed to create locked file/ && ex.message =~ /File exists/ + + Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}" + end + def fresh_worktree?(path) File.exist?(path) && !clean_stuck_worktree(path) end @@ -1488,14 +1491,7 @@ module Gitlab sort_branches(branches, sort_by) end - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/695 - def git_merged_branch_names(branch_names = []) - return [] unless root_ref - - root_sha = find_branch(root_ref)&.target - - return [] unless root_sha - + def git_merged_branch_names(branch_names, root_sha) git_arguments = %W[branch --merged #{root_sha} --format=%(refname:short)\ %(objectname)] + branch_names @@ -1509,6 +1505,14 @@ module Gitlab end end + def gitaly_merged_branch_names(branch_names, root_sha) + qualified_branch_names = branch_names.map { |b| "refs/heads/#{b}" } + + gitaly_ref_client.merged_branches(qualified_branch_names) + .reject { |b| b.target == root_sha } + .map(&:name) + end + def process_count_commits_options(options) if options[:from] || options[:to] ref = @@ -2023,6 +2027,40 @@ module Gitlab tree_id end + def gitaly_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + gitaly_operation_client.user_rebase(user, rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch) + end + + def git_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) + env = git_env_for_user(user) + + if remote_repository.is_a?(RemoteRepository) + env.merge!(remote_repository.fetch_env) + remote_repo_path = GITALY_INTERNAL_URL + else + remote_repo_path = remote_repository.path + end + + with_worktree(rebase_path, branch, env: env) do + run_git!( + %W(pull --rebase #{remote_repo_path} #{remote_branch}), + chdir: rebase_path, env: env + ) + + rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip + + Gitlab::Git::OperationService.new(user, self) + .update_branch(branch, rebase_sha, branch_sha) + + rebase_sha + end + end + def local_fetch_ref(source_path, source_ref:, target_ref:) args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) run_git(args) @@ -2083,6 +2121,39 @@ module Gitlab remove_remote(remote_name) end + def rugged_multi_action( + user, branch_name, message, actions, author_email, author_name, + start_branch_name, start_repository) + + OperationService.new(user, self).with_branch( + branch_name, + start_branch_name: start_branch_name, + start_repository: start_repository + ) do |start_commit| + index = Gitlab::Git::Index.new(self) + parents = [] + + if start_commit + index.read_tree(start_commit.rugged_commit.tree) + parents = [start_commit.sha] + end + + actions.each { |opts| index.apply(opts.delete(:action), opts) } + + committer = user_to_committer(user) + author = Gitlab::Git.committer_hash(email: author_email, name: author_name) || committer + options = { + tree: index.write_tree, + message: message, + parents: parents, + author: author, + committer: committer + } + + create_commit(options) + end + end + def fetch_remote(remote_name = 'origin', env: nil) run_git(['fetch', remote_name], env: env).last.zero? end diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/rev_list.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/rev_list.rb index 4974205b8..f8b2e7e0e 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/rev_list.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/rev_list.rb @@ -95,7 +95,7 @@ module Gitlab object_output.map do |output_line| sha, path = output_line.split(' ', 2) - next if require_path && path.blank? + next if require_path && path.to_s.empty? sha end.reject(&:nil?) diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/storage/forked_storage_check.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/storage/forked_storage_check.rb index 1307f4007..0a4e557b5 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/storage/forked_storage_check.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/storage/forked_storage_check.rb @@ -27,6 +27,7 @@ module Gitlab status = nil while status.nil? + if deadline > Time.now.utc sleep(wait_time) _pid, status = Process.wait2(filesystem_check_pid, Process::WNOHANG) diff --git a/ruby/vendor/gitlab_git/lib/gitlab/git/wiki_page.rb b/ruby/vendor/gitlab_git/lib/gitlab/git/wiki_page.rb index a06bac441..669ae11a4 100644 --- a/ruby/vendor/gitlab_git/lib/gitlab/git/wiki_page.rb +++ b/ruby/vendor/gitlab_git/lib/gitlab/git/wiki_page.rb @@ -1,7 +1,7 @@ module Gitlab module Git class WikiPage - attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical + attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical, :formatted_data # This class is meant to be serializable so that it can be constructed # by Gitaly and sent over the network to GitLab. @@ -21,6 +21,7 @@ module Gitlab @raw_data = gollum_page.raw_data @name = gollum_page.name @historical = gollum_page.historical? + @formatted_data = gollum_page.formatted_data if gollum_page.is_a?(Gollum::Page) @version = version end |