diff options
author | Tim Zallmann <tzallmann@gitlab.com> | 2018-08-06 20:32:12 +0300 |
---|---|---|
committer | Tim Zallmann <tzallmann@gitlab.com> | 2018-08-06 20:32:12 +0300 |
commit | d737abc537476bf2b500f550b0c733d22f338cf1 (patch) | |
tree | 4b865101f1e4eab3ceedd75973098de9f18ac4b6 /lib/gitlab | |
parent | 0ffd79319ffc06f9ab5dcfce4f20a4da2f739577 (diff) | |
parent | ee851e58490004f985f914f3cfa715b03cdf4982 (diff) |
Merge branch 'sh-support-bitbucket-server-import' into 'master'
Add support for Bitbucket Server imports
Closes #25393
See merge request gitlab-org/gitlab-ce!20164
Diffstat (limited to 'lib/gitlab')
-rw-r--r-- | lib/gitlab/bitbucket_server_import/importer.rb | 327 | ||||
-rw-r--r-- | lib/gitlab/bitbucket_server_import/project_creator.rb | 36 | ||||
-rw-r--r-- | lib/gitlab/import_sources.rb | 19 |
3 files changed, 373 insertions, 9 deletions
diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb new file mode 100644 index 00000000000..268d21a77d1 --- /dev/null +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -0,0 +1,327 @@ +module Gitlab + module BitbucketServerImport + class Importer + include Gitlab::ShellAdapter + attr_reader :recover_missing_commits + attr_reader :project, :project_key, :repository_slug, :client, :errors, :users + + REMOTE_NAME = 'bitbucket_server'.freeze + BATCH_SIZE = 100 + + TempBranch = Struct.new(:name, :sha) + + def self.imports_repository? + true + end + + def self.refmap + [:heads, :tags, '+refs/pull-requests/*/to:refs/merge-requests/*/head'] + end + + # Unlike GitHub, you can't grab the commit SHAs for pull requests that + # have been closed but not merged even though Bitbucket has these + # commits internally. We can recover these pull requests by creating a + # branch with the Bitbucket REST API, but by default we turn this + # behavior off. + def initialize(project, recover_missing_commits: false) + @project = project + @recover_missing_commits = recover_missing_commits + @project_key = project.import_data.data['project_key'] + @repository_slug = project.import_data.data['repo_slug'] + @client = BitbucketServer::Client.new(project.import_data.credentials) + @formatter = Gitlab::ImportFormatter.new + @errors = [] + @users = {} + @temp_branches = [] + end + + def execute + import_repository + import_pull_requests + delete_temp_branches + handle_errors + + true + end + + private + + def handle_errors + return unless errors.any? + + project.update_column(:import_error, { + message: 'The remote data could not be fully imported.', + errors: errors + }.to_json) + end + + def gitlab_user_id(email) + find_user_id(email) || project.creator_id + end + + def find_user_id(email) + return nil unless email + + return users[email] if users.key?(email) + + user = User.find_by_any_email(email, confirmed: true) + users[email] = user&.id + + user&.id + end + + def repo + @repo ||= client.repo(project_key, repository_slug) + end + + def sha_exists?(sha) + project.repository.commit(sha) + end + + def temp_branch_name(pull_request, suffix) + "gitlab/import/pull-request/#{pull_request.iid}/#{suffix}" + end + + # This method restores required SHAs that GitLab needs to create diffs + # into branch names as the following: + # + # gitlab/import/pull-request/N/{to,from} + def restore_branches(pull_requests) + shas_to_restore = [] + + pull_requests.each do |pull_request| + shas_to_restore << TempBranch.new(temp_branch_name(pull_request, :from), + pull_request.source_branch_sha) + shas_to_restore << TempBranch.new(temp_branch_name(pull_request, :to), + pull_request.target_branch_sha) + end + + # Create the branches on the Bitbucket Server first + created_branches = restore_branch_shas(shas_to_restore) + + @temp_branches += created_branches + # Now sync the repository so we get the new branches + import_repository unless created_branches.empty? + end + + def restore_branch_shas(shas_to_restore) + shas_to_restore.each_with_object([]) do |temp_branch, branches_created| + branch_name = temp_branch.name + sha = temp_branch.sha + + next if sha_exists?(sha) + + begin + client.create_branch(project_key, repository_slug, branch_name, sha) + branches_created << temp_branch + rescue BitbucketServer::Connection::ConnectionError => e + Rails.logger.warn("BitbucketServerImporter: Unable to recreate branch for SHA #{sha}: #{e}") + end + end + end + + def import_repository + project.ensure_repository + project.repository.fetch_as_mirror(project.import_url, refmap: self.class.refmap, remote_name: REMOTE_NAME) + rescue Gitlab::Shell::Error, Gitlab::Git::RepositoryMirroring::RemoteError => e + # Expire cache to prevent scenarios such as: + # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true + # 2. Retried import, repo is broken or not imported but +exists?+ still returns true + project.repository.expire_content_cache if project.repository_exists? + + raise e.message + end + + # Bitbucket Server keeps tracks of references for open pull requests in + # refs/heads/pull-requests, but closed and merged requests get moved + # into hidden internal refs under stash-refs/pull-requests. Unless the + # SHAs involved are at the tip of a branch or tag, there is no way to + # retrieve the server for those commits. + # + # To avoid losing history, we use the Bitbucket API to re-create the branch + # on the remote server. Then we have to issue a `git fetch` to download these + # branches. + def import_pull_requests + pull_requests = client.pull_requests(project_key, repository_slug).to_a + + # Creating branches on the server and fetching the newly-created branches + # may take a number of network round-trips. Do this in batches so that we can + # avoid doing a git fetch for every new branch. + pull_requests.each_slice(BATCH_SIZE) do |batch| + restore_branches(batch) if recover_missing_commits + + batch.each do |pull_request| + begin + import_bitbucket_pull_request(pull_request) + rescue StandardError => e + errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, trace: e.backtrace.join("\n"), raw_response: pull_request.raw } + end + end + end + end + + def delete_temp_branches + @temp_branches.each do |branch| + begin + client.delete_branch(project_key, repository_slug, branch.name, branch.sha) + project.repository.delete_branch(branch.name) + rescue BitbucketServer::Connection::ConnectionError => e + @errors << { type: :delete_temp_branches, branch_name: branch.name, errors: e.message } + end + end + end + + def import_bitbucket_pull_request(pull_request) + description = '' + description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author_email) + description += pull_request.description if pull_request.description + + source_branch_sha = pull_request.source_branch_sha + target_branch_sha = pull_request.target_branch_sha + author_id = gitlab_user_id(pull_request.author_email) + + attributes = { + iid: pull_request.iid, + title: pull_request.title, + description: description, + source_project: project, + source_branch: Gitlab::Git.ref_name(pull_request.source_branch_name), + source_branch_sha: source_branch_sha, + target_project: project, + target_branch: Gitlab::Git.ref_name(pull_request.target_branch_name), + target_branch_sha: target_branch_sha, + state: pull_request.state, + author_id: author_id, + assignee_id: nil, + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + } + + merge_request = project.merge_requests.create!(attributes) + import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? + end + + def import_pull_request_comments(pull_request, merge_request) + comments, other_activities = client.activities(project_key, repository_slug, pull_request.iid).partition(&:comment?) + + merge_event = other_activities.find(&:merge_event?) + import_merge_event(merge_request, merge_event) if merge_event + + inline_comments, pr_comments = comments.partition(&:inline_comment?) + + import_inline_comments(inline_comments.map(&:comment), merge_request) + import_standalone_pr_comments(pr_comments.map(&:comment), merge_request) + end + + def import_merge_event(merge_request, merge_event) + committer = merge_event.committer_email + + user_id = gitlab_user_id(committer) + timestamp = merge_event.merge_timestamp + merge_request.update({ merge_commit_sha: merge_event.merge_commit }) + metric = MergeRequest::Metrics.find_or_initialize_by(merge_request: merge_request) + metric.update(merged_by_id: user_id, merged_at: timestamp) + end + + def import_inline_comments(inline_comments, merge_request) + inline_comments.each do |comment| + position = build_position(merge_request, comment) + parent = create_diff_note(merge_request, comment, position) + + next unless parent&.persisted? + + discussion_id = parent.discussion_id + + comment.comments.each do |reply| + create_diff_note(merge_request, reply, position, discussion_id) + end + end + end + + def create_diff_note(merge_request, comment, position, discussion_id = nil) + attributes = pull_request_comment_attributes(comment) + attributes.merge!(position: position, type: 'DiffNote') + attributes[:discussion_id] = discussion_id if discussion_id + + note = merge_request.notes.build(attributes) + + if note.valid? + note.save + return note + end + + # Bitbucket Server supports the ability to comment on any line, not just the + # line in the diff. If we can't add the note as a DiffNote, fallback to creating + # a regular note. + create_fallback_diff_note(merge_request, comment, position) + rescue StandardError => e + errors << { type: :pull_request, id: comment.id, errors: e.message } + nil + end + + def create_fallback_diff_note(merge_request, comment, position) + attributes = pull_request_comment_attributes(comment) + note = "*Comment on" + + note += " #{position.old_path}:#{position.old_line} -->" if position.old_line + note += " #{position.new_path}:#{position.new_line}" if position.new_line + note += "*\n\n#{comment.note}" + + attributes[:note] = note + merge_request.notes.create!(attributes) + end + + def build_position(merge_request, pr_comment) + params = { + diff_refs: merge_request.diff_refs, + old_path: pr_comment.file_path, + new_path: pr_comment.file_path, + old_line: pr_comment.old_pos, + new_line: pr_comment.new_pos + } + + Gitlab::Diff::Position.new(params) + end + + def import_standalone_pr_comments(pr_comments, merge_request) + pr_comments.each do |comment| + begin + merge_request.notes.create!(pull_request_comment_attributes(comment)) + + comment.comments.each do |replies| + merge_request.notes.create!(pull_request_comment_attributes(replies)) + end + rescue StandardError => e + errors << { type: :pull_request, iid: comment.id, errors: e.message } + end + end + end + + def pull_request_comment_attributes(comment) + author = find_user_id(comment.author_email) + note = '' + + unless author + author = project.creator_id + note = "*By #{comment.author_username} (#{comment.author_email})*\n\n" + end + + note += + # Provide some context for replying + if comment.parent_comment + "> #{comment.parent_comment.note.truncate(80)}\n\n#{comment.note}" + else + comment.note + end + + { + project: project, + note: note, + author_id: author, + created_at: comment.created_at, + updated_at: comment.updated_at + } + end + end + end +end diff --git a/lib/gitlab/bitbucket_server_import/project_creator.rb b/lib/gitlab/bitbucket_server_import/project_creator.rb new file mode 100644 index 00000000000..35e8cd7e0ab --- /dev/null +++ b/lib/gitlab/bitbucket_server_import/project_creator.rb @@ -0,0 +1,36 @@ +module Gitlab + module BitbucketServerImport + class ProjectCreator + attr_reader :project_key, :repo_slug, :repo, :name, :namespace, :current_user, :session_data + + def initialize(project_key, repo_slug, repo, name, namespace, current_user, session_data) + @project_key = project_key + @repo_slug = repo_slug + @repo = repo + @name = name + @namespace = namespace + @current_user = current_user + @session_data = session_data + end + + def execute + ::Projects::CreateService.new( + current_user, + name: name, + path: name, + description: repo.description, + namespace_id: namespace.id, + visibility_level: repo.visibility_level, + import_type: 'bitbucket_server', + import_source: repo.browse_url, + import_url: repo.clone_url, + import_data: { + credentials: session_data, + data: { project_key: project_key, repo_slug: repo_slug } + }, + skip_wiki: true + ).execute + end + end + end +end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 45816bee176..f7f5c5787f6 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -9,15 +9,16 @@ module Gitlab # We exclude `bare_repository` here as it has no import class associated ImportTable = [ - ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter), - ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), - ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), - ImportSource.new('google_code', 'Google Code', Gitlab::GoogleCodeImport::Importer), - ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), - ImportSource.new('git', 'Repo by URL', nil), - ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), - ImportSource.new('gitea', 'Gitea', Gitlab::LegacyGithubImport::Importer), - ImportSource.new('manifest', 'Manifest file', nil) + ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter), + ImportSource.new('bitbucket', 'Bitbucket Cloud', Gitlab::BitbucketImport::Importer), + ImportSource.new('bitbucket_server', 'Bitbucket Server', Gitlab::BitbucketServerImport::Importer), + ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), + ImportSource.new('google_code', 'Google Code', Gitlab::GoogleCodeImport::Importer), + ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), + ImportSource.new('git', 'Repo by URL', nil), + ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), + ImportSource.new('gitea', 'Gitea', Gitlab::LegacyGithubImport::Importer), + ImportSource.new('manifest', 'Manifest file', nil) ].freeze class << self |