Merge branch 'sh-support-bitbucket-server-import' into 'master'

Add support for Bitbucket Server imports

Closes #25393

See merge request gitlab-org/gitlab-ce!20164

3 files changed, 373 insertions, 9 deletions

diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb
new file mode 100644
index 00000000000..268d21a77d1
--- /dev/null
+++ b/lib/gitlab/bitbucket_server_import/importer.rb
@@ -0,0 +1,327 @@
+module Gitlab
+  module BitbucketServerImport
+    class Importer
+      include Gitlab::ShellAdapter
+      attr_reader :recover_missing_commits
+      attr_reader :project, :project_key, :repository_slug, :client, :errors, :users
+
+      REMOTE_NAME = 'bitbucket_server'.freeze
+      BATCH_SIZE = 100
+
+      TempBranch = Struct.new(:name, :sha)
+
+      def self.imports_repository?
+        true
+      end
+
+      def self.refmap
+        [:heads, :tags, '+refs/pull-requests/*/to:refs/merge-requests/*/head']
+      end
+
+      # Unlike GitHub, you can't grab the commit SHAs for pull requests that
+      # have been closed but not merged even though Bitbucket has these
+      # commits internally. We can recover these pull requests by creating a
+      # branch with the Bitbucket REST API, but by default we turn this
+      # behavior off.
+      def initialize(project, recover_missing_commits: false)
+        @project = project
+        @recover_missing_commits = recover_missing_commits
+        @project_key = project.import_data.data['project_key']
+        @repository_slug = project.import_data.data['repo_slug']
+        @client = BitbucketServer::Client.new(project.import_data.credentials)
+        @formatter = Gitlab::ImportFormatter.new
+        @errors = []
+        @users = {}
+        @temp_branches = []
+      end
+
+      def execute
+        import_repository
+        import_pull_requests
+        delete_temp_branches
+        handle_errors
+
+        true
+      end
+
+      private
+
+      def handle_errors
+        return unless errors.any?
+
+        project.update_column(:import_error, {
+          message: 'The remote data could not be fully imported.',
+          errors: errors
+        }.to_json)
+      end
+
+      def gitlab_user_id(email)
+        find_user_id(email) || project.creator_id
+      end
+
+      def find_user_id(email)
+        return nil unless email
+
+        return users[email] if users.key?(email)
+
+        user = User.find_by_any_email(email, confirmed: true)
+        users[email] = user&.id
+
+        user&.id
+      end
+
+      def repo
+        @repo ||= client.repo(project_key, repository_slug)
+      end
+
+      def sha_exists?(sha)
+        project.repository.commit(sha)
+      end
+
+      def temp_branch_name(pull_request, suffix)
+        "gitlab/import/pull-request/#{pull_request.iid}/#{suffix}"
+      end
+
+      # This method restores required SHAs that GitLab needs to create diffs
+      # into branch names as the following:
+      #
+      # gitlab/import/pull-request/N/{to,from}
+      def restore_branches(pull_requests)
+        shas_to_restore = []
+
+        pull_requests.each do |pull_request|
+          shas_to_restore << TempBranch.new(temp_branch_name(pull_request, :from),
+                                            pull_request.source_branch_sha)
+          shas_to_restore << TempBranch.new(temp_branch_name(pull_request, :to),
+                                            pull_request.target_branch_sha)
+        end
+
+        # Create the branches on the Bitbucket Server first
+        created_branches = restore_branch_shas(shas_to_restore)
+
+        @temp_branches += created_branches
+        # Now sync the repository so we get the new branches
+        import_repository unless created_branches.empty?
+      end
+
+      def restore_branch_shas(shas_to_restore)
+        shas_to_restore.each_with_object([]) do |temp_branch, branches_created|
+          branch_name = temp_branch.name
+          sha = temp_branch.sha
+
+          next if sha_exists?(sha)
+
+          begin
+            client.create_branch(project_key, repository_slug, branch_name, sha)
+            branches_created << temp_branch
+          rescue BitbucketServer::Connection::ConnectionError => e
+            Rails.logger.warn("BitbucketServerImporter: Unable to recreate branch for SHA #{sha}: #{e}")
+          end
+        end
+      end
+
+      def import_repository
+        project.ensure_repository
+        project.repository.fetch_as_mirror(project.import_url, refmap: self.class.refmap, remote_name: REMOTE_NAME)
+      rescue Gitlab::Shell::Error, Gitlab::Git::RepositoryMirroring::RemoteError => e
+        # Expire cache to prevent scenarios such as:
+        # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true
+        # 2. Retried import, repo is broken or not imported but +exists?+ still returns true
+        project.repository.expire_content_cache if project.repository_exists?
+
+        raise e.message
+      end
+
+      # Bitbucket Server keeps tracks of references for open pull requests in
+      # refs/heads/pull-requests, but closed and merged requests get moved
+      # into hidden internal refs under stash-refs/pull-requests. Unless the
+      # SHAs involved are at the tip of a branch or tag, there is no way to
+      # retrieve the server for those commits.
+      #
+      # To avoid losing history, we use the Bitbucket API to re-create the branch
+      # on the remote server. Then we have to issue a `git fetch` to download these
+      # branches.
+      def import_pull_requests
+        pull_requests = client.pull_requests(project_key, repository_slug).to_a
+
+        # Creating branches on the server and fetching the newly-created branches
+        # may take a number of network round-trips. Do this in batches so that we can
+        # avoid doing a git fetch for every new branch.
+        pull_requests.each_slice(BATCH_SIZE) do |batch|
+          restore_branches(batch) if recover_missing_commits
+
+          batch.each do |pull_request|
+            begin
+              import_bitbucket_pull_request(pull_request)
+            rescue StandardError => e
+              errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, trace: e.backtrace.join("\n"), raw_response: pull_request.raw }
+            end
+          end
+        end
+      end
+
+      def delete_temp_branches
+        @temp_branches.each do |branch|
+          begin
+            client.delete_branch(project_key, repository_slug, branch.name, branch.sha)
+            project.repository.delete_branch(branch.name)
+          rescue BitbucketServer::Connection::ConnectionError => e
+            @errors << { type: :delete_temp_branches, branch_name: branch.name, errors: e.message }
+          end
+        end
+      end
+
+      def import_bitbucket_pull_request(pull_request)
+        description = ''
+        description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author_email)
+        description += pull_request.description if pull_request.description
+
+        source_branch_sha = pull_request.source_branch_sha
+        target_branch_sha = pull_request.target_branch_sha
+        author_id = gitlab_user_id(pull_request.author_email)
+
+        attributes = {
+          iid: pull_request.iid,
+          title: pull_request.title,
+          description: description,
+          source_project: project,
+          source_branch: Gitlab::Git.ref_name(pull_request.source_branch_name),
+          source_branch_sha: source_branch_sha,
+          target_project: project,
+          target_branch: Gitlab::Git.ref_name(pull_request.target_branch_name),
+          target_branch_sha: target_branch_sha,
+          state: pull_request.state,
+          author_id: author_id,
+          assignee_id: nil,
+          created_at: pull_request.created_at,
+          updated_at: pull_request.updated_at
+        }
+
+        merge_request = project.merge_requests.create!(attributes)
+        import_pull_request_comments(pull_request, merge_request) if merge_request.persisted?
+      end
+
+      def import_pull_request_comments(pull_request, merge_request)
+        comments, other_activities = client.activities(project_key, repository_slug, pull_request.iid).partition(&:comment?)
+
+        merge_event = other_activities.find(&:merge_event?)
+        import_merge_event(merge_request, merge_event) if merge_event
+
+        inline_comments, pr_comments = comments.partition(&:inline_comment?)
+
+        import_inline_comments(inline_comments.map(&:comment), merge_request)
+        import_standalone_pr_comments(pr_comments.map(&:comment), merge_request)
+      end
+
+      def import_merge_event(merge_request, merge_event)
+        committer = merge_event.committer_email
+
+        user_id = gitlab_user_id(committer)
+        timestamp = merge_event.merge_timestamp
+        merge_request.update({ merge_commit_sha: merge_event.merge_commit })
+        metric = MergeRequest::Metrics.find_or_initialize_by(merge_request: merge_request)
+        metric.update(merged_by_id: user_id, merged_at: timestamp)
+      end
+
+      def import_inline_comments(inline_comments, merge_request)
+        inline_comments.each do |comment|
+          position = build_position(merge_request, comment)
+          parent = create_diff_note(merge_request, comment, position)
+
+          next unless parent&.persisted?
+
+          discussion_id = parent.discussion_id
+
+          comment.comments.each do |reply|
+            create_diff_note(merge_request, reply, position, discussion_id)
+          end
+        end
+      end
+
+      def create_diff_note(merge_request, comment, position, discussion_id = nil)
+        attributes = pull_request_comment_attributes(comment)
+        attributes.merge!(position: position, type: 'DiffNote')
+        attributes[:discussion_id] = discussion_id if discussion_id
+
+        note = merge_request.notes.build(attributes)
+
+        if note.valid?
+          note.save
+          return note
+        end
+
+        # Bitbucket Server supports the ability to comment on any line, not just the
+        # line in the diff. If we can't add the note as a DiffNote, fallback to creating
+        # a regular note.
+        create_fallback_diff_note(merge_request, comment, position)
+      rescue StandardError => e
+        errors << { type: :pull_request, id: comment.id, errors: e.message }
+        nil
+      end
+
+      def create_fallback_diff_note(merge_request, comment, position)
+        attributes = pull_request_comment_attributes(comment)
+        note = "*Comment on"
+
+        note += " #{position.old_path}:#{position.old_line} -->" if position.old_line
+        note += " #{position.new_path}:#{position.new_line}" if position.new_line
+        note += "*\n\n#{comment.note}"
+
+        attributes[:note] = note
+        merge_request.notes.create!(attributes)
+      end
+
+      def build_position(merge_request, pr_comment)
+        params = {
+          diff_refs: merge_request.diff_refs,
+          old_path: pr_comment.file_path,
+          new_path: pr_comment.file_path,
+          old_line: pr_comment.old_pos,
+          new_line: pr_comment.new_pos
+        }
+
+        Gitlab::Diff::Position.new(params)
+      end
+
+      def import_standalone_pr_comments(pr_comments, merge_request)
+        pr_comments.each do |comment|
+          begin
+            merge_request.notes.create!(pull_request_comment_attributes(comment))
+
+            comment.comments.each do |replies|
+              merge_request.notes.create!(pull_request_comment_attributes(replies))
+            end
+          rescue StandardError => e
+            errors << { type: :pull_request, iid: comment.id, errors: e.message }
+          end
+        end
+      end
+
+      def pull_request_comment_attributes(comment)
+        author = find_user_id(comment.author_email)
+        note = ''
+
+        unless author
+          author = project.creator_id
+          note = "*By #{comment.author_username} (#{comment.author_email})*\n\n"
+        end
+
+        note +=
+          # Provide some context for replying
+          if comment.parent_comment
+            "> #{comment.parent_comment.note.truncate(80)}\n\n#{comment.note}"
+          else
+            comment.note
+          end
+
+        {
+          project: project,
+          note: note,
+          author_id: author,
+          created_at: comment.created_at,
+          updated_at: comment.updated_at
+        }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/bitbucket_server_import/project_creator.rb b/lib/gitlab/bitbucket_server_import/project_creator.rb
new file mode 100644
index 00000000000..35e8cd7e0ab
--- /dev/null
+++ b/lib/gitlab/bitbucket_server_import/project_creator.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module BitbucketServerImport
+    class ProjectCreator
+      attr_reader :project_key, :repo_slug, :repo, :name, :namespace, :current_user, :session_data
+
+      def initialize(project_key, repo_slug, repo, name, namespace, current_user, session_data)
+        @project_key = project_key
+        @repo_slug = repo_slug
+        @repo = repo
+        @name = name
+        @namespace = namespace
+        @current_user = current_user
+        @session_data = session_data
+      end
+
+      def execute
+        ::Projects::CreateService.new(
+          current_user,
+          name: name,
+          path: name,
+          description: repo.description,
+          namespace_id: namespace.id,
+          visibility_level: repo.visibility_level,
+          import_type: 'bitbucket_server',
+          import_source: repo.browse_url,
+          import_url: repo.clone_url,
+          import_data: {
+            credentials: session_data,
+            data: { project_key: project_key, repo_slug: repo_slug }
+          },
+          skip_wiki: true
+        ).execute
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb
index 45816bee176..f7f5c5787f6 100644
--- a/lib/gitlab/import_sources.rb
+++ b/lib/gitlab/import_sources.rb
@@ -9,15 +9,16 @@ module Gitlab
 
     # We exclude `bare_repository` here as it has no import class associated
     ImportTable = [
-      ImportSource.new('github',         'GitHub',        Gitlab::GithubImport::ParallelImporter),
-      ImportSource.new('bitbucket',      'Bitbucket',     Gitlab::BitbucketImport::Importer),
-      ImportSource.new('gitlab',         'GitLab.com',    Gitlab::GitlabImport::Importer),
-      ImportSource.new('google_code',    'Google Code',   Gitlab::GoogleCodeImport::Importer),
-      ImportSource.new('fogbugz',        'FogBugz',       Gitlab::FogbugzImport::Importer),
-      ImportSource.new('git',            'Repo by URL',   nil),
-      ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer),
-      ImportSource.new('gitea',          'Gitea',         Gitlab::LegacyGithubImport::Importer),
-      ImportSource.new('manifest',       'Manifest file', nil)
+      ImportSource.new('github',           'GitHub',           Gitlab::GithubImport::ParallelImporter),
+      ImportSource.new('bitbucket',        'Bitbucket Cloud',  Gitlab::BitbucketImport::Importer),
+      ImportSource.new('bitbucket_server', 'Bitbucket Server', Gitlab::BitbucketServerImport::Importer),
+      ImportSource.new('gitlab',           'GitLab.com',       Gitlab::GitlabImport::Importer),
+      ImportSource.new('google_code',      'Google Code',      Gitlab::GoogleCodeImport::Importer),
+      ImportSource.new('fogbugz',          'FogBugz',          Gitlab::FogbugzImport::Importer),
+      ImportSource.new('git',              'Repo by URL',      nil),
+      ImportSource.new('gitlab_project',   'GitLab export',    Gitlab::ImportExport::Importer),
+      ImportSource.new('gitea',            'Gitea',            Gitlab::LegacyGithubImport::Importer),
+      ImportSource.new('manifest',         'Manifest file',    nil)
     ].freeze
 
     class << self