diff options
Diffstat (limited to 'lib/gitlab/bitbucket_server_import/importers')
7 files changed, 487 insertions, 0 deletions
diff --git a/lib/gitlab/bitbucket_server_import/importers/lfs_object_importer.rb b/lib/gitlab/bitbucket_server_import/importers/lfs_object_importer.rb new file mode 100644 index 00000000000..67b8eefc351 --- /dev/null +++ b/lib/gitlab/bitbucket_server_import/importers/lfs_object_importer.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + module Importers + class LfsObjectImporter + include Loggable + + def initialize(project, lfs_attributes) + @project = project + @lfs_download_object = LfsDownloadObject.new(**lfs_attributes.symbolize_keys) + end + + def execute + log_info(import_stage: 'import_lfs_object', message: 'starting', oid: lfs_download_object.oid) + + Projects::LfsPointers::LfsDownloadService.new(project, lfs_download_object).execute + + log_info(import_stage: 'import_lfs_object', message: 'finished', oid: lfs_download_object.oid) + end + + private + + attr_reader :project, :lfs_download_object + end + end + end +end diff --git a/lib/gitlab/bitbucket_server_import/importers/lfs_objects_importer.rb b/lib/gitlab/bitbucket_server_import/importers/lfs_objects_importer.rb new file mode 100644 index 00000000000..d568f60f4fc --- /dev/null +++ b/lib/gitlab/bitbucket_server_import/importers/lfs_objects_importer.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + module Importers + class LfsObjectsImporter + include ParallelScheduling + + def execute + log_info(import_stage: 'import_lfs_objects', message: 'starting') + + download_service = Projects::LfsPointers::LfsObjectDownloadListService.new(project) + + begin + queue_workers(download_service) if project&.lfs_enabled? + rescue StandardError => e + track_import_failure!(project, exception: e) + end + + log_info(import_stage: 'import_lfs_objects', message: 'finished') + + job_waiter + end + + def sidekiq_worker_class + ImportLfsObjectWorker + end + + def collection_method + :lfs_objects + end + + def id_for_already_processed_cache(lfs_download_object) + lfs_download_object.oid + end + + private + + def queue_workers(download_service) + download_service.each_list_item do |lfs_download_object| + # Needs to come before `already_processed?` as `jobs_remaining` resets to zero when the job restarts and + # jobs_remaining needs to be the total amount of enqueued jobs + job_waiter.jobs_remaining += 1 + + next if already_processed?(lfs_download_object) + + job_delay = calculate_job_delay(job_waiter.jobs_remaining) + + sidekiq_worker_class.perform_in(job_delay, project.id, lfs_download_object.as_json, job_waiter.key) + + mark_as_processed(lfs_download_object) + end + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_server_import/importers/notes_importer.rb b/lib/gitlab/bitbucket_server_import/importers/notes_importer.rb new file mode 100644 index 00000000000..07ee9569ab1 --- /dev/null +++ b/lib/gitlab/bitbucket_server_import/importers/notes_importer.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + module Importers + class NotesImporter + include ParallelScheduling + + def execute + project.merge_requests.find_each do |merge_request| + # Needs to come before `already_processed?` as `jobs_remaining` resets to zero when the job restarts and + # jobs_remaining needs to be the total amount of enqueued jobs + job_waiter.jobs_remaining += 1 + + next if already_processed?(merge_request) + + job_delay = calculate_job_delay(job_waiter.jobs_remaining) + + sidekiq_worker_class.perform_in(job_delay, project.id, { iid: merge_request.iid }, job_waiter.key) + + mark_as_processed(merge_request) + end + + job_waiter + end + + private + + attr_reader :project + + def sidekiq_worker_class + ImportPullRequestNotesWorker + end + + def id_for_already_processed_cache(merge_request) + merge_request.iid + end + + def collection_method + :notes + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_server_import/importers/pull_request_importer.rb b/lib/gitlab/bitbucket_server_import/importers/pull_request_importer.rb new file mode 100644 index 00000000000..5d306f98980 --- /dev/null +++ b/lib/gitlab/bitbucket_server_import/importers/pull_request_importer.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + module Importers + class PullRequestImporter + include Loggable + + def initialize(project, hash) + @project = project + @formatter = Gitlab::ImportFormatter.new + @user_finder = UserFinder.new(project) + + # Object should behave as a object so we can remove object.is_a?(Hash) check + # This will be fixed in https://gitlab.com/gitlab-org/gitlab/-/issues/412328 + @object = hash.with_indifferent_access + end + + def execute + log_info(import_stage: 'import_pull_request', message: 'starting', iid: object[:iid]) + + description = '' + description += author_line + description += object[:description] if object[:description] + + attributes = { + iid: object[:iid], + title: object[:title], + description: description, + source_project_id: project.id, + source_branch: Gitlab::Git.ref_name(object[:source_branch_name]), + source_branch_sha: object[:source_branch_sha], + target_project_id: project.id, + target_branch: Gitlab::Git.ref_name(object[:target_branch_name]), + target_branch_sha: object[:target_branch_sha], + state_id: MergeRequest.available_states[object[:state]], + author_id: user_finder.author_id(object), + created_at: object[:created_at], + updated_at: object[:updated_at] + } + + creator = Gitlab::Import::MergeRequestCreator.new(project) + + creator.execute(attributes) + + log_info(import_stage: 'import_pull_request', message: 'finished', iid: object[:iid]) + end + + private + + attr_reader :object, :project, :formatter, :user_finder + + def author_line + return '' if user_finder.uid(object) + + formatter.author_line(object[:author]) + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_server_import/importers/pull_request_notes_importer.rb b/lib/gitlab/bitbucket_server_import/importers/pull_request_notes_importer.rb new file mode 100644 index 00000000000..69de47e2006 --- /dev/null +++ b/lib/gitlab/bitbucket_server_import/importers/pull_request_notes_importer.rb @@ -0,0 +1,183 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + module Importers + class PullRequestNotesImporter + include Loggable + + def initialize(project, hash) + @project = project + @formatter = Gitlab::ImportFormatter.new + @client = BitbucketServer::Client.new(project.import_data.credentials) + @project_key = project.import_data.data['project_key'] + @repository_slug = project.import_data.data['repo_slug'] + @user_finder = UserFinder.new(project) + + # TODO: Convert object into a object instead of using it as a hash + @object = hash.with_indifferent_access + end + + def execute + log_info(import_stage: 'import_pull_request_notes', message: 'starting', iid: object[:iid]) + + merge_request = project.merge_requests.find_by(iid: object[:iid]) # rubocop: disable CodeReuse/ActiveRecord + + if merge_request + activities = client.activities(project_key, repository_slug, merge_request.iid) + + comments, other_activities = activities.partition(&:comment?) + + merge_event = other_activities.find(&:merge_event?) + import_merge_event(merge_request, merge_event) if merge_event + + inline_comments, pr_comments = comments.partition(&:inline_comment?) + + import_inline_comments(inline_comments.map(&:comment), merge_request) + import_standalone_pr_comments(pr_comments.map(&:comment), merge_request) + end + + log_info(import_stage: 'import_pull_request_notes', message: 'finished', iid: object[:iid]) + end + + private + + attr_reader :object, :project, :formatter, :client, :project_key, :repository_slug, :user_finder + + # rubocop: disable CodeReuse/ActiveRecord + def import_merge_event(merge_request, merge_event) + log_info(import_stage: 'import_merge_event', message: 'starting', iid: merge_request.iid) + + committer = merge_event.committer_email + + user_id = user_finder.find_user_id(by: :email, value: committer) || project.creator_id + timestamp = merge_event.merge_timestamp + merge_request.update({ merge_commit_sha: merge_event.merge_commit }) + metric = MergeRequest::Metrics.find_or_initialize_by(merge_request: merge_request) + metric.update(merged_by_id: user_id, merged_at: timestamp) + + log_info(import_stage: 'import_merge_event', message: 'finished', iid: merge_request.iid) + end + # rubocop: enable CodeReuse/ActiveRecord + + def import_inline_comments(inline_comments, merge_request) + log_info(import_stage: 'import_inline_comments', message: 'starting', iid: merge_request.iid) + + inline_comments.each do |comment| + position = build_position(merge_request, comment) + parent = create_diff_note(merge_request, comment, position) + + next unless parent&.persisted? + + discussion_id = parent.discussion_id + + comment.comments.each do |reply| + create_diff_note(merge_request, reply, position, discussion_id) + end + end + + log_info(import_stage: 'import_inline_comments', message: 'finished', iid: merge_request.iid) + end + + def create_diff_note(merge_request, comment, position, discussion_id = nil) + attributes = pull_request_comment_attributes(comment) + attributes.merge!(position: position, type: 'DiffNote') + attributes[:discussion_id] = discussion_id if discussion_id + + note = merge_request.notes.build(attributes) + + if note.valid? + note.save + return note + end + + log_info(import_stage: 'create_diff_note', message: 'creating fallback DiffNote', iid: merge_request.iid) + + # Bitbucket Server supports the ability to comment on any line, not just the + # line in the diff. If we can't add the note as a DiffNote, fallback to creating + # a regular note. + create_fallback_diff_note(merge_request, comment, position) + rescue StandardError => e + Gitlab::ErrorTracking.log_exception( + e, + import_stage: 'create_diff_note', comment_id: comment.id, error: e.message + ) + + nil + end + + def create_fallback_diff_note(merge_request, comment, position) + attributes = pull_request_comment_attributes(comment) + note = "*Comment on" + + note += " #{position.old_path}:#{position.old_line} -->" if position.old_line + note += " #{position.new_path}:#{position.new_line}" if position.new_line + note += "*\n\n#{comment.note}" + + attributes[:note] = note + merge_request.notes.create!(attributes) + end + + def build_position(merge_request, pr_comment) + params = { + diff_refs: merge_request.diff_refs, + old_path: pr_comment.file_path, + new_path: pr_comment.file_path, + old_line: pr_comment.old_pos, + new_line: pr_comment.new_pos + } + + Gitlab::Diff::Position.new(params) + end + + def import_standalone_pr_comments(pr_comments, merge_request) + log_info(import_stage: 'import_standalone_pr_comments', message: 'starting', iid: merge_request.iid) + + pr_comments.each do |comment| + merge_request.notes.create!(pull_request_comment_attributes(comment)) + + comment.comments.each do |replies| + merge_request.notes.create!(pull_request_comment_attributes(replies)) + end + rescue StandardError => e + Gitlab::ErrorTracking.log_exception( + e, + import_stage: 'import_standalone_pr_comments', + merge_request_id: merge_request.id, + comment_id: comment.id, + error: e.message + ) + ensure + log_info(import_stage: 'import_standalone_pr_comments', message: 'finished', iid: merge_request.iid) + end + end + + def pull_request_comment_attributes(comment) + author = user_finder.uid(comment) + note = '' + + unless author + author = project.creator_id + note = "*By #{comment.author_username} (#{comment.author_email})*\n\n" + end + + note += + # Provide some context for replying + if comment.parent_comment + "> #{comment.parent_comment.note.truncate(80)}\n\n#{comment.note}" + else + comment.note + end + + { + project: project, + note: note, + author_id: author, + created_at: comment.created_at, + updated_at: comment.updated_at + } + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb b/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb new file mode 100644 index 00000000000..92ec10bf037 --- /dev/null +++ b/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + module Importers + class PullRequestsImporter + include ParallelScheduling + + def execute + page = 1 + + loop do + log_info( + import_stage: 'import_pull_requests', message: "importing page #{page} using batch-size #{BATCH_SIZE}" + ) + + pull_requests = client.pull_requests( + project_key, repository_slug, page_offset: page, limit: BATCH_SIZE + ).to_a + + break if pull_requests.empty? + + pull_requests.each do |pull_request| + # Needs to come before `already_processed?` as `jobs_remaining` resets to zero when the job restarts and + # jobs_remaining needs to be the total amount of enqueued jobs + job_waiter.jobs_remaining += 1 + + next if already_processed?(pull_request) + + job_delay = calculate_job_delay(job_waiter.jobs_remaining) + + sidekiq_worker_class.perform_in(job_delay, project.id, pull_request.to_hash, job_waiter.key) + + mark_as_processed(pull_request) + end + + page += 1 + end + + job_waiter + end + + private + + def sidekiq_worker_class + ImportPullRequestWorker + end + + def collection_method + :pull_requests + end + + def id_for_already_processed_cache(object) + object.iid + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_server_import/importers/repository_importer.rb b/lib/gitlab/bitbucket_server_import/importers/repository_importer.rb new file mode 100644 index 00000000000..cd09ac40e9f --- /dev/null +++ b/lib/gitlab/bitbucket_server_import/importers/repository_importer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + module Importers + class RepositoryImporter + include Loggable + + def initialize(project) + @project = project + end + + def execute + log_info(import_stage: 'import_repository', message: 'starting import') + + if project.empty_repo? + project.repository.import_repository(project.import_url) + project.repository.fetch_as_mirror(project.import_url, refmap: refmap) + + update_clone_time + end + + log_info(import_stage: 'import_repository', message: 'finished import') + + true + rescue ::Gitlab::Git::CommandError => e + Gitlab::ErrorTracking.log_exception( + e, import_stage: 'import_repository', message: 'failed import', error: e.message + ) + + # Expire cache to prevent scenarios such as: + # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true + # 2. Retried import, repo is broken or not imported but +exists?+ still returns true + project.repository.expire_content_cache if project.repository_exists? + + raise + end + + private + + attr_reader :project + + def refmap + # We omit :heads and :tags since these are fetched in the import_repository + ['+refs/pull-requests/*/to:refs/merge-requests/*/head'] + end + + def update_clone_time + project.touch(:last_repository_updated_at) + end + end + end + end +end |