diff options
Diffstat (limited to 'lib/gitlab/bitbucket_import')
14 files changed, 488 insertions, 33 deletions
diff --git a/lib/gitlab/bitbucket_import/error_tracking.rb b/lib/gitlab/bitbucket_import/error_tracking.rb new file mode 100644 index 00000000000..eaffe34daf8 --- /dev/null +++ b/lib/gitlab/bitbucket_import/error_tracking.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module ErrorTracking + def track_import_failure!(project, exception:, **args) + Gitlab::Import::ImportFailureService.track( + project_id: project.id, + error_source: self.class.name, + exception: exception, + **args + ) + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 7f228c19b6b..9f87bb2347c 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -16,6 +16,7 @@ module Gitlab @project = project @client = Bitbucket::Client.new(project.import_data.credentials) @formatter = Gitlab::ImportFormatter.new + @ref_converter = Gitlab::BitbucketImport::RefConverter.new(project) @labels = {} @errors = [] @users = {} @@ -31,6 +32,26 @@ module Gitlab true end + def create_labels + LABELS.each do |label_params| + label = ::Labels::FindOrCreateService.new(nil, project, label_params).execute(skip_authorization: true) + if label.valid? + @labels[label_params[:title]] = label + else + raise "Failed to create label \"#{label_params[:title]}\" for project \"#{project.full_name}\"" + end + end + end + + def import_pull_request_comments(pull_request, merge_request) + comments = client.pull_request_comments(repo, pull_request.iid) + + inline_comments, pr_comments = comments.partition(&:inline?) + + import_inline_comments(inline_comments, pull_request, merge_request) + import_standalone_pr_comments(pr_comments, merge_request) + end + private def already_imported?(collection, iid) @@ -166,7 +187,7 @@ module Gitlab note = '' note += @formatter.author_line(comment.author) unless find_user_id(comment.author) - note += comment.note + note += @ref_converter.convert_note(comment.note.to_s) begin gitlab_issue.notes.create!( @@ -182,17 +203,6 @@ module Gitlab end end - def create_labels - LABELS.each do |label_params| - label = ::Labels::FindOrCreateService.new(nil, project, label_params).execute(skip_authorization: true) - if label.valid? - @labels[label_params[:title]] = label - else - raise "Failed to create label \"#{label_params[:title]}\" for project \"#{project.full_name}\"" - end - end - end - def import_pull_requests pull_requests = client.pull_requests(repo) @@ -242,15 +252,6 @@ module Gitlab store_pull_request_error(pull_request, e) end - def import_pull_request_comments(pull_request, merge_request) - comments = client.pull_request_comments(repo, pull_request.iid) - - inline_comments, pr_comments = comments.partition(&:inline?) - - import_inline_comments(inline_comments, pull_request, merge_request) - import_standalone_pr_comments(pr_comments, merge_request) - end - def import_inline_comments(inline_comments, pull_request, merge_request) position_map = {} discussion_map = {} @@ -319,8 +320,7 @@ module Gitlab def comment_note(comment) author = @formatter.author_line(comment.author) unless find_user_id(comment.author) - - author.to_s + comment.note.to_s + author.to_s + @ref_converter.convert_note(comment.note.to_s) end def log_base_data diff --git a/lib/gitlab/bitbucket_import/importers/issue_importer.rb b/lib/gitlab/bitbucket_import/importers/issue_importer.rb new file mode 100644 index 00000000000..2c3be67eabc --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/issue_importer.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class IssueImporter + include Loggable + include ErrorTracking + + def initialize(project, hash) + @project = project + @formatter = Gitlab::ImportFormatter.new + @user_finder = UserFinder.new(project) + @object = hash.with_indifferent_access + end + + def execute + log_info(import_stage: 'import_issue', message: 'starting', iid: object[:iid]) + + description = '' + description += author_line + description += object[:description] if object[:description] + + milestone = object[:milestone] ? project.milestones.find_or_create_by(title: object[:milestone]) : nil # rubocop: disable CodeReuse/ActiveRecord + + attributes = { + iid: object[:iid], + title: object[:title], + description: description, + state_id: Issue.available_states[object[:state]], + author_id: author_id, + assignee_ids: [author_id], + namespace_id: project.project_namespace_id, + milestone: milestone, + work_item_type_id: object[:issue_type_id], + label_ids: [object[:label_id]].compact, + created_at: object[:created_at], + updated_at: object[:updated_at] + } + + project.issues.create!(attributes) + + log_info(import_stage: 'import_issue', message: 'finished', iid: object[:iid]) + rescue StandardError => e + track_import_failure!(project, exception: e) + end + + private + + attr_reader :object, :project, :formatter, :user_finder + + def author_line + return '' if find_user_id + + formatter.author_line(object[:author]) + end + + def find_user_id + user_finder.find_user_id(object[:author]) + end + + def author_id + user_finder.gitlab_user_id(project, object[:author]) + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/issue_notes_importer.rb b/lib/gitlab/bitbucket_import/importers/issue_notes_importer.rb new file mode 100644 index 00000000000..ac0e039939f --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/issue_notes_importer.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class IssueNotesImporter + include ParallelScheduling + + def initialize(project, hash) + @project = project + @formatter = Gitlab::ImportFormatter.new + @user_finder = UserFinder.new(project) + @ref_converter = Gitlab::BitbucketImport::RefConverter.new(project) + @object = hash.with_indifferent_access + end + + def execute + log_info(import_stage: 'import_issue_notes', message: 'starting', iid: object[:iid]) + + issue = project.issues.find_by(iid: object[:iid]) # rubocop: disable CodeReuse/ActiveRecord + + if issue + client.issue_comments(project.import_source, issue.iid).each do |comment| + next unless comment.note.present? + + note = '' + note += formatter.author_line(comment.author) unless user_finder.find_user_id(comment.author) + note += ref_converter.convert_note(comment.note) + + issue.notes.create!( + project: project, + note: note, + author_id: user_finder.gitlab_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + ) + end + end + + log_info(import_stage: 'import_issue_notes', message: 'finished', iid: object[:iid]) + rescue StandardError => e + track_import_failure!(project, exception: e) + end + + private + + attr_reader :object, :project, :formatter, :user_finder, :ref_converter + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/issues_importer.rb b/lib/gitlab/bitbucket_import/importers/issues_importer.rb new file mode 100644 index 00000000000..6162433e701 --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/issues_importer.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class IssuesImporter + include ParallelScheduling + + def execute + log_info(import_stage: 'import_issues', message: 'importing issues') + + issues = client.issues(project.import_source) + + labels = build_labels_hash + + issues.each do |issue| + job_waiter.jobs_remaining += 1 + + next if already_enqueued?(issue) + + job_delay = calculate_job_delay(job_waiter.jobs_remaining) + + issue_hash = issue.to_hash.merge({ issue_type_id: default_issue_type_id, label_id: labels[issue.kind] }) + sidekiq_worker_class.perform_in(job_delay, project.id, issue_hash, job_waiter.key) + + mark_as_enqueued(issue) + end + + job_waiter + rescue StandardError => e + track_import_failure!(project, exception: e) + end + + private + + def sidekiq_worker_class + ImportIssueWorker + end + + def collection_method + :issues + end + + def id_for_already_enqueued_cache(object) + object.iid + end + + def default_issue_type_id + ::WorkItems::Type.default_issue_type.id + end + + def build_labels_hash + labels = {} + project.labels.each { |l| labels[l.title.to_s] = l.id } + labels + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/issues_notes_importer.rb b/lib/gitlab/bitbucket_import/importers/issues_notes_importer.rb new file mode 100644 index 00000000000..03dcc645f07 --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/issues_notes_importer.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class IssuesNotesImporter + include ParallelScheduling + + def execute + project.issues.find_each do |issue| + job_waiter.jobs_remaining += 1 + + next if already_enqueued?(issue) + + job_delay = calculate_job_delay(job_waiter.jobs_remaining) + + sidekiq_class.perform_in(job_delay, project.id, { iid: issue.iid }, job_waiter.key) + + mark_as_enqueued(issue) + end + + job_waiter + rescue StandardError => e + track_import_failure!(project, exception: e) + end + + private + + attr_reader :project + + def sidekiq_class + ImportIssueNotesWorker + end + + def id_for_already_enqueued_cache(object) + object.iid + end + + def collection_method + :issues_notes + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/lfs_object_importer.rb b/lib/gitlab/bitbucket_import/importers/lfs_object_importer.rb new file mode 100644 index 00000000000..06b30c7b496 --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/lfs_object_importer.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class LfsObjectImporter + include Loggable + include ErrorTracking + + def initialize(project, lfs_attributes) + @project = project + @lfs_download_object = LfsDownloadObject.new(**lfs_attributes.symbolize_keys) + end + + def execute + log_info(import_stage: 'import_lfs_object', message: 'starting', oid: lfs_download_object.oid) + + lfs_download_object.validate! + Projects::LfsPointers::LfsDownloadService.new(project, lfs_download_object).execute + + log_info(import_stage: 'import_lfs_object', message: 'finished', oid: lfs_download_object.oid) + rescue StandardError => e + track_import_failure!(project, exception: e) + end + + private + + attr_reader :lfs_download_object, :project + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/lfs_objects_importer.rb b/lib/gitlab/bitbucket_import/importers/lfs_objects_importer.rb new file mode 100644 index 00000000000..aa9ff7000f1 --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/lfs_objects_importer.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class LfsObjectsImporter + include ParallelScheduling + + def execute + log_info(import_stage: 'import_lfs_objects', message: 'starting') + + download_service = Projects::LfsPointers::LfsObjectDownloadListService.new(project) + + begin + queue_workers(download_service) if project.lfs_enabled? + rescue StandardError => e + track_import_failure!(project, exception: e) + end + + log_info(import_stage: 'import_lfs_objects', message: 'finished') + + job_waiter + end + + private + + def sidekiq_worker_class + ImportLfsObjectWorker + end + + def collection_method + :lfs_objects + end + + def id_for_already_enqueued_cache(object) + object.oid + end + + def queue_workers(download_service) + download_service.each_list_item do |lfs_download_object| + # Needs to come before `already_enqueued?` as `jobs_remaining` resets to zero when the job restarts and + # jobs_remaining needs to be the total amount of enqueued jobs + job_waiter.jobs_remaining += 1 + + next if already_enqueued?(lfs_download_object) + + job_delay = calculate_job_delay(job_waiter.jobs_remaining) + + sidekiq_worker_class.perform_in(job_delay, project.id, lfs_download_object.to_hash, job_waiter.key) + + mark_as_enqueued(lfs_download_object) + end + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb index d76e08e1039..a18d50e8fce 100644 --- a/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb @@ -5,6 +5,7 @@ module Gitlab module Importers class PullRequestImporter include Loggable + include ErrorTracking def initialize(project, hash) @project = project @@ -48,7 +49,7 @@ module Gitlab log_info(import_stage: 'import_pull_request', message: 'finished', iid: object[:iid]) rescue StandardError => e - Gitlab::Import::ImportFailureService.track(project_id: project.id, exception: e) + track_import_failure!(project, exception: e) end private diff --git a/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb new file mode 100644 index 00000000000..8ea8b1562f2 --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class PullRequestNotesImporter + include Loggable + include ErrorTracking + + def initialize(project, hash) + @project = project + @importer = Gitlab::BitbucketImport::Importer.new(project) + @object = hash.with_indifferent_access + end + + def execute + log_info(import_stage: 'import_pull_request_notes', message: 'starting', iid: object[:iid]) + + merge_request = project.merge_requests.find_by(iid: object[:iid]) # rubocop: disable CodeReuse/ActiveRecord + + importer.import_pull_request_comments(merge_request, merge_request) if merge_request + + log_info(import_stage: 'import_pull_request_notes', message: 'finished', iid: object[:iid]) + rescue StandardError => e + track_import_failure!(project, exception: e) + end + + private + + attr_reader :object, :project, :importer + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/pull_requests_notes_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_requests_notes_importer.rb new file mode 100644 index 00000000000..a1b0c2a5afe --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/pull_requests_notes_importer.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class PullRequestsNotesImporter + include ParallelScheduling + + def execute + project.merge_requests.find_each do |merge_request| + job_waiter.jobs_remaining += 1 + + next if already_enqueued?(merge_request) + + job_delay = calculate_job_delay(job_waiter.jobs_remaining) + + sidekiq_worker_class.perform_in(job_delay, project.id, { iid: merge_request.iid }, job_waiter.key) + + mark_as_enqueued(merge_request) + end + + job_waiter + rescue StandardError => e + track_import_failure!(project, exception: e) + end + + private + + attr_reader :project + + def sidekiq_worker_class + ImportPullRequestNotesWorker + end + + def id_for_already_enqueued_cache(object) + object.iid + end + + def collection_method + :merge_requests_notes + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/repository_importer.rb b/lib/gitlab/bitbucket_import/importers/repository_importer.rb index 7b0362b6ec6..b8c0ba69d37 100644 --- a/lib/gitlab/bitbucket_import/importers/repository_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/repository_importer.rb @@ -23,6 +23,7 @@ module Gitlab end import_wiki + create_labels log_info(import_stage: 'import_repository', message: 'finished import') @@ -59,6 +60,11 @@ module Gitlab ) end + def create_labels + importer = Gitlab::BitbucketImport::Importer.new(project) + importer.create_labels + end + def wiki WikiFormatter.new(project) end diff --git a/lib/gitlab/bitbucket_import/parallel_scheduling.rb b/lib/gitlab/bitbucket_import/parallel_scheduling.rb index f4df9a35526..ca2597ea5cf 100644 --- a/lib/gitlab/bitbucket_import/parallel_scheduling.rb +++ b/lib/gitlab/bitbucket_import/parallel_scheduling.rb @@ -4,6 +4,7 @@ module Gitlab module BitbucketImport module ParallelScheduling include Loggable + include ErrorTracking attr_reader :project, :already_enqueued_cache_key, :job_waiter_cache_key @@ -79,15 +80,6 @@ module Gitlab (multiplier * 1.minute) + 1.second end - - def track_import_failure!(project, exception:, **args) - Gitlab::Import::ImportFailureService.track( - project_id: project.id, - error_source: self.class.name, - exception: exception, - **args - ) - end end end end diff --git a/lib/gitlab/bitbucket_import/ref_converter.rb b/lib/gitlab/bitbucket_import/ref_converter.rb new file mode 100644 index 00000000000..1159159a76d --- /dev/null +++ b/lib/gitlab/bitbucket_import/ref_converter.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + class RefConverter + REPO_MATCHER = 'https://bitbucket.org/%s' + PR_NOTE_ISSUE_NAME_REGEX = '(?<=/)[^/\)]+(?=\)[^/]*$)' + UNWANTED_NOTE_REF_HTML = "{: data-inline-card='' }" + + attr_reader :project + + def initialize(project) + @project = project + end + + def convert_note(note) + repo_matcher = REPO_MATCHER % project.import_source + + return note unless note.match?(repo_matcher) + + note = note.gsub(repo_matcher, url_helpers.project_url(project)) + .gsub(UNWANTED_NOTE_REF_HTML, '') + .strip + + if note.match?('issues') + note.gsub!('issues', '-/issues') + note.gsub!(issue_name(note), '') + else + note.gsub!('pull-requests', '-/merge_requests') + note.gsub!('src', '-/blob') + note.gsub!('lines-', 'L') + end + + note + end + + private + + def url_helpers + Rails.application.routes.url_helpers + end + + def issue_name(note) + note.match(PR_NOTE_ISSUE_NAME_REGEX)[0] + end + end + end +end |