diff options
Diffstat (limited to 'lib/gitlab/github_import/importer')
11 files changed, 96 insertions, 34 deletions
diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index a9f8483d8c3..44ffcd7a1e4 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -18,7 +18,6 @@ module Gitlab def execute return if merge_request_id.blank? - note.project = project note.merge_request = merge_request build_author_attributes @@ -65,7 +64,7 @@ module Gitlab # To work around this we're using bulk_insert with a single row. This # allows us to efficiently insert data (even if it's just 1 row) # without having to use all sorts of hacks to disable callbacks. - ApplicationRecord.legacy_bulk_insert(LegacyDiffNote.table_name, [{ + attributes = { noteable_type: note.noteable_type, system: false, type: 'LegacyDiffNote', @@ -79,7 +78,12 @@ module Gitlab created_at: note.created_at, updated_at: note.updated_at, st_diff: note.diff_hash.to_yaml - }]) + } + + diff_note = LegacyDiffNote.new(attributes.merge(importing: true)) + diff_note.validate! + + ApplicationRecord.legacy_bulk_insert(LegacyDiffNote.table_name, [attributes]) end # rubocop:enabled Gitlab/BulkInsert diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index d964bae3dd2..b477468d327 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -60,6 +60,9 @@ module Gitlab work_item_type_id: issue.work_item_type_id } + issue = project.issues.new(attributes.merge(importing: true)) + issue.validate! + insert_and_return_id(attributes, project.issues) rescue ActiveRecord::InvalidForeignKey # It's possible the project has been deleted since scheduling this diff --git a/lib/gitlab/github_import/importer/label_links_importer.rb b/lib/gitlab/github_import/importer/label_links_importer.rb index 5e248c7cfc5..52c87dda347 100644 --- a/lib/gitlab/github_import/importer/label_links_importer.rb +++ b/lib/gitlab/github_import/importer/label_links_importer.rb @@ -22,7 +22,7 @@ module Gitlab def create_labels time = Time.zone.now - rows = [] + items = [] target_id = find_target_id issue.label_names.each do |label_name| @@ -31,16 +31,16 @@ module Gitlab # the project's labels. next unless (label_id = label_finder.id_for(label_name)) - rows << { + items << LabelLink.new( label_id: label_id, target_id: target_id, target_type: issue.issuable_type, created_at: time, updated_at: time - } + ) end - ApplicationRecord.legacy_bulk_insert(LabelLink.table_name, rows) # rubocop:disable Gitlab/BulkInsert + LabelLink.bulk_insert!(items) end def find_target_id diff --git a/lib/gitlab/github_import/importer/labels_importer.rb b/lib/gitlab/github_import/importer/labels_importer.rb index 9a011f17a18..d5d1cd28b7c 100644 --- a/lib/gitlab/github_import/importer/labels_importer.rb +++ b/lib/gitlab/github_import/importer/labels_importer.rb @@ -13,7 +13,10 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def execute - bulk_insert(Label, build_labels) + rows, validation_errors = build_labels + + bulk_insert(rows) + bulk_insert_failures(validation_errors) if validation_errors.any? build_labels_cache end @@ -29,7 +32,7 @@ module Gitlab LabelFinder.new(project).build_cache end - def build(label) + def build_attributes(label) time = Time.zone.now { @@ -49,6 +52,10 @@ module Gitlab def object_type :label end + + def model + Label + end end end end diff --git a/lib/gitlab/github_import/importer/lfs_objects_importer.rb b/lib/gitlab/github_import/importer/lfs_objects_importer.rb index 775afd5f53a..d064278e4a0 100644 --- a/lib/gitlab/github_import/importer/lfs_objects_importer.rb +++ b/lib/gitlab/github_import/importer/lfs_objects_importer.rb @@ -27,9 +27,9 @@ module Gitlab end def each_object_to_import - lfs_objects = Projects::LfsPointers::LfsObjectDownloadListService.new(project).execute + download_service = Projects::LfsPointers::LfsObjectDownloadListService.new(project) - lfs_objects.each do |object| + download_service.each_list_item do |object| Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) yield object diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb index 1a3a54d0053..560fbdc66e3 100644 --- a/lib/gitlab/github_import/importer/milestones_importer.rb +++ b/lib/gitlab/github_import/importer/milestones_importer.rb @@ -13,7 +13,10 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def execute - bulk_insert(Milestone, build_milestones) + rows, validation_errors = build_milestones + + bulk_insert(rows) + bulk_insert_failures(validation_errors) if validation_errors.any? build_milestones_cache end @@ -29,7 +32,7 @@ module Gitlab MilestoneFinder.new(project).build_cache end - def build(milestone) + def build_attributes(milestone) { iid: milestone[:number], title: milestone[:title], @@ -53,6 +56,10 @@ module Gitlab def object_type :milestone end + + def model + Milestone + end end end end diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 69b7b2c2a38..04da015a33f 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -33,6 +33,9 @@ module Gitlab updated_at: note.updated_at } + note = Note.new(attributes.merge(importing: true)) + note.validate! + # We're using bulk_insert here so we can bypass any validations and # callbacks. Running these would result in a lot of unnecessary SQL # queries being executed when importing large projects. diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index 3c17ea1195e..5690a2cc997 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -61,6 +61,9 @@ module Gitlab updated_at: pull_request.updated_at } + mr = project.merge_requests.new(attributes.merge(importing: true)) + mr.validate! + create_merge_request_without_hooks(project, attributes, pull_request.iid) end @@ -93,7 +96,7 @@ module Gitlab return if project.repository.branch_exists?(source_branch) project.repository.add_branch(project.creator, source_branch, pull_request.source_branch_sha) - rescue Gitlab::Git::CommandError => e + rescue Gitlab::Git::PreReceiveError, Gitlab::Git::CommandError => e Gitlab::ErrorTracking.track_exception(e, source_branch: source_branch, project_id: merge_request.project.id, diff --git a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb index 640914acf4d..f05aa26a449 100644 --- a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb @@ -4,42 +4,62 @@ module Gitlab module GithubImport module Importer class PullRequestMergedByImporter + # pull_request - An instance of + # `Gitlab::GithubImport::Representation::PullRequest` + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` def initialize(pull_request, project, client) - @project = project @pull_request = pull_request + @project = project @client = client end def execute - merge_request = project.merge_requests.find_by_iid(pull_request.iid) - timestamp = Time.new.utc - merged_at = pull_request.merged_at user_finder = GithubImport::UserFinder.new(project, client) - gitlab_user_id = user_finder.user_id_for(pull_request.merged_by) + gitlab_user_id = begin + user_finder.user_id_for(pull_request.merged_by) + rescue ::Octokit::NotFound + nil + end + + metrics_upsert(gitlab_user_id) + + add_note! + end + + private + + attr_reader :project, :pull_request, :client + + def metrics_upsert(gitlab_user_id) MergeRequest::Metrics.upsert({ target_project_id: project.id, merge_request_id: merge_request.id, merged_by_id: gitlab_user_id, - merged_at: merged_at, + merged_at: pull_request.merged_at, created_at: timestamp, updated_at: timestamp }, unique_by: :merge_request_id) + end - unless gitlab_user_id - merge_request.notes.create!( - importing: true, - note: missing_author_note, - author_id: project.creator_id, - project: project, - created_at: merged_at - ) - end + def add_note! + merge_request.notes.create!( + importing: true, + note: missing_author_note, + author_id: project.creator_id, + project: project, + created_at: pull_request.merged_at + ) end - private + def merge_request + @merge_request ||= project.merge_requests.find_by_iid(pull_request.iid) + end - attr_reader :project, :pull_request, :client + def timestamp + @timestamp ||= Time.new.utc + end def missing_author_note s_("GitHubImporter|*Merged by: %{author} at %{timestamp}*") % { diff --git a/lib/gitlab/github_import/importer/pull_request_review_importer.rb b/lib/gitlab/github_import/importer/pull_request_review_importer.rb index b11af90aa6f..de66f310edf 100644 --- a/lib/gitlab/github_import/importer/pull_request_review_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_review_importer.rb @@ -4,6 +4,9 @@ module Gitlab module GithubImport module Importer class PullRequestReviewImporter + # review - An instance of `Gitlab::GithubImport::Representation::PullRequestReview` + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` def initialize(review, project, client) @review = review @project = project @@ -13,7 +16,12 @@ module Gitlab def execute user_finder = GithubImport::UserFinder.new(project, client) - gitlab_user_id = user_finder.user_id_for(review.author) + + gitlab_user_id = begin + user_finder.user_id_for(review.author) + rescue ::Octokit::NotFound + nil + end if gitlab_user_id add_review_note!(gitlab_user_id) diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb index fe6da30bbf8..62d579fda08 100644 --- a/lib/gitlab/github_import/importer/releases_importer.rb +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -16,7 +16,10 @@ module Gitlab # to generate HTML version - you also need to regenerate it in # Gitlab::GithubImport::Importer::NoteAttachmentsImporter. def execute - bulk_insert(Release, build_releases) + rows, validation_errors = build_releases + + bulk_insert(rows) + bulk_insert_failures(validation_errors) if validation_errors.any? end def build_releases @@ -27,7 +30,7 @@ module Gitlab existing_tags.include?(release[:tag_name]) || release[:tag_name].nil? end - def build(release) + def build_attributes(release) existing_tags.add(release[:tag_name]) { @@ -66,6 +69,10 @@ module Gitlab def user_finder @user_finder ||= GithubImport::UserFinder.new(project, client) end + + def model + Release + end end end end |