diff options
Diffstat (limited to 'lib/gitlab/github_import/importer')
6 files changed, 124 insertions, 37 deletions
diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index 4cfc920e2e3..0aa0896aa57 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -4,41 +4,64 @@ module Gitlab module GithubImport module Importer class DiffNoteImporter - attr_reader :note, :project, :client, :user_finder - - # note - An instance of `Gitlab::GithubImport::Representation::DiffNote`. - # project - An instance of `Project`. - # client - An instance of `Gitlab::GithubImport::Client`. + # note - An instance of `Gitlab::GithubImport::Representation::DiffNote` + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` def initialize(note, project, client) @note = note @project = project @client = client - @user_finder = GithubImport::UserFinder.new(project, client) end def execute - return unless (mr_id = find_merge_request_id) + return if merge_request_id.blank? - author_id, author_found = user_finder.author_id_for(note) + note.project = project + note.merge_request = merge_request - note_body = MarkdownText.format(note.note, note.author, author_found) + build_author_attributes - attributes = { - discussion_id: Discussion.discussion_id(note), - noteable_type: 'MergeRequest', - noteable_id: mr_id, - project_id: project.id, - author_id: author_id, - note: note_body, - system: false, - commit_id: note.original_commit_id, - line_code: note.line_code, - type: 'LegacyDiffNote', - created_at: note.created_at, - updated_at: note.updated_at, - st_diff: note.diff_hash.to_yaml - } + # Diff notes with suggestions are imported with DiffNote, which is + # slower to import than LegacyDiffNote. Importing DiffNote is slower + # because it cannot use the BulkImporting strategy, which skips + # callbacks and validations. For this reason, notes that don't have + # suggestions are still imported with LegacyDiffNote + if import_with_diff_note? + import_with_diff_note + else + import_with_legacy_diff_note + end + rescue ActiveRecord::InvalidForeignKey => e + # It's possible the project and the issue have been deleted since + # scheduling this job. In this case we'll just skip creating the note + Logger.info( + message: e.message, + github_identifiers: note.github_identifiers + ) + end + private + + attr_reader :note, :project, :client, :author_id, :author_found + + def import_with_diff_note? + note.contains_suggestion? && use_diff_note_with_suggestions_enabled? + end + + def use_diff_note_with_suggestions_enabled? + Feature.enabled?( + :github_importer_use_diff_note_with_suggestions, + default_enabled: :yaml + ) + end + + def build_author_attributes + @author_id, @author_found = user_finder.author_id_for(note) + end + + # rubocop:disable Gitlab/BulkInsert + def import_with_legacy_diff_note + log_diff_note_creation('LegacyDiffNote') # It's possible that during an import we'll insert tens of thousands # of diff notes. If we were to use the Note/LegacyDiffNote model here # we'd also have to run additional queries for both validations and @@ -47,15 +70,70 @@ module Gitlab # To work around this we're using bulk_insert with a single row. This # allows us to efficiently insert data (even if it's just 1 row) # without having to use all sorts of hacks to disable callbacks. - Gitlab::Database.main.bulk_insert(LegacyDiffNote.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert - rescue ActiveRecord::InvalidForeignKey - # It's possible the project and the issue have been deleted since - # scheduling this job. In this case we'll just skip creating the note. + ApplicationRecord.legacy_bulk_insert(LegacyDiffNote.table_name, [{ + noteable_type: note.noteable_type, + system: false, + type: 'LegacyDiffNote', + discussion_id: note.discussion_id, + noteable_id: merge_request_id, + project_id: project.id, + author_id: author_id, + note: note_body, + commit_id: note.original_commit_id, + line_code: note.line_code, + created_at: note.created_at, + updated_at: note.updated_at, + st_diff: note.diff_hash.to_yaml + }]) + end + # rubocop:enabled Gitlab/BulkInsert + + def import_with_diff_note + log_diff_note_creation('DiffNote') + + ::Import::Github::Notes::CreateService.new(project, author, { + noteable_type: note.noteable_type, + system: false, + type: 'DiffNote', + noteable_id: merge_request_id, + project_id: project.id, + note: note_body, + discussion_id: note.discussion_id, + commit_id: note.original_commit_id, + created_at: note.created_at, + updated_at: note.updated_at, + position: note.diff_position + }).execute + end + + def note_body + @note_body ||= MarkdownText.format(note.note, note.author, author_found) + end + + def author + @author ||= User.find(author_id) + end + + def merge_request + @merge_request ||= MergeRequest.find(merge_request_id) end # Returns the ID of the merge request this note belongs to. - def find_merge_request_id - GithubImport::IssuableFinder.new(project, note).database_id + def merge_request_id + @merge_request_id ||= GithubImport::IssuableFinder.new(project, note).database_id + end + + def user_finder + @user_finder ||= GithubImport::UserFinder.new(project, client) + end + + def log_diff_note_creation(model) + Logger.info( + project_id: project.id, + importer: self.class.name, + github_identifiers: note.github_identifiers, + model: model + ) end end end diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index f8665676ccf..7f46615f17e 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -75,7 +75,7 @@ module Gitlab end end - Gitlab::Database.main.bulk_insert(IssueAssignee.table_name, assignees) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(IssueAssignee.table_name, assignees) # rubocop:disable Gitlab/BulkInsert end end end diff --git a/lib/gitlab/github_import/importer/label_links_importer.rb b/lib/gitlab/github_import/importer/label_links_importer.rb index b608bb48e38..5e248c7cfc5 100644 --- a/lib/gitlab/github_import/importer/label_links_importer.rb +++ b/lib/gitlab/github_import/importer/label_links_importer.rb @@ -40,7 +40,7 @@ module Gitlab } end - Gitlab::Database.main.bulk_insert(LabelLink.table_name, rows) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(LabelLink.table_name, rows) # rubocop:disable Gitlab/BulkInsert end def find_target_id diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 1fd42a69fac..2cc3a82dd9b 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -37,7 +37,7 @@ module Gitlab # We're using bulk_insert here so we can bypass any validations and # callbacks. Running these would result in a lot of unnecessary SQL # queries being executed when importing large projects. - Gitlab::Database.main.bulk_insert(Note.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(Note.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert rescue ActiveRecord::InvalidForeignKey # It's possible the project and the issue have been deleted since # scheduling this job. In this case we'll just skip creating the note. diff --git a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb index 287e0ea7f7f..c56b391cbec 100644 --- a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb @@ -31,9 +31,7 @@ module Gitlab end def each_object_to_import - project.merge_requests.with_state(:merged).find_each do |merge_request| - next if already_imported?(merge_request) - + merge_requests_to_import.find_each do |merge_request| Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) pull_request = client.pull_request(project.import_source, merge_request.iid) @@ -42,6 +40,17 @@ module Gitlab mark_as_imported(merge_request) end end + + private + + # Returns only the merge requests that still have merged_by to be imported. + def merge_requests_to_import + project.merge_requests.id_not_in(already_imported_objects).with_state(:merged) + end + + def already_imported_objects + Gitlab::Cache::Import::Caching.values_from_set(already_imported_cache_key) + end end end end diff --git a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb index bd65eb5899c..5e55d09fe3d 100644 --- a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb @@ -86,7 +86,7 @@ module Gitlab # Returns only the merge requests that still have reviews to be imported. def merge_requests_to_import - project.merge_requests.where.not(id: already_imported_merge_requests) # rubocop: disable CodeReuse/ActiveRecord + project.merge_requests.id_not_in(already_imported_merge_requests) end def already_imported_merge_requests |