diff options
Diffstat (limited to 'lib/gitlab/github_import')
21 files changed, 50 insertions, 65 deletions
diff --git a/lib/gitlab/github_import/bulk_importing.rb b/lib/gitlab/github_import/bulk_importing.rb index d16f4d7587b..47080ea1979 100644 --- a/lib/gitlab/github_import/bulk_importing.rb +++ b/lib/gitlab/github_import/bulk_importing.rb @@ -32,7 +32,7 @@ module Gitlab log_error(github_identifiers, build_record.errors.full_messages) errors << { validation_errors: build_record.errors, - github_identifiers: github_identifiers + external_identifiers: github_identifiers } next end @@ -69,7 +69,7 @@ module Gitlab correlation_id_value: correlation_id_value, retry_count: nil, created_at: Time.zone.now, - external_identifiers: error[:github_identifiers] + external_identifiers: error[:external_identifiers] } end @@ -79,8 +79,7 @@ module Gitlab private def log_and_increment_counter(value, operation) - Gitlab::Import::Logger.info( - import_type: :github, + Logger.info( project_id: project.id, importer: self.class.name, message: "#{value} #{object_type.to_s.pluralize} #{operation}" @@ -95,12 +94,11 @@ module Gitlab end def log_error(github_identifiers, messages) - Gitlab::Import::Logger.error( - import_type: :github, + Logger.error( project_id: project.id, importer: self.class.name, message: messages, - github_identifiers: github_identifiers + external_identifiers: github_identifiers ) end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 23d4faa3dde..5a0ae680ab8 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -284,10 +284,10 @@ module Gitlab def on_retry proc do |exception, try, elapsed_time, next_interval| - Gitlab::Import::Logger.info( + Logger.info( message: "GitHub connection retry triggered", 'error.class': exception.class, - 'error.message': exception.message, + 'exception.message': exception.message, try_count: try, elapsed_time_s: elapsed_time, wait_to_retry_s: next_interval diff --git a/lib/gitlab/github_import/clients/proxy.rb b/lib/gitlab/github_import/clients/proxy.rb index 27030f5382a..a95a8cddc8d 100644 --- a/lib/gitlab/github_import/clients/proxy.rb +++ b/lib/gitlab/github_import/clients/proxy.rb @@ -10,19 +10,15 @@ module Gitlab REPOS_COUNT_CACHE_KEY = 'github-importer/provider-repo-count/%{type}/%{user_id}' - def initialize(access_token, client_options) - @client = pick_client(access_token, client_options) + def initialize(access_token) + @client = Gitlab::GithubImport::Client.new(access_token) end def repos(search_text, options) - return { repos: filtered(client.repos, search_text) } if use_legacy? - fetch_repos_via_graphql(search_text, options) end def count_repos_by(relation_type, user_id) - return if use_legacy? - key = format(REPOS_COUNT_CACHE_KEY, type: relation_type, user_id: user_id) ::Gitlab::Cache::Import::Caching.read_integer(key, timeout: 5.minutes) || @@ -40,22 +36,6 @@ module Gitlab } end - def pick_client(access_token, client_options) - return Gitlab::GithubImport::Client.new(access_token) unless use_legacy? - - Gitlab::LegacyGithubImport::Client.new(access_token, **client_options) - end - - def filtered(collection, search_text) - return collection if search_text.blank? - - collection.select { |item| item[:name].to_s.downcase.include?(search_text) } - end - - def use_legacy? - Feature.disabled?(:remove_legacy_github_client) - end - def fetch_and_cache_repos_count_via_graphql(relation_type, key) response = client.count_repos_by_relation_type_graphql(relation_type: relation_type) count = response.dig(:data, :search, :repositoryCount) diff --git a/lib/gitlab/github_import/exceptions.rb b/lib/gitlab/github_import/exceptions.rb index 3a36b64a11b..b7d93182603 100644 --- a/lib/gitlab/github_import/exceptions.rb +++ b/lib/gitlab/github_import/exceptions.rb @@ -6,6 +6,8 @@ module Gitlab # Sometimes it's not clear which of not implemented interfaces caused this error. # We need custom exception to be able to add text that gives extra context. NotImplementedError = Class.new(StandardError) + + NoteableNotFound = Class.new(StandardError) end end end diff --git a/lib/gitlab/github_import/importer/attachments/issues_importer.rb b/lib/gitlab/github_import/importer/attachments/issues_importer.rb index c8f0b59fd18..a0e1a3f2d25 100644 --- a/lib/gitlab/github_import/importer/attachments/issues_importer.rb +++ b/lib/gitlab/github_import/importer/attachments/issues_importer.rb @@ -24,7 +24,7 @@ module Gitlab private def collection - project.issues.select(:id, :description, :iid) + project.issues.id_not_in(already_imported_ids).select(:id, :description, :iid) end def ordering_column diff --git a/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb b/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb index cd3a327a846..22b3e7c640b 100644 --- a/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb +++ b/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb @@ -24,7 +24,7 @@ module Gitlab private def collection - project.merge_requests.select(:id, :description, :iid) + project.merge_requests.id_not_in(already_imported_ids).select(:id, :description, :iid) end def ordering_column diff --git a/lib/gitlab/github_import/importer/attachments/notes_importer.rb b/lib/gitlab/github_import/importer/attachments/notes_importer.rb index aa38a7a3a3f..5ab0cf5b6b0 100644 --- a/lib/gitlab/github_import/importer/attachments/notes_importer.rb +++ b/lib/gitlab/github_import/importer/attachments/notes_importer.rb @@ -26,7 +26,7 @@ module Gitlab # TODO: exclude :system, :noteable_type from select after removing override Note#note method # https://gitlab.com/gitlab-org/gitlab/-/issues/369923 def collection - project.notes.user.select(:id, :note, :system, :noteable_type) + project.notes.id_not_in(already_imported_ids).user.select(:id, :note, :system, :noteable_type) end end end diff --git a/lib/gitlab/github_import/importer/attachments/releases_importer.rb b/lib/gitlab/github_import/importer/attachments/releases_importer.rb index 7d6dbeb901e..0527170f5e1 100644 --- a/lib/gitlab/github_import/importer/attachments/releases_importer.rb +++ b/lib/gitlab/github_import/importer/attachments/releases_importer.rb @@ -24,7 +24,7 @@ module Gitlab private def collection - project.releases.select(:id, :description, :tag) + project.releases.id_not_in(already_imported_ids).select(:id, :description, :tag) end end end diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index 44ffcd7a1e4..d49180e6927 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -36,13 +36,6 @@ module Gitlab Logger.warn(message: e.message, 'error.class': e.class.name) import_with_legacy_diff_note - rescue ActiveRecord::InvalidForeignKey => e - # It's possible the project and the issue have been deleted since - # scheduling this job. In this case we'll just skip creating the note - Logger.info( - message: e.message, - github_identifiers: note.github_identifiers - ) end private @@ -71,6 +64,7 @@ module Gitlab discussion_id: note.discussion_id, noteable_id: merge_request_id, project_id: project.id, + namespace_id: project.project_namespace_id, author_id: author_id, note: note_body, commit_id: note.original_commit_id, @@ -132,7 +126,7 @@ module Gitlab Logger.info( project_id: project.id, importer: self.class.name, - github_identifiers: note.github_identifiers, + external_identifiers: note.github_identifiers, model: model ) end diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index a537841ecf3..3cf67d7df96 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -29,8 +29,8 @@ module Gitlab def execute Issue.transaction do if (issue_id = create_issue) - create_assignees(issue_id) issuable_finder.cache_database_id(issue_id) + create_assignees(issue_id) update_search_data(issue_id) end end @@ -64,9 +64,6 @@ module Gitlab issue.validate! insert_and_return_id(attributes, project.issues) - rescue ActiveRecord::InvalidForeignKey - # It's possible the project has been deleted since scheduling this - # job. In this case we'll just skip creating the issue. end # Stores all issue assignees in the database. diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 04da015a33f..dc5c6e49b55 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -17,7 +17,9 @@ module Gitlab end def execute - return unless (noteable_id = find_noteable_id) + noteable_id = find_noteable_id + + raise Exceptions::NoteableNotFound, 'Error to find noteable_id for note' unless noteable_id author_id, author_found = user_finder.author_id_for(note) @@ -25,6 +27,7 @@ module Gitlab noteable_type: note.noteable_type, noteable_id: noteable_id, project_id: project.id, + namespace_id: project.project_namespace_id, author_id: author_id, note: note_body(author_found), discussion_id: note.discussion_id, @@ -33,19 +36,15 @@ module Gitlab updated_at: note.updated_at } - note = Note.new(attributes.merge(importing: true)) - note.validate! + Note.new(attributes.merge(importing: true)).validate! - # We're using bulk_insert here so we can bypass any validations and - # callbacks. Running these would result in a lot of unnecessary SQL + # We're using bulk_insert here so we can bypass any callbacks. + # Running these would result in a lot of unnecessary SQL # queries being executed when importing large projects. # Note: if you're going to replace `legacy_bulk_insert` with something that trigger callback # to generate HTML version - you also need to regenerate it in # Gitlab::GithubImport::Importer::NoteAttachmentsImporter. ApplicationRecord.legacy_bulk_insert(Note.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert - rescue ActiveRecord::InvalidForeignKey - # It's possible the project and the issue have been deleted since - # scheduling this job. In this case we'll just skip creating the note. end # Returns the ID of the issue or merge request to create the note for. diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index 5690a2cc997..acdafef670c 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -27,9 +27,9 @@ module Gitlab mr, already_exists = create_merge_request if mr + issuable_finder.cache_database_id(mr.id) set_merge_request_assignees(mr) insert_git_data(mr, already_exists) - issuable_finder.cache_database_id(mr.id) end end diff --git a/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb index 0a92aee801d..7f78df615a2 100644 --- a/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb @@ -19,6 +19,9 @@ module Gitlab review_requests = client.pull_request_review_requests(repo, merge_request.iid) review_requests[:merge_request_id] = merge_request.id review_requests[:merge_request_iid] = merge_request.iid + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + yield review_requests mark_merge_request_imported(merge_request) @@ -42,6 +45,10 @@ module Gitlab :pull_request_review_requests end + def object_type + :pull_request_review_request + end + # rubocop:disable CodeReuse/ActiveRecord def merge_request_collection project.merge_requests diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index 62863ba67fd..671e023e90b 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -44,7 +44,7 @@ module Gitlab pname = project.path_with_namespace - Gitlab::Import::Logger.info( + Logger.info( message: 'GitHub importer finished updating repository', project_name: pname ) diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index cfc1ec526b0..cccd99f48b1 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -211,6 +211,12 @@ module Gitlab private + # Returns the set used to track "already imported" objects. + # Items are the values returned by `#id_for_already_imported_cache`. + def already_imported_ids + Gitlab::Cache::Import::Caching.values_from_set(already_imported_cache_key) + end + def additional_object_data {} end diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index 191e15962a6..e8e515d1f87 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -7,7 +7,7 @@ module Gitlab include ToHash include ExposeAttribute - NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i.freeze + NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i expose_attribute :noteable_id, :commit_id, :file_path, :diff_hunk, :author, :created_at, :updated_at, diff --git a/lib/gitlab/github_import/representation/diff_notes/discussion_id.rb b/lib/gitlab/github_import/representation/diff_notes/discussion_id.rb index 38b560f21c0..db36e81c5b8 100644 --- a/lib/gitlab/github_import/representation/diff_notes/discussion_id.rb +++ b/lib/gitlab/github_import/representation/diff_notes/discussion_id.rb @@ -6,7 +6,7 @@ module Gitlab module DiffNotes class DiscussionId NOTEABLE_TYPE = 'MergeRequest' - DISCUSSION_CACHE_REGEX = %r{/(?<repo>[^/]*)/pull/(?<iid>\d+)}i.freeze + DISCUSSION_CACHE_REGEX = %r{/(?<repo>[^/]*)/pull/(?<iid>\d+)}i DISCUSSION_CACHE_KEY = 'github-importer/discussion-id-map/%{project}/%{noteable_id}/%{original_note_id}' def initialize(note) diff --git a/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb b/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb index 38b15c4b5bb..fdf74fd9c9f 100644 --- a/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb +++ b/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb @@ -16,7 +16,7 @@ module Gitlab # - the ```suggestion tag must be the first text of the line # - it might have up to 3 spaces before the ```suggestion tag # - extra text on the ```suggestion tag line will be ignored - GITHUB_SUGGESTION = /^\ {,3}(?<suggestion>```suggestion\b).*(?<eol>\R)/.freeze + GITHUB_SUGGESTION = /^\ {,3}(?<suggestion>```suggestion\b).*(?<eol>\R)/ def initialize(note:, start_line: nil, end_line: nil) @note = note diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb index 7a8bdfb1c64..76adbb651af 100644 --- a/lib/gitlab/github_import/representation/note.rb +++ b/lib/gitlab/github_import/representation/note.rb @@ -12,7 +12,7 @@ module Gitlab expose_attribute :noteable_id, :noteable_type, :author, :note, :created_at, :updated_at, :note_id - NOTEABLE_TYPE_REGEX = %r{/(?<type>(pull|issues))/(?<iid>\d+)}i.freeze + NOTEABLE_TYPE_REGEX = %r{/(?<type>(pull|issues))/(?<iid>\d+)}i # Builds a note from a GitHub API response. # diff --git a/lib/gitlab/github_import/settings.rb b/lib/gitlab/github_import/settings.rb index 73a5f49a9e3..a4170f4147f 100644 --- a/lib/gitlab/github_import/settings.rb +++ b/lib/gitlab/github_import/settings.rb @@ -61,8 +61,11 @@ module Gitlab additional_access_tokens: user_settings[:additional_access_tokens] ) - import_data = project.create_or_update_import_data( - data: { optional_stages: optional_stages }, + import_data = project.build_or_assign_import_data( + data: { + optional_stages: optional_stages, + timeout_strategy: user_settings[:timeout_strategy] + }, credentials: credentials ) diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 1832f071a44..4bf2d8a0aca 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -271,8 +271,7 @@ module Gitlab end def log(message, username: nil) - Gitlab::Import::Logger.info( - import_type: :github, + Logger.info( project_id: project.id, class: self.class.name, username: username, |