diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-12-20 16:37:47 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-12-20 16:37:47 +0300 |
commit | aee0a117a889461ce8ced6fcf73207fe017f1d99 (patch) | |
tree | 891d9ef189227a8445d83f35c1b0fc99573f4380 /lib/gitlab/github_import | |
parent | 8d46af3258650d305f53b819eabf7ab18d22f59e (diff) |
Add latest changes from gitlab-org/gitlab@14-6-stable-eev14.6.0-rc42
Diffstat (limited to 'lib/gitlab/github_import')
-rw-r--r-- | lib/gitlab/github_import/client.rb | 26 | ||||
-rw-r--r-- | lib/gitlab/github_import/importer/diff_note_importer.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/github_import/importer/note_importer.rb | 1 | ||||
-rw-r--r-- | lib/gitlab/github_import/parallel_importer.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/github_import/representation/diff_note.rb | 21 | ||||
-rw-r--r-- | lib/gitlab/github_import/representation/note.rb | 8 |
6 files changed, 50 insertions, 12 deletions
diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index efa816c5eb0..d2495b32800 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -21,6 +21,7 @@ module Gitlab SEARCH_MAX_REQUESTS_PER_MINUTE = 30 DEFAULT_PER_PAGE = 100 LOWER_PER_PAGE = 50 + CLIENT_CONNECTION_ERROR = ::Faraday::ConnectionFailed # used/set in sawyer agent which octokit uses # A single page of data and the corresponding page number. Page = Struct.new(:objects, :number) @@ -148,14 +149,14 @@ module Gitlab # whether we are running in parallel mode or not. For more information see # `#rate_or_wait_for_rate_limit`. def with_rate_limit - return yield unless rate_limiting_enabled? + return with_retry { yield } unless rate_limiting_enabled? request_count_counter.increment raise_or_wait_for_rate_limit unless requests_remaining? begin - yield + with_retry { yield } rescue ::Octokit::TooManyRequests raise_or_wait_for_rate_limit @@ -166,7 +167,7 @@ module Gitlab end def search_repos_by_name(name, options = {}) - octokit.search_repositories(search_query(str: name, type: :name), options) + with_retry { octokit.search_repositories(search_query(str: name, type: :name), options) } end def search_query(str:, type:, include_collaborations: true, include_orgs: true) @@ -270,6 +271,25 @@ module Gitlab .map { |org| "org:#{org.login}" } .join(' ') end + + def with_retry + Retriable.retriable(on: CLIENT_CONNECTION_ERROR, on_retry: on_retry) do + yield + end + end + + def on_retry + proc do |exception, try, elapsed_time, next_interval| + Gitlab::Import::Logger.info( + message: "GitHub connection retry triggered", + 'error.class': exception.class, + 'error.message': exception.message, + try_count: try, + elapsed_time_s: elapsed_time, + wait_to_retry_s: next_interval + ) + end + end end end end diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index 0aa0896aa57..8a8d23401c1 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -31,6 +31,10 @@ module Gitlab else import_with_legacy_diff_note end + rescue ::DiffNote::NoteDiffFileCreationError => e + Logger.warn(message: e.message, 'error.class': e.class.name) + + import_with_legacy_diff_note rescue ActiveRecord::InvalidForeignKey => e # It's possible the project and the issue have been deleted since # scheduling this job. In this case we'll just skip creating the note diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 2cc3a82dd9b..673f56b5753 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -29,6 +29,7 @@ module Gitlab project_id: project.id, author_id: author_id, note: note_body, + discussion_id: note.discussion_id, system: false, created_at: note.created_at, updated_at: note.updated_at diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb index f72e595e8e9..a71590c02f8 100644 --- a/lib/gitlab/github_import/parallel_importer.rb +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -35,7 +35,9 @@ module Gitlab def execute Gitlab::Import::SetAsyncJid.set_jid(project.import_state) + # We need to track this job's status for use by Gitlab::GithubImport::RefreshImportJidWorker. Stage::ImportRepositoryWorker + .with_status .perform_async(project.id) true diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index fecff0644c2..04f53accfeb 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -4,6 +4,7 @@ module Gitlab module GithubImport module Representation class DiffNote + include Gitlab::Utils::StrongMemoize include ToHash include ExposeAttribute @@ -127,15 +128,17 @@ module Gitlab end def discussion_id - if in_reply_to_id.present? - current_discussion_id - else - Discussion.discussion_id( - Struct - .new(:noteable_id, :noteable_type) - .new(merge_request.id, NOTEABLE_TYPE) - ).tap do |discussion_id| - cache_discussion_id(discussion_id) + strong_memoize(:discussion_id) do + if in_reply_to_id.present? + current_discussion_id + else + Discussion.discussion_id( + Struct + .new(:noteable_id, :noteable_type) + .new(merge_request.id, NOTEABLE_TYPE) + ).tap do |discussion_id| + cache_discussion_id(discussion_id) + end end end end diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb index bcdb1a5459b..bbf20b7e9e6 100644 --- a/lib/gitlab/github_import/representation/note.rb +++ b/lib/gitlab/github_import/representation/note.rb @@ -63,6 +63,14 @@ module Gitlab @attributes = attributes end + def discussion_id + Discussion.discussion_id( + Struct + .new(:noteable_id, :noteable_type) + .new(noteable_id, noteable_type) + ) + end + alias_method :issuable_type, :noteable_type def github_identifiers |