diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-03-18 23:02:30 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-03-18 23:02:30 +0300 |
commit | 41fe97390ceddf945f3d967b8fdb3de4c66b7dea (patch) | |
tree | 9c8d89a8624828992f06d892cd2f43818ff5dcc8 /lib/gitlab/github_import | |
parent | 0804d2dc31052fb45a1efecedc8e06ce9bc32862 (diff) |
Add latest changes from gitlab-org/gitlab@14-9-stable-eev14.9.0-rc42
Diffstat (limited to 'lib/gitlab/github_import')
-rw-r--r-- | lib/gitlab/github_import/importer/diff_note_importer.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/github_import/importer/pull_requests_importer.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/github_import/parallel_scheduling.rb | 41 |
3 files changed, 51 insertions, 2 deletions
diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index 02b582190b6..a9f8483d8c3 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -4,6 +4,8 @@ module Gitlab module GithubImport module Importer class DiffNoteImporter + DiffNoteCreationError = Class.new(ActiveRecord::RecordInvalid) + # note - An instance of `Gitlab::GithubImport::Representation::DiffNote` # project - An instance of `Project` # client - An instance of `Gitlab::GithubImport::Client` @@ -31,7 +33,7 @@ module Gitlab else import_with_legacy_diff_note end - rescue ::DiffNote::NoteDiffFileCreationError => e + rescue ::DiffNote::NoteDiffFileCreationError, DiffNoteCreationError => e Logger.warn(message: e.message, 'error.class': e.class.name) import_with_legacy_diff_note @@ -84,7 +86,7 @@ module Gitlab def import_with_diff_note log_diff_note_creation('DiffNote') - ::Import::Github::Notes::CreateService.new(project, author, { + record = ::Import::Github::Notes::CreateService.new(project, author, { noteable_type: note.noteable_type, system: false, type: 'DiffNote', @@ -97,6 +99,8 @@ module Gitlab updated_at: note.updated_at, position: note.diff_position }).execute + + raise DiffNoteCreationError, record unless record.persisted? end def note_body diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index fc0c099b71c..5d291d9d723 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -74,6 +74,10 @@ module Gitlab { state: 'all', sort: 'created', direction: 'asc' } end + def parallel_import_batch + { size: 200, delay: 1.minute } + end + def repository_updates_counter @repository_updates_counter ||= Gitlab::Metrics.counter( :github_importer_repository_updates, diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index a8e006ea082..4dec9543a13 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -72,6 +72,14 @@ module Gitlab # Imports all objects in parallel by scheduling a Sidekiq job for every # individual object. def parallel_import + if Feature.enabled?(:spread_parallel_import, default_enabled: :yaml) && parallel_import_batch.present? + spread_parallel_import + else + parallel_import_deprecated + end + end + + def parallel_import_deprecated waiter = JobWaiter.new each_object_to_import do |object| @@ -86,6 +94,33 @@ module Gitlab waiter end + def spread_parallel_import + waiter = JobWaiter.new + + import_arguments = [] + + each_object_to_import do |object| + repr = representation_class.from_api_response(object) + + import_arguments << [project.id, repr.to_hash, waiter.key] + + waiter.jobs_remaining += 1 + end + + # rubocop:disable Scalability/BulkPerformWithContext + Gitlab::ApplicationContext.with_context(project: project) do + sidekiq_worker_class.bulk_perform_in( + 1.second, + import_arguments, + batch_size: parallel_import_batch[:size], + batch_delay: parallel_import_batch[:delay] + ) + end + # rubocop:enable Scalability/BulkPerformWithContext + + waiter + end + # The method that will be called for traversing through all the objects to # import, yielding them to the supplied block. def each_object_to_import @@ -171,6 +206,12 @@ module Gitlab raise NotImplementedError end + # Default batch settings for parallel import (can be redefined in Importer classes) + # Example: { size: 100, delay: 1.minute } + def parallel_import_batch + {} + end + def abort_on_failure false end |