Add latest changes from gitlab-org/gitlab@14-9-stable-eev14.9.0-rc42

3 files changed, 51 insertions, 2 deletions

diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb
index 02b582190b6..a9f8483d8c3 100644
--- a/lib/gitlab/github_import/importer/diff_note_importer.rb
+++ b/lib/gitlab/github_import/importer/diff_note_importer.rb
@@ -4,6 +4,8 @@ module Gitlab
   module GithubImport
     module Importer
       class DiffNoteImporter
+        DiffNoteCreationError = Class.new(ActiveRecord::RecordInvalid)
+
         # note - An instance of `Gitlab::GithubImport::Representation::DiffNote`
         # project - An instance of `Project`
         # client - An instance of `Gitlab::GithubImport::Client`
@@ -31,7 +33,7 @@ module Gitlab
           else
             import_with_legacy_diff_note
           end
-        rescue ::DiffNote::NoteDiffFileCreationError => e
+        rescue ::DiffNote::NoteDiffFileCreationError, DiffNoteCreationError => e
           Logger.warn(message: e.message, 'error.class': e.class.name)
 
           import_with_legacy_diff_note
@@ -84,7 +86,7 @@ module Gitlab
         def import_with_diff_note
           log_diff_note_creation('DiffNote')
 
-          ::Import::Github::Notes::CreateService.new(project, author, {
+          record = ::Import::Github::Notes::CreateService.new(project, author, {
             noteable_type: note.noteable_type,
             system: false,
             type: 'DiffNote',
@@ -97,6 +99,8 @@ module Gitlab
             updated_at: note.updated_at,
             position: note.diff_position
           }).execute
+
+          raise DiffNoteCreationError, record unless record.persisted?
         end
 
         def note_body
diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb
index fc0c099b71c..5d291d9d723 100644
--- a/lib/gitlab/github_import/importer/pull_requests_importer.rb
+++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb
@@ -74,6 +74,10 @@ module Gitlab
           { state: 'all', sort: 'created', direction: 'asc' }
         end
 
+        def parallel_import_batch
+          { size: 200, delay: 1.minute }
+        end
+
         def repository_updates_counter
           @repository_updates_counter ||= Gitlab::Metrics.counter(
             :github_importer_repository_updates,
diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb
index a8e006ea082..4dec9543a13 100644
--- a/lib/gitlab/github_import/parallel_scheduling.rb
+++ b/lib/gitlab/github_import/parallel_scheduling.rb
@@ -72,6 +72,14 @@ module Gitlab
       # Imports all objects in parallel by scheduling a Sidekiq job for every
       # individual object.
       def parallel_import
+        if Feature.enabled?(:spread_parallel_import, default_enabled: :yaml) && parallel_import_batch.present?
+          spread_parallel_import
+        else
+          parallel_import_deprecated
+        end
+      end
+
+      def parallel_import_deprecated
         waiter = JobWaiter.new
 
         each_object_to_import do |object|
@@ -86,6 +94,33 @@ module Gitlab
         waiter
       end
 
+      def spread_parallel_import
+        waiter = JobWaiter.new
+
+        import_arguments = []
+
+        each_object_to_import do |object|
+          repr = representation_class.from_api_response(object)
+
+          import_arguments << [project.id, repr.to_hash, waiter.key]
+
+          waiter.jobs_remaining += 1
+        end
+
+        # rubocop:disable Scalability/BulkPerformWithContext
+        Gitlab::ApplicationContext.with_context(project: project) do
+          sidekiq_worker_class.bulk_perform_in(
+            1.second,
+            import_arguments,
+            batch_size: parallel_import_batch[:size],
+            batch_delay: parallel_import_batch[:delay]
+          )
+        end
+        # rubocop:enable Scalability/BulkPerformWithContext
+
+        waiter
+      end
+
       # The method that will be called for traversing through all the objects to
       # import, yielding them to the supplied block.
       def each_object_to_import
@@ -171,6 +206,12 @@ module Gitlab
         raise NotImplementedError
       end
 
+      # Default batch settings for parallel import (can be redefined in Importer classes)
+      # Example: { size: 100, delay: 1.minute }
+      def parallel_import_batch
+        {}
+      end
+
       def abort_on_failure
         false
       end