Add latest changes from gitlab-org/gitlab@master

2 files changed, 23 insertions, 4 deletions

diff --git a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb
index 64fa705329e..772388ffc9e 100644
--- a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb
+++ b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb
@@ -5,6 +5,10 @@ module Gitlab
     # Module that provides methods shared by the various workers used for
     # importing GitHub projects.
     module ReschedulingMethods
+      include JobDelayCalculator
+
+      ENQUEUED_JOB_COUNT = 'github-importer/enqueued_job_count/%{project}/%{collection}'
+
       # project_id - The ID of the GitLab project to import the note into.
       # hash - A Hash containing the details of the GitHub object to import.
       # notify_key - The Redis key to notify upon completion, if any.
@@ -18,10 +22,7 @@ module Gitlab
         if try_import(project, client, hash)
           notify_waiter(notify_key)
         else
-          # In the event of hitting the rate limit we want to reschedule the job
-          # so its retried after our rate limit has been reset.
-          self.class
-            .perform_in(client.rate_limit_resets_in, project.id, hash, notify_key)
+          reschedule_job(project, client, hash, notify_key)
         end
       end
 
@@ -35,6 +36,20 @@ module Gitlab
       def notify_waiter(key = nil)
         JobWaiter.notify(key, jid) if key
       end
+
+      def reschedule_job(project, client, hash, notify_key)
+        # In the event of hitting the rate limit we want to reschedule the job
+        # so its retried after our rate limit has been reset with additional delay
+        # to spread the load.
+        enqueued_job_count_key = format(ENQUEUED_JOB_COUNT, project: project.id, collection: object_type)
+        enqueued_job_counter =
+          Gitlab::Cache::Import::Caching.increment(enqueued_job_count_key, timeout: client.rate_limit_resets_in)
+
+        job_delay = client.rate_limit_resets_in + calculate_job_delay(enqueued_job_counter)
+
+        self.class
+          .perform_in(job_delay, project.id, hash, notify_key)
+      end
     end
   end
 end
diff --git a/app/workers/gitlab/github_import/import_pull_request_worker.rb b/app/workers/gitlab/github_import/import_pull_request_worker.rb
index 79938a157d7..dbc1feca2e0 100644
--- a/app/workers/gitlab/github_import/import_pull_request_worker.rb
+++ b/app/workers/gitlab/github_import/import_pull_request_worker.rb
@@ -16,6 +16,10 @@ module Gitlab
       def object_type
         :pull_request
       end
+
+      def parallel_import_batch
+        { size: 200, delay: 1.minute }
+      end
     end
   end
 end