diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-04-26 06:18:41 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-04-26 06:18:41 +0300 |
commit | 77da08b6e8159daae9b352082bad0c55a003994f (patch) | |
tree | 4606ebcf8e9f864b4bee18f152f8e744fc1befb1 /app/workers | |
parent | 7d4bff6fe8d0716b9634709325e5857452b82ca5 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'app/workers')
-rw-r--r-- | app/workers/concerns/gitlab/github_import/rescheduling_methods.rb | 23 | ||||
-rw-r--r-- | app/workers/gitlab/github_import/import_pull_request_worker.rb | 4 |
2 files changed, 23 insertions, 4 deletions
diff --git a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb index 64fa705329e..772388ffc9e 100644 --- a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb +++ b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb @@ -5,6 +5,10 @@ module Gitlab # Module that provides methods shared by the various workers used for # importing GitHub projects. module ReschedulingMethods + include JobDelayCalculator + + ENQUEUED_JOB_COUNT = 'github-importer/enqueued_job_count/%{project}/%{collection}' + # project_id - The ID of the GitLab project to import the note into. # hash - A Hash containing the details of the GitHub object to import. # notify_key - The Redis key to notify upon completion, if any. @@ -18,10 +22,7 @@ module Gitlab if try_import(project, client, hash) notify_waiter(notify_key) else - # In the event of hitting the rate limit we want to reschedule the job - # so its retried after our rate limit has been reset. - self.class - .perform_in(client.rate_limit_resets_in, project.id, hash, notify_key) + reschedule_job(project, client, hash, notify_key) end end @@ -35,6 +36,20 @@ module Gitlab def notify_waiter(key = nil) JobWaiter.notify(key, jid) if key end + + def reschedule_job(project, client, hash, notify_key) + # In the event of hitting the rate limit we want to reschedule the job + # so its retried after our rate limit has been reset with additional delay + # to spread the load. + enqueued_job_count_key = format(ENQUEUED_JOB_COUNT, project: project.id, collection: object_type) + enqueued_job_counter = + Gitlab::Cache::Import::Caching.increment(enqueued_job_count_key, timeout: client.rate_limit_resets_in) + + job_delay = client.rate_limit_resets_in + calculate_job_delay(enqueued_job_counter) + + self.class + .perform_in(job_delay, project.id, hash, notify_key) + end end end end diff --git a/app/workers/gitlab/github_import/import_pull_request_worker.rb b/app/workers/gitlab/github_import/import_pull_request_worker.rb index 79938a157d7..dbc1feca2e0 100644 --- a/app/workers/gitlab/github_import/import_pull_request_worker.rb +++ b/app/workers/gitlab/github_import/import_pull_request_worker.rb @@ -16,6 +16,10 @@ module Gitlab def object_type :pull_request end + + def parallel_import_batch + { size: 200, delay: 1.minute } + end end end end |