1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
|
# frozen_string_literal: true
module Gitlab
module GithubImport
# Module that provides methods shared by the various workers used for
# importing GitHub projects.
module ReschedulingMethods
extend ActiveSupport::Concern
include JobDelayCalculator
ENQUEUED_JOB_COUNT = 'github-importer/enqueued_job_count/%{project}/%{collection}'
included do
loggable_arguments 2
end
# project_id - The ID of the GitLab project to import the note into.
# hash - A Hash containing the details of the GitHub object to import.
# notify_key - The Redis key to notify upon completion, if any.
def perform(project_id, hash, notify_key = nil)
project = Project.find_by_id(project_id)
return notify_waiter(notify_key) unless project
client = GithubImport.new_client_for(project, parallel: true)
if try_import(project, client, hash)
notify_waiter(notify_key)
else
reschedule_job(project, client, hash, notify_key)
end
end
def try_import(...)
import(...)
true
rescue RateLimitError
false
end
def notify_waiter(key = nil)
JobWaiter.notify(key, jid, ttl: Gitlab::Import::JOB_WAITER_TTL) if key
end
def reschedule_job(project, client, hash, notify_key)
# In the event of hitting the rate limit we want to reschedule the job
# so its retried after our rate limit has been reset with additional delay
# to spread the load.
enqueued_job_count_key = format(ENQUEUED_JOB_COUNT, project: project.id, collection: object_type)
enqueued_job_counter =
Gitlab::Cache::Import::Caching.increment(enqueued_job_count_key, timeout: client.rate_limit_resets_in)
job_delay = client.rate_limit_resets_in + calculate_job_delay(enqueued_job_counter)
self.class
.perform_in(job_delay, project.id, hash, notify_key)
end
end
end
end
|
