blob: 16be7a77ab1726ce60ee51ff4a1490e5f035d32d (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
|
# frozen_string_literal: true
module Gitlab
module Import
module StuckImportJob
extend ActiveSupport::Concern
IMPORT_JOBS_EXPIRATION = 15.hours.seconds.to_i
included do
include ApplicationWorker
# rubocop:disable Scalability/CronWorkerContext
# This worker updates several import states inline and does not schedule
# other jobs. So no context needed
include CronjobQueue
# rubocop:enable Scalability/CronWorkerContext
feature_category :importers
worker_resource_boundary :cpu
end
def perform
stuck_imports_without_jid_count = mark_imports_without_jid_as_failed!
stuck_imports_with_jid_count = mark_imports_with_jid_as_failed!
track_metrics(stuck_imports_with_jid_count, stuck_imports_without_jid_count)
end
private
def track_metrics(with_jid_count, without_jid_count)
raise NotImplementedError
end
def mark_imports_without_jid_as_failed!
enqueued_import_states_without_jid.each do |import_state|
import_state.mark_as_failed(error_message)
end.size
end
def mark_imports_with_jid_as_failed!
jids_and_ids = enqueued_import_states_with_jid.pluck(:jid, :id).to_h # rubocop: disable CodeReuse/ActiveRecord
# Find the jobs that aren't currently running or that exceeded the threshold.
completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys)
return 0 unless completed_jids.any?
completed_import_state_ids = jids_and_ids.values_at(*completed_jids)
# We select the import states again, because they may have transitioned from
# scheduled/started to finished/failed while we were looking up their Sidekiq status.
completed_import_states = enqueued_import_states_with_jid.id_in(completed_import_state_ids)
completed_import_state_jids = completed_import_states.map { |import_state| import_state.jid }.join(', ')
Gitlab::Import::Logger.info(
message: 'Marked stuck import jobs as failed',
job_ids: completed_import_state_jids
)
completed_import_states.each do |import_state|
import_state.mark_as_failed(error_message)
end.size
end
def enqueued_import_states
raise NotImplementedError
end
def enqueued_import_states_with_jid
enqueued_import_states.with_jid
end
def enqueued_import_states_without_jid
enqueued_import_states.without_jid
end
def error_message
_("Import timed out. Import took longer than %{import_jobs_expiration} seconds") % { import_jobs_expiration: IMPORT_JOBS_EXPIRATION }
end
end
end
end
|
