1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
|
# frozen_string_literal: true
module Gitlab
module GithubGistsImport
class ImportGistWorker # rubocop:disable Scalability/IdempotentWorker
include ApplicationWorker
GISTS_ERRORS_BY_ID = 'gitlab:github-gists-import:%{user_id}:errors'
data_consistency :always
queue_namespace :github_gists_importer
feature_category :importers
sidekiq_options dead: false, retry: 5
sidekiq_retries_exhausted do |msg|
args = msg['args']
user_id = args[0]
gist_hash = args[1]
jid = msg['jid']
new.perform_failure(user_id, gist_hash, msg['error_class'], msg['error_message'], msg['correlation_id'])
# If a job is being exhausted we still want to notify the
# Gitlab::GithubGistsImport::FinishImportWorker to prevent
# the entire import from getting stuck
if args.length == 3 && (key = args.last) && key.is_a?(String)
JobWaiter.notify(key, jid, ttl: Gitlab::Import::JOB_WAITER_TTL)
end
end
def perform(user_id, gist_hash, notify_key)
gist = representation_class.from_json_hash(gist_hash)
github_identifiers = gist.github_identifiers
with_logging(user_id, github_identifiers) do
result = importer_class.new(gist, user_id).execute
if result.success?
track_gist_import('success', user_id)
else
error(user_id, result.errors, github_identifiers)
perform_failure(
user_id,
gist_hash,
importer_class::FileCountLimitError.name,
importer_class::FILE_COUNT_LIMIT_MESSAGE
)
end
JobWaiter.notify(notify_key, jid, ttl: Gitlab::Import::JOB_WAITER_TTL)
end
rescue StandardError => e
log_and_track_error(user_id, e, github_identifiers)
raise
end
def perform_failure(user_id, gist_hash, exception_class, exception_message, correlation_id = nil)
track_gist_import('failed', user_id)
github_identifiers = representation_class.from_json_hash(gist_hash).github_identifiers
persist_failure(user_id, exception_class, exception_message, github_identifiers, correlation_id)
end
private
def importer_class
::Gitlab::GithubGistsImport::Importer::GistImporter
end
def representation_class
::Gitlab::GithubGistsImport::Representation::Gist
end
def with_logging(user_id, github_identifiers)
info(user_id, 'start importer', github_identifiers)
yield
info(user_id, 'importer finished', github_identifiers)
end
def track_gist_import(status, user_id)
user = User.find(user_id)
Gitlab::Tracking.event(
self.class.name,
'create',
label: 'github_gist_import',
user: user,
status: status
)
end
def log_and_track_error(user_id, exception, github_identifiers)
error(user_id, exception.message, github_identifiers)
Gitlab::ErrorTracking.track_exception(exception,
import_type: :github_gists,
user_id: user_id
)
end
def error(user_id, error_message, github_identifiers)
attributes = {
user_id: user_id,
github_identifiers: github_identifiers,
message: 'importer failed',
'error.message': error_message
}
Gitlab::GithubImport::Logger.error(structured_payload(attributes))
cache_error_for_email(user_id, github_identifiers[:id], error_message)
end
def info(user_id, message, gist_id)
attributes = {
user_id: user_id,
message: message,
github_identifiers: gist_id
}
Gitlab::GithubImport::Logger.info(structured_payload(attributes))
end
def cache_error_for_email(user_id, gist_id, error_message)
key = format(GISTS_ERRORS_BY_ID, user_id: user_id)
::Gitlab::Cache::Import::Caching.hash_add(key, gist_id, error_message)
end
def persist_failure(user_id, exception_class, exception_message, github_identifiers, correlation_id = nil)
ImportFailure.create!(
source: importer_class.name,
exception_class: exception_class,
exception_message: exception_message.truncate(255),
correlation_id_value: correlation_id || Labkit::Correlation::CorrelationId.current_or_new_id,
user_id: user_id,
external_identifiers: github_identifiers
)
end
end
end
end
|
