1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
|
# frozen_string_literal: true
module Gitlab
module GithubImport
# ObjectImporter defines the base behaviour for every Sidekiq worker that
# imports a single resource such as a note or pull request.
module ObjectImporter
extend ActiveSupport::Concern
included do
include ApplicationWorker
include GithubImport::Queue
include ReschedulingMethods
feature_category :importers
worker_has_external_dependencies!
sidekiq_options retry: 5
sidekiq_retries_exhausted do |msg|
args = msg['args']
jid = msg['jid']
# If a job is being exhausted we still want to notify the
# Gitlab::Import::AdvanceStageWorker to prevent the entire import from getting stuck
if args.length == 3 && (key = args.last) && key.is_a?(String)
JobWaiter.notify(key, jid, ttl: Gitlab::Import::JOB_WAITER_TTL)
end
end
end
NotRetriableError = Class.new(StandardError)
# project - An instance of `Project` to import the data into.
# client - An instance of `Gitlab::GithubImport::Client`
# hash - A Hash containing the details of the object to import.
def import(project, client, hash)
if project.import_state&.completed?
info(
project.id,
message: 'Project import is no longer running. Stopping worker.',
import_status: project.import_state.status
)
return
end
object = representation_class.from_json_hash(hash)
# To better express in the logs what object is being imported.
self.github_identifiers = object.github_identifiers
info(project.id, message: 'starting importer')
importer_class.new(object, project, client).execute
increment_object_counter(object, project) if increment_object_counter?(object)
info(project.id, message: 'importer finished')
rescue ActiveRecord::RecordInvalid, NotRetriableError, NoMethodError => e
# We do not raise exception to prevent job retry
track_exception(project, e)
rescue UserFinder::FailedToObtainLockError
warn(project.id, message: 'Failed to obtaing lock for user finder. Retrying later.')
raise
rescue StandardError => e
track_and_raise_exception(project, e)
end
def increment_object_counter?(_object)
true
end
def increment_object_counter(_object, project)
Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :imported)
end
def object_type
raise NotImplementedError
end
# Returns the representation class to use for the object. This class must
# define the class method `from_json_hash`.
def representation_class
raise NotImplementedError
end
# Returns the class to use for importing the object.
def importer_class
raise NotImplementedError
end
private
attr_accessor :github_identifiers
def info(project_id, extra = {})
Logger.info(log_attributes(project_id, extra))
end
def warn(project_id, extra = {})
Logger.warn(log_attributes(project_id, extra))
end
def log_attributes(project_id, extra = {})
extra.merge(
project_id: project_id,
importer: importer_class.name,
external_identifiers: github_identifiers
)
end
def track_exception(project, exception, fail_import: false)
external_identifiers = github_identifiers || {}
external_identifiers[:object_type] ||= object_type&.to_s
Gitlab::Import::ImportFailureService.track(
project_id: project.id,
error_source: importer_class.name,
exception: exception,
fail_import: fail_import,
external_identifiers: external_identifiers
)
end
def track_and_raise_exception(project, exception, fail_import: false)
track_exception(project, exception, fail_import: fail_import)
raise(exception)
end
end
end
end
|
