1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
|
# frozen_string_literal: true
module BulkImports
class EntityWorker
include ApplicationWorker
idempotent!
deduplicate :until_executed
data_consistency :always
feature_category :importers
sidekiq_options retry: false, dead: false
worker_has_external_dependencies!
PERFORM_DELAY = 5.seconds
# Keep `_current_stage` parameter for backwards compatibility.
# The parameter will be remove in https://gitlab.com/gitlab-org/gitlab/-/issues/426311
def perform(entity_id, _current_stage = nil)
@entity = ::BulkImports::Entity.find(entity_id)
return unless @entity.started?
if running_tracker.present?
log_info(message: 'Stage running', entity_stage: running_tracker.stage)
else
start_next_stage
end
re_enqueue
rescue StandardError => e
Gitlab::ErrorTracking.track_exception(e, log_params(message: 'Entity failed'))
@entity.fail_op!
end
private
attr_reader :entity
def re_enqueue
BulkImports::EntityWorker.perform_in(PERFORM_DELAY, entity.id)
end
def running_tracker
@running_tracker ||= BulkImports::Tracker.running_trackers(entity.id).first
end
def next_pipeline_trackers_for(entity_id)
BulkImports::Tracker.next_pipeline_trackers_for(entity_id).update(status_event: 'enqueue')
end
def start_next_stage
next_pipeline_trackers = next_pipeline_trackers_for(entity.id)
next_pipeline_trackers.each_with_index do |pipeline_tracker, index|
log_info(message: 'Stage starting', entity_stage: pipeline_tracker.stage) if index == 0
BulkImports::PipelineWorker.perform_async(
pipeline_tracker.id,
pipeline_tracker.stage,
entity.id
)
end
end
def source_version
entity.bulk_import.source_version_info.to_s
end
def logger
@logger ||= Gitlab::Import::Logger.build
end
def log_exception(exception, payload)
Gitlab::ExceptionLogFormatter.format!(exception, payload)
logger.error(structured_payload(payload))
end
def log_info(payload)
logger.info(structured_payload(log_params(payload)))
end
def log_params(extra)
defaults = {
bulk_import_entity_id: entity.id,
bulk_import_id: entity.bulk_import_id,
bulk_import_entity_type: entity.source_type,
source_full_path: entity.source_full_path,
source_version: source_version,
importer: 'gitlab_migration'
}
defaults.merge(extra)
end
end
end
|
