1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
|
# frozen_string_literal: true
module BulkImports
class BatchedRelationExportService
include Gitlab::Utils::StrongMemoize
BATCH_SIZE = 1000
BATCH_CACHE_KEY = 'bulk_imports/batched_relation_export/%{export_id}/%{batch_id}'
CACHE_DURATION = 4.hours
def self.cache_key(export_id, batch_id)
Kernel.format(BATCH_CACHE_KEY, export_id: export_id, batch_id: batch_id)
end
def initialize(user, portable, relation, jid)
@user = user
@portable = portable
@relation = relation
@resolved_relation = portable.public_send(relation) # rubocop:disable GitlabSecurity/PublicSend
@jid = jid
end
def execute
return finish_export! if batches_count == 0
start_export!
export.batches.destroy_all # rubocop: disable Cop/DestroyAll
enqueue_batch_exports
ensure
FinishBatchedRelationExportWorker.perform_async(export.id)
end
private
attr_reader :user, :portable, :relation, :jid, :config, :resolved_relation
def export
@export ||= portable.bulk_import_exports.find_or_create_by!(relation: relation) # rubocop:disable CodeReuse/ActiveRecord
end
def objects_count
resolved_relation.count
end
def batches_count
objects_count.fdiv(BATCH_SIZE).ceil
end
def start_export!
update_export!('start')
end
def finish_export!
update_export!('finish')
end
def update_export!(event)
export.update!(
status_event: event,
total_objects_count: objects_count,
batched: true,
batches_count: batches_count,
jid: jid,
error: nil
)
end
def enqueue_batch_exports
resolved_relation.each_batch(of: BATCH_SIZE) do |batch, batch_number|
batch_id = find_or_create_batch(batch_number).id
ids = batch.pluck(batch.model.primary_key) # rubocop:disable CodeReuse/ActiveRecord
Gitlab::Cache::Import::Caching.set_add(self.class.cache_key(export.id, batch_id), ids, timeout: CACHE_DURATION)
RelationBatchExportWorker.perform_async(user.id, batch_id)
end
end
def find_or_create_batch(batch_number)
export.batches.find_or_create_by!(batch_number: batch_number) # rubocop:disable CodeReuse/ActiveRecord
end
end
end
|
