1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
|
# frozen_string_literal: true
module Gitlab
module ImportExport
module JSON
class StreamingSerializer
include Gitlab::ImportExport::CommandLineUtil
BATCH_SIZE = 100
SMALLER_BATCH_SIZE = 20
def self.batch_size(exportable)
if Feature.enabled?(:export_reduce_relation_batch_size, exportable)
SMALLER_BATCH_SIZE
else
BATCH_SIZE
end
end
class Raw < String
def to_json(*_args)
to_s
end
end
def initialize(exportable, relations_schema, json_writer, exportable_path:)
@exportable = exportable
@exportable_path = exportable_path
@relations_schema = relations_schema
@json_writer = json_writer
end
def execute
serialize_root
includes.each do |relation_definition|
serialize_relation(relation_definition)
end
end
private
attr_reader :json_writer, :relations_schema, :exportable
def serialize_root
attributes = exportable.as_json(
relations_schema.merge(include: nil, preloads: nil))
json_writer.write_attributes(@exportable_path, attributes)
end
def serialize_relation(definition)
raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash)
raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one?
key, options = definition.first
record = exportable.public_send(key) # rubocop: disable GitlabSecurity/PublicSend
if record.is_a?(ActiveRecord::Relation)
serialize_many_relations(key, record, options)
elsif record.respond_to?(:each) # this is to support `project_members` that return an Array
serialize_many_each(key, record, options)
else
serialize_single_relation(key, record, options)
end
end
def serialize_many_relations(key, records, options)
enumerator = Enumerator.new do |items|
key_preloads = preloads&.dig(key)
records = records.preload(key_preloads) if key_preloads
records.find_each(batch_size: batch_size) do |record|
items << Raw.new(record.to_json(options))
end
end
json_writer.write_relation_array(@exportable_path, key, enumerator)
end
def serialize_many_each(key, records, options)
enumerator = Enumerator.new do |items|
records.each do |record|
items << Raw.new(record.to_json(options))
end
end
json_writer.write_relation_array(@exportable_path, key, enumerator)
end
def serialize_single_relation(key, record, options)
json = Raw.new(record.to_json(options))
json_writer.write_relation(@exportable_path, key, json)
end
def includes
relations_schema[:include]
end
def preloads
relations_schema[:preload]
end
def batch_size
@batch_size ||= self.class.batch_size(@exportable)
end
end
end
end
end
|
