1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
|
# frozen_string_literal: true
module ImportCsv
class BaseService
include Gitlab::Utils::StrongMemoize
def initialize(user, project, csv_io)
@user = user
@project = project
@csv_io = csv_io
@results = { success: 0, error_lines: [], parse_error: false }
end
PreprocessError = Class.new(StandardError)
def execute
process_csv
email_results_to_user
results
end
def email_results_to_user
raise NotImplementedError
end
private
attr_reader :user, :project, :csv_io, :results
def attributes_for(row)
raise NotImplementedError
end
def validate_headers_presence!(headers)
raise NotImplementedError
end
def create_object_class
raise NotImplementedError
end
def validate_structure!
header_line = csv_data.lines.first
validate_headers_presence!(header_line)
detect_col_sep
end
def preprocess!
# any logic can be added in subclasses if needed
# hence just a no-op rather than NotImplementedError
end
def process_csv
validate_structure!
preprocess!
with_csv_lines.each do |row, line_no|
attributes = attributes_for(row)
if create_object(attributes)&.persisted?
results[:success] += 1
else
results[:error_lines].push(line_no)
end
end
rescue ArgumentError, CSV::MalformedCSVError => e
results[:parse_error] = true
results[:error_lines].push(e.line_number) if e.respond_to?(:line_number)
rescue PreprocessError
results[:parse_error] = false
end
def with_csv_lines
CSV.new(
csv_data,
col_sep: detect_col_sep,
headers: true,
header_converters: :symbol
).each.with_index(2)
end
def csv_data
@csv_io.open(&:read).force_encoding(Encoding::UTF_8)
end
strong_memoize_attr :csv_data
def detect_col_sep
header = csv_data.lines.first
if header.include?(",")
","
elsif header.include?(";")
";"
elsif header.include?("\t")
"\t"
else
raise CSV::MalformedCSVError.new('Invalid CSV format', 1)
end
end
strong_memoize_attr :detect_col_sep
def create_object(attributes)
# NOTE: CSV imports are performed by workers, so we do not have a request context in order
# to create a SpamParams object to pass to the issuable create service.
spam_params = nil
# default_params can be extracted into a method if we need
# to support creation of objects that belongs to groups.
default_params = { container: project,
current_user: user,
params: attributes,
spam_params: spam_params }
create_service = create_object_class.new(**default_params.merge(extra_create_service_params))
create_service.execute_without_rate_limiting
end
# Overidden in subclasses to support specific parameters
def extra_create_service_params
{}
end
end
end
|
