1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
|
# frozen_string_literal: true
module Gitlab
module ImportExport
module CommandLineUtil
UNTAR_MASK = 'u+rwX,go+rX,go-w'
DEFAULT_DIR_MODE = 0700
CLEAN_DIR_IGNORE_FILE_NAMES = %w[. ..].freeze
CommandLineUtilError = Class.new(StandardError)
FileOversizedError = Class.new(CommandLineUtilError)
HardLinkError = Class.new(CommandLineUtilError)
def tar_czf(archive:, dir:)
tar_with_options(archive: archive, dir: dir, options: 'czf')
end
def untar_zxf(archive:, dir:)
untar_with_options(archive: archive, dir: dir, options: 'zxf')
end
def tar_cf(archive:, dir:)
tar_with_options(archive: archive, dir: dir, options: 'cf')
end
def untar_xf(archive:, dir:)
untar_with_options(archive: archive, dir: dir, options: 'xf')
end
def gzip(dir:, filename:)
gzip_with_options(dir: dir, filename: filename)
end
def gunzip(dir:, filename:)
gzip_with_options(dir: dir, filename: filename, options: 'd')
end
def gzip_with_options(dir:, filename:, options: nil)
filepath = File.join(dir, filename)
cmd = %W[gzip #{filepath}]
cmd << "-#{options}" if options
_, status = Gitlab::Popen.popen(cmd)
if status == 0
status
else
raise Gitlab::ImportExport::Error.file_compression_error
end
end
def mkdir_p(path)
FileUtils.mkdir_p(path, mode: DEFAULT_DIR_MODE)
FileUtils.chmod(DEFAULT_DIR_MODE, path)
end
private
def download_or_copy_upload(uploader, upload_path, size_limit: 0)
if uploader.upload.local?
copy_files(uploader.path, upload_path)
else
download(uploader.url, upload_path, size_limit: size_limit)
end
end
def download(url, upload_path, size_limit: 0)
File.open(upload_path, 'wb') do |file|
current_size = 0
# When migrating from Gitlab::HTTP to Gitlab:HTTP_V2, we need to pass `extra_allowed_uris` as an option
# instead of `allow_object_storage`.
Gitlab::HTTP.get(url, stream_body: true, allow_object_storage: true) do |fragment|
if [301, 302, 303, 307].include?(fragment.code)
Gitlab::Import::Logger.warn(message: "received redirect fragment", fragment_code: fragment.code)
elsif fragment.code == 200
current_size += fragment.bytesize
raise FileOversizedError if size_limit > 0 && current_size > size_limit
file.write(fragment)
else
raise Gitlab::ImportExport::Error, "unsupported response downloading fragment #{fragment.code}"
end
end
end
rescue FileOversizedError
nil
end
def tar_with_options(archive:, dir:, options:)
execute_cmd(%W[tar -#{options} #{archive} -C #{dir} .])
end
def untar_with_options(archive:, dir:, options:)
execute_cmd(%W[tar -#{options} #{archive} -C #{dir}])
execute_cmd(%W[chmod -R #{UNTAR_MASK} #{dir}])
clean_extraction_dir!(dir)
end
# rubocop:disable Gitlab/ModuleWithInstanceVariables
def execute_cmd(cmd)
output, status = Gitlab::Popen.popen(cmd)
return true if status == 0
output = output&.strip
message = "command exited with error code #{status}"
message += ": #{output}" if output.present?
if @shared.respond_to?(:error)
@shared.error(Gitlab::ImportExport::Error.new(message))
false
else
raise Gitlab::ImportExport::Error, message
end
end
# rubocop:enable Gitlab/ModuleWithInstanceVariables
def copy_files(source, destination)
# if we are copying files, create the destination folder
destination_folder = File.file?(source) ? File.dirname(destination) : destination
mkdir_p(destination_folder)
FileUtils.copy_entry(source, destination)
true
end
# Scans and cleans the directory tree.
# Symlinks are considered legal but are removed.
# Files sharing hard links are considered illegal and the directory will be removed
# and a `HardLinkError` exception will be raised.
#
# @raise [HardLinkError] if there multiple hard links to the same file detected.
# @return [Boolean] true
def clean_extraction_dir!(dir)
# Using File::FNM_DOTMATCH to also delete symlinks starting with "."
Dir.glob("#{dir}/**/*", File::FNM_DOTMATCH).each do |filepath|
next if CLEAN_DIR_IGNORE_FILE_NAMES.include?(File.basename(filepath))
raise HardLinkError, 'File shares hard link' if Gitlab::Utils::FileInfo.shares_hard_link?(filepath)
FileUtils.rm(filepath) if Gitlab::Utils::FileInfo.linked?(filepath) || File.pipe?(filepath)
end
true
rescue HardLinkError
FileUtils.remove_dir(dir)
raise
end
end
end
end
|
