diff options
Diffstat (limited to 'app/services/bulk_imports')
-rw-r--r-- | app/services/bulk_imports/file_decompression_service.rb | 58 | ||||
-rw-r--r-- | app/services/bulk_imports/file_download_service.rb | 102 | ||||
-rw-r--r-- | app/services/bulk_imports/relation_export_service.rb | 4 |
3 files changed, 162 insertions, 2 deletions
diff --git a/app/services/bulk_imports/file_decompression_service.rb b/app/services/bulk_imports/file_decompression_service.rb new file mode 100644 index 00000000000..fe9017377ec --- /dev/null +++ b/app/services/bulk_imports/file_decompression_service.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module BulkImports + class FileDecompressionService + include Gitlab::ImportExport::CommandLineUtil + + ServiceError = Class.new(StandardError) + + def initialize(dir:, filename:) + @dir = dir + @filename = filename + @filepath = File.join(@dir, @filename) + @decompressed_filename = File.basename(@filename, '.gz') + @decompressed_filepath = File.join(@dir, @decompressed_filename) + end + + def execute + validate_dir + validate_decompressed_file_size if Feature.enabled?(:validate_import_decompressed_archive_size, default_enabled: :yaml) + validate_symlink(filepath) + + decompress_file + + validate_symlink(decompressed_filepath) + + filepath + rescue StandardError => e + File.delete(filepath) if File.exist?(filepath) + File.delete(decompressed_filepath) if File.exist?(decompressed_filepath) + + raise e + end + + private + + attr_reader :dir, :filename, :filepath, :decompressed_filename, :decompressed_filepath + + def validate_dir + raise(ServiceError, 'Invalid target directory') unless dir.start_with?(Dir.tmpdir) + end + + def validate_decompressed_file_size + raise(ServiceError, 'File decompression error') unless size_validator.valid? + end + + def validate_symlink(filepath) + raise(ServiceError, 'Invalid file') if File.lstat(filepath).symlink? + end + + def decompress_file + gunzip(dir: dir, filename: filename) + end + + def size_validator + @size_validator ||= Gitlab::ImportExport::DecompressedArchiveSizeValidator.new(archive_path: filepath) + end + end +end diff --git a/app/services/bulk_imports/file_download_service.rb b/app/services/bulk_imports/file_download_service.rb new file mode 100644 index 00000000000..c5a1241e0a4 --- /dev/null +++ b/app/services/bulk_imports/file_download_service.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +module BulkImports + class FileDownloadService + FILE_SIZE_LIMIT = 5.gigabytes + ALLOWED_CONTENT_TYPES = %w(application/gzip application/octet-stream).freeze + + ServiceError = Class.new(StandardError) + + def initialize(configuration:, relative_url:, dir:, filename:) + @configuration = configuration + @relative_url = relative_url + @filename = filename + @dir = dir + @filepath = File.join(@dir, @filename) + end + + def execute + validate_dir + validate_url + validate_content_type + validate_content_length + + download_file + + validate_symlink + + filepath + end + + private + + attr_reader :configuration, :relative_url, :dir, :filename, :filepath + + def download_file + File.open(filepath, 'wb') do |file| + bytes_downloaded = 0 + + http_client.stream(relative_url) do |chunk| + bytes_downloaded += chunk.size + + raise(ServiceError, 'Invalid downloaded file') if bytes_downloaded > FILE_SIZE_LIMIT + raise(ServiceError, "File download error #{chunk.code}") unless chunk.code == 200 + + file.write(chunk) + end + end + rescue StandardError => e + File.delete(filepath) if File.exist?(filepath) + + raise e + end + + def http_client + @http_client ||= BulkImports::Clients::HTTP.new( + uri: configuration.url, + token: configuration.access_token + ) + end + + def allow_local_requests? + ::Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? + end + + def headers + @headers ||= http_client.head(relative_url).headers + end + + def validate_dir + raise(ServiceError, 'Invalid target directory') unless dir.start_with?(Dir.tmpdir) + end + + def validate_symlink + if File.lstat(filepath).symlink? + File.delete(filepath) + + raise(ServiceError, 'Invalid downloaded file') + end + end + + def validate_url + ::Gitlab::UrlBlocker.validate!( + http_client.resource_url(relative_url), + allow_localhost: allow_local_requests?, + allow_local_network: allow_local_requests?, + schemes: %w(http https) + ) + end + + def validate_content_length + content_size = headers['content-length'] + + raise(ServiceError, 'Invalid content length') if content_size.blank? || content_size.to_i > FILE_SIZE_LIMIT + end + + def validate_content_type + content_type = headers['content-type'] + + raise(ServiceError, 'Invalid content type') if content_type.blank? || ALLOWED_CONTENT_TYPES.exclude?(content_type) + end + end +end diff --git a/app/services/bulk_imports/relation_export_service.rb b/app/services/bulk_imports/relation_export_service.rb index 53952a33b5f..055f9cafd10 100644 --- a/app/services/bulk_imports/relation_export_service.rb +++ b/app/services/bulk_imports/relation_export_service.rb @@ -86,7 +86,7 @@ module BulkImports # rubocop: disable CodeReuse/Serializer def serializer - @serializer ||= ::Gitlab::ImportExport::JSON::StreamingSerializer.new( + @serializer ||= ::Gitlab::ImportExport::Json::StreamingSerializer.new( portable, portable_tree, json_writer, @@ -96,7 +96,7 @@ module BulkImports # rubocop: enable CodeReuse/Serializer def json_writer - @json_writer ||= ::Gitlab::ImportExport::JSON::NdjsonWriter.new(export_path) + @json_writer ||= ::Gitlab::ImportExport::Json::NdjsonWriter.new(export_path) end def ndjson_filename |