diff options
Diffstat (limited to 'app/services/bulk_imports')
5 files changed, 107 insertions, 20 deletions
diff --git a/app/services/bulk_imports/archive_extraction_service.rb b/app/services/bulk_imports/archive_extraction_service.rb index 9fc828b8e34..caa40d98a76 100644 --- a/app/services/bulk_imports/archive_extraction_service.rb +++ b/app/services/bulk_imports/archive_extraction_service.rb @@ -28,8 +28,8 @@ module BulkImports end def execute - validate_filepath validate_tmpdir + validate_filepath validate_symlink extract_archive @@ -46,7 +46,7 @@ module BulkImports end def validate_tmpdir - raise(BulkImports::Error, 'Invalid target directory') unless File.expand_path(tmpdir).start_with?(Dir.tmpdir) + Gitlab::Utils.check_allowed_absolute_path!(tmpdir, [Dir.tmpdir]) end def validate_symlink diff --git a/app/services/bulk_imports/file_decompression_service.rb b/app/services/bulk_imports/file_decompression_service.rb index fe9017377ec..b76746b199f 100644 --- a/app/services/bulk_imports/file_decompression_service.rb +++ b/app/services/bulk_imports/file_decompression_service.rb @@ -1,21 +1,26 @@ # frozen_string_literal: true +# File Decompression Service allows gzipped files decompression into tmp directory. +# +# @param tmpdir [String] Temp directory to store downloaded file to. Must be located under `Dir.tmpdir`. +# @param filename [String] Name of the file to decompress. module BulkImports class FileDecompressionService include Gitlab::ImportExport::CommandLineUtil ServiceError = Class.new(StandardError) - def initialize(dir:, filename:) - @dir = dir + def initialize(tmpdir:, filename:) + @tmpdir = tmpdir @filename = filename - @filepath = File.join(@dir, @filename) + @filepath = File.join(@tmpdir, @filename) @decompressed_filename = File.basename(@filename, '.gz') - @decompressed_filepath = File.join(@dir, @decompressed_filename) + @decompressed_filepath = File.join(@tmpdir, @decompressed_filename) end def execute - validate_dir + validate_tmpdir + validate_filepath validate_decompressed_file_size if Feature.enabled?(:validate_import_decompressed_archive_size, default_enabled: :yaml) validate_symlink(filepath) @@ -33,10 +38,14 @@ module BulkImports private - attr_reader :dir, :filename, :filepath, :decompressed_filename, :decompressed_filepath + attr_reader :tmpdir, :filename, :filepath, :decompressed_filename, :decompressed_filepath - def validate_dir - raise(ServiceError, 'Invalid target directory') unless dir.start_with?(Dir.tmpdir) + def validate_filepath + Gitlab::Utils.check_path_traversal!(filepath) + end + + def validate_tmpdir + Gitlab::Utils.check_allowed_absolute_path!(tmpdir, [Dir.tmpdir]) end def validate_decompressed_file_size @@ -48,7 +57,7 @@ module BulkImports end def decompress_file - gunzip(dir: dir, filename: filename) + gunzip(dir: tmpdir, filename: filename) end def size_validator diff --git a/app/services/bulk_imports/file_download_service.rb b/app/services/bulk_imports/file_download_service.rb index d08dc72e30b..8d6ba54cd50 100644 --- a/app/services/bulk_imports/file_download_service.rb +++ b/app/services/bulk_imports/file_download_service.rb @@ -1,6 +1,13 @@ # frozen_string_literal: true -# Downloads a remote file. If no filename is given, it'll use the remote filename +# File Download Service allows remote file download into tmp directory. +# +# @param configuration [BulkImports::Configuration] Config object containing url and access token +# @param relative_url [String] Relative URL to download the file from +# @param tmpdir [String] Temp directory to store downloaded file to. Must be located under `Dir.tmpdir`. +# @param file_size_limit [Integer] Maximum allowed file size +# @param allowed_content_types [Array<String>] Allowed file content types +# @param filename [String] Name of the file to download, if known. Use remote filename if none given. module BulkImports class FileDownloadService ServiceError = Class.new(StandardError) @@ -13,20 +20,21 @@ module BulkImports def initialize( configuration:, relative_url:, - dir:, + tmpdir:, file_size_limit: DEFAULT_FILE_SIZE_LIMIT, allowed_content_types: DEFAULT_ALLOWED_CONTENT_TYPES, filename: nil) @configuration = configuration @relative_url = relative_url @filename = filename - @dir = dir + @tmpdir = tmpdir @file_size_limit = file_size_limit @allowed_content_types = allowed_content_types end def execute - validate_dir + validate_tmpdir + validate_filepath validate_url validate_content_type validate_content_length @@ -40,7 +48,7 @@ module BulkImports private - attr_reader :configuration, :relative_url, :dir, :file_size_limit, :allowed_content_types + attr_reader :configuration, :relative_url, :tmpdir, :file_size_limit, :allowed_content_types def download_file File.open(filepath, 'wb') do |file| @@ -76,8 +84,12 @@ module BulkImports @headers ||= http_client.head(relative_url).headers end - def validate_dir - raise(ServiceError, 'Invalid target directory') unless dir.start_with?(Dir.tmpdir) + def validate_filepath + Gitlab::Utils.check_path_traversal!(filepath) + end + + def validate_tmpdir + Gitlab::Utils.check_allowed_absolute_path!(tmpdir, [Dir.tmpdir]) end def validate_symlink @@ -119,7 +131,7 @@ module BulkImports end def filepath - @filepath ||= File.join(@dir, filename) + @filepath ||= File.join(@tmpdir, filename) end def filename diff --git a/app/services/bulk_imports/file_export_service.rb b/app/services/bulk_imports/file_export_service.rb index a7e0f998666..a9d06d84277 100644 --- a/app/services/bulk_imports/file_export_service.rb +++ b/app/services/bulk_imports/file_export_service.rb @@ -26,8 +26,10 @@ module BulkImports def export_service case relation - when FileTransfer::ProjectConfig::UPLOADS_RELATION + when FileTransfer::BaseConfig::UPLOADS_RELATION UploadsExportService.new(portable, export_path) + when FileTransfer::ProjectConfig::LFS_OBJECTS_RELATION + LfsObjectsExportService.new(portable, export_path) else raise BulkImports::Error, 'Unsupported relation export type' end diff --git a/app/services/bulk_imports/lfs_objects_export_service.rb b/app/services/bulk_imports/lfs_objects_export_service.rb new file mode 100644 index 00000000000..fa606e4e5a3 --- /dev/null +++ b/app/services/bulk_imports/lfs_objects_export_service.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module BulkImports + class LfsObjectsExportService + include Gitlab::ImportExport::CommandLineUtil + + BATCH_SIZE = 100 + + def initialize(portable, export_path) + @portable = portable + @export_path = export_path + @lfs_json = {} + end + + def execute + portable.lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| # rubocop: disable CodeReuse/ActiveRecord + batch.each do |lfs_object| + save_lfs_object(lfs_object) + end + + append_lfs_json_for_batch(batch) + end + + write_lfs_json + end + + private + + attr_reader :portable, :export_path, :lfs_json + + def save_lfs_object(lfs_object) + destination_filepath = File.join(export_path, lfs_object.oid) + + if lfs_object.local_store? + copy_files(lfs_object.file.path, destination_filepath) + else + download(lfs_object.file.url, destination_filepath) + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def append_lfs_json_for_batch(lfs_objects_batch) + lfs_objects_projects = LfsObjectsProject + .select('lfs_objects.oid, array_agg(distinct lfs_objects_projects.repository_type) as repository_types') + .joins(:lfs_object) + .where(project: portable, lfs_object: lfs_objects_batch) + .group('lfs_objects.oid') + + lfs_objects_projects.each do |group| + oid = group.oid + + lfs_json[oid] ||= [] + lfs_json[oid] += group.repository_types + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def write_lfs_json + filepath = File.join(export_path, "#{BulkImports::FileTransfer::ProjectConfig::LFS_OBJECTS_RELATION}.json") + + File.write(filepath, lfs_json.to_json) + end + end +end |