Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to 'app/services/bulk_imports')
-rw-r--r--app/services/bulk_imports/archive_extraction_service.rb4
-rw-r--r--app/services/bulk_imports/file_decompression_service.rb27
-rw-r--r--app/services/bulk_imports/file_download_service.rb28
-rw-r--r--app/services/bulk_imports/file_export_service.rb4
-rw-r--r--app/services/bulk_imports/lfs_objects_export_service.rb64
5 files changed, 107 insertions, 20 deletions
diff --git a/app/services/bulk_imports/archive_extraction_service.rb b/app/services/bulk_imports/archive_extraction_service.rb
index 9fc828b8e34..caa40d98a76 100644
--- a/app/services/bulk_imports/archive_extraction_service.rb
+++ b/app/services/bulk_imports/archive_extraction_service.rb
@@ -28,8 +28,8 @@ module BulkImports
end
def execute
- validate_filepath
validate_tmpdir
+ validate_filepath
validate_symlink
extract_archive
@@ -46,7 +46,7 @@ module BulkImports
end
def validate_tmpdir
- raise(BulkImports::Error, 'Invalid target directory') unless File.expand_path(tmpdir).start_with?(Dir.tmpdir)
+ Gitlab::Utils.check_allowed_absolute_path!(tmpdir, [Dir.tmpdir])
end
def validate_symlink
diff --git a/app/services/bulk_imports/file_decompression_service.rb b/app/services/bulk_imports/file_decompression_service.rb
index fe9017377ec..b76746b199f 100644
--- a/app/services/bulk_imports/file_decompression_service.rb
+++ b/app/services/bulk_imports/file_decompression_service.rb
@@ -1,21 +1,26 @@
# frozen_string_literal: true
+# File Decompression Service allows gzipped files decompression into tmp directory.
+#
+# @param tmpdir [String] Temp directory to store downloaded file to. Must be located under `Dir.tmpdir`.
+# @param filename [String] Name of the file to decompress.
module BulkImports
class FileDecompressionService
include Gitlab::ImportExport::CommandLineUtil
ServiceError = Class.new(StandardError)
- def initialize(dir:, filename:)
- @dir = dir
+ def initialize(tmpdir:, filename:)
+ @tmpdir = tmpdir
@filename = filename
- @filepath = File.join(@dir, @filename)
+ @filepath = File.join(@tmpdir, @filename)
@decompressed_filename = File.basename(@filename, '.gz')
- @decompressed_filepath = File.join(@dir, @decompressed_filename)
+ @decompressed_filepath = File.join(@tmpdir, @decompressed_filename)
end
def execute
- validate_dir
+ validate_tmpdir
+ validate_filepath
validate_decompressed_file_size if Feature.enabled?(:validate_import_decompressed_archive_size, default_enabled: :yaml)
validate_symlink(filepath)
@@ -33,10 +38,14 @@ module BulkImports
private
- attr_reader :dir, :filename, :filepath, :decompressed_filename, :decompressed_filepath
+ attr_reader :tmpdir, :filename, :filepath, :decompressed_filename, :decompressed_filepath
- def validate_dir
- raise(ServiceError, 'Invalid target directory') unless dir.start_with?(Dir.tmpdir)
+ def validate_filepath
+ Gitlab::Utils.check_path_traversal!(filepath)
+ end
+
+ def validate_tmpdir
+ Gitlab::Utils.check_allowed_absolute_path!(tmpdir, [Dir.tmpdir])
end
def validate_decompressed_file_size
@@ -48,7 +57,7 @@ module BulkImports
end
def decompress_file
- gunzip(dir: dir, filename: filename)
+ gunzip(dir: tmpdir, filename: filename)
end
def size_validator
diff --git a/app/services/bulk_imports/file_download_service.rb b/app/services/bulk_imports/file_download_service.rb
index d08dc72e30b..8d6ba54cd50 100644
--- a/app/services/bulk_imports/file_download_service.rb
+++ b/app/services/bulk_imports/file_download_service.rb
@@ -1,6 +1,13 @@
# frozen_string_literal: true
-# Downloads a remote file. If no filename is given, it'll use the remote filename
+# File Download Service allows remote file download into tmp directory.
+#
+# @param configuration [BulkImports::Configuration] Config object containing url and access token
+# @param relative_url [String] Relative URL to download the file from
+# @param tmpdir [String] Temp directory to store downloaded file to. Must be located under `Dir.tmpdir`.
+# @param file_size_limit [Integer] Maximum allowed file size
+# @param allowed_content_types [Array<String>] Allowed file content types
+# @param filename [String] Name of the file to download, if known. Use remote filename if none given.
module BulkImports
class FileDownloadService
ServiceError = Class.new(StandardError)
@@ -13,20 +20,21 @@ module BulkImports
def initialize(
configuration:,
relative_url:,
- dir:,
+ tmpdir:,
file_size_limit: DEFAULT_FILE_SIZE_LIMIT,
allowed_content_types: DEFAULT_ALLOWED_CONTENT_TYPES,
filename: nil)
@configuration = configuration
@relative_url = relative_url
@filename = filename
- @dir = dir
+ @tmpdir = tmpdir
@file_size_limit = file_size_limit
@allowed_content_types = allowed_content_types
end
def execute
- validate_dir
+ validate_tmpdir
+ validate_filepath
validate_url
validate_content_type
validate_content_length
@@ -40,7 +48,7 @@ module BulkImports
private
- attr_reader :configuration, :relative_url, :dir, :file_size_limit, :allowed_content_types
+ attr_reader :configuration, :relative_url, :tmpdir, :file_size_limit, :allowed_content_types
def download_file
File.open(filepath, 'wb') do |file|
@@ -76,8 +84,12 @@ module BulkImports
@headers ||= http_client.head(relative_url).headers
end
- def validate_dir
- raise(ServiceError, 'Invalid target directory') unless dir.start_with?(Dir.tmpdir)
+ def validate_filepath
+ Gitlab::Utils.check_path_traversal!(filepath)
+ end
+
+ def validate_tmpdir
+ Gitlab::Utils.check_allowed_absolute_path!(tmpdir, [Dir.tmpdir])
end
def validate_symlink
@@ -119,7 +131,7 @@ module BulkImports
end
def filepath
- @filepath ||= File.join(@dir, filename)
+ @filepath ||= File.join(@tmpdir, filename)
end
def filename
diff --git a/app/services/bulk_imports/file_export_service.rb b/app/services/bulk_imports/file_export_service.rb
index a7e0f998666..a9d06d84277 100644
--- a/app/services/bulk_imports/file_export_service.rb
+++ b/app/services/bulk_imports/file_export_service.rb
@@ -26,8 +26,10 @@ module BulkImports
def export_service
case relation
- when FileTransfer::ProjectConfig::UPLOADS_RELATION
+ when FileTransfer::BaseConfig::UPLOADS_RELATION
UploadsExportService.new(portable, export_path)
+ when FileTransfer::ProjectConfig::LFS_OBJECTS_RELATION
+ LfsObjectsExportService.new(portable, export_path)
else
raise BulkImports::Error, 'Unsupported relation export type'
end
diff --git a/app/services/bulk_imports/lfs_objects_export_service.rb b/app/services/bulk_imports/lfs_objects_export_service.rb
new file mode 100644
index 00000000000..fa606e4e5a3
--- /dev/null
+++ b/app/services/bulk_imports/lfs_objects_export_service.rb
@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module BulkImports
+ class LfsObjectsExportService
+ include Gitlab::ImportExport::CommandLineUtil
+
+ BATCH_SIZE = 100
+
+ def initialize(portable, export_path)
+ @portable = portable
+ @export_path = export_path
+ @lfs_json = {}
+ end
+
+ def execute
+ portable.lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| # rubocop: disable CodeReuse/ActiveRecord
+ batch.each do |lfs_object|
+ save_lfs_object(lfs_object)
+ end
+
+ append_lfs_json_for_batch(batch)
+ end
+
+ write_lfs_json
+ end
+
+ private
+
+ attr_reader :portable, :export_path, :lfs_json
+
+ def save_lfs_object(lfs_object)
+ destination_filepath = File.join(export_path, lfs_object.oid)
+
+ if lfs_object.local_store?
+ copy_files(lfs_object.file.path, destination_filepath)
+ else
+ download(lfs_object.file.url, destination_filepath)
+ end
+ end
+
+ # rubocop: disable CodeReuse/ActiveRecord
+ def append_lfs_json_for_batch(lfs_objects_batch)
+ lfs_objects_projects = LfsObjectsProject
+ .select('lfs_objects.oid, array_agg(distinct lfs_objects_projects.repository_type) as repository_types')
+ .joins(:lfs_object)
+ .where(project: portable, lfs_object: lfs_objects_batch)
+ .group('lfs_objects.oid')
+
+ lfs_objects_projects.each do |group|
+ oid = group.oid
+
+ lfs_json[oid] ||= []
+ lfs_json[oid] += group.repository_types
+ end
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+
+ def write_lfs_json
+ filepath = File.join(export_path, "#{BulkImports::FileTransfer::ProjectConfig::LFS_OBJECTS_RELATION}.json")
+
+ File.write(filepath, lfs_json.to_json)
+ end
+ end
+end