Add latest changes from gitlab-org/gitlab@14-0-stable-eev14.0.0-rc42

3 files changed, 162 insertions, 2 deletions

diff --git a/app/services/bulk_imports/file_decompression_service.rb b/app/services/bulk_imports/file_decompression_service.rb
new file mode 100644
index 00000000000..fe9017377ec
--- /dev/null
+++ b/app/services/bulk_imports/file_decompression_service.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module BulkImports
+  class FileDecompressionService
+    include Gitlab::ImportExport::CommandLineUtil
+
+    ServiceError = Class.new(StandardError)
+
+    def initialize(dir:, filename:)
+      @dir = dir
+      @filename = filename
+      @filepath = File.join(@dir, @filename)
+      @decompressed_filename = File.basename(@filename, '.gz')
+      @decompressed_filepath = File.join(@dir, @decompressed_filename)
+    end
+
+    def execute
+      validate_dir
+      validate_decompressed_file_size if Feature.enabled?(:validate_import_decompressed_archive_size, default_enabled: :yaml)
+      validate_symlink(filepath)
+
+      decompress_file
+
+      validate_symlink(decompressed_filepath)
+
+      filepath
+    rescue StandardError => e
+      File.delete(filepath) if File.exist?(filepath)
+      File.delete(decompressed_filepath) if File.exist?(decompressed_filepath)
+
+      raise e
+    end
+
+    private
+
+    attr_reader :dir, :filename, :filepath, :decompressed_filename, :decompressed_filepath
+
+    def validate_dir
+      raise(ServiceError, 'Invalid target directory') unless dir.start_with?(Dir.tmpdir)
+    end
+
+    def validate_decompressed_file_size
+      raise(ServiceError, 'File decompression error') unless size_validator.valid?
+    end
+
+    def validate_symlink(filepath)
+      raise(ServiceError, 'Invalid file') if File.lstat(filepath).symlink?
+    end
+
+    def decompress_file
+      gunzip(dir: dir, filename: filename)
+    end
+
+    def size_validator
+      @size_validator ||= Gitlab::ImportExport::DecompressedArchiveSizeValidator.new(archive_path: filepath)
+    end
+  end
+end
diff --git a/app/services/bulk_imports/file_download_service.rb b/app/services/bulk_imports/file_download_service.rb
new file mode 100644
index 00000000000..c5a1241e0a4
--- /dev/null
+++ b/app/services/bulk_imports/file_download_service.rb
@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module BulkImports
+  class FileDownloadService
+    FILE_SIZE_LIMIT = 5.gigabytes
+    ALLOWED_CONTENT_TYPES = %w(application/gzip application/octet-stream).freeze
+
+    ServiceError = Class.new(StandardError)
+
+    def initialize(configuration:, relative_url:, dir:, filename:)
+      @configuration = configuration
+      @relative_url = relative_url
+      @filename = filename
+      @dir = dir
+      @filepath = File.join(@dir, @filename)
+    end
+
+    def execute
+      validate_dir
+      validate_url
+      validate_content_type
+      validate_content_length
+
+      download_file
+
+      validate_symlink
+
+      filepath
+    end
+
+    private
+
+    attr_reader :configuration, :relative_url, :dir, :filename, :filepath
+
+    def download_file
+      File.open(filepath, 'wb') do |file|
+        bytes_downloaded = 0
+
+        http_client.stream(relative_url) do |chunk|
+          bytes_downloaded += chunk.size
+
+          raise(ServiceError, 'Invalid downloaded file') if bytes_downloaded > FILE_SIZE_LIMIT
+          raise(ServiceError, "File download error #{chunk.code}") unless chunk.code == 200
+
+          file.write(chunk)
+        end
+      end
+    rescue StandardError => e
+      File.delete(filepath) if File.exist?(filepath)
+
+      raise e
+    end
+
+    def http_client
+      @http_client ||= BulkImports::Clients::HTTP.new(
+        uri: configuration.url,
+        token: configuration.access_token
+      )
+    end
+
+    def allow_local_requests?
+      ::Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services?
+    end
+
+    def headers
+      @headers ||= http_client.head(relative_url).headers
+    end
+
+    def validate_dir
+      raise(ServiceError, 'Invalid target directory') unless dir.start_with?(Dir.tmpdir)
+    end
+
+    def validate_symlink
+      if File.lstat(filepath).symlink?
+        File.delete(filepath)
+
+        raise(ServiceError, 'Invalid downloaded file')
+      end
+    end
+
+    def validate_url
+      ::Gitlab::UrlBlocker.validate!(
+        http_client.resource_url(relative_url),
+        allow_localhost: allow_local_requests?,
+        allow_local_network: allow_local_requests?,
+        schemes: %w(http https)
+      )
+    end
+
+    def validate_content_length
+      content_size = headers['content-length']
+
+      raise(ServiceError, 'Invalid content length') if content_size.blank? || content_size.to_i > FILE_SIZE_LIMIT
+    end
+
+    def validate_content_type
+      content_type = headers['content-type']
+
+      raise(ServiceError, 'Invalid content type') if content_type.blank? || ALLOWED_CONTENT_TYPES.exclude?(content_type)
+    end
+  end
+end
diff --git a/app/services/bulk_imports/relation_export_service.rb b/app/services/bulk_imports/relation_export_service.rb
index 53952a33b5f..055f9cafd10 100644
--- a/app/services/bulk_imports/relation_export_service.rb
+++ b/app/services/bulk_imports/relation_export_service.rb
@@ -86,7 +86,7 @@ module BulkImports
 
     # rubocop: disable CodeReuse/Serializer
     def serializer
-      @serializer ||= ::Gitlab::ImportExport::JSON::StreamingSerializer.new(
+      @serializer ||= ::Gitlab::ImportExport::Json::StreamingSerializer.new(
         portable,
         portable_tree,
         json_writer,
@@ -96,7 +96,7 @@ module BulkImports
     # rubocop: enable CodeReuse/Serializer
 
     def json_writer
-      @json_writer ||= ::Gitlab::ImportExport::JSON::NdjsonWriter.new(export_path)
+      @json_writer ||= ::Gitlab::ImportExport::Json::NdjsonWriter.new(export_path)
     end
 
     def ndjson_filename