From 4fc46d75644b28789e83c95ec4d1309498bb4ba3 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 22 Aug 2023 12:09:21 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- .../packages/nuget/check_duplicates_service.rb | 88 ++++++++++++++++++++++ .../nuget/extract_metadata_file_service.rb | 15 +--- .../nuget/extract_remote_metadata_file_service.rb | 82 ++++++++++++++++++++ .../packages/nuget/metadata_extraction_service.rb | 18 ++--- .../nuget/update_package_from_metadata_service.rb | 2 +- 5 files changed, 184 insertions(+), 21 deletions(-) create mode 100644 app/services/packages/nuget/check_duplicates_service.rb create mode 100644 app/services/packages/nuget/extract_remote_metadata_file_service.rb (limited to 'app/services/packages') diff --git a/app/services/packages/nuget/check_duplicates_service.rb b/app/services/packages/nuget/check_duplicates_service.rb new file mode 100644 index 00000000000..7ad9038d7c1 --- /dev/null +++ b/app/services/packages/nuget/check_duplicates_service.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module Packages + module Nuget + class CheckDuplicatesService < BaseService + include Gitlab::Utils::StrongMemoize + + ExtractionError = Class.new(StandardError) + + def execute + return ServiceResponse.success if package_settings_allow_duplicates? || !target_package_is_duplicate? + + ServiceResponse.error( + message: 'A package with the same name and version already exists', + reason: :conflict + ) + rescue ExtractionError => e + ServiceResponse.error(message: e.message, reason: :bad_request) + end + + private + + def package_settings_allow_duplicates? + package_settings.nuget_duplicates_allowed? || package_settings.class.duplicates_allowed?(existing_package) + end + + def target_package_is_duplicate? + existing_package.name.casecmp(metadata[:package_name]) == 0 && + (existing_package.version.casecmp(metadata[:package_version]) == 0 || + existing_package.normalized_nuget_version&.casecmp(metadata[:package_version]) == 0) + end + + def package_settings + project.namespace.package_settings + end + strong_memoize_attr :package_settings + + def existing_package + ::Packages::Nuget::PackageFinder + .new( + current_user, + project, + package_name: metadata[:package_name], + package_version: metadata[:package_version] + ) + .execute + .first + end + strong_memoize_attr :existing_package + + def metadata + if remote_package_file? + ExtractMetadataContentService + .new(nuspec_file_content) + .execute + .payload + else # to cover the case when package file is on disk not in object storage + MetadataExtractionService + .new(mock_package_file) + .execute + .payload + end + end + strong_memoize_attr :metadata + + def remote_package_file? + params[:remote_url].present? + end + + def nuspec_file_content + ExtractRemoteMetadataFileService + .new(params[:remote_url]) + .execute + .payload + rescue ExtractRemoteMetadataFileService::ExtractionError => e + raise ExtractionError, e.message + end + + def mock_package_file + ::Packages::PackageFile.new( + params + .slice(:file, :file_name) + .merge(package: ::Packages::Package.nuget.build) + ) + end + end + end +end diff --git a/app/services/packages/nuget/extract_metadata_file_service.rb b/app/services/packages/nuget/extract_metadata_file_service.rb index 61e4892fee7..cc040a45016 100644 --- a/app/services/packages/nuget/extract_metadata_file_service.rb +++ b/app/services/packages/nuget/extract_metadata_file_service.rb @@ -3,14 +3,12 @@ module Packages module Nuget class ExtractMetadataFileService - include Gitlab::Utils::StrongMemoize - ExtractionError = Class.new(StandardError) MAX_FILE_SIZE = 4.megabytes.freeze - def initialize(package_file_id) - @package_file_id = package_file_id + def initialize(package_file) + @package_file = package_file end def execute @@ -21,12 +19,7 @@ module Packages private - attr_reader :package_file_id - - def package_file - ::Packages::PackageFile.find_by_id(package_file_id) - end - strong_memoize_attr :package_file + attr_reader :package_file def valid_package_file? package_file && @@ -41,7 +34,7 @@ module Packages raise ExtractionError, 'nuspec file not found' unless entry raise ExtractionError, 'nuspec file too big' if MAX_FILE_SIZE < entry.size - Tempfile.open("nuget_extraction_package_file_#{package_file_id}") do |file| + Tempfile.open("nuget_extraction_package_file_#{package_file.id}") do |file| entry.extract(file.path) { true } # allow #extract to overwrite the file file.unlink file.read diff --git a/app/services/packages/nuget/extract_remote_metadata_file_service.rb b/app/services/packages/nuget/extract_remote_metadata_file_service.rb new file mode 100644 index 00000000000..37624002ce7 --- /dev/null +++ b/app/services/packages/nuget/extract_remote_metadata_file_service.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Packages + module Nuget + class ExtractRemoteMetadataFileService + include Gitlab::Utils::StrongMemoize + + ExtractionError = Class.new(StandardError) + + MAX_FILE_SIZE = 4.megabytes.freeze + METADATA_FILE_EXTENSION = '.nuspec' + MAX_FRAGMENTS = 5 # nuspec file is usually in the first 2 fragments but we buffer 5 max + + def initialize(remote_url) + @remote_url = remote_url + end + + def execute + raise ExtractionError, 'invalid file url' if remote_url.blank? + + if nuspec_file_content.blank? || !nuspec_file_content.instance_of?(String) + raise ExtractionError, 'nuspec file not found' + end + + ServiceResponse.success(payload: nuspec_file_content) + end + + private + + attr_reader :remote_url + + def nuspec_file_content + fragments = [] + + Gitlab::HTTP.get(remote_url, stream_body: true, allow_object_storage: true) do |fragment| + break if fragments.size >= MAX_FRAGMENTS + + fragments << fragment + joined_fragments = fragments.join + + next if joined_fragments.exclude?(METADATA_FILE_EXTENSION) + + nuspec_content = extract_nuspec_file(joined_fragments) + + break nuspec_content if nuspec_content.present? + end + end + strong_memoize_attr :nuspec_file_content + + def extract_nuspec_file(fragments) + StringIO.open(fragments) do |io| + Zip::InputStream.open(io) do |zip| + process_zip_entries(zip) + end + rescue Zip::Error => e + raise ExtractionError, "Error opening zip stream: #{e.message}" + end + end + + def process_zip_entries(zip) + while (entry = zip.get_next_entry) # rubocop:disable Lint/AssignmentInCondition + next unless entry.name.end_with?(METADATA_FILE_EXTENSION) + + raise ExtractionError, 'nuspec file too big' if entry.size > MAX_FILE_SIZE + + return extract_file_content(entry) + end + end + + def extract_file_content(entry) + Tempfile.create('extract_remote_metadata_file_service') do |file| + entry.extract(file.path) { true } # allow #extract to overwrite the file + file.read + end + rescue Zip::DecompressionError + '' # Ignore decompression errors and continue reading the next fragment + rescue Zip::EntrySizeError => e + raise ExtractionError, "nuspec file has the wrong entry size: #{e.message}" + end + end + end +end diff --git a/app/services/packages/nuget/metadata_extraction_service.rb b/app/services/packages/nuget/metadata_extraction_service.rb index e1ee29ef2c6..2c758a5ec20 100644 --- a/app/services/packages/nuget/metadata_extraction_service.rb +++ b/app/services/packages/nuget/metadata_extraction_service.rb @@ -3,8 +3,8 @@ module Packages module Nuget class MetadataExtractionService - def initialize(package_file_id) - @package_file_id = package_file_id + def initialize(package_file) + @package_file = package_file end def execute @@ -13,18 +13,18 @@ module Packages private - attr_reader :package_file_id + attr_reader :package_file - def nuspec_file_content - ExtractMetadataFileService - .new(package_file_id) + def metadata + ExtractMetadataContentService + .new(nuspec_file_content) .execute .payload end - def metadata - ExtractMetadataContentService - .new(nuspec_file_content) + def nuspec_file_content + ExtractMetadataFileService + .new(package_file) .execute .payload end diff --git a/app/services/packages/nuget/update_package_from_metadata_service.rb b/app/services/packages/nuget/update_package_from_metadata_service.rb index 73a52ea569f..258f8c8f6aa 100644 --- a/app/services/packages/nuget/update_package_from_metadata_service.rb +++ b/app/services/packages/nuget/update_package_from_metadata_service.rb @@ -148,7 +148,7 @@ module Packages end def metadata - ::Packages::Nuget::MetadataExtractionService.new(@package_file.id).execute.payload + ::Packages::Nuget::MetadataExtractionService.new(@package_file).execute.payload end strong_memoize_attr :metadata -- cgit v1.2.3