diff options
Diffstat (limited to 'app/services/packages')
10 files changed, 164 insertions, 265 deletions
diff --git a/app/services/packages/debian/find_or_create_package_service.rb b/app/services/packages/debian/find_or_create_package_service.rb deleted file mode 100644 index a9481504d2b..00000000000 --- a/app/services/packages/debian/find_or_create_package_service.rb +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true - -module Packages - module Debian - class FindOrCreatePackageService < ::Packages::CreatePackageService - include Gitlab::Utils::StrongMemoize - - def execute - packages = project.packages - .existing_debian_packages_with(name: params[:name], version: params[:version]) - - package = packages.with_debian_codename_or_suite(params[:distribution_name]).first - - unless package - package_in_other_distribution = packages.first - - if package_in_other_distribution - raise ArgumentError, "Debian package #{params[:name]} #{params[:version]} exists " \ - "in distribution #{package_in_other_distribution.debian_distribution.codename}" - end - end - - package ||= create_package!( - :debian, - debian_publication_attributes: { distribution_id: distribution.id } - ) - - ServiceResponse.success(payload: { package: package }) - end - - private - - def distribution - Packages::Debian::DistributionsFinder.new( - project, - codename_or_suite: params[:distribution_name] - ).execute.last! - end - strong_memoize_attr :distribution - end - end -end diff --git a/app/services/packages/debian/process_changes_service.rb b/app/services/packages/debian/process_changes_service.rb deleted file mode 100644 index eb88e7c9b59..00000000000 --- a/app/services/packages/debian/process_changes_service.rb +++ /dev/null @@ -1,113 +0,0 @@ -# frozen_string_literal: true - -module Packages - module Debian - class ProcessChangesService - include ExclusiveLeaseGuard - include Gitlab::Utils::StrongMemoize - - # used by ExclusiveLeaseGuard - DEFAULT_LEASE_TIMEOUT = 1.hour.to_i.freeze - - def initialize(package_file, creator) - @package_file = package_file - @creator = creator - end - - def execute - # return if changes file has already been processed - return if package_file.debian_file_metadatum&.changes? - - validate! - - try_obtain_lease do - package_file.transaction do - update_files_metadata - update_changes_metadata - end - - ::Packages::Debian::GenerateDistributionWorker.perform_async(:project, package.debian_distribution.id) - end - end - - private - - attr_reader :package_file, :creator - - def validate! - raise ArgumentError, 'invalid package file' unless package_file.debian_file_metadatum - raise ArgumentError, 'invalid package file' unless package_file.debian_file_metadatum.unknown? - raise ArgumentError, 'invalid package file' unless metadata[:file_type] == :changes - raise ArgumentError, 'missing Source field' unless metadata.dig(:fields, 'Source').present? - raise ArgumentError, 'missing Version field' unless metadata.dig(:fields, 'Version').present? - raise ArgumentError, 'missing Distribution field' unless metadata.dig(:fields, 'Distribution').present? - end - - def update_files_metadata - files.each do |filename, entry| - file_metadata = ::Packages::Debian::ExtractMetadataService.new(entry.package_file).execute - - ::Packages::UpdatePackageFileService.new(entry.package_file, package_id: package.id) - .execute - - # Force reload from database, as package has changed - entry.package_file.reload_package - - entry.package_file.debian_file_metadatum.update!( - file_type: file_metadata[:file_type], - component: files[filename].component, - architecture: file_metadata[:architecture], - fields: file_metadata[:fields] - ) - end - end - - def update_changes_metadata - ::Packages::UpdatePackageFileService.new(package_file, package_id: package.id) - .execute - - # Force reload from database, as package has changed - package_file.reload_package - - package_file.debian_file_metadatum.update!( - file_type: metadata[:file_type], - fields: metadata[:fields] - ) - end - - def metadata - ::Packages::Debian::ExtractChangesMetadataService.new(package_file).execute - end - strong_memoize_attr :metadata - - def files - metadata[:files] - end - - def project - package_file.package.project - end - - def package - params = { - name: metadata[:fields]['Source'], - version: metadata[:fields]['Version'], - distribution_name: metadata[:fields]['Distribution'] - } - response = Packages::Debian::FindOrCreatePackageService.new(project, creator, params).execute - response.payload[:package] - end - strong_memoize_attr :package - - # used by ExclusiveLeaseGuard - def lease_key - "packages:debian:process_changes_service:package_file:#{package_file.id}" - end - - # used by ExclusiveLeaseGuard - def lease_timeout - DEFAULT_LEASE_TIMEOUT - end - end - end -end diff --git a/app/services/packages/npm/create_metadata_cache_service.rb b/app/services/packages/npm/create_metadata_cache_service.rb index 75cff5c5453..f470b9f1202 100644 --- a/app/services/packages/npm/create_metadata_cache_service.rb +++ b/app/services/packages/npm/create_metadata_cache_service.rb @@ -30,7 +30,7 @@ module Packages attr_reader :package_name, :project def metadata_content - metadata.payload.to_json + ::API::Entities::NpmPackage.represent(metadata.payload).to_json end strong_memoize_attr :metadata_content diff --git a/app/services/packages/npm/create_package_service.rb b/app/services/packages/npm/create_package_service.rb index 2c578760cc5..f6f2dbb8415 100644 --- a/app/services/packages/npm/create_package_service.rb +++ b/app/services/packages/npm/create_package_service.rb @@ -18,7 +18,7 @@ module Packages ApplicationRecord.transaction { create_npm_package! } end - return error('Could not obtain package lease.', 400) unless package + return error('Could not obtain package lease. Please try again.', 400) unless package package end @@ -40,7 +40,7 @@ module Packages def create_npm_metadatum!(package) package.create_npm_metadatum!(package_json: package_json) rescue ActiveRecord::RecordInvalid => e - if package.npm_metadatum && package.npm_metadatum.errors.added?(:package_json, 'structure is too large') + if package.npm_metadatum && package.npm_metadatum.errors.where(:package_json, :too_large).any? # rubocop: disable CodeReuse/ActiveRecord Gitlab::ErrorTracking.track_exception(e, field_sizes: field_sizes_for_error_tracking) end diff --git a/app/services/packages/npm/deprecate_package_service.rb b/app/services/packages/npm/deprecate_package_service.rb index 2633e9f877c..bca81ebe1de 100644 --- a/app/services/packages/npm/deprecate_package_service.rb +++ b/app/services/packages/npm/deprecate_package_service.rb @@ -31,7 +31,7 @@ module Packages def packages ::Packages::Npm::PackageFinder - .new(params['package_name'], project: project, last_of_each_version: false) + .new(params['package_name'], project: project) .execute end diff --git a/app/services/packages/npm/generate_metadata_service.rb b/app/services/packages/npm/generate_metadata_service.rb index 800c3ce19b4..e1795079513 100644 --- a/app/services/packages/npm/generate_metadata_service.rb +++ b/app/services/packages/npm/generate_metadata_service.rb @@ -98,7 +98,7 @@ module Packages end def package_tags - Packages::Tag.for_package_ids(packages.last_of_each_version_ids) + Packages::Tag.for_package_ids(packages) .preload_package end diff --git a/app/services/packages/nuget/extract_metadata_content_service.rb b/app/services/packages/nuget/extract_metadata_content_service.rb new file mode 100644 index 00000000000..28653654018 --- /dev/null +++ b/app/services/packages/nuget/extract_metadata_content_service.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module Packages + module Nuget + class ExtractMetadataContentService + ROOT_XPATH = '//xmlns:package/xmlns:metadata/xmlns' + + XPATHS = { + package_name: "#{ROOT_XPATH}:id", + package_version: "#{ROOT_XPATH}:version", + authors: "#{ROOT_XPATH}:authors", + description: "#{ROOT_XPATH}:description", + license_url: "#{ROOT_XPATH}:licenseUrl", + project_url: "#{ROOT_XPATH}:projectUrl", + icon_url: "#{ROOT_XPATH}:iconUrl" + }.freeze + + XPATH_DEPENDENCIES = "#{ROOT_XPATH}:dependencies/xmlns:dependency".freeze + XPATH_DEPENDENCY_GROUPS = "#{ROOT_XPATH}:dependencies/xmlns:group".freeze + XPATH_TAGS = "#{ROOT_XPATH}:tags".freeze + XPATH_PACKAGE_TYPES = "#{ROOT_XPATH}:packageTypes/xmlns:packageType".freeze + + def initialize(nuspec_file_content) + @nuspec_file_content = nuspec_file_content + end + + def execute + ServiceResponse.success(payload: extract_metadata(nuspec_file_content)) + end + + private + + attr_reader :nuspec_file_content + + def extract_metadata(file) + doc = Nokogiri::XML(file) + + XPATHS.transform_values { |query| doc.xpath(query).text.presence } + .compact + .tap do |metadata| + metadata[:package_dependencies] = extract_dependencies(doc) + metadata[:package_tags] = extract_tags(doc) + metadata[:package_types] = extract_package_types(doc) + end + end + + def extract_dependencies(doc) + dependencies = [] + + doc.xpath(XPATH_DEPENDENCIES).each do |node| + dependencies << extract_dependency(node) + end + + doc.xpath(XPATH_DEPENDENCY_GROUPS).each do |group_node| + target_framework = group_node.attr('targetFramework') + + group_node.xpath('xmlns:dependency').each do |node| + dependencies << extract_dependency(node).merge(target_framework: target_framework) + end + end + + dependencies + end + + def extract_dependency(node) + { + name: node.attr('id'), + version: node.attr('version') + }.compact + end + + def extract_tags(doc) + tags = doc.xpath(XPATH_TAGS).text + + return [] if tags.blank? + + tags.split(::Packages::Tag::NUGET_TAGS_SEPARATOR) + end + + def extract_package_types(doc) + doc.xpath(XPATH_PACKAGE_TYPES).map { |node| node.attr('name') }.uniq + end + end + end +end diff --git a/app/services/packages/nuget/extract_metadata_file_service.rb b/app/services/packages/nuget/extract_metadata_file_service.rb new file mode 100644 index 00000000000..61e4892fee7 --- /dev/null +++ b/app/services/packages/nuget/extract_metadata_file_service.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Packages + module Nuget + class ExtractMetadataFileService + include Gitlab::Utils::StrongMemoize + + ExtractionError = Class.new(StandardError) + + MAX_FILE_SIZE = 4.megabytes.freeze + + def initialize(package_file_id) + @package_file_id = package_file_id + end + + def execute + raise ExtractionError, 'invalid package file' unless valid_package_file? + + ServiceResponse.success(payload: nuspec_file_content) + end + + private + + attr_reader :package_file_id + + def package_file + ::Packages::PackageFile.find_by_id(package_file_id) + end + strong_memoize_attr :package_file + + def valid_package_file? + package_file && + package_file.package&.nuget? && + package_file.file.size > 0 # rubocop:disable Style/ZeroLengthPredicate + end + + def nuspec_file_content + with_zip_file do |zip_file| + entry = zip_file.glob('*.nuspec').first + + raise ExtractionError, 'nuspec file not found' unless entry + raise ExtractionError, 'nuspec file too big' if MAX_FILE_SIZE < entry.size + + Tempfile.open("nuget_extraction_package_file_#{package_file_id}") do |file| + entry.extract(file.path) { true } # allow #extract to overwrite the file + file.unlink + file.read + end + rescue Zip::EntrySizeError => e + raise ExtractionError, "nuspec file has the wrong entry size: #{e.message}" + end + end + + def with_zip_file + package_file.file.use_open_file do |open_file| + zip_file = Zip::File.new(open_file, false, true) # rubocop:disable Performance/Rubyzip + yield(zip_file) + end + end + end + end +end diff --git a/app/services/packages/nuget/metadata_extraction_service.rb b/app/services/packages/nuget/metadata_extraction_service.rb index 5c60a2912ae..e1ee29ef2c6 100644 --- a/app/services/packages/nuget/metadata_extraction_service.rb +++ b/app/services/packages/nuget/metadata_extraction_service.rb @@ -3,123 +3,30 @@ module Packages module Nuget class MetadataExtractionService - include Gitlab::Utils::StrongMemoize - - ExtractionError = Class.new(StandardError) - - ROOT_XPATH = '//xmlns:package/xmlns:metadata/xmlns' - - XPATHS = { - package_name: "#{ROOT_XPATH}:id", - package_version: "#{ROOT_XPATH}:version", - authors: "#{ROOT_XPATH}:authors", - description: "#{ROOT_XPATH}:description", - license_url: "#{ROOT_XPATH}:licenseUrl", - project_url: "#{ROOT_XPATH}:projectUrl", - icon_url: "#{ROOT_XPATH}:iconUrl" - }.freeze - - XPATH_DEPENDENCIES = "#{ROOT_XPATH}:dependencies/xmlns:dependency".freeze - XPATH_DEPENDENCY_GROUPS = "#{ROOT_XPATH}:dependencies/xmlns:group".freeze - XPATH_TAGS = "#{ROOT_XPATH}:tags".freeze - XPATH_PACKAGE_TYPES = "#{ROOT_XPATH}:packageTypes/xmlns:packageType".freeze - - MAX_FILE_SIZE = 4.megabytes.freeze - def initialize(package_file_id) @package_file_id = package_file_id end def execute - raise ExtractionError, 'invalid package file' unless valid_package_file? - - extract_metadata(nuspec_file_content) + ServiceResponse.success(payload: metadata) end private - def package_file - ::Packages::PackageFile.find_by_id(@package_file_id) - end - strong_memoize_attr :package_file - - def valid_package_file? - package_file && - package_file.package&.nuget? && - package_file.file.size > 0 # rubocop:disable Style/ZeroLengthPredicate - end - - def extract_metadata(file) - doc = Nokogiri::XML(file) - - XPATHS.transform_values { |query| doc.xpath(query).text.presence } - .compact - .tap do |metadata| - metadata[:package_dependencies] = extract_dependencies(doc) - metadata[:package_tags] = extract_tags(doc) - metadata[:package_types] = extract_package_types(doc) - end - end - - def extract_dependencies(doc) - dependencies = [] - - doc.xpath(XPATH_DEPENDENCIES).each do |node| - dependencies << extract_dependency(node) - end - - doc.xpath(XPATH_DEPENDENCY_GROUPS).each do |group_node| - target_framework = group_node.attr("targetFramework") - - group_node.xpath("xmlns:dependency").each do |node| - dependencies << extract_dependency(node).merge(target_framework: target_framework) - end - end - - dependencies - end - - def extract_dependency(node) - { - name: node.attr('id'), - version: node.attr('version') - }.compact - end - - def extract_package_types(doc) - doc.xpath(XPATH_PACKAGE_TYPES).map { |node| node.attr('name') }.uniq - end - - def extract_tags(doc) - tags = doc.xpath(XPATH_TAGS).text - - return [] if tags.blank? - - tags.split(::Packages::Tag::NUGET_TAGS_SEPARATOR) - end + attr_reader :package_file_id def nuspec_file_content - with_zip_file do |zip_file| - entry = zip_file.glob('*.nuspec').first - - raise ExtractionError, 'nuspec file not found' unless entry - raise ExtractionError, 'nuspec file too big' if MAX_FILE_SIZE < entry.size - - Tempfile.open("nuget_extraction_package_file_#{@package_file_id}") do |file| - entry.extract(file.path) { true } # allow #extract to overwrite the file - file.unlink - file.read - end - rescue Zip::EntrySizeError => e - raise ExtractionError, "nuspec file has the wrong entry size: #{e.message}" - end + ExtractMetadataFileService + .new(package_file_id) + .execute + .payload end - def with_zip_file(&block) - package_file.file.use_open_file do |open_file| - zip_file = Zip::File.new(open_file, false, true) - yield(zip_file) - end + def metadata + ExtractMetadataContentService + .new(nuspec_file_content) + .execute + .payload end end end diff --git a/app/services/packages/nuget/update_package_from_metadata_service.rb b/app/services/packages/nuget/update_package_from_metadata_service.rb index 8e2679db31b..d82509fff5e 100644 --- a/app/services/packages/nuget/update_package_from_metadata_service.rb +++ b/app/services/packages/nuget/update_package_from_metadata_service.rb @@ -145,7 +145,7 @@ module Packages end def metadata - ::Packages::Nuget::MetadataExtractionService.new(@package_file.id).execute + ::Packages::Nuget::MetadataExtractionService.new(@package_file.id).execute.payload end strong_memoize_attr :metadata |