From 9dc93a4519d9d5d7be48ff274127136236a3adb3 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 20 Apr 2021 23:50:22 +0000 Subject: Add latest changes from gitlab-org/gitlab@13-11-stable-ee --- .../packages/composer/composer_json_service.rb | 3 +- .../packages/composer/version_parser_service.rb | 3 +- .../packages/debian/create_distribution_service.rb | 3 +- .../debian/extract_changes_metadata_service.rb | 112 +++++++++++++++++++ .../packages/debian/extract_metadata_service.rb | 21 ++-- .../packages/debian/parse_debian822_service.rb | 2 +- .../packages/debian/process_changes_service.rb | 102 +++++++++++++++++ .../packages/debian/update_distribution_service.rb | 3 +- app/services/packages/go/create_package_service.rb | 70 ++++++++++++ app/services/packages/go/sync_packages_service.rb | 24 ++++ .../maven/find_or_create_package_service.rb | 3 +- .../packages/maven/metadata/sync_service.rb | 15 ++- .../packages/nuget/create_dependency_service.rb | 4 +- .../rubygems/create_dependencies_service.rb | 44 ++++++++ .../packages/rubygems/create_gemspec_service.rb | 42 +++++++ .../rubygems/metadata_extraction_service.rb | 56 ++++++++++ .../packages/rubygems/process_gem_service.rb | 124 +++++++++++++++++++++ 17 files changed, 610 insertions(+), 21 deletions(-) create mode 100644 app/services/packages/debian/extract_changes_metadata_service.rb create mode 100644 app/services/packages/debian/process_changes_service.rb create mode 100644 app/services/packages/go/create_package_service.rb create mode 100644 app/services/packages/go/sync_packages_service.rb create mode 100644 app/services/packages/rubygems/create_dependencies_service.rb create mode 100644 app/services/packages/rubygems/create_gemspec_service.rb create mode 100644 app/services/packages/rubygems/metadata_extraction_service.rb create mode 100644 app/services/packages/rubygems/process_gem_service.rb (limited to 'app/services/packages') diff --git a/app/services/packages/composer/composer_json_service.rb b/app/services/packages/composer/composer_json_service.rb index 98aabd84d3d..f346b654c59 100644 --- a/app/services/packages/composer/composer_json_service.rb +++ b/app/services/packages/composer/composer_json_service.rb @@ -6,7 +6,8 @@ module Packages InvalidJson = Class.new(StandardError) def initialize(project, target) - @project, @target = project, target + @project = project + @target = target end def execute diff --git a/app/services/packages/composer/version_parser_service.rb b/app/services/packages/composer/version_parser_service.rb index 811cac0b3b7..36275d1b680 100644 --- a/app/services/packages/composer/version_parser_service.rb +++ b/app/services/packages/composer/version_parser_service.rb @@ -4,7 +4,8 @@ module Packages module Composer class VersionParserService def initialize(tag_name: nil, branch_name: nil) - @tag_name, @branch_name = tag_name, branch_name + @tag_name = tag_name + @branch_name = branch_name end def execute diff --git a/app/services/packages/debian/create_distribution_service.rb b/app/services/packages/debian/create_distribution_service.rb index c6df033e3c1..f947d2e4293 100644 --- a/app/services/packages/debian/create_distribution_service.rb +++ b/app/services/packages/debian/create_distribution_service.rb @@ -4,7 +4,8 @@ module Packages module Debian class CreateDistributionService def initialize(container, user, params) - @container, @params = container, params + @container = container + @params = params @params[:creator] = user @components = params.delete(:components) || ['main'] diff --git a/app/services/packages/debian/extract_changes_metadata_service.rb b/app/services/packages/debian/extract_changes_metadata_service.rb new file mode 100644 index 00000000000..eb5baa7e53f --- /dev/null +++ b/app/services/packages/debian/extract_changes_metadata_service.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +module Packages + module Debian + class ExtractChangesMetadataService + include Gitlab::Utils::StrongMemoize + + ExtractionError = Class.new(StandardError) + + def initialize(package_file) + @package_file = package_file + @entries = {} + end + + def execute + { + file_type: file_type, + architecture: metadata[:architecture], + fields: fields, + files: files + } + rescue ActiveModel::ValidationError => e + raise ExtractionError.new(e.message) + end + + private + + def metadata + strong_memoize(:metadata) do + ::Packages::Debian::ExtractMetadataService.new(@package_file).execute + end + end + + def file_type + metadata[:file_type] + end + + def fields + metadata[:fields] + end + + def files + strong_memoize(:files) do + raise ExtractionError.new("is not a changes file") unless file_type == :changes + raise ExtractionError.new("Files field is missing") if fields['Files'].blank? + raise ExtractionError.new("Checksums-Sha1 field is missing") if fields['Checksums-Sha1'].blank? + raise ExtractionError.new("Checksums-Sha256 field is missing") if fields['Checksums-Sha256'].blank? + + init_entries_from_files + entries_from_checksums_sha1 + entries_from_checksums_sha256 + entries_from_package_files + + @entries + end + end + + def init_entries_from_files + each_lines_for('Files') do |line| + md5sum, size, section, priority, filename = line.split + entry = FileEntry.new( + filename: filename, + size: size.to_i, + md5sum: md5sum, + section: section, + priority: priority) + + @entries[filename] = entry + end + end + + def entries_from_checksums_sha1 + each_lines_for('Checksums-Sha1') do |line| + sha1sum, size, filename = line.split + entry = @entries[filename] + raise ExtractionError.new("#{filename} is listed in Checksums-Sha1 but not in Files") unless entry + raise ExtractionError.new("Size for #{filename} in Files and Checksums-Sha1 differ") unless entry.size == size.to_i + + entry.sha1sum = sha1sum + end + end + + def entries_from_checksums_sha256 + each_lines_for('Checksums-Sha256') do |line| + sha256sum, size, filename = line.split + entry = @entries[filename] + raise ExtractionError.new("#{filename} is listed in Checksums-Sha256 but not in Files") unless entry + raise ExtractionError.new("Size for #{filename} in Files and Checksums-Sha256 differ") unless entry.size == size.to_i + + entry.sha256sum = sha256sum + end + end + + def each_lines_for(field) + fields[field].split("\n").each do |line| + next if line.blank? + + yield(line) + end + end + + def entries_from_package_files + @entries.each do |filename, entry| + entry.package_file = ::Packages::PackageFileFinder.new(@package_file.package, filename).execute! + entry.validate! + rescue ActiveRecord::RecordNotFound + raise ExtractionError.new("#{filename} is listed in Files but was not uploaded") + end + end + end + end +end diff --git a/app/services/packages/debian/extract_metadata_service.rb b/app/services/packages/debian/extract_metadata_service.rb index fd5832bc0ba..015f472c7c9 100644 --- a/app/services/packages/debian/extract_metadata_service.rb +++ b/app/services/packages/debian/extract_metadata_service.rb @@ -58,21 +58,22 @@ module Packages file_type == :dsc || file_type == :buildinfo || file_type == :changes end - def extracted_fields - if file_type_debian? - package_file.file.use_file do |file_path| - ::Packages::Debian::ExtractDebMetadataService.new(file_path).execute - end - elsif file_type_meta? - package_file.file.use_file do |file_path| - ::Packages::Debian::ParseDebian822Service.new(File.read(file_path)).execute.each_value.first + def fields + strong_memoize(:fields) do + if file_type_debian? + package_file.file.use_file do |file_path| + ::Packages::Debian::ExtractDebMetadataService.new(file_path).execute + end + elsif file_type_meta? + package_file.file.use_file do |file_path| + ::Packages::Debian::ParseDebian822Service.new(File.read(file_path)).execute.each_value.first + end end end end def extract_metadata - fields = extracted_fields - architecture = fields.delete(:Architecture) if file_type_debian? + architecture = fields['Architecture'] if file_type_debian? { file_type: file_type, diff --git a/app/services/packages/debian/parse_debian822_service.rb b/app/services/packages/debian/parse_debian822_service.rb index 665929d2324..8be5fdf3b66 100644 --- a/app/services/packages/debian/parse_debian822_service.rb +++ b/app/services/packages/debian/parse_debian822_service.rb @@ -26,7 +26,7 @@ module Packages section[field] += line[1..] unless paragraph_separator?(line) elsif match = match_section_line(line) section_name = match[:name] if section_name.nil? - field = match[:field].to_sym + field = match[:field] raise InvalidDebian822Error, "Duplicate field '#{field}' in section '#{section_name}'" if section.include?(field) diff --git a/app/services/packages/debian/process_changes_service.rb b/app/services/packages/debian/process_changes_service.rb new file mode 100644 index 00000000000..881ad2c46f4 --- /dev/null +++ b/app/services/packages/debian/process_changes_service.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +module Packages + module Debian + class ProcessChangesService + include ExclusiveLeaseGuard + include Gitlab::Utils::StrongMemoize + + # used by ExclusiveLeaseGuard + DEFAULT_LEASE_TIMEOUT = 1.hour.to_i.freeze + + def initialize(package_file, creator) + @package_file = package_file + @creator = creator + end + + def execute + try_obtain_lease do + # return if changes file has already been processed + break if package_file.debian_file_metadatum&.changes? + + validate! + + package_file.transaction do + update_files_metadata + update_changes_metadata + end + end + end + + private + + attr_reader :package_file, :creator + + def validate! + raise ArgumentError, 'invalid package file' unless package_file.debian_file_metadatum + raise ArgumentError, 'invalid package file' unless package_file.debian_file_metadatum.unknown? + raise ArgumentError, 'invalid package file' unless metadata[:file_type] == :changes + end + + def update_files_metadata + files.each do |filename, entry| + entry.package_file.package = package + + file_metadata = ::Packages::Debian::ExtractMetadataService.new(entry.package_file).execute + + entry.package_file.debian_file_metadatum.update!( + file_type: file_metadata[:file_type], + component: files[filename].component, + architecture: file_metadata[:architecture], + fields: file_metadata[:fields] + ) + entry.package_file.save! + end + end + + def update_changes_metadata + package_file.update!(package: package) + package_file.debian_file_metadatum.update!( + file_type: metadata[:file_type], + fields: metadata[:fields] + ) + end + + def metadata + strong_memoize(:metadata) do + ::Packages::Debian::ExtractChangesMetadataService.new(package_file).execute + end + end + + def files + metadata[:files] + end + + def project + package_file.package.project + end + + def package + strong_memoize(:package) do + params = { + 'name': metadata[:fields]['Source'], + 'version': metadata[:fields]['Version'], + 'distribution_name': metadata[:fields]['Distribution'] + } + response = Packages::Debian::FindOrCreatePackageService.new(project, creator, params).execute + response.payload[:package] + end + end + + # used by ExclusiveLeaseGuard + def lease_key + "packages:debian:process_changes_service:package_file:#{package_file.id}" + end + + # used by ExclusiveLeaseGuard + def lease_timeout + DEFAULT_LEASE_TIMEOUT + end + end + end +end diff --git a/app/services/packages/debian/update_distribution_service.rb b/app/services/packages/debian/update_distribution_service.rb index 5bb59b854e9..95face912d5 100644 --- a/app/services/packages/debian/update_distribution_service.rb +++ b/app/services/packages/debian/update_distribution_service.rb @@ -4,7 +4,8 @@ module Packages module Debian class UpdateDistributionService def initialize(distribution, params) - @distribution, @params = distribution, params + @distribution = distribution + @params = params @components = params.delete(:components) diff --git a/app/services/packages/go/create_package_service.rb b/app/services/packages/go/create_package_service.rb new file mode 100644 index 00000000000..4e8b8ef8d6b --- /dev/null +++ b/app/services/packages/go/create_package_service.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Packages + module Go + class CreatePackageService < BaseService + GoZipSizeError = Class.new(StandardError) + + attr_accessor :version + + def initialize(project, user = nil, version:) + super(project, user) + + @version = version + end + + def execute + # check for existing package to avoid SQL errors due to the index + package = ::Packages::Go::PackageFinder.new(version.mod.project, version.mod.name, version.name).execute + return package if package + + # this can be expensive, so do it outside the transaction + files = {} + files[:mod] = prepare_file(version, :mod, version.gomod) + files[:zip] = prepare_file(version, :zip, version.archive.string) + + ActiveRecord::Base.transaction do + # create new package and files + package = create_package + files.each { |type, (file, digests)| create_file(package, type, file, digests) } + package + end + end + + private + + def prepare_file(version, type, content) + file = CarrierWaveStringFile.new(content) + raise GoZipSizeError, "#{version.mod.name}@#{version.name}.#{type} exceeds size limit" if file.size > project.actual_limits.golang_max_file_size + + digests = { + md5: Digest::MD5.hexdigest(content), + sha1: Digest::SHA1.hexdigest(content), + sha256: Digest::SHA256.hexdigest(content) + } + + [file, digests] + end + + def create_package + version.mod.project.packages.create!( + name: version.mod.name, + version: version.name, + package_type: :golang, + created_at: version.commit.committed_date + ) + end + + def create_file(package, type, file, digests) + CreatePackageFileService.new(package, + file: file, + size: file.size, + file_name: "#{version.name}.#{type}", + file_md5: digests[:md5], + file_sha1: digests[:sha1], + file_sha256: digests[:sha256] + ).execute + end + end + end +end diff --git a/app/services/packages/go/sync_packages_service.rb b/app/services/packages/go/sync_packages_service.rb new file mode 100644 index 00000000000..c35d3600388 --- /dev/null +++ b/app/services/packages/go/sync_packages_service.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Packages + module Go + class SyncPackagesService < BaseService + include Gitlab::Golang + + def initialize(project, ref, path = '') + super(project) + + @ref = ref + @path = path + + raise ArgumentError, 'project is required' unless project + raise ArgumentError, 'ref is required' unless ref + raise ArgumentError, "ref #{ref} not found" unless project.repository.find_tag(ref) || project.repository.find_branch(ref) + end + + def execute_async + Packages::Go::SyncPackagesWorker.perform_async(project.id, @ref, @path) + end + end + end +end diff --git a/app/services/packages/maven/find_or_create_package_service.rb b/app/services/packages/maven/find_or_create_package_service.rb index 401e52f7e51..a6cffa3038c 100644 --- a/app/services/packages/maven/find_or_create_package_service.rb +++ b/app/services/packages/maven/find_or_create_package_service.rb @@ -33,7 +33,8 @@ module Packages # # The first upload has to create the proper package (the one with the version set). if params[:file_name] == Packages::Maven::Metadata.filename && !params[:path]&.ends_with?(SNAPSHOT_TERM) - package_name, version = params[:path], nil + package_name = params[:path] + version = nil else package_name, _, version = params[:path].rpartition('/') end diff --git a/app/services/packages/maven/metadata/sync_service.rb b/app/services/packages/maven/metadata/sync_service.rb index a6534aa706d..48e157d4930 100644 --- a/app/services/packages/maven/metadata/sync_service.rb +++ b/app/services/packages/maven/metadata/sync_service.rb @@ -13,16 +13,20 @@ module Packages def execute return error('Blank package name') unless package_name return error('Not allowed') unless Ability.allowed?(current_user, :destroy_package, project) - return error('Non existing versionless package') unless versionless_package_for_versions - return error('Non existing metadata file for versions') unless metadata_package_file_for_versions + result = success('Non existing versionless package(s). Nothing to do.') + + # update versionless package for plugins if it exists if metadata_package_file_for_plugins result = update_plugins_xml return result if result.error? end - update_versions_xml + # update versionless_package for versions if it exists + return update_versions_xml if metadata_package_file_for_versions + + result end private @@ -79,6 +83,9 @@ module Packages def metadata_package_file_for_plugins strong_memoize(:metadata_package_file_for_plugins) do + pkg_name = package_name_for_plugins + next unless pkg_name + metadata_package_file_for(versionless_package_named(package_name_for_plugins)) end end @@ -106,6 +113,8 @@ module Packages end def package_name_for_plugins + return unless versionless_package_for_versions + group = versionless_package_for_versions.maven_metadatum.app_group group.tr('.', '/') end diff --git a/app/services/packages/nuget/create_dependency_service.rb b/app/services/packages/nuget/create_dependency_service.rb index 19143fe3778..62ab485c0fc 100644 --- a/app/services/packages/nuget/create_dependency_service.rb +++ b/app/services/packages/nuget/create_dependency_service.rb @@ -54,9 +54,9 @@ module Packages end def dependencies_for_create_dependency_service - names_and_versions = @dependencies.map do |dependency| + names_and_versions = @dependencies.to_h do |dependency| [dependency[:name], version_or_empty_string(dependency[:version])] - end.to_h + end { 'dependencies' => names_and_versions } end diff --git a/app/services/packages/rubygems/create_dependencies_service.rb b/app/services/packages/rubygems/create_dependencies_service.rb new file mode 100644 index 00000000000..dea429148cf --- /dev/null +++ b/app/services/packages/rubygems/create_dependencies_service.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Packages + module Rubygems + class CreateDependenciesService + include BulkInsertSafe + + def initialize(package, gemspec) + @package = package + @gemspec = gemspec + end + + def execute + set_dependencies + end + + private + + attr_reader :package, :gemspec + + def set_dependencies + Packages::Dependency.transaction do + dependency_type_rows = gemspec.dependencies.map do |dependency| + dependency = Packages::Dependency.safe_find_or_create_by!( + name: dependency.name, + version_pattern: dependency.requirement.to_s + ) + + { + dependency_id: dependency.id, + package_id: package.id, + dependency_type: :dependencies + } + end + + package.dependency_links.upsert_all( + dependency_type_rows, + unique_by: %i[package_id dependency_id dependency_type] + ) + end + end + end + end +end diff --git a/app/services/packages/rubygems/create_gemspec_service.rb b/app/services/packages/rubygems/create_gemspec_service.rb new file mode 100644 index 00000000000..22533264480 --- /dev/null +++ b/app/services/packages/rubygems/create_gemspec_service.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Packages + module Rubygems + class CreateGemspecService + def initialize(package, gemspec) + @package = package + @gemspec = gemspec + end + + def execute + write_gemspec_to_file + end + + private + + attr_reader :package, :gemspec + + def write_gemspec_to_file + file = Tempfile.new + + begin + content = gemspec.to_ruby + file.write(content) + file.flush + + package.package_files.create!( + file: file, + size: file.size, + file_name: "#{gemspec.name}.gemspec", + file_sha1: Digest::SHA1.hexdigest(content), + file_md5: Digest::MD5.hexdigest(content), + file_sha256: Digest::SHA256.hexdigest(content) + ) + ensure + file.close + file.unlink + end + end + end + end +end diff --git a/app/services/packages/rubygems/metadata_extraction_service.rb b/app/services/packages/rubygems/metadata_extraction_service.rb new file mode 100644 index 00000000000..b3bac1854d7 --- /dev/null +++ b/app/services/packages/rubygems/metadata_extraction_service.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Packages + module Rubygems + class MetadataExtractionService + def initialize(package, gemspec) + @package = package + @gemspec = gemspec + end + + def execute + write_metadata + end + + private + + attr_reader :package, :gemspec + + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/PerceivedComplexity + # rubocop:disable Metrics/CyclomaticComplexity + def write_metadata + metadatum.update!( + authors: gemspec&.authors, + files: gemspec&.files&.to_json, + summary: gemspec&.summary, + description: gemspec&.description, + email: gemspec&.email, + homepage: gemspec&.homepage, + licenses: gemspec&.licenses&.to_json, + metadata: gemspec&.metadata&.to_json, + author: gemspec&.author, + bindir: gemspec&.bindir, + executables: gemspec&.executables&.to_json, + extensions: gemspec&.extensions&.to_json, + extra_rdoc_files: gemspec&.extra_rdoc_files&.to_json, + platform: gemspec&.platform, + post_install_message: gemspec&.post_install_message, + rdoc_options: gemspec&.rdoc_options&.to_json, + require_paths: gemspec&.require_paths&.to_json, + required_ruby_version: gemspec&.required_ruby_version&.to_s, + required_rubygems_version: gemspec&.required_rubygems_version&.to_s, + requirements: gemspec&.requirements&.to_json, + rubygems_version: gemspec&.rubygems_version + ) + end + # rubocop:enable Metrics/AbcSize + # rubocop:enable Metrics/PerceivedComplexity + # rubocop:enable Metrics/CyclomaticComplexity + + def metadatum + Packages::Rubygems::Metadatum.safe_find_or_create_by!(package: package) + end + end + end +end diff --git a/app/services/packages/rubygems/process_gem_service.rb b/app/services/packages/rubygems/process_gem_service.rb new file mode 100644 index 00000000000..59bf2a1ec28 --- /dev/null +++ b/app/services/packages/rubygems/process_gem_service.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +require 'rubygems/package' + +module Packages + module Rubygems + class ProcessGemService + include Gitlab::Utils::StrongMemoize + include ExclusiveLeaseGuard + + ExtractionError = Class.new(StandardError) + DEFAULT_LEASE_TIMEOUT = 1.hour.to_i.freeze + + def initialize(package_file) + @package_file = package_file + end + + def execute + return success if process_gem + + error('Gem was not processed') + end + + private + + attr_reader :package_file + + def process_gem + return false unless package_file + + try_obtain_lease do + package.transaction do + rename_package_and_set_version + rename_package_file + ::Packages::Rubygems::MetadataExtractionService.new(package, gemspec).execute + ::Packages::Rubygems::CreateGemspecService.new(package, gemspec).execute + ::Packages::Rubygems::CreateDependenciesService.new(package, gemspec).execute + cleanup_temp_package + end + end + + true + end + + def rename_package_and_set_version + package.update!( + name: gemspec.name, + version: gemspec.version, + status: :default + ) + end + + def rename_package_file + # Updating file_name updates the path where the file is stored. + # We must pass the file again so that CarrierWave can handle the update + package_file.update!( + file_name: "#{gemspec.name}-#{gemspec.version}.gem", + file: package_file.file, + package_id: package.id + ) + end + + def cleanup_temp_package + temp_package.destroy if package.id != temp_package.id + end + + def gemspec + strong_memoize(:gemspec) do + gem.spec + end + end + + def success + ServiceResponse.success(payload: { package: package }) + end + + def error(message) + ServiceResponse.error(message: message) + end + + def temp_package + strong_memoize(:temp_package) do + package_file.package + end + end + + def package + strong_memoize(:package) do + # if package with name/version already exists, use that package + package = temp_package.project + .packages + .rubygems + .with_name(gemspec.name) + .with_version(gemspec.version.to_s) + .last + package || temp_package + end + end + + def gem + # use_file will set an exclusive lease on the file for as long as + # the resulting gem object is being used. This means we are not + # able to rename the package_file while also using the gem object. + # We need to use a separate AR object to create the gem file to allow + # `package_file` to be free for update so we re-find the file here. + Packages::PackageFile.find(package_file.id).file.use_file do |file_path| + Gem::Package.new(File.open(file_path)) + end + rescue + raise ExtractionError.new('Unable to read gem file') + end + + # used by ExclusiveLeaseGuard + def lease_key + "packages:rubygems:process_gem_service:package:#{package.id}" + end + + # used by ExclusiveLeaseGuard + def lease_timeout + DEFAULT_LEASE_TIMEOUT + end + end + end +end -- cgit v1.2.3