From ee664acb356f8123f4f6b00b73c1e1cf0866c7fb Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 20 Oct 2022 09:40:42 +0000 Subject: Add latest changes from gitlab-org/gitlab@15-5-stable-ee --- .../packages/debian/create_package_file_service.rb | 14 +++- .../mark_packages_for_destruction_service.rb | 79 ++++++++++++++++++++ app/services/packages/rpm/parse_package_service.rb | 84 ++++++++++++++++++++++ .../rpm/repository_metadata/base_builder.rb | 30 +++++++- .../rpm/repository_metadata/build_primary_xml.rb | 73 +++++++++++++++++++ .../rpm/repository_metadata/build_repomd_xml.rb | 5 +- 6 files changed, 278 insertions(+), 7 deletions(-) create mode 100644 app/services/packages/mark_packages_for_destruction_service.rb create mode 100644 app/services/packages/rpm/parse_package_service.rb (limited to 'app/services/packages') diff --git a/app/services/packages/debian/create_package_file_service.rb b/app/services/packages/debian/create_package_file_service.rb index 53275fdc9bb..19e68183ea2 100644 --- a/app/services/packages/debian/create_package_file_service.rb +++ b/app/services/packages/debian/create_package_file_service.rb @@ -5,18 +5,20 @@ module Packages class CreatePackageFileService include ::Packages::FIPS - def initialize(package, params) + def initialize(package:, current_user:, params: {}) @package = package + @current_user = current_user @params = params end def execute raise DisabledError, 'Debian registry is not FIPS compliant' if Gitlab::FIPS.enabled? raise ArgumentError, "Invalid package" unless package.present? + raise ArgumentError, "Invalid user" unless current_user.present? # Debian package file are first uploaded to incoming with empty metadata, # and are moved later by Packages::Debian::ProcessChangesService - package.package_files.create!( + package_file = package.package_files.create!( file: params[:file], size: params[:file]&.size, file_name: params[:file_name], @@ -29,11 +31,17 @@ module Packages fields: nil } ) + + if params[:file_name].end_with? '.changes' + ::Packages::Debian::ProcessChangesWorker.perform_async(package_file.id, current_user.id) + end + + package_file end private - attr_reader :package, :params + attr_reader :package, :current_user, :params end end end diff --git a/app/services/packages/mark_packages_for_destruction_service.rb b/app/services/packages/mark_packages_for_destruction_service.rb new file mode 100644 index 00000000000..023392cf2d9 --- /dev/null +++ b/app/services/packages/mark_packages_for_destruction_service.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module Packages + class MarkPackagesForDestructionService + include BaseServiceUtility + + BATCH_SIZE = 20 + + UNAUTHORIZED_RESPONSE = ServiceResponse.error( + message: "You don't have the permission to perform this action", + reason: :unauthorized + ).freeze + + ERROR_RESPONSE = ServiceResponse.error( + message: 'Failed to mark the packages as pending destruction' + ).freeze + + SUCCESS_RESPONSE = ServiceResponse.success( + message: 'Packages were successfully marked as pending destruction' + ).freeze + + # Initialize this service with the given packages and user. + # + # * `packages`: must be an ActiveRecord relationship. + # * `current_user`: an User object. Could be nil. + def initialize(packages:, current_user: nil) + @packages = packages + @current_user = current_user + end + + def execute(batch_size: BATCH_SIZE) + no_access = false + min_batch_size = [batch_size, BATCH_SIZE].min + + @packages.each_batch(of: min_batch_size) do |batched_packages| + loaded_packages = batched_packages.including_project_route.to_a + + break no_access = true unless can_destroy_packages?(loaded_packages) + + ::Packages::Package.id_in(loaded_packages.map(&:id)) + .update_all(status: :pending_destruction) + + sync_maven_metadata(loaded_packages) + mark_package_files_for_destruction(loaded_packages) + end + + return UNAUTHORIZED_RESPONSE if no_access + + SUCCESS_RESPONSE + rescue StandardError + ERROR_RESPONSE + end + + private + + def mark_package_files_for_destruction(packages) + ::Packages::MarkPackageFilesForDestructionWorker.bulk_perform_async_with_contexts( + packages, + arguments_proc: -> (package) { package.id }, + context_proc: -> (package) { { project: package.project, user: @current_user } } + ) + end + + def sync_maven_metadata(packages) + maven_packages_with_version = packages.select { |pkg| pkg.maven? && pkg.version? } + ::Packages::Maven::Metadata::SyncWorker.bulk_perform_async_with_contexts( + maven_packages_with_version, + arguments_proc: -> (package) { [@current_user.id, package.project_id, package.name] }, + context_proc: -> (package) { { project: package.project, user: @current_user } } + ) + end + + def can_destroy_packages?(packages) + packages.all? do |package| + can?(@current_user, :destroy_package, package) + end + end + end +end diff --git a/app/services/packages/rpm/parse_package_service.rb b/app/services/packages/rpm/parse_package_service.rb new file mode 100644 index 00000000000..689a161a81a --- /dev/null +++ b/app/services/packages/rpm/parse_package_service.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +module Packages + module Rpm + class ParsePackageService + include ::Gitlab::Utils::StrongMemoize + + BUILD_ATTRIBUTES_METHOD_NAMES = %i[changelogs requirements provides].freeze + STATIC_ATTRIBUTES = %i[name version release summary description arch + license sourcerpm group buildhost packager vendor].freeze + + CHANGELOGS_RPM_KEYS = %i[changelogtext changelogtime].freeze + REQUIREMENTS_RPM_KEYS = %i[requirename requireversion requireflags].freeze + PROVIDES_RPM_KEYS = %i[providename provideflags provideversion].freeze + + def initialize(package_file) + @rpm = RPM::File.new(package_file) + end + + def execute + raise ArgumentError, 'Unable to parse package' unless valid_package? + + { + files: rpm.files || [], + epoch: package_tags[:epoch] || '0', + changelogs: build_changelogs, + requirements: build_requirements, + provides: build_provides + }.merge(extract_static_attributes) + end + + private + + attr_reader :rpm + + def valid_package? + rpm.files && package_tags && true + rescue RuntimeError + # if arr-pm throws an error due to an incorrect file format, + # we just want this validation to fail rather than throw an exception + false + end + + def package_tags + strong_memoize(:package_tags) do + rpm.tags + end + end + + def extract_static_attributes + STATIC_ATTRIBUTES.each_with_object({}) do |attribute, hash| + hash[attribute] = package_tags[attribute] + end + end + + # Define methods for building RPM attribute data from parsed package + # Transform + # changelogtime: [123, 234], + # changelogname: ["First", "Second"] + # changelogtext: ["Work1", "Work2"] + # Into + # changelog: [ + # {changelogname: "First", changelogtext: "Work1", changelogtime: 123}, + # {changelogname: "Second", changelogtext: "Work2", changelogtime: 234} + # ] + BUILD_ATTRIBUTES_METHOD_NAMES.each do |resource| + define_method("build_#{resource}") do + resource_keys = self.class.const_get("#{resource.upcase}_RPM_KEYS", false).dup + return [] if resource_keys.any? { package_tags[_1].blank? } + + first_attributes = package_tags[resource_keys.first] + zipped_data = first_attributes.zip(*resource_keys[1..].map { package_tags[_1] }) + build_hashes(resource_keys, zipped_data) + end + end + + def build_hashes(resource_keys, zipped_data) + zipped_data.map do |data| + resource_keys.zip(data).to_h + end + end + end + end +end diff --git a/app/services/packages/rpm/repository_metadata/base_builder.rb b/app/services/packages/rpm/repository_metadata/base_builder.rb index 9d76336d764..2c0a11457ec 100644 --- a/app/services/packages/rpm/repository_metadata/base_builder.rb +++ b/app/services/packages/rpm/repository_metadata/base_builder.rb @@ -3,17 +3,43 @@ module Packages module Rpm module RepositoryMetadata class BaseBuilder + def initialize(xml: nil, data: {}) + @xml = Nokogiri::XML(xml) if xml.present? + @data = data + end + def execute - build_empty_structure + return build_empty_structure if xml.blank? + + update_xml_document + update_package_count + xml.to_xml end private + attr_reader :xml, :data + def build_empty_structure Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| - xml.public_send(self.class::ROOT_TAG, self.class::ROOT_ATTRIBUTES) # rubocop:disable GitlabSecurity/PublicSend + xml.method_missing(self.class::ROOT_TAG, self.class::ROOT_ATTRIBUTES) end.to_xml end + + def update_xml_document + # Add to the root xml element a new package metadata node + xml.at(self.class::ROOT_TAG).add_child(build_new_node) + end + + def update_package_count + packages_count = xml.css("//#{self.class::ROOT_TAG}/package").count + + xml.at(self.class::ROOT_TAG).attributes["packages"].value = packages_count.to_s + end + + def build_new_node + raise NotImplementedError, "#{self.class} should implement #{__method__}" + end end end end diff --git a/app/services/packages/rpm/repository_metadata/build_primary_xml.rb b/app/services/packages/rpm/repository_metadata/build_primary_xml.rb index affb41677c2..580bf844a0c 100644 --- a/app/services/packages/rpm/repository_metadata/build_primary_xml.rb +++ b/app/services/packages/rpm/repository_metadata/build_primary_xml.rb @@ -9,6 +9,79 @@ module Packages 'xmlns:rpm': 'http://linux.duke.edu/metadata/rpm', packages: '0' }.freeze + + # Nodes that have only text without attributes + REQUIRED_BASE_ATTRIBUTES = %i[name arch summary description].freeze + NOT_REQUIRED_BASE_ATTRIBUTES = %i[url packager].freeze + FORMAT_NODE_BASE_ATTRIBUTES = %i[license vendor group buildhost sourcerpm].freeze + + private + + def build_new_node + builder = Nokogiri::XML::Builder.new do |xml| + xml.package(type: :rpm, 'xmlns:rpm': 'http://linux.duke.edu/metadata/rpm') do + build_required_base_attributes(xml) + build_not_required_base_attributes(xml) + xml.version epoch: data[:epoch], ver: data[:version], rel: data[:release] + xml.checksum data[:checksum], type: 'sha256', pkgid: 'YES' + xml.size package: data[:packagesize], installed: data[:installedsize], archive: data[:archivesize] + xml.time file: data[:filetime], build: data[:buildtime] + xml.location href: data[:location] if data[:location].present? + build_format_node(xml) + end + end + + Nokogiri::XML(builder.to_xml).at('package') + end + + def build_required_base_attributes(xml) + REQUIRED_BASE_ATTRIBUTES.each do |attribute| + xml.method_missing(attribute, data[attribute]) + end + end + + def build_not_required_base_attributes(xml) + NOT_REQUIRED_BASE_ATTRIBUTES.each do |attribute| + xml.method_missing(attribute, data[attribute]) if data[attribute].present? + end + end + + def build_format_node(xml) + xml.format do + build_base_format_attributes(xml) + build_provides_node(xml) + build_requires_node(xml) + end + end + + def build_base_format_attributes(xml) + FORMAT_NODE_BASE_ATTRIBUTES.each do |attribute| + xml[:rpm].method_missing(attribute, data[attribute]) if data[attribute].present? + end + end + + def build_requires_node(xml) + xml[:rpm].requires do + data[:requirements].each do |requires| + xml[:rpm].entry( + name: requires[:requirename], + flags: requires[:requireflags], + ver: requires[:requireversion] + ) + end + end + end + + def build_provides_node(xml) + xml[:rpm].provides do + data[:provides].each do |provides| + xml[:rpm].entry( + name: provides[:providename], + flags: provides[:provideflags], + ver: provides[:provideversion]) + end + end + end end end end diff --git a/app/services/packages/rpm/repository_metadata/build_repomd_xml.rb b/app/services/packages/rpm/repository_metadata/build_repomd_xml.rb index c6cfd77815d..84614196254 100644 --- a/app/services/packages/rpm/repository_metadata/build_repomd_xml.rb +++ b/app/services/packages/rpm/repository_metadata/build_repomd_xml.rb @@ -9,6 +9,7 @@ module Packages xmlns: 'http://linux.duke.edu/metadata/repo', 'xmlns:rpm': 'http://linux.duke.edu/metadata/rpm' }.freeze + ALLOWED_DATA_VALUE_KEYS = %i[checksum open-checksum location timestamp size open-size].freeze # Expected `data` structure # @@ -48,9 +49,9 @@ module Packages end def build_file_info(info, xml) - info.each do |key, attributes| + info.slice(*ALLOWED_DATA_VALUE_KEYS).each do |key, attributes| value = attributes.delete(:value) - xml.public_send(key, value, attributes) # rubocop:disable GitlabSecurity/PublicSend + xml.method_missing(key, value, attributes) end end end -- cgit v1.2.3