From 6438df3a1e0fb944485cebf07976160184697d72 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Wed, 20 Jan 2021 13:34:23 -0600 Subject: Add latest changes from gitlab-org/gitlab@13-8-stable-ee --- app/services/packages/create_event_service.rb | 16 ++- .../packages/debian/create_package_file_service.rb | 36 +++++++ .../packages/debian/extract_metadata_service.rb | 85 ++++++++++++++++ .../debian/get_or_create_incoming_service.rb | 11 +++ .../maven/find_or_create_package_service.rb | 23 ++++- app/services/packages/nuget/search_service.rb | 110 ++++++++++++++++----- 6 files changed, 244 insertions(+), 37 deletions(-) create mode 100644 app/services/packages/debian/create_package_file_service.rb create mode 100644 app/services/packages/debian/extract_metadata_service.rb create mode 100644 app/services/packages/debian/get_or_create_incoming_service.rb (limited to 'app/services/packages') diff --git a/app/services/packages/create_event_service.rb b/app/services/packages/create_event_service.rb index f0328ceb08a..63248ef07c9 100644 --- a/app/services/packages/create_event_service.rb +++ b/app/services/packages/create_event_service.rb @@ -3,11 +3,13 @@ module Packages class CreateEventService < BaseService def execute - if Feature.enabled?(:collect_package_events_redis) && redis_event_name - if guest? - ::Gitlab::UsageDataCounters::GuestPackageEventCounter.count(redis_event_name) - else - ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event(current_user.id, redis_event_name) + if Feature.enabled?(:collect_package_events_redis, default_enabled: true) + ::Packages::Event.unique_counters_for(event_scope, event_name, originator_type).each do |event_name| + ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: current_user.id) + end + + ::Packages::Event.counters_for(event_scope, event_name, originator_type).each do |event_name| + ::Gitlab::UsageDataCounters::PackageEventCounter.count(event_name) end end @@ -23,10 +25,6 @@ module Packages private - def redis_event_name - @redis_event_name ||= ::Packages::Event.allowed_event_name(event_scope, event_name, originator_type) - end - def event_scope @event_scope ||= scope.is_a?(::Packages::Package) ? scope.package_type : scope end diff --git a/app/services/packages/debian/create_package_file_service.rb b/app/services/packages/debian/create_package_file_service.rb new file mode 100644 index 00000000000..2022a63a725 --- /dev/null +++ b/app/services/packages/debian/create_package_file_service.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Packages + module Debian + class CreatePackageFileService + def initialize(package, params) + @package = package + @params = params + end + + def execute + raise ArgumentError, "Invalid package" unless package.present? + + # Debian package file are first uploaded to incoming with empty metadata, + # and are moved later by Packages::Debian::ProcessChangesService + package.package_files.create!( + file: params[:file], + size: params[:file]&.size, + file_name: params[:file_name], + file_sha1: params[:file_sha1], + file_sha256: params[:file]&.sha256, + file_md5: params[:file_md5], + debian_file_metadatum_attributes: { + file_type: 'unknown', + architecture: nil, + fields: nil + } + ) + end + + private + + attr_reader :package, :params + end + end +end diff --git a/app/services/packages/debian/extract_metadata_service.rb b/app/services/packages/debian/extract_metadata_service.rb new file mode 100644 index 00000000000..fd5832bc0ba --- /dev/null +++ b/app/services/packages/debian/extract_metadata_service.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module Packages + module Debian + class ExtractMetadataService + include Gitlab::Utils::StrongMemoize + + ExtractionError = Class.new(StandardError) + + def initialize(package_file) + @package_file = package_file + end + + def execute + raise ExtractionError.new('invalid package file') unless valid_package_file? + + extract_metadata + end + + private + + attr_reader :package_file + + def valid_package_file? + package_file && + package_file.package&.debian? && + package_file.file.size > 0 # rubocop:disable Style/ZeroLengthPredicate + end + + def file_type_basic + %i[dsc deb udeb buildinfo changes].each do |format| + return format if package_file.file_name.end_with?(".#{format}") + end + + nil + end + + def file_type_source + # https://manpages.debian.org/buster/dpkg-dev/dpkg-source.1.en.html + %i[gzip bzip2 lzma xz].each do |format| + return :source if package_file.file_name.end_with?(".tar.#{format}") + end + + nil + end + + def file_type + strong_memoize(:file_type) do + file_type_basic || file_type_source || :unknown + end + end + + def file_type_debian? + file_type == :deb || file_type == :udeb + end + + def file_type_meta? + file_type == :dsc || file_type == :buildinfo || file_type == :changes + end + + def extracted_fields + if file_type_debian? + package_file.file.use_file do |file_path| + ::Packages::Debian::ExtractDebMetadataService.new(file_path).execute + end + elsif file_type_meta? + package_file.file.use_file do |file_path| + ::Packages::Debian::ParseDebian822Service.new(File.read(file_path)).execute.each_value.first + end + end + end + + def extract_metadata + fields = extracted_fields + architecture = fields.delete(:Architecture) if file_type_debian? + + { + file_type: file_type, + architecture: architecture, + fields: fields + } + end + end + end +end diff --git a/app/services/packages/debian/get_or_create_incoming_service.rb b/app/services/packages/debian/get_or_create_incoming_service.rb new file mode 100644 index 00000000000..09e7877a2b4 --- /dev/null +++ b/app/services/packages/debian/get_or_create_incoming_service.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Packages + module Debian + class GetOrCreateIncomingService < ::Packages::CreatePackageService + def execute + find_or_create_package!(:debian, name: 'incoming', version: nil) + end + end + end +end diff --git a/app/services/packages/maven/find_or_create_package_service.rb b/app/services/packages/maven/find_or_create_package_service.rb index f598b5e7cd4..8ee449cbfdc 100644 --- a/app/services/packages/maven/find_or_create_package_service.rb +++ b/app/services/packages/maven/find_or_create_package_service.rb @@ -2,14 +2,23 @@ module Packages module Maven class FindOrCreatePackageService < BaseService - MAVEN_METADATA_FILE = 'maven-metadata.xml'.freeze - SNAPSHOT_TERM = '-SNAPSHOT'.freeze + MAVEN_METADATA_FILE = 'maven-metadata.xml' + SNAPSHOT_TERM = '-SNAPSHOT' def execute package = ::Packages::Maven::PackageFinder.new(params[:path], current_user, project: project) .execute + unless Namespace::PackageSetting.duplicates_allowed?(package) + files = package&.package_files || [] + current_maven_files = files.map { |file| extname(file.file_name) } + + if current_maven_files.compact.include?(extname(params[:file_name])) + return ServiceResponse.error(message: 'Duplicate package is not allowed') + end + end + unless package # Maven uploads several files during `mvn deploy` in next order: # - my-company/my-app/1.0-SNAPSHOT/my-app.jar @@ -48,7 +57,15 @@ module Packages package.build_infos.safe_find_or_create_by!(pipeline: params[:build].pipeline) if params[:build].present? - package + ServiceResponse.success(payload: { package: package }) + end + + private + + def extname(filename) + return if filename.blank? + + File.extname(filename) end end end diff --git a/app/services/packages/nuget/search_service.rb b/app/services/packages/nuget/search_service.rb index b95aa30bec1..1eead1e62b3 100644 --- a/app/services/packages/nuget/search_service.rb +++ b/app/services/packages/nuget/search_service.rb @@ -3,6 +3,7 @@ module Packages module Nuget class SearchService < BaseService + include ::Packages::FinderHelper include Gitlab::Utils::StrongMemoize include ActiveRecord::ConnectionAdapters::Quoting @@ -16,8 +17,9 @@ module Packages padding: 0 }.freeze - def initialize(project, search_term, options = {}) - @project = project + def initialize(current_user, project_or_group, search_term, options = {}) + @current_user = current_user + @project_or_group = project_or_group @search_term = search_term @options = DEFAULT_OPTIONS.merge(options) @@ -26,8 +28,8 @@ module Packages end def execute - OpenStruct.new( - total_count: package_names.total_count, + Result.new( + total_count: non_paginated_matching_package_names.count, results: search_packages ) end @@ -39,52 +41,104 @@ module Packages # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24182#technical-notes # and https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource subquery_name = :partition_subquery - arel_table = Arel::Table.new(:partition_subquery) + arel_table = Arel::Table.new(subquery_name) column_names = Packages::Package.column_names.map do |cn| "#{subquery_name}.#{quote_column_name(cn)}" end # rubocop: disable CodeReuse/ActiveRecord - pkgs = Packages::Package.select(column_names.join(',')) - .from(package_names_partition, subquery_name) - .where(arel_table[:row_number].lteq(MAX_VERSIONS_PER_PACKAGE)) + pkgs = Packages::Package + pkgs = pkgs.with(project_ids_cte.to_arel) if use_project_ids_cte? + pkgs = pkgs.select(column_names.join(',')) + .from(package_names_partition, subquery_name) + .where(arel_table[:row_number].lteq(MAX_VERSIONS_PER_PACKAGE)) return pkgs if include_prerelease_versions? # we can't use pkgs.without_version_like since we have a custom from pkgs.where.not(arel_table[:version].matches(PRE_RELEASE_VERSION_MATCHING_TERM)) + # rubocop: enable CodeReuse/ActiveRecord end def package_names_partition + # rubocop: disable CodeReuse/ActiveRecord table_name = quote_table_name(Packages::Package.table_name) name_column = "#{table_name}.#{quote_column_name('name')}" created_at_column = "#{table_name}.#{quote_column_name('created_at')}" select_sql = "ROW_NUMBER() OVER (PARTITION BY #{name_column} ORDER BY #{created_at_column} DESC) AS row_number, #{table_name}.*" - @project.packages - .select(select_sql) - .nuget - .has_version - .without_nuget_temporary_name - .with_name(package_names) + nuget_packages.select(select_sql) + .with_name(paginated_matching_package_names) + .where(project_id: project_ids) + # rubocop: enable CodeReuse/ActiveRecord end - def package_names - strong_memoize(:package_names) do - pkgs = @project.packages - .nuget - .has_version - .without_nuget_temporary_name - .order_name - .select_distinct_name + def paginated_matching_package_names + pkgs = base_matching_package_names + pkgs.page(0) # we're using a padding + .per(per_page) + .padding(padding) + end + + def non_paginated_matching_package_names + # rubocop: disable CodeReuse/ActiveRecord + pkgs = base_matching_package_names + pkgs = pkgs.with(project_ids_cte.to_arel) if use_project_ids_cte? + pkgs + # rubocop: enable CodeReuse/ActiveRecord + end + + def base_matching_package_names + strong_memoize(:base_matching_package_names) do + # rubocop: disable CodeReuse/ActiveRecord + pkgs = nuget_packages.order_name + .select_distinct_name + .where(project_id: project_ids) pkgs = pkgs.without_version_like(PRE_RELEASE_VERSION_MATCHING_TERM) unless include_prerelease_versions? pkgs = pkgs.search_by_name(@search_term) if @search_term.present? - pkgs.page(0) # we're using a padding - .per(per_page) - .padding(padding) + pkgs + # rubocop: enable CodeReuse/ActiveRecord end end + def nuget_packages + Packages::Package.nuget + .has_version + .without_nuget_temporary_name + end + + def project_ids_cte + return unless use_project_ids_cte? + + strong_memoize(:project_ids_cte) do + query = projects_visible_to_user(@current_user, within_group: @project_or_group) + Gitlab::SQL::CTE.new(:project_ids, query.select(:id)) + end + end + + def project_ids + return @project_or_group.id if project? + + if use_project_ids_cte? + # rubocop: disable CodeReuse/ActiveRecord + Project.select(:id) + .from(project_ids_cte.table) + # rubocop: enable CodeReuse/ActiveRecord + end + end + + def use_project_ids_cte? + group? + end + + def project? + @project_or_group.is_a?(::Project) + end + + def group? + @project_or_group.is_a?(::Group) + end + def include_prerelease_versions? @options[:include_prerelease_versions] end @@ -96,6 +150,12 @@ module Packages def per_page [@options[:per_page], MAX_PER_PAGE].min end + + class Result + include ActiveModel::Model + + attr_accessor :results, :total_count + end end end end -- cgit v1.2.3