From a09983ae35713f5a2bbb100981116d31ce99826e Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Mon, 20 Jul 2020 12:26:25 +0000 Subject: Add latest changes from gitlab-org/gitlab@13-2-stable-ee --- .../packages/composer/composer_json_service.rb | 31 +++++ .../packages/composer/create_package_service.rb | 57 ++++++++++ .../packages/composer/version_parser_service.rb | 33 ++++++ .../packages/conan/create_package_file_service.rb | 31 +++++ .../packages/conan/create_package_service.rb | 19 ++++ app/services/packages/conan/search_service.rb | 58 ++++++++++ app/services/packages/create_dependency_service.rb | 82 ++++++++++++++ .../packages/create_package_file_service.rb | 22 ++++ .../packages/maven/create_package_service.rb | 28 +++++ .../maven/find_or_create_package_service.rb | 41 +++++++ .../packages/npm/create_package_service.rb | 91 +++++++++++++++ app/services/packages/npm/create_tag_service.rb | 34 ++++++ .../packages/nuget/create_dependency_service.rb | 71 ++++++++++++ .../packages/nuget/create_package_service.rb | 23 ++++ .../packages/nuget/metadata_extraction_service.rb | 106 +++++++++++++++++ app/services/packages/nuget/search_service.rb | 101 +++++++++++++++++ .../packages/nuget/sync_metadatum_service.rb | 50 +++++++++ .../nuget/update_package_from_metadata_service.rb | 125 +++++++++++++++++++++ .../packages/pypi/create_package_service.rb | 40 +++++++ app/services/packages/remove_tag_service.rb | 16 +++ app/services/packages/update_tags_service.rb | 41 +++++++ 21 files changed, 1100 insertions(+) create mode 100644 app/services/packages/composer/composer_json_service.rb create mode 100644 app/services/packages/composer/create_package_service.rb create mode 100644 app/services/packages/composer/version_parser_service.rb create mode 100644 app/services/packages/conan/create_package_file_service.rb create mode 100644 app/services/packages/conan/create_package_service.rb create mode 100644 app/services/packages/conan/search_service.rb create mode 100644 app/services/packages/create_dependency_service.rb create mode 100644 app/services/packages/create_package_file_service.rb create mode 100644 app/services/packages/maven/create_package_service.rb create mode 100644 app/services/packages/maven/find_or_create_package_service.rb create mode 100644 app/services/packages/npm/create_package_service.rb create mode 100644 app/services/packages/npm/create_tag_service.rb create mode 100644 app/services/packages/nuget/create_dependency_service.rb create mode 100644 app/services/packages/nuget/create_package_service.rb create mode 100644 app/services/packages/nuget/metadata_extraction_service.rb create mode 100644 app/services/packages/nuget/search_service.rb create mode 100644 app/services/packages/nuget/sync_metadatum_service.rb create mode 100644 app/services/packages/nuget/update_package_from_metadata_service.rb create mode 100644 app/services/packages/pypi/create_package_service.rb create mode 100644 app/services/packages/remove_tag_service.rb create mode 100644 app/services/packages/update_tags_service.rb (limited to 'app/services/packages') diff --git a/app/services/packages/composer/composer_json_service.rb b/app/services/packages/composer/composer_json_service.rb new file mode 100644 index 00000000000..6ffb5a77da3 --- /dev/null +++ b/app/services/packages/composer/composer_json_service.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Packages + module Composer + class ComposerJsonService + def initialize(project, target) + @project, @target = project, target + end + + def execute + composer_json + end + + private + + def composer_json + composer_file = @project.repository.blob_at(@target, 'composer.json') + + composer_file_not_found! unless composer_file + + Gitlab::Json.parse(composer_file.data) + rescue JSON::ParserError + raise 'Could not parse composer.json file. Invalid JSON.' + end + + def composer_file_not_found! + raise 'The file composer.json was not found.' + end + end + end +end diff --git a/app/services/packages/composer/create_package_service.rb b/app/services/packages/composer/create_package_service.rb new file mode 100644 index 00000000000..ad5d267698b --- /dev/null +++ b/app/services/packages/composer/create_package_service.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Packages + module Composer + class CreatePackageService < BaseService + include ::Gitlab::Utils::StrongMemoize + + def execute + # fetches json outside of transaction + composer_json + + ::Packages::Package.transaction do + ::Packages::Composer::Metadatum.upsert( + package_id: created_package.id, + target_sha: target, + composer_json: composer_json + ) + end + end + + private + + def created_package + project + .packages + .composer + .safe_find_or_create_by!(name: package_name, version: package_version) + end + + def composer_json + strong_memoize(:composer_json) do + ::Packages::Composer::ComposerJsonService.new(project, target).execute + end + end + + def package_name + composer_json['name'] + end + + def target + (branch || tag).target + end + + def branch + params[:branch] + end + + def tag + params[:tag] + end + + def package_version + ::Packages::Composer::VersionParserService.new(tag_name: tag&.name, branch_name: branch&.name).execute + end + end + end +end diff --git a/app/services/packages/composer/version_parser_service.rb b/app/services/packages/composer/version_parser_service.rb new file mode 100644 index 00000000000..76dfd7a14bd --- /dev/null +++ b/app/services/packages/composer/version_parser_service.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Packages + module Composer + class VersionParserService + def initialize(tag_name: nil, branch_name: nil) + @tag_name, @branch_name = tag_name, branch_name + end + + def execute + if @tag_name.present? + @tag_name.match(Gitlab::Regex.composer_package_version_regex).captures[0] + elsif @branch_name.present? + branch_sufix_or_prefix(@branch_name.match(Gitlab::Regex.composer_package_version_regex)) + end + end + + private + + def branch_sufix_or_prefix(match) + if match + if match.captures[1] == '.x' + match.captures[0] + '-dev' + else + match.captures[0] + '.x-dev' + end + else + "dev-#{@branch_name}" + end + end + end + end +end diff --git a/app/services/packages/conan/create_package_file_service.rb b/app/services/packages/conan/create_package_file_service.rb new file mode 100644 index 00000000000..2db5c4e507b --- /dev/null +++ b/app/services/packages/conan/create_package_file_service.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Packages + module Conan + class CreatePackageFileService + attr_reader :package, :file, :params + + def initialize(package, file, params) + @package = package + @file = file + @params = params + end + + def execute + package.package_files.create!( + file: file, + size: params['file.size'], + file_name: params[:file_name], + file_sha1: params['file.sha1'], + file_md5: params['file.md5'], + conan_file_metadatum_attributes: { + recipe_revision: params[:recipe_revision], + package_revision: params[:package_revision], + conan_package_reference: params[:conan_package_reference], + conan_file_type: params[:conan_file_type] + } + ) + end + end + end +end diff --git a/app/services/packages/conan/create_package_service.rb b/app/services/packages/conan/create_package_service.rb new file mode 100644 index 00000000000..22a0436c5fb --- /dev/null +++ b/app/services/packages/conan/create_package_service.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Packages + module Conan + class CreatePackageService < BaseService + def execute + project.packages.create!( + name: params[:package_name], + version: params[:package_version], + package_type: :conan, + conan_metadatum_attributes: { + package_username: params[:package_username], + package_channel: params[:package_channel] + } + ) + end + end + end +end diff --git a/app/services/packages/conan/search_service.rb b/app/services/packages/conan/search_service.rb new file mode 100644 index 00000000000..4513616bad2 --- /dev/null +++ b/app/services/packages/conan/search_service.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Packages + module Conan + class SearchService < BaseService + include ActiveRecord::Sanitization::ClassMethods + + WILDCARD = '*' + RECIPE_SEPARATOR = '@' + + def initialize(user, params) + super(nil, user, params) + end + + def execute + ServiceResponse.success(payload: { results: search_results }) + end + + private + + def search_results + return [] if wildcard_query? + + return search_for_single_package(sanitized_query) if params[:query].include?(RECIPE_SEPARATOR) + + search_packages(build_query) + end + + def wildcard_query? + params[:query] == WILDCARD + end + + def build_query + return "#{sanitized_query}%" if params[:query].end_with?(WILDCARD) + + sanitized_query + end + + def search_packages(query) + ::Packages::Conan::PackageFinder.new(current_user, query: query).execute.map(&:conan_recipe) + end + + def search_for_single_package(query) + name, version, username, _ = query.split(/[@\/]/) + full_path = Packages::Conan::Metadatum.full_path_from(package_username: username) + project = Project.find_by_full_path(full_path) + return unless current_user.can?(:read_package, project) + + result = project.packages.with_name(name).with_version(version).order_created.last + [result&.conan_recipe].compact + end + + def sanitized_query + @sanitized_query ||= sanitize_sql_like(params[:query].delete(WILDCARD)) + end + end + end +end diff --git a/app/services/packages/create_dependency_service.rb b/app/services/packages/create_dependency_service.rb new file mode 100644 index 00000000000..2999885d55d --- /dev/null +++ b/app/services/packages/create_dependency_service.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true +module Packages + class CreateDependencyService < BaseService + attr_reader :package, :dependencies + + def initialize(package, dependencies) + @package = package + @dependencies = dependencies + end + + def execute + Packages::DependencyLink.dependency_types.each_key do |type| + create_dependency(type) + end + end + + private + + def create_dependency(type) + return unless dependencies[type].is_a?(Hash) + + names_and_version_patterns = dependencies[type] + existing_ids, existing_names = find_existing_ids_and_names(names_and_version_patterns) + dependencies_to_insert = names_and_version_patterns + + if existing_names.any? + dependencies_to_insert = names_and_version_patterns.reject { |k, _| k.in?(existing_names) } + end + + ActiveRecord::Base.transaction do + inserted_ids = bulk_insert_package_dependencies(dependencies_to_insert) + bulk_insert_package_dependency_links(type, (existing_ids + inserted_ids)) + end + end + + def find_existing_ids_and_names(names_and_version_patterns) + ids_and_names = Packages::Dependency.for_package_names_and_version_patterns(names_and_version_patterns) + .pluck_ids_and_names + ids = ids_and_names.map(&:first) || [] + names = ids_and_names.map(&:second) || [] + [ids, names] + end + + def bulk_insert_package_dependencies(names_and_version_patterns) + return [] if names_and_version_patterns.empty? + + rows = names_and_version_patterns.map do |name, version_pattern| + { + name: name, + version_pattern: version_pattern + } + end + + ids = database.bulk_insert(Packages::Dependency.table_name, rows, return_ids: true, on_conflict: :do_nothing) + return ids if ids.size == names_and_version_patterns.size + + Packages::Dependency.uncached do + # The bulk_insert statement above do not dirty the query cache. To make + # sure that the results are fresh from the database and not from a stalled + # and potentially wrong cache, this query has to be done with the query + # chache disabled. + Packages::Dependency.ids_for_package_names_and_version_patterns(names_and_version_patterns) + end + end + + def bulk_insert_package_dependency_links(type, dependency_ids) + rows = dependency_ids.map do |dependency_id| + { + package_id: package.id, + dependency_id: dependency_id, + dependency_type: Packages::DependencyLink.dependency_types[type.to_s] + } + end + + database.bulk_insert(Packages::DependencyLink.table_name, rows) + end + + def database + ::Gitlab::Database + end + end +end diff --git a/app/services/packages/create_package_file_service.rb b/app/services/packages/create_package_file_service.rb new file mode 100644 index 00000000000..0ebceeee779 --- /dev/null +++ b/app/services/packages/create_package_file_service.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true +module Packages + class CreatePackageFileService + attr_reader :package, :params + + def initialize(package, params) + @package = package + @params = params + end + + def execute + package.package_files.create!( + file: params[:file], + size: params[:size], + file_name: params[:file_name], + file_sha1: params[:file_sha1], + file_sha256: params[:file_sha256], + file_md5: params[:file_md5] + ) + end + end +end diff --git a/app/services/packages/maven/create_package_service.rb b/app/services/packages/maven/create_package_service.rb new file mode 100644 index 00000000000..aca5d28ca98 --- /dev/null +++ b/app/services/packages/maven/create_package_service.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true +module Packages + module Maven + class CreatePackageService < BaseService + def execute + app_group, _, app_name = params[:name].rpartition('/') + app_group.tr!('/', '.') + + package = project.packages.create!( + name: params[:name], + version: params[:version], + package_type: :maven, + maven_metadatum_attributes: { + path: params[:path], + app_group: app_group, + app_name: app_name, + app_version: params[:version] + } + ) + + build = params[:build] + package.create_build_info!(pipeline: build.pipeline) if build.present? + + package + end + end + end +end diff --git a/app/services/packages/maven/find_or_create_package_service.rb b/app/services/packages/maven/find_or_create_package_service.rb new file mode 100644 index 00000000000..50a008843ad --- /dev/null +++ b/app/services/packages/maven/find_or_create_package_service.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true +module Packages + module Maven + class FindOrCreatePackageService < BaseService + MAVEN_METADATA_FILE = 'maven-metadata.xml'.freeze + + def execute + package = ::Packages::Maven::PackageFinder + .new(params[:path], current_user, project: project).execute + + unless package + if params[:file_name] == MAVEN_METADATA_FILE + # Maven uploads several files during `mvn deploy` in next order: + # - my-company/my-app/1.0-SNAPSHOT/my-app.jar + # - my-company/my-app/1.0-SNAPSHOT/my-app.pom + # - my-company/my-app/1.0-SNAPSHOT/maven-metadata.xml + # - my-company/my-app/maven-metadata.xml + # + # The last xml file does not have VERSION in URL because it contains + # information about all versions. + package_name, version = params[:path], nil + else + package_name, _, version = params[:path].rpartition('/') + end + + package_params = { + name: package_name, + path: params[:path], + version: version, + build: params[:build] + } + + package = ::Packages::Maven::CreatePackageService + .new(project, current_user, package_params).execute + end + + package + end + end + end +end diff --git a/app/services/packages/npm/create_package_service.rb b/app/services/packages/npm/create_package_service.rb new file mode 100644 index 00000000000..cf927683ce9 --- /dev/null +++ b/app/services/packages/npm/create_package_service.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true +module Packages + module Npm + class CreatePackageService < BaseService + include Gitlab::Utils::StrongMemoize + + def execute + return error('Version is empty.', 400) if version.blank? + return error('Package already exists.', 403) if current_package_exists? + + ActiveRecord::Base.transaction { create_package! } + end + + private + + def create_package! + package = project.packages.create!( + name: name, + version: version, + package_type: 'npm' + ) + + if build.present? + package.create_build_info!(pipeline: build.pipeline) + end + + ::Packages::CreatePackageFileService.new(package, file_params).execute + ::Packages::CreateDependencyService.new(package, package_dependencies).execute + ::Packages::Npm::CreateTagService.new(package, dist_tag).execute + + package + end + + def current_package_exists? + project.packages + .npm + .with_name(name) + .with_version(version) + .exists? + end + + def name + params[:name] + end + + def version + strong_memoize(:version) do + params[:versions].each_key.first + end + end + + def version_data + params[:versions][version] + end + + def build + params[:build] + end + + def dist_tag + params['dist-tags'].each_key.first + end + + def package_file_name + strong_memoize(:package_file_name) do + "#{name}-#{version}.tgz" + end + end + + def attachment + strong_memoize(:attachment) do + params['_attachments'][package_file_name] + end + end + + def file_params + { + file: CarrierWaveStringFile.new(Base64.decode64(attachment['data'])), + size: attachment['length'], + file_sha1: version_data[:dist][:shasum], + file_name: package_file_name + } + end + + def package_dependencies + _version, versions_data = params[:versions].first + versions_data + end + end + end +end diff --git a/app/services/packages/npm/create_tag_service.rb b/app/services/packages/npm/create_tag_service.rb new file mode 100644 index 00000000000..82974d0ca4b --- /dev/null +++ b/app/services/packages/npm/create_tag_service.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true +module Packages + module Npm + class CreateTagService + include Gitlab::Utils::StrongMemoize + + attr_reader :package, :tag_name + + def initialize(package, tag_name) + @package = package + @tag_name = tag_name + end + + def execute + if existing_tag.present? + existing_tag.update_column(:package_id, package.id) + existing_tag + else + package.tags.create!(name: tag_name) + end + end + + private + + def existing_tag + strong_memoize(:existing_tag) do + Packages::TagsFinder + .new(package.project, package.name, package_type: package.package_type) + .find_by_name(tag_name) + end + end + end + end +end diff --git a/app/services/packages/nuget/create_dependency_service.rb b/app/services/packages/nuget/create_dependency_service.rb new file mode 100644 index 00000000000..2be5db732f6 --- /dev/null +++ b/app/services/packages/nuget/create_dependency_service.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true +module Packages + module Nuget + class CreateDependencyService < BaseService + def initialize(package, dependencies = []) + @package = package + @dependencies = dependencies + end + + def execute + return if @dependencies.empty? + + @package.transaction do + create_dependency_links + create_dependency_link_metadata + end + end + + private + + def create_dependency_links + ::Packages::CreateDependencyService + .new(@package, dependencies_for_create_dependency_service) + .execute + end + + def create_dependency_link_metadata + inserted_links = ::Packages::DependencyLink.preload_dependency + .for_package(@package) + + return if inserted_links.empty? + + rows = inserted_links.map do |dependency_link| + raw_dependency = raw_dependency_for(dependency_link.dependency) + + next if raw_dependency[:target_framework].blank? + + { + dependency_link_id: dependency_link.id, + target_framework: raw_dependency[:target_framework] + } + end + + ::Gitlab::Database.bulk_insert(::Packages::Nuget::DependencyLinkMetadatum.table_name, rows.compact) + end + + def raw_dependency_for(dependency) + name = dependency.name + version = dependency.version_pattern.presence + + @dependencies.find do |raw_dependency| + raw_dependency[:name] == name && raw_dependency[:version] == version + end + end + + def dependencies_for_create_dependency_service + names_and_versions = @dependencies.map do |dependency| + [dependency[:name], version_or_empty_string(dependency[:version])] + end.to_h + + { 'dependencies' => names_and_versions } + end + + def version_or_empty_string(version) + return '' if version.blank? + + version + end + end + end +end diff --git a/app/services/packages/nuget/create_package_service.rb b/app/services/packages/nuget/create_package_service.rb new file mode 100644 index 00000000000..68ad7f028e4 --- /dev/null +++ b/app/services/packages/nuget/create_package_service.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Packages + module Nuget + class CreatePackageService < BaseService + TEMPORARY_PACKAGE_NAME = 'NuGet.Temporary.Package' + PACKAGE_VERSION = '0.0.0' + + def execute + project.packages.nuget.create!( + name: TEMPORARY_PACKAGE_NAME, + version: "#{PACKAGE_VERSION}-#{uuid}" + ) + end + + private + + def uuid + SecureRandom.uuid + end + end + end +end diff --git a/app/services/packages/nuget/metadata_extraction_service.rb b/app/services/packages/nuget/metadata_extraction_service.rb new file mode 100644 index 00000000000..6fec398fab0 --- /dev/null +++ b/app/services/packages/nuget/metadata_extraction_service.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +module Packages + module Nuget + class MetadataExtractionService + include Gitlab::Utils::StrongMemoize + + ExtractionError = Class.new(StandardError) + + XPATHS = { + package_name: '//xmlns:package/xmlns:metadata/xmlns:id', + package_version: '//xmlns:package/xmlns:metadata/xmlns:version', + license_url: '//xmlns:package/xmlns:metadata/xmlns:licenseUrl', + project_url: '//xmlns:package/xmlns:metadata/xmlns:projectUrl', + icon_url: '//xmlns:package/xmlns:metadata/xmlns:iconUrl' + }.freeze + + XPATH_DEPENDENCIES = '//xmlns:package/xmlns:metadata/xmlns:dependencies/xmlns:dependency' + XPATH_DEPENDENCY_GROUPS = '//xmlns:package/xmlns:metadata/xmlns:dependencies/xmlns:group' + XPATH_TAGS = '//xmlns:package/xmlns:metadata/xmlns:tags' + + MAX_FILE_SIZE = 4.megabytes.freeze + + def initialize(package_file_id) + @package_file_id = package_file_id + end + + def execute + raise ExtractionError.new('invalid package file') unless valid_package_file? + + extract_metadata(nuspec_file) + end + + private + + def package_file + strong_memoize(:package_file) do + ::Packages::PackageFile.find_by_id(@package_file_id) + end + end + + def valid_package_file? + package_file && + package_file.package&.nuget? && + package_file.file.size.positive? + end + + def extract_metadata(file) + doc = Nokogiri::XML(file) + + XPATHS.transform_values { |query| doc.xpath(query).text.presence } + .compact + .tap do |metadata| + metadata[:package_dependencies] = extract_dependencies(doc) + metadata[:package_tags] = extract_tags(doc) + end + end + + def extract_dependencies(doc) + dependencies = [] + + doc.xpath(XPATH_DEPENDENCIES).each do |node| + dependencies << extract_dependency(node) + end + + doc.xpath(XPATH_DEPENDENCY_GROUPS).each do |group_node| + target_framework = group_node.attr("targetFramework") + + group_node.xpath("xmlns:dependency").each do |node| + dependencies << extract_dependency(node).merge(target_framework: target_framework) + end + end + + dependencies + end + + def extract_dependency(node) + { + name: node.attr('id'), + version: node.attr('version') + }.compact + end + + def extract_tags(doc) + tags = doc.xpath(XPATH_TAGS).text + + return [] if tags.blank? + + tags.split(::Packages::Tag::NUGET_TAGS_SEPARATOR) + end + + def nuspec_file + package_file.file.use_file do |file_path| + Zip::File.open(file_path) do |zip_file| + entry = zip_file.glob('*.nuspec').first + + raise ExtractionError.new('nuspec file not found') unless entry + raise ExtractionError.new('nuspec file too big') if entry.size > MAX_FILE_SIZE + + entry.get_input_stream.read + end + end + end + end + end +end diff --git a/app/services/packages/nuget/search_service.rb b/app/services/packages/nuget/search_service.rb new file mode 100644 index 00000000000..f7e09e11819 --- /dev/null +++ b/app/services/packages/nuget/search_service.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +module Packages + module Nuget + class SearchService < BaseService + include Gitlab::Utils::StrongMemoize + include ActiveRecord::ConnectionAdapters::Quoting + + MAX_PER_PAGE = 30 + MAX_VERSIONS_PER_PACKAGE = 10 + PRE_RELEASE_VERSION_MATCHING_TERM = '%-%' + + DEFAULT_OPTIONS = { + include_prerelease_versions: true, + per_page: Kaminari.config.default_per_page, + padding: 0 + }.freeze + + def initialize(project, search_term, options = {}) + @project = project + @search_term = search_term + @options = DEFAULT_OPTIONS.merge(options) + + raise ArgumentError, 'negative per_page' if per_page.negative? + raise ArgumentError, 'negative padding' if padding.negative? + end + + def execute + OpenStruct.new( + total_count: package_names.total_count, + results: search_packages + ) + end + + private + + def search_packages + # custom query to get package names and versions as expected from the nuget search api + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24182#technical-notes + # and https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource + subquery_name = :partition_subquery + arel_table = Arel::Table.new(:partition_subquery) + column_names = Packages::Package.column_names.map do |cn| + "#{subquery_name}.#{quote_column_name(cn)}" + end + + # rubocop: disable CodeReuse/ActiveRecord + pkgs = Packages::Package.select(column_names.join(',')) + .from(package_names_partition, subquery_name) + .where(arel_table[:row_number].lteq(MAX_VERSIONS_PER_PACKAGE)) + + return pkgs if include_prerelease_versions? + + # we can't use pkgs.without_version_like since we have a custom from + pkgs.where.not(arel_table[:version].matches(PRE_RELEASE_VERSION_MATCHING_TERM)) + end + + def package_names_partition + table_name = quote_table_name(Packages::Package.table_name) + name_column = "#{table_name}.#{quote_column_name('name')}" + created_at_column = "#{table_name}.#{quote_column_name('created_at')}" + select_sql = "ROW_NUMBER() OVER (PARTITION BY #{name_column} ORDER BY #{created_at_column} DESC) AS row_number, #{table_name}.*" + + @project.packages + .select(select_sql) + .nuget + .has_version + .without_nuget_temporary_name + .with_name(package_names) + end + + def package_names + strong_memoize(:package_names) do + pkgs = @project.packages + .nuget + .has_version + .without_nuget_temporary_name + .order_name + .select_distinct_name + pkgs = pkgs.without_version_like(PRE_RELEASE_VERSION_MATCHING_TERM) unless include_prerelease_versions? + pkgs = pkgs.search_by_name(@search_term) if @search_term.present? + pkgs.page(0) # we're using a padding + .per(per_page) + .padding(padding) + end + end + + def include_prerelease_versions? + @options[:include_prerelease_versions] + end + + def padding + @options[:padding] + end + + def per_page + [@options[:per_page], MAX_PER_PAGE].min + end + end + end +end diff --git a/app/services/packages/nuget/sync_metadatum_service.rb b/app/services/packages/nuget/sync_metadatum_service.rb new file mode 100644 index 00000000000..ca9cc4d5b78 --- /dev/null +++ b/app/services/packages/nuget/sync_metadatum_service.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Packages + module Nuget + class SyncMetadatumService + include Gitlab::Utils::StrongMemoize + + def initialize(package, metadata) + @package = package + @metadata = metadata + end + + def execute + if blank_metadata? + metadatum.destroy! if metadatum.persisted? + else + metadatum.update!( + license_url: license_url, + project_url: project_url, + icon_url: icon_url + ) + end + end + + private + + def metadatum + strong_memoize(:metadatum) do + @package.nuget_metadatum || @package.build_nuget_metadatum + end + end + + def blank_metadata? + project_url.blank? && license_url.blank? && icon_url.blank? + end + + def project_url + @metadata[:project_url] + end + + def license_url + @metadata[:license_url] + end + + def icon_url + @metadata[:icon_url] + end + end + end +end diff --git a/app/services/packages/nuget/update_package_from_metadata_service.rb b/app/services/packages/nuget/update_package_from_metadata_service.rb new file mode 100644 index 00000000000..f72b1386985 --- /dev/null +++ b/app/services/packages/nuget/update_package_from_metadata_service.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: true + +module Packages + module Nuget + class UpdatePackageFromMetadataService + include Gitlab::Utils::StrongMemoize + include ExclusiveLeaseGuard + + # used by ExclusiveLeaseGuard + DEFAULT_LEASE_TIMEOUT = 1.hour.to_i.freeze + + InvalidMetadataError = Class.new(StandardError) + + def initialize(package_file) + @package_file = package_file + end + + def execute + raise InvalidMetadataError.new('package name and/or package version not found in metadata') unless valid_metadata? + + try_obtain_lease do + @package_file.transaction do + package = existing_package ? link_to_existing_package : update_linked_package + + update_package(package) + + # Updating file_name updates the path where the file is stored. + # We must pass the file again so that CarrierWave can handle the update + @package_file.update!( + file_name: package_filename, + file: @package_file.file + ) + end + end + end + + private + + def update_package(package) + ::Packages::Nuget::SyncMetadatumService + .new(package, metadata.slice(:project_url, :license_url, :icon_url)) + .execute + ::Packages::UpdateTagsService + .new(package, package_tags) + .execute + rescue => e + raise InvalidMetadataError, e.message + end + + def valid_metadata? + package_name.present? && package_version.present? + end + + def link_to_existing_package + package_to_destroy = @package_file.package + # Updating package_id updates the path where the file is stored. + # We must pass the file again so that CarrierWave can handle the update + @package_file.update!( + package_id: existing_package.id, + file: @package_file.file + ) + package_to_destroy.destroy! + existing_package + end + + def update_linked_package + @package_file.package.update!( + name: package_name, + version: package_version + ) + + ::Packages::Nuget::CreateDependencyService.new(@package_file.package, package_dependencies) + .execute + @package_file.package + end + + def existing_package + strong_memoize(:existing_package) do + @package_file.project.packages + .nuget + .with_name(package_name) + .with_version(package_version) + .first + end + end + + def package_name + metadata[:package_name] + end + + def package_version + metadata[:package_version] + end + + def package_dependencies + metadata.fetch(:package_dependencies, []) + end + + def package_tags + metadata.fetch(:package_tags, []) + end + + def metadata + strong_memoize(:metadata) do + ::Packages::Nuget::MetadataExtractionService.new(@package_file.id).execute + end + end + + def package_filename + "#{package_name.downcase}.#{package_version.downcase}.nupkg" + end + + # used by ExclusiveLeaseGuard + def lease_key + package_id = existing_package ? existing_package.id : @package_file.package_id + "packages:nuget:update_package_from_metadata_service:package:#{package_id}" + end + + # used by ExclusiveLeaseGuard + def lease_timeout + DEFAULT_LEASE_TIMEOUT + end + end + end +end diff --git a/app/services/packages/pypi/create_package_service.rb b/app/services/packages/pypi/create_package_service.rb new file mode 100644 index 00000000000..1313fc80e33 --- /dev/null +++ b/app/services/packages/pypi/create_package_service.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Packages + module Pypi + class CreatePackageService < BaseService + include ::Gitlab::Utils::StrongMemoize + + def execute + ::Packages::Package.transaction do + Packages::Pypi::Metadatum.upsert( + package_id: created_package.id, + required_python: params[:requires_python] + ) + + ::Packages::CreatePackageFileService.new(created_package, file_params).execute + end + end + + private + + def created_package + strong_memoize(:created_package) do + project + .packages + .pypi + .safe_find_or_create_by!(name: params[:name], version: params[:version]) + end + end + + def file_params + { + file: params[:content], + file_name: params[:content].original_filename, + file_md5: params[:md5_digest], + file_sha256: params[:sha256_digest] + } + end + end + end +end diff --git a/app/services/packages/remove_tag_service.rb b/app/services/packages/remove_tag_service.rb new file mode 100644 index 00000000000..465b85506a6 --- /dev/null +++ b/app/services/packages/remove_tag_service.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true +module Packages + class RemoveTagService < BaseService + attr_reader :package_tag + + def initialize(package_tag) + raise ArgumentError, "Package tag must be set" if package_tag.blank? + + @package_tag = package_tag + end + + def execute + package_tag.delete + end + end +end diff --git a/app/services/packages/update_tags_service.rb b/app/services/packages/update_tags_service.rb new file mode 100644 index 00000000000..da50cd3479e --- /dev/null +++ b/app/services/packages/update_tags_service.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true +module Packages + class UpdateTagsService + include Gitlab::Utils::StrongMemoize + + def initialize(package, tags = []) + @package = package + @tags = tags + end + + def execute + return if @tags.empty? + + tags_to_destroy = existing_tags - @tags + tags_to_create = @tags - existing_tags + + @package.tags.with_name(tags_to_destroy).delete_all if tags_to_destroy.any? + ::Gitlab::Database.bulk_insert(Packages::Tag.table_name, rows(tags_to_create)) if tags_to_create.any? + end + + private + + def existing_tags + strong_memoize(:existing_tags) do + @package.tag_names + end + end + + def rows(tags) + now = Time.zone.now + tags.map do |tag| + { + package_id: @package.id, + name: tag, + created_at: now, + updated_at: now + } + end + end + end +end -- cgit v1.2.3