Add latest changes from gitlab-org/gitlab@13-2-stable-ee

21 files changed, 1100 insertions, 0 deletions

diff --git a/app/services/packages/composer/composer_json_service.rb b/app/services/packages/composer/composer_json_service.rb
new file mode 100644
index 00000000000..6ffb5a77da3
--- /dev/null
+++ b/app/services/packages/composer/composer_json_service.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Packages
+  module Composer
+    class ComposerJsonService
+      def initialize(project, target)
+        @project, @target = project, target
+      end
+
+      def execute
+        composer_json
+      end
+
+      private
+
+      def composer_json
+        composer_file = @project.repository.blob_at(@target, 'composer.json')
+
+        composer_file_not_found! unless composer_file
+
+        Gitlab::Json.parse(composer_file.data)
+      rescue JSON::ParserError
+        raise 'Could not parse composer.json file. Invalid JSON.'
+      end
+
+      def composer_file_not_found!
+        raise 'The file composer.json was not found.'
+      end
+    end
+  end
+end
diff --git a/app/services/packages/composer/create_package_service.rb b/app/services/packages/composer/create_package_service.rb
new file mode 100644
index 00000000000..ad5d267698b
--- /dev/null
+++ b/app/services/packages/composer/create_package_service.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Packages
+  module Composer
+    class CreatePackageService < BaseService
+      include ::Gitlab::Utils::StrongMemoize
+
+      def execute
+        # fetches json outside of transaction
+        composer_json
+
+        ::Packages::Package.transaction do
+          ::Packages::Composer::Metadatum.upsert(
+            package_id: created_package.id,
+            target_sha: target,
+            composer_json: composer_json
+          )
+        end
+      end
+
+      private
+
+      def created_package
+        project
+          .packages
+          .composer
+          .safe_find_or_create_by!(name: package_name, version: package_version)
+      end
+
+      def composer_json
+        strong_memoize(:composer_json) do
+          ::Packages::Composer::ComposerJsonService.new(project, target).execute
+        end
+      end
+
+      def package_name
+        composer_json['name']
+      end
+
+      def target
+        (branch || tag).target
+      end
+
+      def branch
+        params[:branch]
+      end
+
+      def tag
+        params[:tag]
+      end
+
+      def package_version
+        ::Packages::Composer::VersionParserService.new(tag_name: tag&.name, branch_name: branch&.name).execute
+      end
+    end
+  end
+end
diff --git a/app/services/packages/composer/version_parser_service.rb b/app/services/packages/composer/version_parser_service.rb
new file mode 100644
index 00000000000..76dfd7a14bd
--- /dev/null
+++ b/app/services/packages/composer/version_parser_service.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Packages
+  module Composer
+    class VersionParserService
+      def initialize(tag_name: nil, branch_name: nil)
+        @tag_name, @branch_name = tag_name, branch_name
+      end
+
+      def execute
+        if @tag_name.present?
+          @tag_name.match(Gitlab::Regex.composer_package_version_regex).captures[0]
+        elsif @branch_name.present?
+          branch_sufix_or_prefix(@branch_name.match(Gitlab::Regex.composer_package_version_regex))
+        end
+      end
+
+      private
+
+      def branch_sufix_or_prefix(match)
+        if match
+          if match.captures[1] == '.x'
+            match.captures[0] + '-dev'
+          else
+            match.captures[0] + '.x-dev'
+          end
+        else
+          "dev-#{@branch_name}"
+        end
+      end
+    end
+  end
+end
diff --git a/app/services/packages/conan/create_package_file_service.rb b/app/services/packages/conan/create_package_file_service.rb
new file mode 100644
index 00000000000..2db5c4e507b
--- /dev/null
+++ b/app/services/packages/conan/create_package_file_service.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Packages
+  module Conan
+    class CreatePackageFileService
+      attr_reader :package, :file, :params
+
+      def initialize(package, file, params)
+        @package = package
+        @file = file
+        @params = params
+      end
+
+      def execute
+        package.package_files.create!(
+          file:      file,
+          size:      params['file.size'],
+          file_name: params[:file_name],
+          file_sha1: params['file.sha1'],
+          file_md5:  params['file.md5'],
+          conan_file_metadatum_attributes: {
+            recipe_revision: params[:recipe_revision],
+            package_revision: params[:package_revision],
+            conan_package_reference: params[:conan_package_reference],
+            conan_file_type: params[:conan_file_type]
+          }
+        )
+      end
+    end
+  end
+end
diff --git a/app/services/packages/conan/create_package_service.rb b/app/services/packages/conan/create_package_service.rb
new file mode 100644
index 00000000000..22a0436c5fb
--- /dev/null
+++ b/app/services/packages/conan/create_package_service.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Packages
+  module Conan
+    class CreatePackageService < BaseService
+      def execute
+        project.packages.create!(
+          name: params[:package_name],
+          version: params[:package_version],
+          package_type: :conan,
+          conan_metadatum_attributes: {
+            package_username: params[:package_username],
+            package_channel: params[:package_channel]
+          }
+        )
+      end
+    end
+  end
+end
diff --git a/app/services/packages/conan/search_service.rb b/app/services/packages/conan/search_service.rb
new file mode 100644
index 00000000000..4513616bad2
--- /dev/null
+++ b/app/services/packages/conan/search_service.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Packages
+  module Conan
+    class SearchService < BaseService
+      include ActiveRecord::Sanitization::ClassMethods
+
+      WILDCARD = '*'
+      RECIPE_SEPARATOR = '@'
+
+      def initialize(user, params)
+        super(nil, user, params)
+      end
+
+      def execute
+        ServiceResponse.success(payload: { results: search_results })
+      end
+
+      private
+
+      def search_results
+        return [] if wildcard_query?
+
+        return search_for_single_package(sanitized_query) if params[:query].include?(RECIPE_SEPARATOR)
+
+        search_packages(build_query)
+      end
+
+      def wildcard_query?
+        params[:query] == WILDCARD
+      end
+
+      def build_query
+        return "#{sanitized_query}%" if params[:query].end_with?(WILDCARD)
+
+        sanitized_query
+      end
+
+      def search_packages(query)
+        ::Packages::Conan::PackageFinder.new(current_user, query: query).execute.map(&:conan_recipe)
+      end
+
+      def search_for_single_package(query)
+        name, version, username, _ = query.split(/[@\/]/)
+        full_path = Packages::Conan::Metadatum.full_path_from(package_username: username)
+        project = Project.find_by_full_path(full_path)
+        return unless current_user.can?(:read_package, project)
+
+        result = project.packages.with_name(name).with_version(version).order_created.last
+        [result&.conan_recipe].compact
+      end
+
+      def sanitized_query
+        @sanitized_query ||= sanitize_sql_like(params[:query].delete(WILDCARD))
+      end
+    end
+  end
+end
diff --git a/app/services/packages/create_dependency_service.rb b/app/services/packages/create_dependency_service.rb
new file mode 100644
index 00000000000..2999885d55d
--- /dev/null
+++ b/app/services/packages/create_dependency_service.rb
@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+module Packages
+  class CreateDependencyService < BaseService
+    attr_reader :package, :dependencies
+
+    def initialize(package, dependencies)
+      @package = package
+      @dependencies = dependencies
+    end
+
+    def execute
+      Packages::DependencyLink.dependency_types.each_key do |type|
+        create_dependency(type)
+      end
+    end
+
+    private
+
+    def create_dependency(type)
+      return unless dependencies[type].is_a?(Hash)
+
+      names_and_version_patterns = dependencies[type]
+      existing_ids, existing_names = find_existing_ids_and_names(names_and_version_patterns)
+      dependencies_to_insert = names_and_version_patterns
+
+      if existing_names.any?
+        dependencies_to_insert = names_and_version_patterns.reject { |k, _| k.in?(existing_names) }
+      end
+
+      ActiveRecord::Base.transaction do
+        inserted_ids = bulk_insert_package_dependencies(dependencies_to_insert)
+        bulk_insert_package_dependency_links(type, (existing_ids + inserted_ids))
+      end
+    end
+
+    def find_existing_ids_and_names(names_and_version_patterns)
+      ids_and_names = Packages::Dependency.for_package_names_and_version_patterns(names_and_version_patterns)
+                                          .pluck_ids_and_names
+      ids = ids_and_names.map(&:first) || []
+      names = ids_and_names.map(&:second) || []
+      [ids, names]
+    end
+
+    def bulk_insert_package_dependencies(names_and_version_patterns)
+      return [] if names_and_version_patterns.empty?
+
+      rows = names_and_version_patterns.map do |name, version_pattern|
+        {
+          name: name,
+          version_pattern: version_pattern
+        }
+      end
+
+      ids = database.bulk_insert(Packages::Dependency.table_name, rows, return_ids: true, on_conflict: :do_nothing)
+      return ids if ids.size == names_and_version_patterns.size
+
+      Packages::Dependency.uncached do
+        # The bulk_insert statement above do not dirty the query cache. To make
+        # sure that the results are fresh from the database and not from a stalled
+        # and potentially wrong cache, this query has to be done with the query
+        # chache disabled.
+        Packages::Dependency.ids_for_package_names_and_version_patterns(names_and_version_patterns)
+      end
+    end
+
+    def bulk_insert_package_dependency_links(type, dependency_ids)
+      rows = dependency_ids.map do |dependency_id|
+        {
+          package_id: package.id,
+          dependency_id: dependency_id,
+          dependency_type: Packages::DependencyLink.dependency_types[type.to_s]
+        }
+      end
+
+      database.bulk_insert(Packages::DependencyLink.table_name, rows)
+    end
+
+    def database
+      ::Gitlab::Database
+    end
+  end
+end
diff --git a/app/services/packages/create_package_file_service.rb b/app/services/packages/create_package_file_service.rb
new file mode 100644
index 00000000000..0ebceeee779
--- /dev/null
+++ b/app/services/packages/create_package_file_service.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+module Packages
+  class CreatePackageFileService
+    attr_reader :package, :params
+
+    def initialize(package, params)
+      @package = package
+      @params = params
+    end
+
+    def execute
+      package.package_files.create!(
+        file:        params[:file],
+        size:        params[:size],
+        file_name:   params[:file_name],
+        file_sha1:   params[:file_sha1],
+        file_sha256: params[:file_sha256],
+        file_md5:    params[:file_md5]
+      )
+    end
+  end
+end
diff --git a/app/services/packages/maven/create_package_service.rb b/app/services/packages/maven/create_package_service.rb
new file mode 100644
index 00000000000..aca5d28ca98
--- /dev/null
+++ b/app/services/packages/maven/create_package_service.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+module Packages
+  module Maven
+    class CreatePackageService < BaseService
+      def execute
+        app_group, _, app_name = params[:name].rpartition('/')
+        app_group.tr!('/', '.')
+
+        package = project.packages.create!(
+          name: params[:name],
+          version: params[:version],
+          package_type: :maven,
+          maven_metadatum_attributes: {
+            path: params[:path],
+            app_group: app_group,
+            app_name: app_name,
+            app_version: params[:version]
+          }
+        )
+
+        build = params[:build]
+        package.create_build_info!(pipeline: build.pipeline) if build.present?
+
+        package
+      end
+    end
+  end
+end
diff --git a/app/services/packages/maven/find_or_create_package_service.rb b/app/services/packages/maven/find_or_create_package_service.rb
new file mode 100644
index 00000000000..50a008843ad
--- /dev/null
+++ b/app/services/packages/maven/find_or_create_package_service.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+module Packages
+  module Maven
+    class FindOrCreatePackageService < BaseService
+      MAVEN_METADATA_FILE = 'maven-metadata.xml'.freeze
+
+      def execute
+        package = ::Packages::Maven::PackageFinder
+          .new(params[:path], current_user, project: project).execute
+
+        unless package
+          if params[:file_name] == MAVEN_METADATA_FILE
+            # Maven uploads several files during `mvn deploy` in next order:
+            #   - my-company/my-app/1.0-SNAPSHOT/my-app.jar
+            #   - my-company/my-app/1.0-SNAPSHOT/my-app.pom
+            #   - my-company/my-app/1.0-SNAPSHOT/maven-metadata.xml
+            #   - my-company/my-app/maven-metadata.xml
+            #
+            # The last xml file does not have VERSION in URL because it contains
+            # information about all versions.
+            package_name, version = params[:path], nil
+          else
+            package_name, _, version = params[:path].rpartition('/')
+          end
+
+          package_params = {
+            name: package_name,
+            path: params[:path],
+            version: version,
+            build: params[:build]
+          }
+
+          package = ::Packages::Maven::CreatePackageService
+            .new(project, current_user, package_params).execute
+        end
+
+        package
+      end
+    end
+  end
+end
diff --git a/app/services/packages/npm/create_package_service.rb b/app/services/packages/npm/create_package_service.rb
new file mode 100644
index 00000000000..cf927683ce9
--- /dev/null
+++ b/app/services/packages/npm/create_package_service.rb
@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+module Packages
+  module Npm
+    class CreatePackageService < BaseService
+      include Gitlab::Utils::StrongMemoize
+
+      def execute
+        return error('Version is empty.', 400) if version.blank?
+        return error('Package already exists.', 403) if current_package_exists?
+
+        ActiveRecord::Base.transaction { create_package! }
+      end
+
+      private
+
+      def create_package!
+        package = project.packages.create!(
+          name: name,
+          version: version,
+          package_type: 'npm'
+        )
+
+        if build.present?
+          package.create_build_info!(pipeline: build.pipeline)
+        end
+
+        ::Packages::CreatePackageFileService.new(package, file_params).execute
+        ::Packages::CreateDependencyService.new(package, package_dependencies).execute
+        ::Packages::Npm::CreateTagService.new(package, dist_tag).execute
+
+        package
+      end
+
+      def current_package_exists?
+        project.packages
+               .npm
+               .with_name(name)
+               .with_version(version)
+               .exists?
+      end
+
+      def name
+        params[:name]
+      end
+
+      def version
+        strong_memoize(:version) do
+          params[:versions].each_key.first
+        end
+      end
+
+      def version_data
+        params[:versions][version]
+      end
+
+      def build
+        params[:build]
+      end
+
+      def dist_tag
+        params['dist-tags'].each_key.first
+      end
+
+      def package_file_name
+        strong_memoize(:package_file_name) do
+          "#{name}-#{version}.tgz"
+        end
+      end
+
+      def attachment
+        strong_memoize(:attachment) do
+          params['_attachments'][package_file_name]
+        end
+      end
+
+      def file_params
+        {
+          file:      CarrierWaveStringFile.new(Base64.decode64(attachment['data'])),
+          size:      attachment['length'],
+          file_sha1: version_data[:dist][:shasum],
+          file_name: package_file_name
+        }
+      end
+
+      def package_dependencies
+        _version, versions_data = params[:versions].first
+        versions_data
+      end
+    end
+  end
+end
diff --git a/app/services/packages/npm/create_tag_service.rb b/app/services/packages/npm/create_tag_service.rb
new file mode 100644
index 00000000000..82974d0ca4b
--- /dev/null
+++ b/app/services/packages/npm/create_tag_service.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+module Packages
+  module Npm
+    class CreateTagService
+      include Gitlab::Utils::StrongMemoize
+
+      attr_reader :package, :tag_name
+
+      def initialize(package, tag_name)
+        @package = package
+        @tag_name = tag_name
+      end
+
+      def execute
+        if existing_tag.present?
+          existing_tag.update_column(:package_id, package.id)
+          existing_tag
+        else
+          package.tags.create!(name: tag_name)
+        end
+      end
+
+      private
+
+      def existing_tag
+        strong_memoize(:existing_tag) do
+          Packages::TagsFinder
+            .new(package.project, package.name, package_type: package.package_type)
+            .find_by_name(tag_name)
+        end
+      end
+    end
+  end
+end
diff --git a/app/services/packages/nuget/create_dependency_service.rb b/app/services/packages/nuget/create_dependency_service.rb
new file mode 100644
index 00000000000..2be5db732f6
--- /dev/null
+++ b/app/services/packages/nuget/create_dependency_service.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+module Packages
+  module Nuget
+    class CreateDependencyService < BaseService
+      def initialize(package, dependencies = [])
+        @package = package
+        @dependencies = dependencies
+      end
+
+      def execute
+        return if @dependencies.empty?
+
+        @package.transaction do
+          create_dependency_links
+          create_dependency_link_metadata
+        end
+      end
+
+      private
+
+      def create_dependency_links
+        ::Packages::CreateDependencyService
+          .new(@package, dependencies_for_create_dependency_service)
+          .execute
+      end
+
+      def create_dependency_link_metadata
+        inserted_links = ::Packages::DependencyLink.preload_dependency
+                                                   .for_package(@package)
+
+        return if inserted_links.empty?
+
+        rows = inserted_links.map do |dependency_link|
+          raw_dependency = raw_dependency_for(dependency_link.dependency)
+
+          next if raw_dependency[:target_framework].blank?
+
+          {
+            dependency_link_id: dependency_link.id,
+            target_framework: raw_dependency[:target_framework]
+          }
+        end
+
+        ::Gitlab::Database.bulk_insert(::Packages::Nuget::DependencyLinkMetadatum.table_name, rows.compact)
+      end
+
+      def raw_dependency_for(dependency)
+        name = dependency.name
+        version = dependency.version_pattern.presence
+
+        @dependencies.find do |raw_dependency|
+          raw_dependency[:name] == name && raw_dependency[:version] == version
+        end
+      end
+
+      def dependencies_for_create_dependency_service
+        names_and_versions = @dependencies.map do |dependency|
+          [dependency[:name], version_or_empty_string(dependency[:version])]
+        end.to_h
+
+        { 'dependencies' => names_and_versions }
+      end
+
+      def version_or_empty_string(version)
+        return '' if version.blank?
+
+        version
+      end
+    end
+  end
+end
diff --git a/app/services/packages/nuget/create_package_service.rb b/app/services/packages/nuget/create_package_service.rb
new file mode 100644
index 00000000000..68ad7f028e4
--- /dev/null
+++ b/app/services/packages/nuget/create_package_service.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Packages
+  module Nuget
+    class CreatePackageService < BaseService
+      TEMPORARY_PACKAGE_NAME = 'NuGet.Temporary.Package'
+      PACKAGE_VERSION = '0.0.0'
+
+      def execute
+        project.packages.nuget.create!(
+          name: TEMPORARY_PACKAGE_NAME,
+          version: "#{PACKAGE_VERSION}-#{uuid}"
+        )
+      end
+
+      private
+
+      def uuid
+        SecureRandom.uuid
+      end
+    end
+  end
+end
diff --git a/app/services/packages/nuget/metadata_extraction_service.rb b/app/services/packages/nuget/metadata_extraction_service.rb
new file mode 100644
index 00000000000..6fec398fab0
--- /dev/null
+++ b/app/services/packages/nuget/metadata_extraction_service.rb
@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Packages
+  module Nuget
+    class MetadataExtractionService
+      include Gitlab::Utils::StrongMemoize
+
+      ExtractionError = Class.new(StandardError)
+
+      XPATHS = {
+        package_name: '//xmlns:package/xmlns:metadata/xmlns:id',
+        package_version: '//xmlns:package/xmlns:metadata/xmlns:version',
+        license_url: '//xmlns:package/xmlns:metadata/xmlns:licenseUrl',
+        project_url: '//xmlns:package/xmlns:metadata/xmlns:projectUrl',
+        icon_url: '//xmlns:package/xmlns:metadata/xmlns:iconUrl'
+      }.freeze
+
+      XPATH_DEPENDENCIES = '//xmlns:package/xmlns:metadata/xmlns:dependencies/xmlns:dependency'
+      XPATH_DEPENDENCY_GROUPS = '//xmlns:package/xmlns:metadata/xmlns:dependencies/xmlns:group'
+      XPATH_TAGS = '//xmlns:package/xmlns:metadata/xmlns:tags'
+
+      MAX_FILE_SIZE = 4.megabytes.freeze
+
+      def initialize(package_file_id)
+        @package_file_id = package_file_id
+      end
+
+      def execute
+        raise ExtractionError.new('invalid package file') unless valid_package_file?
+
+        extract_metadata(nuspec_file)
+      end
+
+      private
+
+      def package_file
+        strong_memoize(:package_file) do
+          ::Packages::PackageFile.find_by_id(@package_file_id)
+        end
+      end
+
+      def valid_package_file?
+        package_file &&
+          package_file.package&.nuget? &&
+          package_file.file.size.positive?
+      end
+
+      def extract_metadata(file)
+        doc = Nokogiri::XML(file)
+
+        XPATHS.transform_values { |query| doc.xpath(query).text.presence }
+              .compact
+              .tap do |metadata|
+                metadata[:package_dependencies] = extract_dependencies(doc)
+                metadata[:package_tags] = extract_tags(doc)
+              end
+      end
+
+      def extract_dependencies(doc)
+        dependencies = []
+
+        doc.xpath(XPATH_DEPENDENCIES).each do |node|
+          dependencies << extract_dependency(node)
+        end
+
+        doc.xpath(XPATH_DEPENDENCY_GROUPS).each do |group_node|
+          target_framework = group_node.attr("targetFramework")
+
+          group_node.xpath("xmlns:dependency").each do |node|
+            dependencies << extract_dependency(node).merge(target_framework: target_framework)
+          end
+        end
+
+        dependencies
+      end
+
+      def extract_dependency(node)
+        {
+          name: node.attr('id'),
+          version: node.attr('version')
+        }.compact
+      end
+
+      def extract_tags(doc)
+        tags = doc.xpath(XPATH_TAGS).text
+
+        return [] if tags.blank?
+
+        tags.split(::Packages::Tag::NUGET_TAGS_SEPARATOR)
+      end
+
+      def nuspec_file
+        package_file.file.use_file do |file_path|
+          Zip::File.open(file_path) do |zip_file|
+            entry = zip_file.glob('*.nuspec').first
+
+            raise ExtractionError.new('nuspec file not found') unless entry
+            raise ExtractionError.new('nuspec file too big') if entry.size > MAX_FILE_SIZE
+
+            entry.get_input_stream.read
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/app/services/packages/nuget/search_service.rb b/app/services/packages/nuget/search_service.rb
new file mode 100644
index 00000000000..f7e09e11819
--- /dev/null
+++ b/app/services/packages/nuget/search_service.rb
@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Packages
+  module Nuget
+    class SearchService < BaseService
+      include Gitlab::Utils::StrongMemoize
+      include ActiveRecord::ConnectionAdapters::Quoting
+
+      MAX_PER_PAGE = 30
+      MAX_VERSIONS_PER_PACKAGE = 10
+      PRE_RELEASE_VERSION_MATCHING_TERM = '%-%'
+
+      DEFAULT_OPTIONS = {
+        include_prerelease_versions: true,
+        per_page: Kaminari.config.default_per_page,
+        padding: 0
+      }.freeze
+
+      def initialize(project, search_term, options = {})
+        @project = project
+        @search_term = search_term
+        @options = DEFAULT_OPTIONS.merge(options)
+
+        raise ArgumentError, 'negative per_page' if per_page.negative?
+        raise ArgumentError, 'negative padding' if padding.negative?
+      end
+
+      def execute
+        OpenStruct.new(
+          total_count: package_names.total_count,
+          results: search_packages
+        )
+      end
+
+      private
+
+      def search_packages
+        # custom query to get package names and versions as expected from the nuget search api
+        # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24182#technical-notes
+        # and https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource
+        subquery_name = :partition_subquery
+        arel_table = Arel::Table.new(:partition_subquery)
+        column_names = Packages::Package.column_names.map do |cn|
+          "#{subquery_name}.#{quote_column_name(cn)}"
+        end
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        pkgs = Packages::Package.select(column_names.join(','))
+                                .from(package_names_partition, subquery_name)
+                                .where(arel_table[:row_number].lteq(MAX_VERSIONS_PER_PACKAGE))
+
+        return pkgs if include_prerelease_versions?
+
+        # we can't use pkgs.without_version_like since we have a custom from
+        pkgs.where.not(arel_table[:version].matches(PRE_RELEASE_VERSION_MATCHING_TERM))
+      end
+
+      def package_names_partition
+        table_name = quote_table_name(Packages::Package.table_name)
+        name_column = "#{table_name}.#{quote_column_name('name')}"
+        created_at_column = "#{table_name}.#{quote_column_name('created_at')}"
+        select_sql = "ROW_NUMBER() OVER (PARTITION BY #{name_column} ORDER BY #{created_at_column} DESC) AS row_number, #{table_name}.*"
+
+        @project.packages
+                .select(select_sql)
+                .nuget
+                .has_version
+                .without_nuget_temporary_name
+                .with_name(package_names)
+      end
+
+      def package_names
+        strong_memoize(:package_names) do
+          pkgs = @project.packages
+                         .nuget
+                         .has_version
+                         .without_nuget_temporary_name
+                         .order_name
+                         .select_distinct_name
+          pkgs = pkgs.without_version_like(PRE_RELEASE_VERSION_MATCHING_TERM) unless include_prerelease_versions?
+          pkgs = pkgs.search_by_name(@search_term) if @search_term.present?
+          pkgs.page(0) # we're using a padding
+              .per(per_page)
+              .padding(padding)
+        end
+      end
+
+      def include_prerelease_versions?
+        @options[:include_prerelease_versions]
+      end
+
+      def padding
+        @options[:padding]
+      end
+
+      def per_page
+        [@options[:per_page], MAX_PER_PAGE].min
+      end
+    end
+  end
+end
diff --git a/app/services/packages/nuget/sync_metadatum_service.rb b/app/services/packages/nuget/sync_metadatum_service.rb
new file mode 100644
index 00000000000..ca9cc4d5b78
--- /dev/null
+++ b/app/services/packages/nuget/sync_metadatum_service.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Packages
+  module Nuget
+    class SyncMetadatumService
+      include Gitlab::Utils::StrongMemoize
+
+      def initialize(package, metadata)
+        @package = package
+        @metadata = metadata
+      end
+
+      def execute
+        if blank_metadata?
+          metadatum.destroy! if metadatum.persisted?
+        else
+          metadatum.update!(
+            license_url: license_url,
+            project_url: project_url,
+            icon_url: icon_url
+          )
+        end
+      end
+
+      private
+
+      def metadatum
+        strong_memoize(:metadatum) do
+          @package.nuget_metadatum || @package.build_nuget_metadatum
+        end
+      end
+
+      def blank_metadata?
+        project_url.blank? && license_url.blank? && icon_url.blank?
+      end
+
+      def project_url
+        @metadata[:project_url]
+      end
+
+      def license_url
+        @metadata[:license_url]
+      end
+
+      def icon_url
+        @metadata[:icon_url]
+      end
+    end
+  end
+end
diff --git a/app/services/packages/nuget/update_package_from_metadata_service.rb b/app/services/packages/nuget/update_package_from_metadata_service.rb
new file mode 100644
index 00000000000..f72b1386985
--- /dev/null
+++ b/app/services/packages/nuget/update_package_from_metadata_service.rb
@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Packages
+  module Nuget
+    class UpdatePackageFromMetadataService
+      include Gitlab::Utils::StrongMemoize
+      include ExclusiveLeaseGuard
+
+      # used by ExclusiveLeaseGuard
+      DEFAULT_LEASE_TIMEOUT = 1.hour.to_i.freeze
+
+      InvalidMetadataError = Class.new(StandardError)
+
+      def initialize(package_file)
+        @package_file = package_file
+      end
+
+      def execute
+        raise InvalidMetadataError.new('package name and/or package version not found in metadata') unless valid_metadata?
+
+        try_obtain_lease do
+          @package_file.transaction do
+            package = existing_package ? link_to_existing_package : update_linked_package
+
+            update_package(package)
+
+            # Updating file_name updates the path where the file is stored.
+            # We must pass the file again so that CarrierWave can handle the update
+            @package_file.update!(
+              file_name: package_filename,
+              file: @package_file.file
+            )
+          end
+        end
+      end
+
+      private
+
+      def update_package(package)
+        ::Packages::Nuget::SyncMetadatumService
+          .new(package, metadata.slice(:project_url, :license_url, :icon_url))
+          .execute
+        ::Packages::UpdateTagsService
+          .new(package, package_tags)
+          .execute
+      rescue => e
+        raise InvalidMetadataError, e.message
+      end
+
+      def valid_metadata?
+        package_name.present? && package_version.present?
+      end
+
+      def link_to_existing_package
+        package_to_destroy = @package_file.package
+        # Updating package_id updates the path where the file is stored.
+        # We must pass the file again so that CarrierWave can handle the update
+        @package_file.update!(
+          package_id: existing_package.id,
+          file: @package_file.file
+        )
+        package_to_destroy.destroy!
+        existing_package
+      end
+
+      def update_linked_package
+        @package_file.package.update!(
+          name: package_name,
+          version: package_version
+        )
+
+        ::Packages::Nuget::CreateDependencyService.new(@package_file.package, package_dependencies)
+                                                  .execute
+        @package_file.package
+      end
+
+      def existing_package
+        strong_memoize(:existing_package) do
+          @package_file.project.packages
+                               .nuget
+                               .with_name(package_name)
+                               .with_version(package_version)
+                               .first
+        end
+      end
+
+      def package_name
+        metadata[:package_name]
+      end
+
+      def package_version
+        metadata[:package_version]
+      end
+
+      def package_dependencies
+        metadata.fetch(:package_dependencies, [])
+      end
+
+      def package_tags
+        metadata.fetch(:package_tags, [])
+      end
+
+      def metadata
+        strong_memoize(:metadata) do
+          ::Packages::Nuget::MetadataExtractionService.new(@package_file.id).execute
+        end
+      end
+
+      def package_filename
+        "#{package_name.downcase}.#{package_version.downcase}.nupkg"
+      end
+
+      # used by ExclusiveLeaseGuard
+      def lease_key
+        package_id = existing_package ? existing_package.id : @package_file.package_id
+        "packages:nuget:update_package_from_metadata_service:package:#{package_id}"
+      end
+
+      # used by ExclusiveLeaseGuard
+      def lease_timeout
+        DEFAULT_LEASE_TIMEOUT
+      end
+    end
+  end
+end
diff --git a/app/services/packages/pypi/create_package_service.rb b/app/services/packages/pypi/create_package_service.rb
new file mode 100644
index 00000000000..1313fc80e33
--- /dev/null
+++ b/app/services/packages/pypi/create_package_service.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Packages
+  module Pypi
+    class CreatePackageService < BaseService
+      include ::Gitlab::Utils::StrongMemoize
+
+      def execute
+        ::Packages::Package.transaction do
+          Packages::Pypi::Metadatum.upsert(
+            package_id: created_package.id,
+            required_python: params[:requires_python]
+          )
+
+          ::Packages::CreatePackageFileService.new(created_package, file_params).execute
+        end
+      end
+
+      private
+
+      def created_package
+        strong_memoize(:created_package) do
+          project
+            .packages
+            .pypi
+            .safe_find_or_create_by!(name: params[:name], version: params[:version])
+        end
+      end
+
+      def file_params
+        {
+          file: params[:content],
+          file_name: params[:content].original_filename,
+          file_md5: params[:md5_digest],
+          file_sha256: params[:sha256_digest]
+        }
+      end
+    end
+  end
+end
diff --git a/app/services/packages/remove_tag_service.rb b/app/services/packages/remove_tag_service.rb
new file mode 100644
index 00000000000..465b85506a6
--- /dev/null
+++ b/app/services/packages/remove_tag_service.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+module Packages
+  class RemoveTagService < BaseService
+    attr_reader :package_tag
+
+    def initialize(package_tag)
+      raise ArgumentError, "Package tag must be set" if package_tag.blank?
+
+      @package_tag = package_tag
+    end
+
+    def execute
+      package_tag.delete
+    end
+  end
+end
diff --git a/app/services/packages/update_tags_service.rb b/app/services/packages/update_tags_service.rb
new file mode 100644
index 00000000000..da50cd3479e
--- /dev/null
+++ b/app/services/packages/update_tags_service.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+module Packages
+  class UpdateTagsService
+    include Gitlab::Utils::StrongMemoize
+
+    def initialize(package, tags = [])
+      @package = package
+      @tags = tags
+    end
+
+    def execute
+      return if @tags.empty?
+
+      tags_to_destroy = existing_tags - @tags
+      tags_to_create = @tags - existing_tags
+
+      @package.tags.with_name(tags_to_destroy).delete_all if tags_to_destroy.any?
+      ::Gitlab::Database.bulk_insert(Packages::Tag.table_name, rows(tags_to_create)) if tags_to_create.any?
+    end
+
+    private
+
+    def existing_tags
+      strong_memoize(:existing_tags) do
+        @package.tag_names
+      end
+    end
+
+    def rows(tags)
+      now = Time.zone.now
+      tags.map do |tag|
+        {
+          package_id: @package.id,
+          name: tag,
+          created_at: now,
+          updated_at: now
+        }
+      end
+    end
+  end
+end