From 7e9c479f7de77702622631cff2628a9c8dcbc627 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 19 Nov 2020 08:27:35 +0000 Subject: Add latest changes from gitlab-org/gitlab@13-6-stable-ee --- .../packages/composer/create_package_service.rb | 4 +- .../packages/composer/version_parser_service.rb | 2 +- app/services/packages/create_event_service.rb | 28 ++++++--- .../packages/create_package_file_service.rb | 9 ++- .../debian/extract_deb_metadata_service.rb | 41 ++++++++++++++ .../packages/debian/parse_debian822_service.rb | 66 ++++++++++++++++++++++ .../generic/create_package_file_service.rb | 3 +- .../generic/find_or_create_package_service.rb | 2 +- .../packages/maven/create_package_service.rb | 7 +-- .../maven/find_or_create_package_service.rb | 5 +- .../packages/npm/create_package_service.rb | 5 +- 11 files changed, 148 insertions(+), 24 deletions(-) create mode 100644 app/services/packages/debian/extract_deb_metadata_service.rb create mode 100644 app/services/packages/debian/parse_debian822_service.rb (limited to 'app/services/packages') diff --git a/app/services/packages/composer/create_package_service.rb b/app/services/packages/composer/create_package_service.rb index 7e16fc78599..2d2f1568187 100644 --- a/app/services/packages/composer/create_package_service.rb +++ b/app/services/packages/composer/create_package_service.rb @@ -10,11 +10,11 @@ module Packages composer_json ::Packages::Package.transaction do - ::Packages::Composer::Metadatum.upsert( + ::Packages::Composer::Metadatum.upsert({ package_id: created_package.id, target_sha: target, composer_json: composer_json - ) + }) end end diff --git a/app/services/packages/composer/version_parser_service.rb b/app/services/packages/composer/version_parser_service.rb index 76dfd7a14bd..811cac0b3b7 100644 --- a/app/services/packages/composer/version_parser_service.rb +++ b/app/services/packages/composer/version_parser_service.rb @@ -9,7 +9,7 @@ module Packages def execute if @tag_name.present? - @tag_name.match(Gitlab::Regex.composer_package_version_regex).captures[0] + @tag_name.delete_prefix('v') elsif @branch_name.present? branch_sufix_or_prefix(@branch_name.match(Gitlab::Regex.composer_package_version_regex)) end diff --git a/app/services/packages/create_event_service.rb b/app/services/packages/create_event_service.rb index d009cba2812..8350ff993bf 100644 --- a/app/services/packages/create_event_service.rb +++ b/app/services/packages/create_event_service.rb @@ -3,18 +3,30 @@ module Packages class CreateEventService < BaseService def execute - event_scope = scope.is_a?(::Packages::Package) ? scope.package_type : scope - - ::Packages::Event.create!( - event_type: event_name, - originator: current_user&.id, - originator_type: originator_type, - event_scope: event_scope - ) + if Feature.enabled?(:collect_package_events_redis) && redis_event_name + ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event(current_user.id, redis_event_name) + end + + if Feature.enabled?(:collect_package_events) + ::Packages::Event.create!( + event_type: event_name, + originator: current_user&.id, + originator_type: originator_type, + event_scope: event_scope + ) + end end private + def redis_event_name + @redis_event_name ||= ::Packages::Event.allowed_event_name(event_scope, event_name, originator_type) + end + + def event_scope + @event_scope ||= scope.is_a?(::Packages::Package) ? scope.package_type : scope + end + def scope params[:scope] end diff --git a/app/services/packages/create_package_file_service.rb b/app/services/packages/create_package_file_service.rb index 0ebceeee779..5723b0b4717 100644 --- a/app/services/packages/create_package_file_service.rb +++ b/app/services/packages/create_package_file_service.rb @@ -9,7 +9,7 @@ module Packages end def execute - package.package_files.create!( + package_file = package.package_files.build( file: params[:file], size: params[:size], file_name: params[:file_name], @@ -17,6 +17,13 @@ module Packages file_sha256: params[:file_sha256], file_md5: params[:file_md5] ) + + if params[:build].present? + package_file.package_file_build_infos << package_file.package_file_build_infos.build(pipeline: params[:build].pipeline) + end + + package_file.save! + package_file end end end diff --git a/app/services/packages/debian/extract_deb_metadata_service.rb b/app/services/packages/debian/extract_deb_metadata_service.rb new file mode 100644 index 00000000000..eb106f4cd30 --- /dev/null +++ b/app/services/packages/debian/extract_deb_metadata_service.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Packages + module Debian + # Returns .deb file metadata + class ExtractDebMetadataService + CommandFailedError = Class.new(StandardError) + + def initialize(file_path) + @file_path = file_path + end + + def execute + unless success? + raise CommandFailedError, "The `#{cmd}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}" + end + + sections = ParseDebian822Service.new(result.stdout).execute + + sections.each_value.first + end + + private + + def cmd + @cmd ||= begin + dpkg_deb_path = Gitlab.config.packages.dpkg_deb_path + [dpkg_deb_path, '--field', @file_path] + end + end + + def result + @result ||= Gitlab::Popen.popen_with_detail(cmd) + end + + def success? + result.status&.exitstatus == 0 + end + end + end +end diff --git a/app/services/packages/debian/parse_debian822_service.rb b/app/services/packages/debian/parse_debian822_service.rb new file mode 100644 index 00000000000..665929d2324 --- /dev/null +++ b/app/services/packages/debian/parse_debian822_service.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Packages + module Debian + # Parse String as Debian RFC822 control data format + # https://manpages.debian.org/unstable/dpkg-dev/deb822.5 + class ParseDebian822Service + InvalidDebian822Error = Class.new(StandardError) + + def initialize(input) + @input = input + end + + def execute + output = {} + @input.each_line('', chomp: true) do |block| + section = {} + section_name, field = nil + block.each_line(chomp: true) do |line| + next if comment_line?(line) + + if continuation_line?(line) + raise InvalidDebian822Error, "Parse error. Unexpected continuation line" if field.nil? + + section[field] += "\n" + section[field] += line[1..] unless paragraph_separator?(line) + elsif match = match_section_line(line) + section_name = match[:name] if section_name.nil? + field = match[:field].to_sym + + raise InvalidDebian822Error, "Duplicate field '#{field}' in section '#{section_name}'" if section.include?(field) + + section[field] = match[:value] + else + raise InvalidDebian822Error, "Parse error on line #{line}" + end + end + + raise InvalidDebian822Error, "Duplicate section '#{section_name}'" if output[section_name] + + output[section_name] = section + end + + output + end + + private + + def comment_line?(line) + line.match?(/^#/) + end + + def continuation_line?(line) + line.match?(/^ /) + end + + def paragraph_separator?(line) + line == ' .' + end + + def match_section_line(line) + line.match(/(?(?^\S+):\s*(?.*))/) + end + end + end +end diff --git a/app/services/packages/generic/create_package_file_service.rb b/app/services/packages/generic/create_package_file_service.rb index 4d49c63799f..f25e8b0ae56 100644 --- a/app/services/packages/generic/create_package_file_service.rb +++ b/app/services/packages/generic/create_package_file_service.rb @@ -28,7 +28,8 @@ module Packages file: params[:file], size: params[:file].size, file_sha256: params[:file].sha256, - file_name: params[:file_name] + file_name: params[:file_name], + build: params[:build] } ::Packages::CreatePackageFileService.new(package, file_params).execute diff --git a/app/services/packages/generic/find_or_create_package_service.rb b/app/services/packages/generic/find_or_create_package_service.rb index 8a8459d167e..97f774a836b 100644 --- a/app/services/packages/generic/find_or_create_package_service.rb +++ b/app/services/packages/generic/find_or_create_package_service.rb @@ -6,7 +6,7 @@ module Packages def execute find_or_create_package!(::Packages::Package.package_types['generic']) do |package| if params[:build].present? - package.build_info = Packages::BuildInfo.new(pipeline: params[:build].pipeline) + package.build_infos.new(pipeline: params[:build].pipeline) end end end diff --git a/app/services/packages/maven/create_package_service.rb b/app/services/packages/maven/create_package_service.rb index 3df17021499..540c7b1d4da 100644 --- a/app/services/packages/maven/create_package_service.rb +++ b/app/services/packages/maven/create_package_service.rb @@ -6,7 +6,7 @@ module Packages app_group, _, app_name = params[:name].rpartition('/') app_group.tr!('/', '.') - package = create_package!(:maven, + create_package!(:maven, maven_metadatum_attributes: { path: params[:path], app_group: app_group, @@ -14,11 +14,6 @@ module Packages app_version: params[:version] } ) - - build = params[:build] - package.create_build_info!(pipeline: build.pipeline) if build.present? - - package end end end diff --git a/app/services/packages/maven/find_or_create_package_service.rb b/app/services/packages/maven/find_or_create_package_service.rb index 505f45a7b21..a2a61ff8d93 100644 --- a/app/services/packages/maven/find_or_create_package_service.rb +++ b/app/services/packages/maven/find_or_create_package_service.rb @@ -38,8 +38,7 @@ module Packages package_params = { name: package_name, path: params[:path], - version: version, - build: params[:build] + version: version } package = @@ -47,6 +46,8 @@ module Packages .execute end + package.build_infos.create!(pipeline: params[:build].pipeline) if params[:build].present? + package end end diff --git a/app/services/packages/npm/create_package_service.rb b/app/services/packages/npm/create_package_service.rb index 7f868b71734..c4b75348bba 100644 --- a/app/services/packages/npm/create_package_service.rb +++ b/app/services/packages/npm/create_package_service.rb @@ -18,7 +18,7 @@ module Packages package = create_package!(:npm, name: name, version: version) if build.present? - package.create_build_info!(pipeline: build.pipeline) + package.build_infos.create!(pipeline: build.pipeline) end ::Packages::CreatePackageFileService.new(package, file_params).execute @@ -75,7 +75,8 @@ module Packages file: CarrierWaveStringFile.new(Base64.decode64(attachment['data'])), size: attachment['length'], file_sha1: version_data[:dist][:shasum], - file_name: package_file_name + file_name: package_file_name, + build: params[:build] } end -- cgit v1.2.3