From a5f4bba440d7f9ea47046a0a561d49adf0a1e6d4 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 16 Jun 2021 18:25:58 +0000 Subject: Add latest changes from gitlab-org/gitlab@14-0-stable-ee --- .../packages/debian/create_distribution_service.rb | 21 +++++---- .../debian/destroy_distribution_service.rb | 33 -------------- .../debian/generate_distribution_service.rb | 53 +++++++++++++++------- .../packages/debian/process_changes_service.rb | 2 + .../packages/debian/update_distribution_service.rb | 10 +++- .../packages/helm/extract_file_metadata_service.rb | 53 ++++++++++++++++++++++ .../packages/nuget/metadata_extraction_service.rb | 27 +++++++---- 7 files changed, 130 insertions(+), 69 deletions(-) delete mode 100644 app/services/packages/debian/destroy_distribution_service.rb create mode 100644 app/services/packages/helm/extract_file_metadata_service.rb (limited to 'app/services/packages') diff --git a/app/services/packages/debian/create_distribution_service.rb b/app/services/packages/debian/create_distribution_service.rb index f947d2e4293..b4b1ec952ef 100644 --- a/app/services/packages/debian/create_distribution_service.rb +++ b/app/services/packages/debian/create_distribution_service.rb @@ -38,14 +38,19 @@ module Packages append_errors(distribution) return error unless errors.empty? - distribution.transaction do - if distribution.save - create_components - create_architectures - - success - end - end || error + result = distribution.transaction do + next unless distribution.save + + create_components + create_architectures + success + end + + result ||= error + + ::Packages::Debian::GenerateDistributionWorker.perform_async(distribution.class.container_type, distribution.reset.id) if result.success? + + result end def create_components diff --git a/app/services/packages/debian/destroy_distribution_service.rb b/app/services/packages/debian/destroy_distribution_service.rb deleted file mode 100644 index bef1127fece..00000000000 --- a/app/services/packages/debian/destroy_distribution_service.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Packages - module Debian - class DestroyDistributionService - def initialize(distribution) - @distribution = distribution - end - - def execute - destroy_distribution - end - - private - - def destroy_distribution - if @distribution.destroy - success - else - error("Unable to destroy Debian #{@distribution.model_name.human.downcase}") - end - end - - def success - ServiceResponse.success - end - - def error(message) - ServiceResponse.error(message: message, payload: { distribution: @distribution }) - end - end - end -end diff --git a/app/services/packages/debian/generate_distribution_service.rb b/app/services/packages/debian/generate_distribution_service.rb index 67348af1a49..651325c49a0 100644 --- a/app/services/packages/debian/generate_distribution_service.rb +++ b/app/services/packages/debian/generate_distribution_service.rb @@ -6,6 +6,8 @@ module Packages include Gitlab::Utils::StrongMemoize include ExclusiveLeaseGuard + ONE_HOUR = 1.hour.freeze + # used by ExclusiveLeaseGuard DEFAULT_LEASE_TIMEOUT = 1.hour.to_i.freeze @@ -62,7 +64,7 @@ module Packages def initialize(distribution) @distribution = distribution - @last_generated_at = nil + @oldest_kept_generated_at = nil @md5sum = [] @sha256 = [] end @@ -70,7 +72,10 @@ module Packages def execute try_obtain_lease do @distribution.transaction do - @last_generated_at = @distribution.component_files.maximum(:created_at) + # We consider `apt-get update` can take at most one hour + # We keep all generations younger than one hour + # and the previous generation + @oldest_kept_generated_at = @distribution.component_files.updated_before(release_date - ONE_HOUR).maximum(:updated_at) generate_component_files generate_release destroy_old_component_files @@ -96,7 +101,7 @@ module Packages .with_debian_file_type(package_file_type) .find_each .map(&method(:package_stanza_from_fields)) - create_component_file(component, component_file_type, architecture, package_file_type, paragraphs.join("\n")) + reuse_or_create_component_file(component, component_file_type, architecture, paragraphs.join("\n")) end def package_stanza_from_fields(package_file) @@ -127,17 +132,32 @@ module Packages end end - def create_component_file(component, component_file_type, architecture, package_file_type, content) - component_file = component.files.create!( - file_type: component_file_type, - architecture: architecture, - compression_type: nil, - file: CarrierWaveStringFile.new(content), - file_md5: Digest::MD5.hexdigest(content), - file_sha256: Digest::SHA256.hexdigest(content) - ) - @md5sum.append(" #{component_file.file_md5} #{component_file.size.to_s.rjust(8)} #{component_file.relative_path}") - @sha256.append(" #{component_file.file_sha256} #{component_file.size.to_s.rjust(8)} #{component_file.relative_path}") + def reuse_or_create_component_file(component, component_file_type, architecture, content) + file_md5 = Digest::MD5.hexdigest(content) + file_sha256 = Digest::SHA256.hexdigest(content) + component_file = component.files + .with_file_type(component_file_type) + .with_architecture(architecture) + .with_compression_type(nil) + .with_file_sha256(file_sha256) + .last + + if component_file + component_file.touch(time: release_date) + else + component_file = component.files.create!( + updated_at: release_date, + file_type: component_file_type, + architecture: architecture, + compression_type: nil, + file: CarrierWaveStringFile.new(content), + file_md5: file_md5, + file_sha256: file_sha256 + ) + end + + @md5sum.append(" #{file_md5} #{component_file.size.to_s.rjust(8)} #{component_file.relative_path}") + @sha256.append(" #{file_sha256} #{component_file.size.to_s.rjust(8)} #{component_file.relative_path}") end def generate_release @@ -187,10 +207,9 @@ module Packages end def destroy_old_component_files - # Only keep the last generation and one hour before - return if @last_generated_at.nil? + return if @oldest_kept_generated_at.nil? - @distribution.component_files.created_before(@last_generated_at - 1.hour).destroy_all # rubocop:disable Cop/DestroyAll + @distribution.component_files.updated_before(@oldest_kept_generated_at).destroy_all # rubocop:disable Cop/DestroyAll end # used by ExclusiveLeaseGuard diff --git a/app/services/packages/debian/process_changes_service.rb b/app/services/packages/debian/process_changes_service.rb index 881ad2c46f4..b6e81012656 100644 --- a/app/services/packages/debian/process_changes_service.rb +++ b/app/services/packages/debian/process_changes_service.rb @@ -25,6 +25,8 @@ module Packages update_files_metadata update_changes_metadata end + + ::Packages::Debian::GenerateDistributionWorker.perform_async(:project, package.debian_distribution.id) end end diff --git a/app/services/packages/debian/update_distribution_service.rb b/app/services/packages/debian/update_distribution_service.rb index 95face912d5..218167ecdc5 100644 --- a/app/services/packages/debian/update_distribution_service.rb +++ b/app/services/packages/debian/update_distribution_service.rb @@ -31,7 +31,7 @@ module Packages end def update_distribution - distribution.transaction do + result = distribution.transaction do if distribution.update(params) update_components if components update_architectures if architectures @@ -41,7 +41,13 @@ module Packages append_errors(distribution) error end - end || error + end + + result ||= error + + ::Packages::Debian::GenerateDistributionWorker.perform_async(distribution.class.container_type, distribution.id) if result.success? + + result end def update_components diff --git a/app/services/packages/helm/extract_file_metadata_service.rb b/app/services/packages/helm/extract_file_metadata_service.rb new file mode 100644 index 00000000000..e7373d8ea8f --- /dev/null +++ b/app/services/packages/helm/extract_file_metadata_service.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require 'rubygems/package' + +module Packages + module Helm + class ExtractFileMetadataService + ExtractionError = Class.new(StandardError) + + def initialize(package_file) + @package_file = package_file + end + + def execute + raise ExtractionError, 'invalid package file' unless valid_package_file? + + metadata + end + + private + + def valid_package_file? + @package_file && + @package_file.package&.helm? && + @package_file.file.size > 0 # rubocop:disable Style/ZeroLengthPredicate + end + + def metadata + YAML.safe_load(chart_yaml_content) + rescue Psych::Exception => e + raise ExtractionError, "Error while parsing Chart.yaml: #{e.message}" + end + + def chart_yaml_content + @package_file.file.use_open_file do |file| + tar_reader = Gem::Package::TarReader.new(Zlib::GzipReader.new(file)) + + chart_yaml = tar_reader.find do |entry| + next unless entry.file? + + entry.full_name.end_with?('/Chart.yaml') + end + + raise ExtractionError, 'Chart.yaml not found within a directory' unless chart_yaml + + chart_yaml.read + ensure + tar_reader.close + end + end + end + end +end diff --git a/app/services/packages/nuget/metadata_extraction_service.rb b/app/services/packages/nuget/metadata_extraction_service.rb index dd5f1bcc936..63da98dde43 100644 --- a/app/services/packages/nuget/metadata_extraction_service.rb +++ b/app/services/packages/nuget/metadata_extraction_service.rb @@ -28,7 +28,7 @@ module Packages def execute raise ExtractionError, 'invalid package file' unless valid_package_file? - extract_metadata(nuspec_file) + extract_metadata(nuspec_file_content) end private @@ -39,6 +39,10 @@ module Packages end end + def project + package_file.package.project + end + def valid_package_file? package_file && package_file.package&.nuget? && @@ -89,16 +93,21 @@ module Packages tags.split(::Packages::Tag::NUGET_TAGS_SEPARATOR) end - def nuspec_file - package_file.file.use_file do |file_path| - Zip::File.open(file_path) do |zip_file| - entry = zip_file.glob('*.nuspec').first + def nuspec_file_content + with_zip_file do |zip_file| + entry = zip_file.glob('*.nuspec').first - raise ExtractionError, 'nuspec file not found' unless entry - raise ExtractionError, 'nuspec file too big' if entry.size > MAX_FILE_SIZE + raise ExtractionError, 'nuspec file not found' unless entry + raise ExtractionError, 'nuspec file too big' if entry.size > MAX_FILE_SIZE - entry.get_input_stream.read - end + entry.get_input_stream.read + end + end + + def with_zip_file(&block) + package_file.file.use_open_file do |open_file| + zip_file = Zip::File.new(open_file, false, true) + yield(zip_file) end end end -- cgit v1.2.3