Add latest changes from gitlab-org/gitlab@14-0-stable-eev14.0.0-rc42

7 files changed, 130 insertions, 69 deletions

diff --git a/app/services/packages/debian/create_distribution_service.rb b/app/services/packages/debian/create_distribution_service.rb
index f947d2e4293..b4b1ec952ef 100644
--- a/app/services/packages/debian/create_distribution_service.rb
+++ b/app/services/packages/debian/create_distribution_service.rb
@@ -38,14 +38,19 @@ module Packages
         append_errors(distribution)
         return error unless errors.empty?
 
-        distribution.transaction do
-          if distribution.save
-            create_components
-            create_architectures
-
-            success
-          end
-        end || error
+        result = distribution.transaction do
+          next unless distribution.save
+
+          create_components
+          create_architectures
+          success
+        end
+
+        result ||= error
+
+        ::Packages::Debian::GenerateDistributionWorker.perform_async(distribution.class.container_type, distribution.reset.id) if result.success?
+
+        result
       end
 
       def create_components
diff --git a/app/services/packages/debian/destroy_distribution_service.rb b/app/services/packages/debian/destroy_distribution_service.rb
deleted file mode 100644
index bef1127fece..00000000000
--- a/app/services/packages/debian/destroy_distribution_service.rb
+++ /dev/null
@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-
-module Packages
-  module Debian
-    class DestroyDistributionService
-      def initialize(distribution)
-        @distribution = distribution
-      end
-
-      def execute
-        destroy_distribution
-      end
-
-      private
-
-      def destroy_distribution
-        if @distribution.destroy
-          success
-        else
-          error("Unable to destroy Debian #{@distribution.model_name.human.downcase}")
-        end
-      end
-
-      def success
-        ServiceResponse.success
-      end
-
-      def error(message)
-        ServiceResponse.error(message: message, payload: { distribution: @distribution })
-      end
-    end
-  end
-end
diff --git a/app/services/packages/debian/generate_distribution_service.rb b/app/services/packages/debian/generate_distribution_service.rb
index 67348af1a49..651325c49a0 100644
--- a/app/services/packages/debian/generate_distribution_service.rb
+++ b/app/services/packages/debian/generate_distribution_service.rb
@@ -6,6 +6,8 @@ module Packages
       include Gitlab::Utils::StrongMemoize
       include ExclusiveLeaseGuard
 
+      ONE_HOUR = 1.hour.freeze
+
       # used by ExclusiveLeaseGuard
       DEFAULT_LEASE_TIMEOUT = 1.hour.to_i.freeze
 
@@ -62,7 +64,7 @@ module Packages
 
       def initialize(distribution)
         @distribution = distribution
-        @last_generated_at = nil
+        @oldest_kept_generated_at = nil
         @md5sum = []
         @sha256 = []
       end
@@ -70,7 +72,10 @@ module Packages
       def execute
         try_obtain_lease do
           @distribution.transaction do
-            @last_generated_at = @distribution.component_files.maximum(:created_at)
+            # We consider `apt-get update` can take at most one hour
+            # We keep all generations younger than one hour
+            # and the previous generation
+            @oldest_kept_generated_at = @distribution.component_files.updated_before(release_date - ONE_HOUR).maximum(:updated_at)
             generate_component_files
             generate_release
             destroy_old_component_files
@@ -96,7 +101,7 @@ module Packages
                                   .with_debian_file_type(package_file_type)
                                   .find_each
                                   .map(&method(:package_stanza_from_fields))
-        create_component_file(component, component_file_type, architecture, package_file_type, paragraphs.join("\n"))
+        reuse_or_create_component_file(component, component_file_type, architecture, paragraphs.join("\n"))
       end
 
       def package_stanza_from_fields(package_file)
@@ -127,17 +132,32 @@ module Packages
         end
       end
 
-      def create_component_file(component, component_file_type, architecture, package_file_type, content)
-        component_file = component.files.create!(
-          file_type: component_file_type,
-          architecture: architecture,
-          compression_type: nil,
-          file: CarrierWaveStringFile.new(content),
-          file_md5: Digest::MD5.hexdigest(content),
-          file_sha256: Digest::SHA256.hexdigest(content)
-        )
-        @md5sum.append(" #{component_file.file_md5} #{component_file.size.to_s.rjust(8)} #{component_file.relative_path}")
-        @sha256.append(" #{component_file.file_sha256} #{component_file.size.to_s.rjust(8)} #{component_file.relative_path}")
+      def reuse_or_create_component_file(component, component_file_type, architecture, content)
+        file_md5 = Digest::MD5.hexdigest(content)
+        file_sha256 = Digest::SHA256.hexdigest(content)
+        component_file = component.files
+                                  .with_file_type(component_file_type)
+                                  .with_architecture(architecture)
+                                  .with_compression_type(nil)
+                                  .with_file_sha256(file_sha256)
+                                  .last
+
+        if component_file
+          component_file.touch(time: release_date)
+        else
+          component_file = component.files.create!(
+            updated_at: release_date,
+            file_type: component_file_type,
+            architecture: architecture,
+            compression_type: nil,
+            file: CarrierWaveStringFile.new(content),
+            file_md5: file_md5,
+            file_sha256: file_sha256
+          )
+        end
+
+        @md5sum.append(" #{file_md5} #{component_file.size.to_s.rjust(8)} #{component_file.relative_path}")
+        @sha256.append(" #{file_sha256} #{component_file.size.to_s.rjust(8)} #{component_file.relative_path}")
       end
 
       def generate_release
@@ -187,10 +207,9 @@ module Packages
       end
 
       def destroy_old_component_files
-        # Only keep the last generation and one hour before
-        return if @last_generated_at.nil?
+        return if @oldest_kept_generated_at.nil?
 
-        @distribution.component_files.created_before(@last_generated_at - 1.hour).destroy_all # rubocop:disable Cop/DestroyAll
+        @distribution.component_files.updated_before(@oldest_kept_generated_at).destroy_all # rubocop:disable Cop/DestroyAll
       end
 
       # used by ExclusiveLeaseGuard
diff --git a/app/services/packages/debian/process_changes_service.rb b/app/services/packages/debian/process_changes_service.rb
index 881ad2c46f4..b6e81012656 100644
--- a/app/services/packages/debian/process_changes_service.rb
+++ b/app/services/packages/debian/process_changes_service.rb
@@ -25,6 +25,8 @@ module Packages
             update_files_metadata
             update_changes_metadata
           end
+
+          ::Packages::Debian::GenerateDistributionWorker.perform_async(:project, package.debian_distribution.id)
         end
       end
 
diff --git a/app/services/packages/debian/update_distribution_service.rb b/app/services/packages/debian/update_distribution_service.rb
index 95face912d5..218167ecdc5 100644
--- a/app/services/packages/debian/update_distribution_service.rb
+++ b/app/services/packages/debian/update_distribution_service.rb
@@ -31,7 +31,7 @@ module Packages
       end
 
       def update_distribution
-        distribution.transaction do
+        result = distribution.transaction do
           if distribution.update(params)
             update_components if components
             update_architectures if architectures
@@ -41,7 +41,13 @@ module Packages
             append_errors(distribution)
             error
           end
-        end || error
+        end
+
+        result ||= error
+
+        ::Packages::Debian::GenerateDistributionWorker.perform_async(distribution.class.container_type, distribution.id) if result.success?
+
+        result
       end
 
       def update_components
diff --git a/app/services/packages/helm/extract_file_metadata_service.rb b/app/services/packages/helm/extract_file_metadata_service.rb
new file mode 100644
index 00000000000..e7373d8ea8f
--- /dev/null
+++ b/app/services/packages/helm/extract_file_metadata_service.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'rubygems/package'
+
+module Packages
+  module Helm
+    class ExtractFileMetadataService
+      ExtractionError = Class.new(StandardError)
+
+      def initialize(package_file)
+        @package_file = package_file
+      end
+
+      def execute
+        raise ExtractionError, 'invalid package file' unless valid_package_file?
+
+        metadata
+      end
+
+      private
+
+      def valid_package_file?
+        @package_file &&
+          @package_file.package&.helm? &&
+          @package_file.file.size > 0 # rubocop:disable Style/ZeroLengthPredicate
+      end
+
+      def metadata
+        YAML.safe_load(chart_yaml_content)
+      rescue Psych::Exception => e
+        raise ExtractionError, "Error while parsing Chart.yaml: #{e.message}"
+      end
+
+      def chart_yaml_content
+        @package_file.file.use_open_file do |file|
+          tar_reader = Gem::Package::TarReader.new(Zlib::GzipReader.new(file))
+
+          chart_yaml = tar_reader.find do |entry|
+            next unless entry.file?
+
+            entry.full_name.end_with?('/Chart.yaml')
+          end
+
+          raise ExtractionError, 'Chart.yaml not found within a directory' unless chart_yaml
+
+          chart_yaml.read
+        ensure
+          tar_reader.close
+        end
+      end
+    end
+  end
+end
diff --git a/app/services/packages/nuget/metadata_extraction_service.rb b/app/services/packages/nuget/metadata_extraction_service.rb
index dd5f1bcc936..63da98dde43 100644
--- a/app/services/packages/nuget/metadata_extraction_service.rb
+++ b/app/services/packages/nuget/metadata_extraction_service.rb
@@ -28,7 +28,7 @@ module Packages
       def execute
         raise ExtractionError, 'invalid package file' unless valid_package_file?
 
-        extract_metadata(nuspec_file)
+        extract_metadata(nuspec_file_content)
       end
 
       private
@@ -39,6 +39,10 @@ module Packages
         end
       end
 
+      def project
+        package_file.package.project
+      end
+
       def valid_package_file?
         package_file &&
           package_file.package&.nuget? &&
@@ -89,16 +93,21 @@ module Packages
         tags.split(::Packages::Tag::NUGET_TAGS_SEPARATOR)
       end
 
-      def nuspec_file
-        package_file.file.use_file do |file_path|
-          Zip::File.open(file_path) do |zip_file|
-            entry = zip_file.glob('*.nuspec').first
+      def nuspec_file_content
+        with_zip_file do |zip_file|
+          entry = zip_file.glob('*.nuspec').first
 
-            raise ExtractionError, 'nuspec file not found' unless entry
-            raise ExtractionError, 'nuspec file too big' if entry.size > MAX_FILE_SIZE
+          raise ExtractionError, 'nuspec file not found' unless entry
+          raise ExtractionError, 'nuspec file too big' if entry.size > MAX_FILE_SIZE
 
-            entry.get_input_stream.read
-          end
+          entry.get_input_stream.read
+        end
+      end
+
+      def with_zip_file(&block)
+        package_file.file.use_open_file do |open_file|
+          zip_file = Zip::File.new(open_file, false, true)
+          yield(zip_file)
         end
       end
     end