Merge branch 'container-repository-cleanup-api' into 'master'

Container repository cleanup API

Closes #55978

See merge request gitlab-org/gitlab-ce!24303

9 files changed, 178 insertions, 8 deletions

diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb
index 2c08a8e1acf..cf057d774cf 100644
--- a/app/models/container_repository.rb
+++ b/app/models/container_repository.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class ContainerRepository < ActiveRecord::Base
+  include Gitlab::Utils::StrongMemoize
+
   belongs_to :project
 
   validates :name, length: { minimum: 0, allow_nil: false }
@@ -8,6 +10,8 @@ class ContainerRepository < ActiveRecord::Base
 
   delegate :client, to: :registry
 
+  scope :ordered, -> { order(:name) }
+
   # rubocop: disable CodeReuse/ServiceClass
   def registry
     @registry ||= begin
@@ -39,11 +43,12 @@ class ContainerRepository < ActiveRecord::Base
   end
 
   def tags
-    return @tags if defined?(@tags)
     return [] unless manifest && manifest['tags']
 
-    @tags = manifest['tags'].map do |tag|
-      ContainerRegistry::Tag.new(self, tag)
+    strong_memoize(:tags) do
+      manifest['tags'].sort.map do |tag|
+        ContainerRegistry::Tag.new(self, tag)
+      end
     end
   end
 
diff --git a/app/policies/container_repository_policy.rb b/app/policies/container_repository_policy.rb
new file mode 100644
index 00000000000..6781c845142
--- /dev/null
+++ b/app/policies/container_repository_policy.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class ContainerRepositoryPolicy < BasePolicy
+  delegate { @subject.project }
+end
diff --git a/app/serializers/container_repository_entity.rb b/app/serializers/container_repository_entity.rb
index 59bf35f5aff..cc746698a05 100644
--- a/app/serializers/container_repository_entity.rb
+++ b/app/serializers/container_repository_entity.rb
@@ -3,7 +3,7 @@
 class ContainerRepositoryEntity < Grape::Entity
   include RequestAwareEntity
 
-  expose :id, :path, :location
+  expose :id, :name, :path, :location, :created_at
 
   expose :tags_path do |repository|
     project_registry_repository_tags_path(project, repository, format: :json)
diff --git a/app/serializers/container_tag_entity.rb b/app/serializers/container_tag_entity.rb
index 637294877f8..361c073e22e 100644
--- a/app/serializers/container_tag_entity.rb
+++ b/app/serializers/container_tag_entity.rb
@@ -3,7 +3,7 @@
 class ContainerTagEntity < Grape::Entity
   include RequestAwareEntity
 
-  expose :name, :location, :revision, :short_revision, :total_size, :created_at
+  expose :name, :path, :location, :digest, :revision, :short_revision, :total_size, :created_at
 
   expose :destroy_path, if: -> (*) { can_destroy? } do |tag|
     project_registry_repository_tag_path(project, tag.repository, tag.name)
diff --git a/app/services/concerns/exclusive_lease_guard.rb b/app/services/concerns/exclusive_lease_guard.rb
index f102e00d150..28879d2d67f 100644
--- a/app/services/concerns/exclusive_lease_guard.rb
+++ b/app/services/concerns/exclusive_lease_guard.rb
@@ -6,9 +6,14 @@
 #
 # `#try_obtain_lease` takes a block which will be run if it was able to
 # obtain  the lease. Implement `#lease_timeout` to configure the timeout
-# for the exclusive lease. Optionally override `#lease_key` to set the
+# for the exclusive lease.
+#
+# Optionally override `#lease_key` to set the
 # lease key, it defaults to the class name with underscores.
 #
+# Optionally override `#lease_release?` to prevent the job to
+# be re-executed more often than LEASE_TIMEOUT.
+#
 module ExclusiveLeaseGuard
   extend ActiveSupport::Concern
 
@@ -23,7 +28,7 @@ module ExclusiveLeaseGuard
     begin
       yield lease
     ensure
-      release_lease(lease)
+      release_lease(lease) if lease_release?
     end
   end
 
@@ -40,6 +45,10 @@ module ExclusiveLeaseGuard
           "#{self.class.name} does not implement #{__method__}"
   end
 
+  def lease_release?
+    true
+  end
+
   def release_lease(uuid)
     Gitlab::ExclusiveLease.cancel(lease_key, uuid)
   end
diff --git a/app/services/projects/container_repository/cleanup_tags_service.rb b/app/services/projects/container_repository/cleanup_tags_service.rb
new file mode 100644
index 00000000000..488290db824
--- /dev/null
+++ b/app/services/projects/container_repository/cleanup_tags_service.rb
@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Projects
+  module ContainerRepository
+    class CleanupTagsService < BaseService
+      def execute(container_repository)
+        return error('feature disabled') unless can_use?
+        return error('access denied') unless can_admin?
+
+        tags = container_repository.tags
+        tags_by_digest = group_by_digest(tags)
+
+        tags = without_latest(tags)
+        tags = filter_by_name(tags)
+        tags = with_manifest(tags)
+        tags = order_by_date(tags)
+        tags = filter_keep_n(tags)
+        tags = filter_by_older_than(tags)
+
+        deleted_tags = delete_tags(tags, tags_by_digest)
+
+        success(deleted: deleted_tags.map(&:name))
+      end
+
+      private
+
+      def delete_tags(tags_to_delete, tags_by_digest)
+        deleted_digests = group_by_digest(tags_to_delete).select do |digest, tags|
+          delete_tag_digest(digest, tags, tags_by_digest[digest])
+        end
+
+        deleted_digests.values.flatten
+      end
+
+      def delete_tag_digest(digest, tags, other_tags)
+        # Issue: https://gitlab.com/gitlab-org/gitlab-ce/issues/21405
+        # we have to remove all tags due
+        # to Docker Distribution bug unable
+        # to delete single tag
+        return unless tags.count == other_tags.count
+
+        # delete all tags
+        tags.map(&:delete)
+      end
+
+      def group_by_digest(tags)
+        tags.group_by(&:digest)
+      end
+
+      def without_latest(tags)
+        tags.reject(&:latest?)
+      end
+
+      def with_manifest(tags)
+        tags.select(&:valid?)
+      end
+
+      def order_by_date(tags)
+        now = DateTime.now
+        tags.sort_by { |tag| tag.created_at || now }.reverse
+      end
+
+      def filter_by_name(tags)
+        regex = Gitlab::UntrustedRegexp.new("\\A#{params['name_regex']}\\z")
+
+        tags.select do |tag|
+          regex.scan(tag.name).any?
+        end
+      end
+
+      def filter_keep_n(tags)
+        tags.drop(params['keep_n'].to_i)
+      end
+
+      def filter_by_older_than(tags)
+        return tags unless params['older_than']
+
+        older_than = ChronicDuration.parse(params['older_than']).seconds.ago
+
+        tags.select do |tag|
+          tag.created_at && tag.created_at < older_than
+        end
+      end
+
+      def can_admin?
+        can?(current_user, :admin_container_image, project)
+      end
+
+      def can_use?
+        Feature.enabled?(:container_registry_cleanup, project, default_enabled: true)
+      end
+    end
+  end
+end
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 223ddc80c88..4a306fc7a26 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -90,13 +90,15 @@
 - object_pool:object_pool_join
 - object_pool:object_pool_destroy
 
+- container_repository:delete_container_repository
+- container_repository:cleanup_container_repository
+
 - default
 - mailers # ActionMailer::DeliveryJob.queue_name
 
 - authorized_projects
 - background_migration
 - create_gpg_signature
-- delete_container_repository
 - delete_merged_branches
 - delete_user
 - email_receiver
diff --git a/app/workers/cleanup_container_repository_worker.rb b/app/workers/cleanup_container_repository_worker.rb
new file mode 100644
index 00000000000..974ee8c8146
--- /dev/null
+++ b/app/workers/cleanup_container_repository_worker.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+class CleanupContainerRepositoryWorker
+  include ApplicationWorker
+  include ExclusiveLeaseGuard
+
+  queue_namespace :container_repository
+
+  LEASE_TIMEOUT = 1.hour
+
+  attr_reader :container_repository, :current_user
+
+  def perform(current_user_id, container_repository_id, params)
+    @current_user = User.find_by_id(current_user_id)
+    @container_repository = ContainerRepository.find_by_id(container_repository_id)
+
+    return unless valid?
+
+    try_obtain_lease do
+      Projects::ContainerRepository::CleanupTagsService
+        .new(project, current_user, params)
+        .execute(container_repository)
+    end
+  end
+
+  private
+
+  def valid?
+    current_user && container_repository && project
+  end
+
+  def project
+    container_repository&.project
+  end
+
+  # For ExclusiveLeaseGuard concern
+  def lease_key
+    @lease_key ||= "container_repository:cleanup_tags:#{container_repository.id}"
+  end
+
+  # For ExclusiveLeaseGuard concern
+  def lease_timeout
+    LEASE_TIMEOUT
+  end
+
+  # For ExclusiveLeaseGuard concern
+  def lease_release?
+    # we don't allow to execute this worker
+    # more often than LEASE_TIMEOUT
+    # for given container repository
+    false
+  end
+end
diff --git a/app/workers/delete_container_repository_worker.rb b/app/workers/delete_container_repository_worker.rb
index e8fe9d82797..42e66513ff1 100644
--- a/app/workers/delete_container_repository_worker.rb
+++ b/app/workers/delete_container_repository_worker.rb
@@ -4,6 +4,8 @@ class DeleteContainerRepositoryWorker
   include ApplicationWorker
   include ExclusiveLeaseGuard
 
+  queue_namespace :container_repository
+
   LEASE_TIMEOUT = 1.hour
 
   attr_reader :container_repository