From 045d07bab37df2020f650f7354157f5267f57c8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Thu, 10 Jan 2019 15:22:58 +0100 Subject: Add Container Registry API This includes a set of APIs to manipulate container registry. This includes also an ability to delete tags based on requested criteria, like keep-last-n, matching-name, older-than. --- app/models/container_repository.rb | 11 ++- app/policies/container_repository_policy.rb | 5 ++ app/serializers/container_repository_entity.rb | 2 +- app/serializers/container_tag_entity.rb | 2 +- app/services/concerns/exclusive_lease_guard.rb | 13 ++- .../container_repository/cleanup_tags_service.rb | 94 ++++++++++++++++++++++ app/workers/all_queues.yml | 4 +- app/workers/cleanup_container_repository_worker.rb | 53 ++++++++++++ app/workers/delete_container_repository_worker.rb | 2 + 9 files changed, 178 insertions(+), 8 deletions(-) create mode 100644 app/policies/container_repository_policy.rb create mode 100644 app/services/projects/container_repository/cleanup_tags_service.rb create mode 100644 app/workers/cleanup_container_repository_worker.rb (limited to 'app') diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 2c08a8e1acf..cf057d774cf 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class ContainerRepository < ActiveRecord::Base + include Gitlab::Utils::StrongMemoize + belongs_to :project validates :name, length: { minimum: 0, allow_nil: false } @@ -8,6 +10,8 @@ class ContainerRepository < ActiveRecord::Base delegate :client, to: :registry + scope :ordered, -> { order(:name) } + # rubocop: disable CodeReuse/ServiceClass def registry @registry ||= begin @@ -39,11 +43,12 @@ class ContainerRepository < ActiveRecord::Base end def tags - return @tags if defined?(@tags) return [] unless manifest && manifest['tags'] - @tags = manifest['tags'].map do |tag| - ContainerRegistry::Tag.new(self, tag) + strong_memoize(:tags) do + manifest['tags'].sort.map do |tag| + ContainerRegistry::Tag.new(self, tag) + end end end diff --git a/app/policies/container_repository_policy.rb b/app/policies/container_repository_policy.rb new file mode 100644 index 00000000000..6781c845142 --- /dev/null +++ b/app/policies/container_repository_policy.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class ContainerRepositoryPolicy < BasePolicy + delegate { @subject.project } +end diff --git a/app/serializers/container_repository_entity.rb b/app/serializers/container_repository_entity.rb index 59bf35f5aff..cc746698a05 100644 --- a/app/serializers/container_repository_entity.rb +++ b/app/serializers/container_repository_entity.rb @@ -3,7 +3,7 @@ class ContainerRepositoryEntity < Grape::Entity include RequestAwareEntity - expose :id, :path, :location + expose :id, :name, :path, :location, :created_at expose :tags_path do |repository| project_registry_repository_tags_path(project, repository, format: :json) diff --git a/app/serializers/container_tag_entity.rb b/app/serializers/container_tag_entity.rb index 637294877f8..361c073e22e 100644 --- a/app/serializers/container_tag_entity.rb +++ b/app/serializers/container_tag_entity.rb @@ -3,7 +3,7 @@ class ContainerTagEntity < Grape::Entity include RequestAwareEntity - expose :name, :location, :revision, :short_revision, :total_size, :created_at + expose :name, :path, :location, :digest, :revision, :short_revision, :total_size, :created_at expose :destroy_path, if: -> (*) { can_destroy? } do |tag| project_registry_repository_tag_path(project, tag.repository, tag.name) diff --git a/app/services/concerns/exclusive_lease_guard.rb b/app/services/concerns/exclusive_lease_guard.rb index f102e00d150..28879d2d67f 100644 --- a/app/services/concerns/exclusive_lease_guard.rb +++ b/app/services/concerns/exclusive_lease_guard.rb @@ -6,9 +6,14 @@ # # `#try_obtain_lease` takes a block which will be run if it was able to # obtain the lease. Implement `#lease_timeout` to configure the timeout -# for the exclusive lease. Optionally override `#lease_key` to set the +# for the exclusive lease. +# +# Optionally override `#lease_key` to set the # lease key, it defaults to the class name with underscores. # +# Optionally override `#lease_release?` to prevent the job to +# be re-executed more often than LEASE_TIMEOUT. +# module ExclusiveLeaseGuard extend ActiveSupport::Concern @@ -23,7 +28,7 @@ module ExclusiveLeaseGuard begin yield lease ensure - release_lease(lease) + release_lease(lease) if lease_release? end end @@ -40,6 +45,10 @@ module ExclusiveLeaseGuard "#{self.class.name} does not implement #{__method__}" end + def lease_release? + true + end + def release_lease(uuid) Gitlab::ExclusiveLease.cancel(lease_key, uuid) end diff --git a/app/services/projects/container_repository/cleanup_tags_service.rb b/app/services/projects/container_repository/cleanup_tags_service.rb new file mode 100644 index 00000000000..488290db824 --- /dev/null +++ b/app/services/projects/container_repository/cleanup_tags_service.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module Projects + module ContainerRepository + class CleanupTagsService < BaseService + def execute(container_repository) + return error('feature disabled') unless can_use? + return error('access denied') unless can_admin? + + tags = container_repository.tags + tags_by_digest = group_by_digest(tags) + + tags = without_latest(tags) + tags = filter_by_name(tags) + tags = with_manifest(tags) + tags = order_by_date(tags) + tags = filter_keep_n(tags) + tags = filter_by_older_than(tags) + + deleted_tags = delete_tags(tags, tags_by_digest) + + success(deleted: deleted_tags.map(&:name)) + end + + private + + def delete_tags(tags_to_delete, tags_by_digest) + deleted_digests = group_by_digest(tags_to_delete).select do |digest, tags| + delete_tag_digest(digest, tags, tags_by_digest[digest]) + end + + deleted_digests.values.flatten + end + + def delete_tag_digest(digest, tags, other_tags) + # Issue: https://gitlab.com/gitlab-org/gitlab-ce/issues/21405 + # we have to remove all tags due + # to Docker Distribution bug unable + # to delete single tag + return unless tags.count == other_tags.count + + # delete all tags + tags.map(&:delete) + end + + def group_by_digest(tags) + tags.group_by(&:digest) + end + + def without_latest(tags) + tags.reject(&:latest?) + end + + def with_manifest(tags) + tags.select(&:valid?) + end + + def order_by_date(tags) + now = DateTime.now + tags.sort_by { |tag| tag.created_at || now }.reverse + end + + def filter_by_name(tags) + regex = Gitlab::UntrustedRegexp.new("\\A#{params['name_regex']}\\z") + + tags.select do |tag| + regex.scan(tag.name).any? + end + end + + def filter_keep_n(tags) + tags.drop(params['keep_n'].to_i) + end + + def filter_by_older_than(tags) + return tags unless params['older_than'] + + older_than = ChronicDuration.parse(params['older_than']).seconds.ago + + tags.select do |tag| + tag.created_at && tag.created_at < older_than + end + end + + def can_admin? + can?(current_user, :admin_container_image, project) + end + + def can_use? + Feature.enabled?(:container_registry_cleanup, project, default_enabled: true) + end + end + end +end diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 223ddc80c88..4a306fc7a26 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -90,13 +90,15 @@ - object_pool:object_pool_join - object_pool:object_pool_destroy +- container_repository:delete_container_repository +- container_repository:cleanup_container_repository + - default - mailers # ActionMailer::DeliveryJob.queue_name - authorized_projects - background_migration - create_gpg_signature -- delete_container_repository - delete_merged_branches - delete_user - email_receiver diff --git a/app/workers/cleanup_container_repository_worker.rb b/app/workers/cleanup_container_repository_worker.rb new file mode 100644 index 00000000000..974ee8c8146 --- /dev/null +++ b/app/workers/cleanup_container_repository_worker.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +class CleanupContainerRepositoryWorker + include ApplicationWorker + include ExclusiveLeaseGuard + + queue_namespace :container_repository + + LEASE_TIMEOUT = 1.hour + + attr_reader :container_repository, :current_user + + def perform(current_user_id, container_repository_id, params) + @current_user = User.find_by_id(current_user_id) + @container_repository = ContainerRepository.find_by_id(container_repository_id) + + return unless valid? + + try_obtain_lease do + Projects::ContainerRepository::CleanupTagsService + .new(project, current_user, params) + .execute(container_repository) + end + end + + private + + def valid? + current_user && container_repository && project + end + + def project + container_repository&.project + end + + # For ExclusiveLeaseGuard concern + def lease_key + @lease_key ||= "container_repository:cleanup_tags:#{container_repository.id}" + end + + # For ExclusiveLeaseGuard concern + def lease_timeout + LEASE_TIMEOUT + end + + # For ExclusiveLeaseGuard concern + def lease_release? + # we don't allow to execute this worker + # more often than LEASE_TIMEOUT + # for given container repository + false + end +end diff --git a/app/workers/delete_container_repository_worker.rb b/app/workers/delete_container_repository_worker.rb index e8fe9d82797..42e66513ff1 100644 --- a/app/workers/delete_container_repository_worker.rb +++ b/app/workers/delete_container_repository_worker.rb @@ -4,6 +4,8 @@ class DeleteContainerRepositoryWorker include ApplicationWorker include ExclusiveLeaseGuard + queue_namespace :container_repository + LEASE_TIMEOUT = 1.hour attr_reader :container_repository -- cgit v1.2.3