diff options
Diffstat (limited to 'app/services/projects/container_repository')
5 files changed, 254 insertions, 139 deletions
diff --git a/app/services/projects/container_repository/base_container_repository_service.rb b/app/services/projects/container_repository/base_container_repository_service.rb new file mode 100644 index 00000000000..d7539737e78 --- /dev/null +++ b/app/services/projects/container_repository/base_container_repository_service.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Projects + module ContainerRepository + class BaseContainerRepositoryService < ::BaseContainerService + include ::Gitlab::Utils::StrongMemoize + + alias_method :container_repository, :container + + def initialize(container_repository:, current_user: nil, params: {}) + super(container: container_repository, current_user: current_user, params: params) + end + + delegate :project, to: :container_repository + end + end +end diff --git a/app/services/projects/container_repository/cleanup_tags_base_service.rb b/app/services/projects/container_repository/cleanup_tags_base_service.rb new file mode 100644 index 00000000000..8ea4ae4830a --- /dev/null +++ b/app/services/projects/container_repository/cleanup_tags_base_service.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +module Projects + module ContainerRepository + class CleanupTagsBaseService < BaseContainerRepositoryService + private + + def filter_out_latest!(tags) + tags.reject!(&:latest?) + end + + def filter_by_name!(tags) + regex_delete = ::Gitlab::UntrustedRegexp.new("\\A#{name_regex_delete || name_regex}\\z") + regex_retain = ::Gitlab::UntrustedRegexp.new("\\A#{name_regex_keep}\\z") + + tags.select! do |tag| + # regex_retain will override any overlapping matches by regex_delete + regex_delete.match?(tag.name) && !regex_retain.match?(tag.name) + end + end + + # Should return [tags_to_delete, tags_to_keep] + def partition_by_keep_n(tags) + return [tags, []] unless keep_n + + tags = order_by_date_desc(tags) + + tags.partition.with_index { |_, index| index >= keep_n_as_integer } + end + + # Should return [tags_to_delete, tags_to_keep] + def partition_by_older_than(tags) + return [tags, []] unless older_than + + older_than_timestamp = older_than_in_seconds.ago + + tags.partition do |tag| + timestamp = pushed_at(tag) + + timestamp && timestamp < older_than_timestamp + end + end + + def order_by_date_desc(tags) + now = DateTime.current + tags.sort_by! { |tag| pushed_at(tag) || now } + .reverse! + end + + def delete_tags(tags) + return success(deleted: []) unless tags.any? + + service = Projects::ContainerRepository::DeleteTagsService.new( + project, + current_user, + tags: tags.map(&:name), + container_expiration_policy: container_expiration_policy + ) + + service.execute(container_repository) + end + + def can_destroy? + return true if container_expiration_policy + + can?(current_user, :destroy_container_image, project) + end + + def valid_regex? + %w[name_regex_delete name_regex name_regex_keep].each do |param_name| + regex = params[param_name] + ::Gitlab::UntrustedRegexp.new(regex) unless regex.blank? + end + true + rescue RegexpError => e + ::Gitlab::ErrorTracking.log_exception(e, project_id: project.id) + false + end + + def older_than + params['older_than'] + end + + def name_regex_delete + params['name_regex_delete'] + end + + def name_regex + params['name_regex'] + end + + def name_regex_keep + params['name_regex_keep'] + end + + def container_expiration_policy + params['container_expiration_policy'] + end + + def keep_n + params['keep_n'] + end + + def project + container_repository.project + end + + def keep_n_as_integer + keep_n.to_i + end + + def older_than_in_seconds + strong_memoize(:older_than_in_seconds) do + ChronicDuration.parse(older_than).seconds + end + end + end + end +end diff --git a/app/services/projects/container_repository/cleanup_tags_service.rb b/app/services/projects/container_repository/cleanup_tags_service.rb index 0a8e8e72766..285c3e252ef 100644 --- a/app/services/projects/container_repository/cleanup_tags_service.rb +++ b/app/services/projects/container_repository/cleanup_tags_service.rb @@ -2,39 +2,33 @@ module Projects module ContainerRepository - class CleanupTagsService - include BaseServiceUtility - include ::Gitlab::Utils::StrongMemoize + class CleanupTagsService < CleanupTagsBaseService + def initialize(container_repository:, current_user: nil, params: {}) + super - def initialize(container_repository, user = nil, params = {}) - @container_repository = container_repository - @current_user = user @params = params.dup - - @project = container_repository.project - @tags = container_repository.tags - tags_size = @tags.size - @counts = { - original_size: tags_size, - cached_tags_count: 0 - } + @counts = { cached_tags_count: 0 } end def execute return error('access denied') unless can_destroy? return error('invalid regex') unless valid_regex? - filter_out_latest - filter_by_name + tags = container_repository.tags + @counts[:original_size] = tags.size + + filter_out_latest!(tags) + filter_by_name!(tags) + + tags = truncate(tags) + populate_from_cache(tags) - truncate - populate_from_cache + tags = filter_keep_n(tags) + tags = filter_by_older_than(tags) - filter_keep_n - filter_by_older_than + @counts[:before_delete_size] = tags.size - delete_tags.merge(@counts).tap do |result| - result[:before_delete_size] = @tags.size + delete_tags(tags).merge(@counts).tap do |result| result[:deleted_size] = result[:deleted]&.size result[:status] = :error if @counts[:before_truncate_size] != @counts[:after_truncate_size] @@ -43,94 +37,45 @@ module Projects private - def delete_tags - return success(deleted: []) unless @tags.any? - - service = Projects::ContainerRepository::DeleteTagsService.new( - @project, - @current_user, - tags: @tags.map(&:name), - container_expiration_policy: container_expiration_policy - ) - - service.execute(@container_repository) - end - - def filter_out_latest - @tags.reject!(&:latest?) - end - - def order_by_date - now = DateTime.current - @tags.sort_by! { |tag| tag.created_at || now } - .reverse! - end + def filter_keep_n(tags) + tags, tags_to_keep = partition_by_keep_n(tags) - def filter_by_name - regex_delete = ::Gitlab::UntrustedRegexp.new("\\A#{name_regex_delete || name_regex}\\z") - regex_retain = ::Gitlab::UntrustedRegexp.new("\\A#{name_regex_keep}\\z") - - @tags.select! do |tag| - # regex_retain will override any overlapping matches by regex_delete - regex_delete.match?(tag.name) && !regex_retain.match?(tag.name) - end - end - - def filter_keep_n - return unless keep_n + cache_tags(tags_to_keep) - order_by_date - cache_tags(@tags.first(keep_n_as_integer)) - @tags = @tags.drop(keep_n_as_integer) + tags end - def filter_by_older_than - return unless older_than - - older_than_timestamp = older_than_in_seconds.ago - - @tags, tags_to_keep = @tags.partition do |tag| - tag.created_at && tag.created_at < older_than_timestamp - end + def filter_by_older_than(tags) + tags, tags_to_keep = partition_by_older_than(tags) cache_tags(tags_to_keep) - end - def can_destroy? - return true if container_expiration_policy - - can?(@current_user, :destroy_container_image, @project) + tags end - def valid_regex? - %w(name_regex_delete name_regex name_regex_keep).each do |param_name| - regex = @params[param_name] - ::Gitlab::UntrustedRegexp.new(regex) unless regex.blank? - end - true - rescue RegexpError => e - ::Gitlab::ErrorTracking.log_exception(e, project_id: @project.id) - false + def pushed_at(tag) + tag.created_at end - def truncate - @counts[:before_truncate_size] = @tags.size - @counts[:after_truncate_size] = @tags.size + def truncate(tags) + @counts[:before_truncate_size] = tags.size + @counts[:after_truncate_size] = tags.size - return if max_list_size == 0 + return tags if max_list_size == 0 # truncate the list to make sure that after the #filter_keep_n # execution, the resulting list will be max_list_size truncated_size = max_list_size + keep_n_as_integer - return if @tags.size <= truncated_size + return tags if tags.size <= truncated_size - @tags = @tags.sample(truncated_size) - @counts[:after_truncate_size] = @tags.size + tags = tags.sample(truncated_size) + @counts[:after_truncate_size] = tags.size + tags end - def populate_from_cache - @counts[:cached_tags_count] = cache.populate(@tags) if caching_enabled? + def populate_from_cache(tags) + @counts[:cached_tags_count] = cache.populate(tags) if caching_enabled? end def cache_tags(tags) @@ -139,7 +84,7 @@ module Projects def cache strong_memoize(:cache) do - ::Gitlab::ContainerRepository::Tags::Cache.new(@container_repository) + ::Gitlab::ContainerRepository::Tags::Cache.new(container_repository) end end @@ -153,40 +98,6 @@ module Projects def max_list_size ::Gitlab::CurrentSettings.current_application_settings.container_registry_cleanup_tags_service_max_list_size.to_i end - - def keep_n - @params['keep_n'] - end - - def keep_n_as_integer - keep_n.to_i - end - - def older_than_in_seconds - strong_memoize(:older_than_in_seconds) do - ChronicDuration.parse(older_than).seconds - end - end - - def older_than - @params['older_than'] - end - - def name_regex_delete - @params['name_regex_delete'] - end - - def name_regex - @params['name_regex'] - end - - def name_regex_keep - @params['name_regex_keep'] - end - - def container_expiration_policy - @params['container_expiration_policy'] - end end end end diff --git a/app/services/projects/container_repository/gitlab/cleanup_tags_service.rb b/app/services/projects/container_repository/gitlab/cleanup_tags_service.rb new file mode 100644 index 00000000000..81bb94c867a --- /dev/null +++ b/app/services/projects/container_repository/gitlab/cleanup_tags_service.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Projects + module ContainerRepository + module Gitlab + class CleanupTagsService < CleanupTagsBaseService + include ::Projects::ContainerRepository::Gitlab::Timeoutable + + TAGS_PAGE_SIZE = 1000 + + def initialize(container_repository:, current_user: nil, params: {}) + super + @params = params.dup + end + + def execute + return error('access denied') unless can_destroy? + return error('invalid regex') unless valid_regex? + + with_timeout do |start_time, result| + container_repository.each_tags_page(page_size: TAGS_PAGE_SIZE) do |tags| + execute_for_tags(tags, result) + + raise TimeoutError if timeout?(start_time) + end + end + end + + private + + def execute_for_tags(tags, overall_result) + original_size = tags.size + + filter_out_latest!(tags) + filter_by_name!(tags) + + tags = filter_by_keep_n(tags) + tags = filter_by_older_than(tags) + + overall_result[:before_delete_size] += tags.size + overall_result[:original_size] += original_size + + result = delete_tags(tags) + + overall_result[:deleted_size] += result[:deleted]&.size + overall_result[:deleted] += result[:deleted] + overall_result[:status] = result[:status] unless overall_result[:status] == :error + end + + def with_timeout + result = { + original_size: 0, + before_delete_size: 0, + deleted_size: 0, + deleted: [] + } + + yield Time.zone.now, result + + result + rescue TimeoutError + result[:status] = :error + + result + end + + def filter_by_keep_n(tags) + partition_by_keep_n(tags).first + end + + def filter_by_older_than(tags) + partition_by_older_than(tags).first + end + + def pushed_at(tag) + tag.updated_at || tag.created_at + end + end + end + end +end diff --git a/app/services/projects/container_repository/gitlab/delete_tags_service.rb b/app/services/projects/container_repository/gitlab/delete_tags_service.rb index 81cef554dec..530cf87c338 100644 --- a/app/services/projects/container_repository/gitlab/delete_tags_service.rb +++ b/app/services/projects/container_repository/gitlab/delete_tags_service.rb @@ -6,10 +6,7 @@ module Projects class DeleteTagsService include BaseServiceUtility include ::Gitlab::Utils::StrongMemoize - - DISABLED_TIMEOUTS = [nil, 0].freeze - - TimeoutError = Class.new(StandardError) + include ::Projects::ContainerRepository::Gitlab::Timeoutable def initialize(container_repository, tag_names) @container_repository = container_repository @@ -44,16 +41,6 @@ module Projects @deleted_tags.any? ? success(deleted: @deleted_tags) : error('could not delete tags') end - - def timeout?(start_time) - return false if service_timeout.in?(DISABLED_TIMEOUTS) - - (Time.zone.now - start_time) > service_timeout - end - - def service_timeout - ::Gitlab::CurrentSettings.current_application_settings.container_registry_delete_tags_service_timeout - end end end end |