diff options
Diffstat (limited to 'spec/services/projects/container_repository')
-rw-r--r-- | spec/services/projects/container_repository/cleanup_tags_service_spec.rb | 456 | ||||
-rw-r--r-- | spec/services/projects/container_repository/gitlab/cleanup_tags_service_spec.rb | 183 |
2 files changed, 341 insertions, 298 deletions
diff --git a/spec/services/projects/container_repository/cleanup_tags_service_spec.rb b/spec/services/projects/container_repository/cleanup_tags_service_spec.rb index 79904e2bf72..2008de195ab 100644 --- a/spec/services/projects/container_repository/cleanup_tags_service_spec.rb +++ b/spec/services/projects/container_repository/cleanup_tags_service_spec.rb @@ -5,11 +5,13 @@ require 'spec_helper' RSpec.describe Projects::ContainerRepository::CleanupTagsService, :clean_gitlab_redis_cache do using RSpec::Parameterized::TableSyntax + include_context 'for a cleanup tags service' + let_it_be(:user) { create(:user) } let_it_be(:project, reload: true) { create(:project, :private) } let(:repository) { create(:container_repository, :root, project: project) } - let(:service) { described_class.new(repository, user, params) } + let(:service) { described_class.new(container_repository: repository, current_user: user, params: params) } let(:tags) { %w[latest A Ba Bb C D E] } before do @@ -39,268 +41,141 @@ RSpec.describe Projects::ContainerRepository::CleanupTagsService, :clean_gitlab_ describe '#execute' do subject { service.execute } - context 'when no params are specified' do - let(:params) { {} } - - it 'does not remove anything' do - expect_any_instance_of(Projects::ContainerRepository::DeleteTagsService) - .not_to receive(:execute) - expect_no_caching - - is_expected.to eq(expected_service_response(before_truncate_size: 0, after_truncate_size: 0, before_delete_size: 0)) - end - end - - context 'when regex matching everything is specified' do - shared_examples 'removes all matches' do - it 'does remove all tags except latest' do - expect_no_caching - - expect_delete(%w(A Ba Bb C D E)) - - is_expected.to eq(expected_service_response(deleted: %w(A Ba Bb C D E))) - end - end - - let(:params) do - { 'name_regex_delete' => '.*' } - end - - it_behaves_like 'removes all matches' - - context 'with deprecated name_regex param' do - let(:params) do - { 'name_regex' => '.*' } - end - - it_behaves_like 'removes all matches' - end - end - - context 'with invalid regular expressions' do - shared_examples 'handling an invalid regex' do - it 'keeps all tags' do - expect_no_caching - - expect(Projects::ContainerRepository::DeleteTagsService) - .not_to receive(:new) - - subject - end - - it { is_expected.to eq(status: :error, message: 'invalid regex') } - - it 'calls error tracking service' do - expect(Gitlab::ErrorTracking).to receive(:log_exception).and_call_original - - subject - end - end - - context 'when name_regex_delete is invalid' do - let(:params) { { 'name_regex_delete' => '*test*' } } - - it_behaves_like 'handling an invalid regex' - end - - context 'when name_regex is invalid' do - let(:params) { { 'name_regex' => '*test*' } } - - it_behaves_like 'handling an invalid regex' - end - - context 'when name_regex_keep is invalid' do - let(:params) { { 'name_regex_keep' => '*test*' } } - - it_behaves_like 'handling an invalid regex' - end - end - - context 'when delete regex matching specific tags is used' do - let(:params) do - { 'name_regex_delete' => 'C|D' } - end - - it 'does remove C and D' do - expect_delete(%w(C D)) - - expect_no_caching - - is_expected.to eq(expected_service_response(deleted: %w(C D), before_truncate_size: 2, after_truncate_size: 2, before_delete_size: 2)) - end - - context 'with overriding allow regex' do - let(:params) do - { 'name_regex_delete' => 'C|D', - 'name_regex_keep' => 'C' } - end - - it 'does not remove C' do - expect_delete(%w(D)) - - expect_no_caching - - is_expected.to eq(expected_service_response(deleted: %w(D), before_truncate_size: 1, after_truncate_size: 1, before_delete_size: 1)) - end - end - - context 'with name_regex_delete overriding deprecated name_regex' do - let(:params) do - { 'name_regex' => 'C|D', - 'name_regex_delete' => 'D' } - end - - it 'does not remove C' do - expect_delete(%w(D)) - - expect_no_caching - - is_expected.to eq(expected_service_response(deleted: %w(D), before_truncate_size: 1, after_truncate_size: 1, before_delete_size: 1)) - end - end - end - - context 'with allow regex value' do - let(:params) do - { 'name_regex_delete' => '.*', - 'name_regex_keep' => 'B.*' } - end - - it 'does not remove B*' do - expect_delete(%w(A C D E)) - - expect_no_caching - - is_expected.to eq(expected_service_response(deleted: %w(A C D E), before_truncate_size: 4, after_truncate_size: 4, before_delete_size: 4)) - end - end - - context 'when keeping only N tags' do - let(:params) do - { 'name_regex' => 'A|B.*|C', - 'keep_n' => 1 } - end - - it 'sorts tags by date' do - expect_delete(%w(Bb Ba C)) - - expect_no_caching - - expect(service).to receive(:order_by_date).and_call_original - - is_expected.to eq(expected_service_response(deleted: %w(Bb Ba C), before_truncate_size: 4, after_truncate_size: 4, before_delete_size: 3)) - end - end - - context 'when not keeping N tags' do - let(:params) do - { 'name_regex' => 'A|B.*|C' } - end - - it 'does not sort tags by date' do - expect_delete(%w(A Ba Bb C)) - - expect_no_caching - - expect(service).not_to receive(:order_by_date) - - is_expected.to eq(expected_service_response(deleted: %w(A Ba Bb C), before_truncate_size: 4, after_truncate_size: 4, before_delete_size: 4)) - end - end - - context 'when removing keeping only 3' do - let(:params) do - { 'name_regex_delete' => '.*', - 'keep_n' => 3 } - end - - it 'does remove B* and C as they are the oldest' do - expect_delete(%w(Bb Ba C)) - - expect_no_caching - - is_expected.to eq(expected_service_response(deleted: %w(Bb Ba C), before_delete_size: 3)) - end - end - - context 'when removing older than 1 day' do - let(:params) do - { 'name_regex_delete' => '.*', - 'older_than' => '1 day' } - end - - it 'does remove B* and C as they are older than 1 day' do - expect_delete(%w(Ba Bb C)) - - expect_no_caching - - is_expected.to eq(expected_service_response(deleted: %w(Ba Bb C), before_delete_size: 3)) - end - end - - context 'when combining all parameters' do + it_behaves_like 'handling invalid params', + service_response_extra: { + before_truncate_size: 0, + after_truncate_size: 0, + before_delete_size: 0, + cached_tags_count: 0 + }, + supports_caching: true + + it_behaves_like 'when regex matching everything is specified', + delete_expectations: [%w(A Ba Bb C D E)], + service_response_extra: { + before_truncate_size: 6, + after_truncate_size: 6, + before_delete_size: 6, + cached_tags_count: 0 + }, + supports_caching: true + + it_behaves_like 'when delete regex matching specific tags is used', + service_response_extra: { + before_truncate_size: 2, + after_truncate_size: 2, + before_delete_size: 2, + cached_tags_count: 0 + }, + supports_caching: true + + it_behaves_like 'when delete regex matching specific tags is used with overriding allow regex', + service_response_extra: { + before_truncate_size: 1, + after_truncate_size: 1, + before_delete_size: 1, + cached_tags_count: 0 + }, + supports_caching: true + + it_behaves_like 'with allow regex value', + delete_expectations: [%w(A C D E)], + service_response_extra: { + before_truncate_size: 4, + after_truncate_size: 4, + before_delete_size: 4, + cached_tags_count: 0 + }, + supports_caching: true + + it_behaves_like 'when keeping only N tags', + delete_expectations: [%w(Bb Ba C)], + service_response_extra: { + before_truncate_size: 4, + after_truncate_size: 4, + before_delete_size: 3, + cached_tags_count: 0 + }, + supports_caching: true + + it_behaves_like 'when not keeping N tags', + delete_expectations: [%w(A Ba Bb C)], + service_response_extra: { + before_truncate_size: 4, + after_truncate_size: 4, + before_delete_size: 4, + cached_tags_count: 0 + }, + supports_caching: true + + it_behaves_like 'when removing keeping only 3', + delete_expectations: [%w(Bb Ba C)], + service_response_extra: { + before_truncate_size: 6, + after_truncate_size: 6, + before_delete_size: 3, + cached_tags_count: 0 + }, + supports_caching: true + + it_behaves_like 'when removing older than 1 day', + delete_expectations: [%w(Ba Bb C)], + service_response_extra: { + before_truncate_size: 6, + after_truncate_size: 6, + before_delete_size: 3, + cached_tags_count: 0 + }, + supports_caching: true + + it_behaves_like 'when combining all parameters', + delete_expectations: [%w(Bb Ba C)], + service_response_extra: { + before_truncate_size: 6, + after_truncate_size: 6, + before_delete_size: 3, + cached_tags_count: 0 + }, + supports_caching: true + + it_behaves_like 'when running a container_expiration_policy', + delete_expectations: [%w(Bb Ba C)], + service_response_extra: { + before_truncate_size: 6, + after_truncate_size: 6, + before_delete_size: 3, + cached_tags_count: 0 + }, + supports_caching: true + + context 'when running a container_expiration_policy with caching' do + let(:user) { nil } let(:params) do - { 'name_regex_delete' => '.*', + { + 'name_regex_delete' => '.*', 'keep_n' => 1, - 'older_than' => '1 day' } + 'older_than' => '1 day', + 'container_expiration_policy' => true + } end - it 'does remove B* and C' do - expect_delete(%w(Bb Ba C)) - - expect_no_caching + it 'expects caching to be used' do + expect_delete(%w(Bb Ba C), container_expiration_policy: true) + expect_caching - is_expected.to eq(expected_service_response(deleted: %w(Bb Ba C), before_delete_size: 3)) + subject end - end - - context 'when running a container_expiration_policy' do - let(:user) { nil } - - context 'with valid container_expiration_policy param' do - let(:params) do - { 'name_regex_delete' => '.*', - 'keep_n' => 1, - 'older_than' => '1 day', - 'container_expiration_policy' => true } - end + context 'when setting set to false' do before do - expect_delete(%w(Bb Ba C), container_expiration_policy: true) - end - - it { is_expected.to eq(expected_service_response(deleted: %w(Bb Ba C), before_delete_size: 3)) } - - context 'caching' do - it 'expects caching to be used' do - expect_caching - - subject - end - - context 'when setting set to false' do - before do - stub_application_setting(container_registry_expiration_policies_caching: false) - end - - it 'does not use caching' do - expect_no_caching - - subject - end - end + stub_application_setting(container_registry_expiration_policies_caching: false) end - end - context 'without container_expiration_policy param' do - let(:params) do - { 'name_regex_delete' => '.*', - 'keep_n' => 1, - 'older_than' => '1 day' } - end + it 'does not use caching' do + expect_delete(%w(Bb Ba C), container_expiration_policy: true) + expect_no_caching - it 'fails' do - is_expected.to eq(status: :error, message: 'access denied') + subject end end end @@ -322,10 +197,12 @@ RSpec.describe Projects::ContainerRepository::CleanupTagsService, :clean_gitlab_ service_response = expected_service_response( status: status, original_size: original_size, + deleted: nil + ).merge( before_truncate_size: before_truncate_size, after_truncate_size: after_truncate_size, before_delete_size: before_delete_size, - deleted: nil + cached_tags_count: 0 ) expect(result).to eq(service_response) @@ -395,11 +272,8 @@ RSpec.describe Projects::ContainerRepository::CleanupTagsService, :clean_gitlab_ end it 'caches the created_at values' do - ::Gitlab::Redis::Cache.with do |redis| - expect_mget(redis, tags_and_created_ats.keys) - - expect_set(redis, cacheable_tags) - end + expect_mget(tags_and_created_ats.keys) + expect_set(cacheable_tags) expect(subject).to include(cached_tags_count: 0) end @@ -412,12 +286,10 @@ RSpec.describe Projects::ContainerRepository::CleanupTagsService, :clean_gitlab_ end it 'uses them' do - ::Gitlab::Redis::Cache.with do |redis| - expect_mget(redis, tags_and_created_ats.keys) + expect_mget(tags_and_created_ats.keys) - # because C is already in cache, it should not be cached again - expect_set(redis, cacheable_tags.except('C')) - end + # because C is already in cache, it should not be cached again + expect_set(cacheable_tags.except('C')) # We will ping the container registry for all tags *except* for C because it's cached expect(ContainerRegistry::Blob).to receive(:new).with(repository, { "digest" => "sha256:configA" }).and_call_original @@ -429,15 +301,27 @@ RSpec.describe Projects::ContainerRepository::CleanupTagsService, :clean_gitlab_ end end - def expect_mget(redis, keys) - expect(redis).to receive(:mget).with(keys.map(&method(:cache_key))).and_call_original + def expect_mget(keys) + Gitlab::Redis::Cache.with do |redis| + expect(redis).to receive(:mget).with(keys.map(&method(:cache_key))).and_call_original + end end - def expect_set(redis, tags) - tags.each do |tag_name, created_at| + def expect_set(tags) + selected_tags = tags.map do |tag_name, created_at| ex = 1.day.seconds - (Time.zone.now - created_at).seconds - if ex > 0 - expect(redis).to receive(:set).with(cache_key(tag_name), rfc3339(created_at), ex: ex.to_i) + [tag_name, created_at, ex.to_i] if ex.positive? + end.compact + + return if selected_tags.count.zero? + + Gitlab::Redis::Cache.with do |redis| + expect(redis).to receive(:pipelined).and_call_original + + expect_next_instance_of(Redis::PipelinedConnection) do |pipeline| + selected_tags.each do |tag_name, created_at, ex| + expect(pipeline).to receive(:set).with(cache_key(tag_name), rfc3339(created_at), ex: ex) + end end end end @@ -476,38 +360,14 @@ RSpec.describe Projects::ContainerRepository::CleanupTagsService, :clean_gitlab_ end end - def expect_delete(tags, container_expiration_policy: nil) - expect(Projects::ContainerRepository::DeleteTagsService) - .to receive(:new) - .with(repository.project, user, tags: tags, container_expiration_policy: container_expiration_policy) - .and_call_original - - expect_any_instance_of(Projects::ContainerRepository::DeleteTagsService) - .to receive(:execute) - .with(repository) { { status: :success, deleted: tags } } - end - - # all those -1 because the default tags on L13 have a "latest" that will be filtered out - def expected_service_response(status: :success, deleted: [], original_size: tags.size, before_truncate_size: tags.size - 1, after_truncate_size: tags.size - 1, before_delete_size: tags.size - 1) - { - status: status, - deleted: deleted, - original_size: original_size, - before_truncate_size: before_truncate_size, - after_truncate_size: after_truncate_size, - before_delete_size: before_delete_size, - cached_tags_count: 0 - }.compact.merge(deleted_size: deleted&.size) - end - - def expect_no_caching - expect(::Gitlab::Redis::Cache).not_to receive(:with) - end - def expect_caching ::Gitlab::Redis::Cache.with do |redis| expect(redis).to receive(:mget).and_call_original - expect(redis).to receive(:set).and_call_original + expect(redis).to receive(:pipelined).and_call_original + + expect_next_instance_of(Redis::PipelinedConnection) do |pipeline| + expect(pipeline).to receive(:set).and_call_original + end end end end diff --git a/spec/services/projects/container_repository/gitlab/cleanup_tags_service_spec.rb b/spec/services/projects/container_repository/gitlab/cleanup_tags_service_spec.rb new file mode 100644 index 00000000000..d2cdb667659 --- /dev/null +++ b/spec/services/projects/container_repository/gitlab/cleanup_tags_service_spec.rb @@ -0,0 +1,183 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Projects::ContainerRepository::Gitlab::CleanupTagsService do + using RSpec::Parameterized::TableSyntax + + include_context 'for a cleanup tags service' + + let_it_be(:user) { create(:user) } + let_it_be(:user) { create(:user) } + let_it_be(:project, reload: true) { create(:project, :private) } + + let(:repository) { create(:container_repository, :root, :import_done, project: project) } + let(:service) { described_class.new(container_repository: repository, current_user: user, params: params) } + let(:tags) { %w[latest A Ba Bb C D E] } + + before do + project.add_maintainer(user) if user + + stub_container_registry_config(enabled: true) + + stub_const("#{described_class}::TAGS_PAGE_SIZE", tags_page_size) + + one_hour_ago = 1.hour.ago + five_days_ago = 5.days.ago + six_days_ago = 6.days.ago + one_month_ago = 1.month.ago + + stub_tags( + { + 'latest' => one_hour_ago, + 'A' => one_hour_ago, + 'Ba' => five_days_ago, + 'Bb' => six_days_ago, + 'C' => one_month_ago, + 'D' => nil, + 'E' => nil + } + ) + end + + describe '#execute' do + subject { service.execute } + + context 'with several tags pages' do + let(:tags_page_size) { 2 } + + it_behaves_like 'handling invalid params' + + it_behaves_like 'when regex matching everything is specified', + delete_expectations: [%w[A], %w[Ba Bb], %w[C D], %w[E]] + + it_behaves_like 'when delete regex matching specific tags is used' + + it_behaves_like 'when delete regex matching specific tags is used with overriding allow regex' + + it_behaves_like 'with allow regex value', + delete_expectations: [%w[A], %w[C D], %w[E]] + + it_behaves_like 'when keeping only N tags', + delete_expectations: [%w[Bb]] + + it_behaves_like 'when not keeping N tags', + delete_expectations: [%w[A], %w[Ba Bb], %w[C]] + + context 'when removing keeping only 3' do + let(:params) do + { + 'name_regex_delete' => '.*', + 'keep_n' => 3 + } + end + + it_behaves_like 'not removing anything' + end + + it_behaves_like 'when removing older than 1 day', + delete_expectations: [%w[Ba Bb], %w[C]] + + it_behaves_like 'when combining all parameters', + delete_expectations: [%w[Bb], %w[C]] + + it_behaves_like 'when running a container_expiration_policy', + delete_expectations: [%w[Bb], %w[C]] + + context 'with a timeout' do + let(:params) do + { 'name_regex_delete' => '.*' } + end + + it 'removes the first few pages' do + expect(service).to receive(:timeout?).and_return(false, true) + + expect_delete(%w[A]) + expect_delete(%w[Ba Bb]) + + response = expected_service_response(status: :error, deleted: %w[A Ba Bb], original_size: 4) + + is_expected.to eq(response) + end + end + end + + context 'with a single tags page' do + let(:tags_page_size) { 1000 } + + it_behaves_like 'handling invalid params' + + it_behaves_like 'when regex matching everything is specified', + delete_expectations: [%w[A Ba Bb C D E]] + + it_behaves_like 'when delete regex matching specific tags is used' + + it_behaves_like 'when delete regex matching specific tags is used with overriding allow regex' + + it_behaves_like 'with allow regex value', + delete_expectations: [%w[A C D E]] + + it_behaves_like 'when keeping only N tags', + delete_expectations: [%w[Ba Bb C]] + + it_behaves_like 'when not keeping N tags', + delete_expectations: [%w[A Ba Bb C]] + + it_behaves_like 'when removing keeping only 3', + delete_expectations: [%w[Ba Bb C]] + + it_behaves_like 'when removing older than 1 day', + delete_expectations: [%w[Ba Bb C]] + + it_behaves_like 'when combining all parameters', + delete_expectations: [%w[Ba Bb C]] + + it_behaves_like 'when running a container_expiration_policy', + delete_expectations: [%w[Ba Bb C]] + end + end + + private + + def stub_tags(tags) + chunked = tags_page_size < tags.size + previous_last = nil + max_chunk_index = tags.size / tags_page_size + + tags.keys.in_groups_of(tags_page_size, false).each_with_index do |chunked_tag_names, index| + last = index == max_chunk_index + pagination_needed = chunked && !last + + response = { + pagination: pagination_needed ? pagination_with(last: chunked_tag_names.last) : {}, + response_body: chunked_tag_names.map do |name| + tag_raw_response(name, tags[name]) + end + } + + allow(repository.gitlab_api_client) + .to receive(:tags) + .with(repository.path, page_size: described_class::TAGS_PAGE_SIZE, last: previous_last) + .and_return(response) + previous_last = chunked_tag_names.last + end + end + + def pagination_with(last:) + { + next: { + uri: URI("http://test.org?last=#{last}") + } + } + end + + def tag_raw_response(name, timestamp) + timestamp_field = name.start_with?('B') ? 'updated_at' : 'created_at' + { + 'name' => name, + 'digest' => 'sha256:1234567890', + 'media_type' => 'application/vnd.oci.image.manifest.v1+json', + timestamp_field => timestamp&.iso8601 + } + end +end |