Add latest changes from gitlab-org/gitlab@master

7 files changed, 172 insertions, 11 deletions

diff --git a/spec/workers/concerns/packages/cleanup_artifact_worker_spec.rb b/spec/workers/concerns/packages/cleanup_artifact_worker_spec.rb
new file mode 100644
index 00000000000..95962d4810e
--- /dev/null
+++ b/spec/workers/concerns/packages/cleanup_artifact_worker_spec.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe ::Packages::CleanupArtifactWorker do
+  let_it_be(:worker_class) do
+    Class.new do
+      def self.name
+        'Gitlab::Foo::Bar::DummyWorker'
+      end
+
+      include ApplicationWorker
+      include ::Packages::CleanupArtifactWorker
+    end
+  end
+
+  let(:worker) { worker_class.new }
+
+  describe '#model' do
+    subject { worker.send(:model) }
+
+    it { expect { subject }.to raise_error(NotImplementedError) }
+  end
+
+  describe '#log_metadata' do
+    subject { worker.send(:log_metadata) }
+
+    it { expect { subject }.to raise_error(NotImplementedError) }
+  end
+
+  describe '#log_cleanup_item' do
+    subject { worker.send(:log_cleanup_item) }
+
+    it { expect { subject }.to raise_error(NotImplementedError) }
+  end
+end
diff --git a/spec/workers/dependency_proxy/cleanup_dependency_proxy_worker_spec.rb b/spec/workers/dependency_proxy/cleanup_dependency_proxy_worker_spec.rb
index ed0bdefbdb8..1100f9a7fae 100644
--- a/spec/workers/dependency_proxy/cleanup_dependency_proxy_worker_spec.rb
+++ b/spec/workers/dependency_proxy/cleanup_dependency_proxy_worker_spec.rb
@@ -9,8 +9,8 @@ RSpec.describe DependencyProxy::CleanupDependencyProxyWorker do
     context 'when there are records to be deleted' do
       it_behaves_like 'an idempotent worker' do
         it 'queues the cleanup jobs', :aggregate_failures do
-          create(:dependency_proxy_blob, :expired)
-          create(:dependency_proxy_manifest, :expired)
+          create(:dependency_proxy_blob, :pending_destruction)
+          create(:dependency_proxy_manifest, :pending_destruction)
 
           expect(DependencyProxy::CleanupBlobWorker).to receive(:perform_with_capacity).twice
           expect(DependencyProxy::CleanupManifestWorker).to receive(:perform_with_capacity).twice
diff --git a/spec/workers/dependency_proxy/image_ttl_group_policy_worker_spec.rb b/spec/workers/dependency_proxy/image_ttl_group_policy_worker_spec.rb
index b035a2ec0b7..6a2fdfbe8f5 100644
--- a/spec/workers/dependency_proxy/image_ttl_group_policy_worker_spec.rb
+++ b/spec/workers/dependency_proxy/image_ttl_group_policy_worker_spec.rb
@@ -17,19 +17,19 @@ RSpec.describe DependencyProxy::ImageTtlGroupPolicyWorker do
       let_it_be_with_reload(:new_blob) { create(:dependency_proxy_blob, group: group) }
       let_it_be_with_reload(:new_manifest) { create(:dependency_proxy_manifest, group: group) }
 
-      it 'updates the old images to expired' do
+      it 'updates the old images to pending_destruction' do
         expect { subject }
-          .to change { old_blob.reload.status }.from('default').to('expired')
-          .and change { old_manifest.reload.status }.from('default').to('expired')
+          .to change { old_blob.reload.status }.from('default').to('pending_destruction')
+          .and change { old_manifest.reload.status }.from('default').to('pending_destruction')
           .and not_change { new_blob.reload.status }
           .and not_change { new_manifest.reload.status }
       end
     end
 
     context 'counts logging' do
-      let_it_be(:expired_blob) { create(:dependency_proxy_blob, :expired, group: group) }
-      let_it_be(:expired_blob2) { create(:dependency_proxy_blob, :expired, group: group) }
-      let_it_be(:expired_manifest) { create(:dependency_proxy_manifest, :expired, group: group) }
+      let_it_be(:expired_blob) { create(:dependency_proxy_blob, :pending_destruction, group: group) }
+      let_it_be(:expired_blob2) { create(:dependency_proxy_blob, :pending_destruction, group: group) }
+      let_it_be(:expired_manifest) { create(:dependency_proxy_manifest, :pending_destruction, group: group) }
       let_it_be(:processing_blob) { create(:dependency_proxy_blob, status: :processing, group: group) }
       let_it_be(:processing_manifest) { create(:dependency_proxy_manifest, status: :processing, group: group) }
       let_it_be(:error_blob) { create(:dependency_proxy_blob, status: :error, group: group) }
diff --git a/spec/workers/every_sidekiq_worker_spec.rb b/spec/workers/every_sidekiq_worker_spec.rb
index 30bde2a2791..bb4e2981070 100644
--- a/spec/workers/every_sidekiq_worker_spec.rb
+++ b/spec/workers/every_sidekiq_worker_spec.rb
@@ -361,6 +361,7 @@ RSpec.describe 'Every Sidekiq worker' do
         'ObjectPool::ScheduleJoinWorker' => 3,
         'ObjectStorage::BackgroundMoveWorker' => 5,
         'ObjectStorage::MigrateUploadsWorker' => 3,
+        'Packages::CleanupPackageFileWorker' => 0,
         'Packages::Composer::CacheUpdateWorker' => false,
         'Packages::Go::SyncPackagesWorker' => 3,
         'Packages::Maven::Metadata::SyncWorker' => 3,
diff --git a/spec/workers/packages/cleanup_package_file_worker_spec.rb b/spec/workers/packages/cleanup_package_file_worker_spec.rb
new file mode 100644
index 00000000000..4dd06f13aff
--- /dev/null
+++ b/spec/workers/packages/cleanup_package_file_worker_spec.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Packages::CleanupPackageFileWorker do
+  let_it_be(:package) { create(:package) }
+
+  let(:worker) { described_class.new }
+
+  describe '#perform_work' do
+    subject { worker.perform_work }
+
+    context 'with no work to do' do
+      it { is_expected.to be_nil }
+    end
+
+    context 'with work to do' do
+      let_it_be(:package_file1) { create(:package_file, package: package) }
+      let_it_be(:package_file2) { create(:package_file, :pending_destruction, package: package) }
+      let_it_be(:package_file3) { create(:package_file, :pending_destruction, package: package, updated_at: 1.year.ago, created_at: 1.year.ago) }
+
+      it 'deletes the oldest package file pending destruction based on id', :aggregate_failures do
+        # NOTE: The worker doesn't explicitly look for the lower id value, but this is how PostgreSQL works when
+        # using LIMIT without ORDER BY.
+        expect(worker).to receive(:log_extra_metadata_on_done).with(:package_file_id, package_file2.id)
+        expect(worker).to receive(:log_extra_metadata_on_done).with(:package_id, package.id)
+
+        expect { subject }.to change { Packages::PackageFile.count }.by(-1)
+      end
+    end
+
+    context 'with an error during the destroy' do
+      let_it_be(:package_file) { create(:package_file, :pending_destruction) }
+
+      before do
+        expect(worker).to receive(:log_metadata).and_raise('Error!')
+      end
+
+      it 'handles the error' do
+        expect { subject }.to change { Packages::PackageFile.error.count }.from(0).to(1)
+        expect(package_file.reload).to be_error
+      end
+    end
+  end
+
+  describe '#max_running_jobs' do
+    let(:capacity) { 5 }
+
+    subject { worker.max_running_jobs }
+
+    before do
+      stub_application_setting(packages_cleanup_package_file_worker_capacity: capacity)
+    end
+
+    it { is_expected.to eq(capacity) }
+  end
+
+  describe '#remaining_work_count' do
+    before(:context) do
+      create_list(:package_file, 3, :pending_destruction, package: package)
+    end
+
+    subject { worker.remaining_work_count }
+
+    it { is_expected.to eq(3) }
+  end
+end
diff --git a/spec/workers/packages/cleanup_package_registry_worker_spec.rb b/spec/workers/packages/cleanup_package_registry_worker_spec.rb
new file mode 100644
index 00000000000..e43864975f6
--- /dev/null
+++ b/spec/workers/packages/cleanup_package_registry_worker_spec.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Packages::CleanupPackageRegistryWorker do
+  describe '#perform' do
+    let_it_be_with_reload(:package_files) { create_list(:package_file, 2, :pending_destruction) }
+
+    let(:worker) { described_class.new }
+
+    subject(:perform) { worker.perform }
+
+    context 'with package files pending destruction' do
+      it_behaves_like 'an idempotent worker'
+
+      it 'queues the cleanup job' do
+        expect(Packages::CleanupPackageFileWorker).to receive(:perform_with_capacity)
+
+        perform
+      end
+    end
+
+    context 'with no package files pending destruction' do
+      before do
+        ::Packages::PackageFile.update_all(status: :default)
+      end
+
+      it_behaves_like 'an idempotent worker'
+
+      it 'does not queue the cleanup job' do
+        expect(Packages::CleanupPackageFileWorker).not_to receive(:perform_with_capacity)
+
+        perform
+      end
+    end
+
+    describe 'counts logging' do
+      let_it_be(:processing_package_file) { create(:package_file, status: :processing) }
+
+      it 'logs all the counts', :aggregate_failures do
+        expect(worker).to receive(:log_extra_metadata_on_done).with(:pending_destruction_package_files_count, 2)
+        expect(worker).to receive(:log_extra_metadata_on_done).with(:processing_package_files_count, 1)
+        expect(worker).to receive(:log_extra_metadata_on_done).with(:error_package_files_count, 0)
+
+        perform
+      end
+
+      context 'with load balancing enabled', :db_load_balancing do
+        it 'reads the count from the replica' do
+          expect(Gitlab::Database::LoadBalancing::Session.current).to receive(:use_replicas_for_read_queries).and_call_original
+
+          perform
+        end
+      end
+    end
+  end
+end
diff --git a/spec/workers/purge_dependency_proxy_cache_worker_spec.rb b/spec/workers/purge_dependency_proxy_cache_worker_spec.rb
index b928104fb58..3de59670f8d 100644
--- a/spec/workers/purge_dependency_proxy_cache_worker_spec.rb
+++ b/spec/workers/purge_dependency_proxy_cache_worker_spec.rb
@@ -25,11 +25,11 @@ RSpec.describe PurgeDependencyProxyCacheWorker do
         include_examples 'an idempotent worker' do
           let(:job_args) { [user.id, group_id] }
 
-          it 'expires the blobs and returns ok', :aggregate_failures do
+          it 'marks the blobs as pending_destruction and returns ok', :aggregate_failures do
             subject
 
-            expect(blob).to be_expired
-            expect(manifest).to be_expired
+            expect(blob).to be_pending_destruction
+            expect(manifest).to be_pending_destruction
           end
         end
       end