Merge branch 'mk/add-orphan-upload-file-task-tests' into 'master'

Add object storage related tests for `gitlab:cleanup:project_uploads` task

See merge request gitlab-org/gitlab-ce!20950

1 files changed, 45 insertions, 276 deletions

diff --git a/spec/tasks/gitlab/cleanup_rake_spec.rb b/spec/tasks/gitlab/cleanup_rake_spec.rb
index ba08ece1b4b..cc2cca10f58 100644
--- a/spec/tasks/gitlab/cleanup_rake_spec.rb
+++ b/spec/tasks/gitlab/cleanup_rake_spec.rb
@@ -68,317 +68,86 @@ describe 'gitlab:cleanup rake tasks' do
     end
   end
 
+  # A single integration test that is redundant with one part of the
+  # Gitlab::Cleanup::ProjectUploads spec.
+  #
+  # Additionally, this tests DRY_RUN env var values, and the extra line of
+  # output that says you can disable DRY_RUN if it's enabled.
   describe 'cleanup:project_uploads' do
-    context 'orphaned project upload file' do
-      context 'when an upload record matching the secret and filename is found' do
-        context 'when the project is still in legacy storage' do
-          let!(:orphaned) { create(:upload, :issuable_upload, :with_file, model: build(:project, :legacy_storage)) }
-          let!(:correct_path) { orphaned.absolute_path }
-          let!(:other_project) { create(:project, :legacy_storage) }
-          let!(:orphaned_path) { correct_path.sub(/#{orphaned.model.full_path}/, other_project.full_path) }
+    let!(:logger) { double(:logger) }
 
-          before do
-            FileUtils.mkdir_p(File.dirname(orphaned_path))
-            FileUtils.mv(correct_path, orphaned_path)
-          end
-
-          it 'moves the file to its proper location' do
-            expect(Rails.logger).to receive(:info).twice
-            expect(Rails.logger).to receive(:info).with("Did fix #{orphaned_path} -> #{correct_path}")
-
-            expect(File.exist?(orphaned_path)).to be_truthy
-            expect(File.exist?(correct_path)).to be_falsey
-
-            stub_env('DRY_RUN', 'false')
-            run_rake_task('gitlab:cleanup:project_uploads')
-
-            expect(File.exist?(orphaned_path)).to be_falsey
-            expect(File.exist?(correct_path)).to be_truthy
-          end
-
-          it 'a dry run does not move the file' do
-            expect(Rails.logger).to receive(:info).twice
-            expect(Rails.logger).to receive(:info).with("Can fix #{orphaned_path} -> #{correct_path}")
-            expect(Rails.logger).to receive(:info)
-
-            expect(File.exist?(orphaned_path)).to be_truthy
-            expect(File.exist?(correct_path)).to be_falsey
-
-            run_rake_task('gitlab:cleanup:project_uploads')
-
-            expect(File.exist?(orphaned_path)).to be_truthy
-            expect(File.exist?(correct_path)).to be_falsey
-          end
-
-          context 'when the project record is missing (Upload#absolute_path raises error)' do
-            let!(:lost_and_found_path) { File.join(FileUploader.root, '-', 'project-lost-found', other_project.full_path, orphaned.path) }
-
-            before do
-              orphaned.model.delete
-            end
-
-            it 'moves the file to lost and found' do
-              expect(Rails.logger).to receive(:info).twice
-              expect(Rails.logger).to receive(:info).with("Did move to lost and found #{orphaned_path} -> #{lost_and_found_path}")
-
-              expect(File.exist?(orphaned_path)).to be_truthy
-              expect(File.exist?(lost_and_found_path)).to be_falsey
-
-              stub_env('DRY_RUN', 'false')
-              run_rake_task('gitlab:cleanup:project_uploads')
-
-              expect(File.exist?(orphaned_path)).to be_falsey
-              expect(File.exist?(lost_and_found_path)).to be_truthy
-            end
-
-            it 'a dry run does not move the file' do
-              expect(Rails.logger).to receive(:info).twice
-              expect(Rails.logger).to receive(:info).with("Can move to lost and found #{orphaned_path} -> #{lost_and_found_path}")
-              expect(Rails.logger).to receive(:info)
-
-              expect(File.exist?(orphaned_path)).to be_truthy
-              expect(File.exist?(lost_and_found_path)).to be_falsey
-
-              run_rake_task('gitlab:cleanup:project_uploads')
-
-              expect(File.exist?(orphaned_path)).to be_truthy
-              expect(File.exist?(lost_and_found_path)).to be_falsey
-            end
-          end
-        end
-
-        context 'when the project was moved to hashed storage' do
-          let!(:orphaned) { create(:upload, :issuable_upload, :with_file) }
-          let!(:correct_path) { orphaned.absolute_path }
-          let!(:orphaned_path) { File.join(FileUploader.root, 'foo', 'bar', orphaned.path) }
-
-          before do
-            FileUtils.mkdir_p(File.dirname(orphaned_path))
-            FileUtils.mv(correct_path, orphaned_path)
-          end
-
-          it 'moves the file to its proper location' do
-            expect(Rails.logger).to receive(:info).twice
-            expect(Rails.logger).to receive(:info).with("Did fix #{orphaned_path} -> #{correct_path}")
-
-            expect(File.exist?(orphaned_path)).to be_truthy
-            expect(File.exist?(correct_path)).to be_falsey
-
-            stub_env('DRY_RUN', 'false')
-            run_rake_task('gitlab:cleanup:project_uploads')
-
-            expect(File.exist?(orphaned_path)).to be_falsey
-            expect(File.exist?(correct_path)).to be_truthy
-          end
-
-          it 'a dry run does not move the file' do
-            expect(Rails.logger).to receive(:info).twice
-            expect(Rails.logger).to receive(:info).with("Can fix #{orphaned_path} -> #{correct_path}")
-            expect(Rails.logger).to receive(:info)
+    before do
+      expect(main_object).to receive(:logger).and_return(logger).at_least(1).times
 
-            expect(File.exist?(orphaned_path)).to be_truthy
-            expect(File.exist?(correct_path)).to be_falsey
+      allow(logger).to receive(:info).at_least(1).times
+      allow(logger).to receive(:debug).at_least(1).times
+    end
 
-            run_rake_task('gitlab:cleanup:project_uploads')
+    context 'with a fixable orphaned project upload file' do
+      let(:orphaned) { create(:upload, :issuable_upload, :with_file, model: build(:project, :legacy_storage)) }
+      let(:new_path) { orphaned.absolute_path }
+      let(:path) { File.join(FileUploader.root, 'some', 'wrong', 'location', orphaned.path) }
 
-            expect(File.exist?(orphaned_path)).to be_truthy
-            expect(File.exist?(correct_path)).to be_falsey
-          end
-        end
+      before do
+        FileUtils.mkdir_p(File.dirname(path))
+        FileUtils.mv(new_path, path)
       end
 
-      context 'when a matching upload record can not be found' do
-        context 'when the file path fits the known pattern' do
-          let!(:orphaned) { create(:upload, :issuable_upload, :with_file, model: build(:project, :legacy_storage)) }
-          let!(:orphaned_path) { orphaned.absolute_path }
-          let!(:lost_and_found_path) { File.join(FileUploader.root, '-', 'project-lost-found', orphaned.model.full_path, orphaned.path) }
-
-          before do
-            orphaned.delete
-          end
-
-          it 'moves the file to lost and found' do
-            expect(Rails.logger).to receive(:info).twice
-            expect(Rails.logger).to receive(:info).with("Did move to lost and found #{orphaned_path} -> #{lost_and_found_path}")
-
-            expect(File.exist?(orphaned_path)).to be_truthy
-            expect(File.exist?(lost_and_found_path)).to be_falsey
-
-            stub_env('DRY_RUN', 'false')
-            run_rake_task('gitlab:cleanup:project_uploads')
-
-            expect(File.exist?(orphaned_path)).to be_falsey
-            expect(File.exist?(lost_and_found_path)).to be_truthy
-          end
-
-          it 'a dry run does not move the file' do
-            expect(Rails.logger).to receive(:info).twice
-            expect(Rails.logger).to receive(:info).with("Can move to lost and found #{orphaned_path} -> #{lost_and_found_path}")
-            expect(Rails.logger).to receive(:info)
-
-            expect(File.exist?(orphaned_path)).to be_truthy
-            expect(File.exist?(lost_and_found_path)).to be_falsey
-
-            run_rake_task('gitlab:cleanup:project_uploads')
-
-            expect(File.exist?(orphaned_path)).to be_truthy
-            expect(File.exist?(lost_and_found_path)).to be_falsey
-          end
+      context 'with DRY_RUN disabled' do
+        before do
+          stub_env('DRY_RUN', 'false')
         end
 
-        context 'when the file path does not fit the known pattern' do
-          let!(:invalid_path) { File.join('group', 'file.jpg') }
-          let!(:orphaned_path) { File.join(FileUploader.root, invalid_path) }
-          let!(:lost_and_found_path) { File.join(FileUploader.root, '-', 'project-lost-found', invalid_path) }
-
-          before do
-            FileUtils.mkdir_p(File.dirname(orphaned_path))
-            FileUtils.touch(orphaned_path)
-          end
-
-          after do
-            File.delete(orphaned_path) if File.exist?(orphaned_path)
-          end
-
-          it 'moves the file to lost and found' do
-            expect(Rails.logger).to receive(:info).twice
-            expect(Rails.logger).to receive(:info).with("Did move to lost and found #{orphaned_path} -> #{lost_and_found_path}")
-
-            expect(File.exist?(orphaned_path)).to be_truthy
-            expect(File.exist?(lost_and_found_path)).to be_falsey
-
-            stub_env('DRY_RUN', 'false')
-            run_rake_task('gitlab:cleanup:project_uploads')
-
-            expect(File.exist?(orphaned_path)).to be_falsey
-            expect(File.exist?(lost_and_found_path)).to be_truthy
-          end
-
-          it 'a dry run does not move the file' do
-            expect(Rails.logger).to receive(:info).twice
-            expect(Rails.logger).to receive(:info).with("Can move to lost and found #{orphaned_path} -> #{lost_and_found_path}")
-            expect(Rails.logger).to receive(:info)
-
-            expect(File.exist?(orphaned_path)).to be_truthy
-            expect(File.exist?(lost_and_found_path)).to be_falsey
-
-            run_rake_task('gitlab:cleanup:project_uploads')
+        it 'moves the file to its proper location' do
+          run_rake_task('gitlab:cleanup:project_uploads')
 
-            expect(File.exist?(orphaned_path)).to be_truthy
-            expect(File.exist?(lost_and_found_path)).to be_falsey
-          end
+          expect(File.exist?(path)).to be_falsey
+          expect(File.exist?(new_path)).to be_truthy
         end
-      end
-    end
-
-    context 'non-orphaned project upload file' do
-      it 'does not move the file' do
-        tracked = create(:upload, :issuable_upload, :with_file, model: build(:project, :legacy_storage))
-        tracked_path = tracked.absolute_path
 
-        expect(Rails.logger).not_to receive(:info).with(/move|fix/i)
-        expect(File.exist?(tracked_path)).to be_truthy
-
-        stub_env('DRY_RUN', 'false')
-        run_rake_task('gitlab:cleanup:project_uploads')
-
-        expect(File.exist?(tracked_path)).to be_truthy
-      end
-    end
-
-    context 'ignorable cases' do
-      shared_examples_for 'does not move anything' do
-        it 'does not move even an orphan file' do
-          orphaned = create(:upload, :issuable_upload, :with_file, model: project)
-          orphaned_path = orphaned.absolute_path
-          orphaned.delete
-
-          expect(File.exist?(orphaned_path)).to be_truthy
+        it 'logs action as done' do
+          expect(logger).to receive(:info).with("Looking for orphaned project uploads to clean up...")
+          expect(logger).to receive(:info).with("Did fix #{path} -> #{new_path}")
 
           run_rake_task('gitlab:cleanup:project_uploads')
-
-          expect(File.exist?(orphaned_path)).to be_truthy
         end
       end
 
-      # Because we aren't concerned about these, and can save a lot of
-      # processing time by ignoring them. If we wish to cleanup hashed storage
-      # directories, it should simply require removing this test and modifying
-      # the find command.
-      context 'when the file is already in hashed storage' do
-        let(:project) { create(:project) }
+      shared_examples_for 'does not move the file' do
+        it 'does not move the file' do
+          run_rake_task('gitlab:cleanup:project_uploads')
 
-        before do
-          stub_env('DRY_RUN', 'false')
-          expect(Rails.logger).not_to receive(:info).with(/move|fix/i)
+          expect(File.exist?(path)).to be_truthy
+          expect(File.exist?(new_path)).to be_falsey
         end
 
-        it_behaves_like 'does not move anything'
-      end
-
-      context 'when DRY_RUN env var is unset' do
-        let(:project) { create(:project, :legacy_storage) }
+        it 'logs action as able to be done' do
+          expect(logger).to receive(:info).with("Looking for orphaned project uploads to clean up. Dry run...")
+          expect(logger).to receive(:info).with("Can fix #{path} -> #{new_path}")
+          expect(logger).to receive(:info).with(/To clean up these files run this command with DRY_RUN=false/)
 
-        it_behaves_like 'does not move anything'
+          run_rake_task('gitlab:cleanup:project_uploads')
+        end
       end
 
-      context 'when DRY_RUN env var is true' do
-        let(:project) { create(:project, :legacy_storage) }
-
+      context 'with DRY_RUN explicitly enabled' do
         before do
           stub_env('DRY_RUN', 'true')
         end
 
-        it_behaves_like 'does not move anything'
+        it_behaves_like 'does not move the file'
       end
 
-      context 'when DRY_RUN env var is foo' do
-        let(:project) { create(:project, :legacy_storage) }
-
+      context 'with DRY_RUN set to an unknown value' do
         before do
           stub_env('DRY_RUN', 'foo')
         end
 
-        it_behaves_like 'does not move anything'
+        it_behaves_like 'does not move the file'
       end
 
-      it 'does not move any non-project (FileUploader) uploads' do
-        stub_env('DRY_RUN', 'false')
-
-        paths = []
-        orphaned1 = create(:upload, :personal_snippet_upload, :with_file)
-        orphaned2 = create(:upload, :namespace_upload, :with_file)
-        orphaned3 = create(:upload, :attachment_upload, :with_file)
-        paths << orphaned1.absolute_path
-        paths << orphaned2.absolute_path
-        paths << orphaned3.absolute_path
-        Upload.delete_all
-
-        expect(Rails.logger).not_to receive(:info).with(/move|fix/i)
-        paths.each do |path|
-          expect(File.exist?(path)).to be_truthy
-        end
-
-        run_rake_task('gitlab:cleanup:project_uploads')
-
-        paths.each do |path|
-          expect(File.exist?(path)).to be_truthy
-        end
-      end
-
-      it 'does not move any uploads in tmp (which would interfere with ongoing upload activity)' do
-        stub_env('DRY_RUN', 'false')
-
-        path = File.join(FileUploader.root, 'tmp', 'foo.jpg')
-        FileUtils.mkdir_p(File.dirname(path))
-        FileUtils.touch(path)
-
-        expect(Rails.logger).not_to receive(:info).with(/move|fix/i)
-        expect(File.exist?(path)).to be_truthy
-
-        run_rake_task('gitlab:cleanup:project_uploads')
-
-        expect(File.exist?(path)).to be_truthy
+      context 'with DRY_RUN unset' do
+        it_behaves_like 'does not move the file'
       end
     end
   end