Kick off follow up background migration jobs

To process the unhashed_upload_files table.

3 files changed, 105 insertions, 17 deletions

diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb
new file mode 100644
index 00000000000..6dbef41cff8
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb
@@ -0,0 +1,51 @@
+module Gitlab
+  module BackgroundMigration
+    class PopulateUntrackedUploads
+      class UnhashedUploadFile < ActiveRecord::Base
+        self.table_name = 'unhashed_upload_files'
+
+        scope :untracked, -> { where(tracked: false) }
+
+        def ensure_tracked!
+          # TODO
+          # unless unhashed_upload_file.in_uploads?
+          #   unhashed_upload_file.add_to_uploads
+          # end
+          #
+          # unhashed_upload_file.mark_as_tracked
+        end
+
+        def model_id
+          # TODO
+        end
+
+        def model_type
+          # TODO
+        end
+
+        def uploader
+          # TODO
+        end
+      end
+
+      class Upload < ActiveRecord::Base
+        self.table_name = 'uploads'
+      end
+
+      def perform(start_id, end_id)
+        return unless migrate?
+
+        files = UnhashedUploadFile.untracked.where(id: start_id..end_id)
+        files.each do |unhashed_upload_file|
+          unhashed_upload_file.ensure_tracked!
+        end
+      end
+
+      private
+
+      def migrate?
+        UnhashedUploadFile.table_exists? && Upload.table_exists?
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb
index 11f43044829..556457039fa 100644
--- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb
+++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb
@@ -1,10 +1,16 @@
 module Gitlab
   module BackgroundMigration
     class PrepareUnhashedUploads
+      # For bulk_queue_background_migration_jobs_by_range
+      include Database::MigrationHelpers
+
       FILE_PATH_BATCH_SIZE = 500
       UPLOAD_DIR = "#{CarrierWave.root}/uploads"
+      FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'
 
       class UnhashedUploadFile < ActiveRecord::Base
+        include EachBatch
+
         self.table_name = 'unhashed_upload_files'
       end
 
@@ -74,7 +80,7 @@ module Gitlab
       end
 
       def schedule_populate_untracked_uploads_jobs
-        # TODO
+        bulk_queue_background_migration_jobs_by_range(UnhashedUploadFile, FOLLOW_UP_MIGRATION)
       end
     end
   end
diff --git a/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb
index 2f641a5deed..76d126e8f00 100644
--- a/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb
+++ b/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb
@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema: 20171103140253 do
+describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, :sidekiq, schema: 20171103140253 do
   let!(:unhashed_upload_files) { table(:unhashed_upload_files) }
 
   let(:user1) { create(:user) }
@@ -9,6 +9,18 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema
   let(:project2) { create(:project) }
   let(:appearance) { create(:appearance) }
 
+  matcher :be_scheduled_migration do |*expected|
+    match do |migration|
+      BackgroundMigrationWorker.jobs.any? do |job|
+        job['args'] == [migration, expected]
+      end
+    end
+
+    failure_message do |migration|
+      "Migration `#{migration}` with args `#{expected.inspect}` not scheduled!"
+    end
+  end
+
   context 'when files were uploaded before and after hashed storage was enabled' do
     before do
       fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg')
@@ -28,16 +40,29 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema
     end
 
     it 'adds unhashed files to the unhashed_upload_files table' do
-      expect do
-        described_class.new.perform
-      end.to change { unhashed_upload_files.count }.from(0).to(5)
+      Sidekiq::Testing.fake! do
+        expect do
+          described_class.new.perform
+        end.to change { unhashed_upload_files.count }.from(0).to(5)
+      end
     end
 
     it 'does not add hashed files to the unhashed_upload_files table' do
-      described_class.new.perform
+      Sidekiq::Testing.fake! do
+        described_class.new.perform
 
-      hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path
-      expect(unhashed_upload_files.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey
+        hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path
+        expect(unhashed_upload_files.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey
+      end
+    end
+
+    it 'correctly schedules the follow-up background migration jobs' do
+      Sidekiq::Testing.fake! do
+        described_class.new.perform
+
+        expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5)
+        expect(BackgroundMigrationWorker.jobs.size).to eq(1)
+      end
     end
 
     # E.g. from a previous failed run of this background migration
@@ -47,9 +72,11 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema
       end
 
       it 'clears existing data before adding new data' do
-        expect do
-          described_class.new.perform
-        end.to change { unhashed_upload_files.count }.from(1).to(5)
+        Sidekiq::Testing.fake! do
+          expect do
+            described_class.new.perform
+          end.to change { unhashed_upload_files.count }.from(1).to(5)
+        end
       end
     end
 
@@ -61,9 +88,11 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema
       end
 
       it 'does not add files from /uploads/tmp' do
-        expect do
-          described_class.new.perform
-        end.to change { unhashed_upload_files.count }.from(0).to(5)
+        Sidekiq::Testing.fake! do
+          expect do
+            described_class.new.perform
+          end.to change { unhashed_upload_files.count }.from(0).to(5)
+        end
       end
     end
   end
@@ -72,9 +101,11 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema
   # may not have an upload directory because they have no uploads.
   context 'when no files were ever uploaded' do
     it 'does not add to the unhashed_upload_files table (and does not raise error)' do
-      expect do
-        described_class.new.perform
-      end.not_to change { unhashed_upload_files.count }.from(0)
+      Sidekiq::Testing.fake! do
+        expect do
+          described_class.new.perform
+        end.not_to change { unhashed_upload_files.count }.from(0)
+      end
     end
   end
 end