diff options
author | Michael Kozono <mkozono@gmail.com> | 2017-11-07 23:53:24 +0300 |
---|---|---|
committer | Michael Kozono <mkozono@gmail.com> | 2017-12-02 02:26:40 +0300 |
commit | 8315c66a569bbc1b4806762e4da49c22813fc523 (patch) | |
tree | ab1662d21a2a296d7dcfefa8c92d46e178b1cdcc | |
parent | b6ea41d13073ce8b4d16b2edb602c82aae10ea0b (diff) |
Kick off follow up background migration jobs
To process the unhashed_upload_files table.
3 files changed, 105 insertions, 17 deletions
diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb new file mode 100644 index 00000000000..6dbef41cff8 --- /dev/null +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -0,0 +1,51 @@ +module Gitlab + module BackgroundMigration + class PopulateUntrackedUploads + class UnhashedUploadFile < ActiveRecord::Base + self.table_name = 'unhashed_upload_files' + + scope :untracked, -> { where(tracked: false) } + + def ensure_tracked! + # TODO + # unless unhashed_upload_file.in_uploads? + # unhashed_upload_file.add_to_uploads + # end + # + # unhashed_upload_file.mark_as_tracked + end + + def model_id + # TODO + end + + def model_type + # TODO + end + + def uploader + # TODO + end + end + + class Upload < ActiveRecord::Base + self.table_name = 'uploads' + end + + def perform(start_id, end_id) + return unless migrate? + + files = UnhashedUploadFile.untracked.where(id: start_id..end_id) + files.each do |unhashed_upload_file| + unhashed_upload_file.ensure_tracked! + end + end + + private + + def migrate? + UnhashedUploadFile.table_exists? && Upload.table_exists? + end + end + end +end diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb index 11f43044829..556457039fa 100644 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -1,10 +1,16 @@ module Gitlab module BackgroundMigration class PrepareUnhashedUploads + # For bulk_queue_background_migration_jobs_by_range + include Database::MigrationHelpers + FILE_PATH_BATCH_SIZE = 500 UPLOAD_DIR = "#{CarrierWave.root}/uploads" + FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads' class UnhashedUploadFile < ActiveRecord::Base + include EachBatch + self.table_name = 'unhashed_upload_files' end @@ -74,7 +80,7 @@ module Gitlab end def schedule_populate_untracked_uploads_jobs - # TODO + bulk_queue_background_migration_jobs_by_range(UnhashedUploadFile, FOLLOW_UP_MIGRATION) end end end diff --git a/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb index 2f641a5deed..76d126e8f00 100644 --- a/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema: 20171103140253 do +describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, :sidekiq, schema: 20171103140253 do let!(:unhashed_upload_files) { table(:unhashed_upload_files) } let(:user1) { create(:user) } @@ -9,6 +9,18 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema let(:project2) { create(:project) } let(:appearance) { create(:appearance) } + matcher :be_scheduled_migration do |*expected| + match do |migration| + BackgroundMigrationWorker.jobs.any? do |job| + job['args'] == [migration, expected] + end + end + + failure_message do |migration| + "Migration `#{migration}` with args `#{expected.inspect}` not scheduled!" + end + end + context 'when files were uploaded before and after hashed storage was enabled' do before do fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') @@ -28,16 +40,29 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema end it 'adds unhashed files to the unhashed_upload_files table' do - expect do - described_class.new.perform - end.to change { unhashed_upload_files.count }.from(0).to(5) + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.to change { unhashed_upload_files.count }.from(0).to(5) + end end it 'does not add hashed files to the unhashed_upload_files table' do - described_class.new.perform + Sidekiq::Testing.fake! do + described_class.new.perform - hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path - expect(unhashed_upload_files.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey + hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path + expect(unhashed_upload_files.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey + end + end + + it 'correctly schedules the follow-up background migration jobs' do + Sidekiq::Testing.fake! do + described_class.new.perform + + expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5) + expect(BackgroundMigrationWorker.jobs.size).to eq(1) + end end # E.g. from a previous failed run of this background migration @@ -47,9 +72,11 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema end it 'clears existing data before adding new data' do - expect do - described_class.new.perform - end.to change { unhashed_upload_files.count }.from(1).to(5) + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.to change { unhashed_upload_files.count }.from(1).to(5) + end end end @@ -61,9 +88,11 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema end it 'does not add files from /uploads/tmp' do - expect do - described_class.new.perform - end.to change { unhashed_upload_files.count }.from(0).to(5) + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.to change { unhashed_upload_files.count }.from(0).to(5) + end end end end @@ -72,9 +101,11 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema # may not have an upload directory because they have no uploads. context 'when no files were ever uploaded' do it 'does not add to the unhashed_upload_files table (and does not raise error)' do - expect do - described_class.new.perform - end.not_to change { unhashed_upload_files.count }.from(0) + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.not_to change { unhashed_upload_files.count }.from(0) + end end end end |