diff options
Diffstat (limited to 'lib/gitlab/background_migration/populate_untracked_uploads.rb')
-rw-r--r-- | lib/gitlab/background_migration/populate_untracked_uploads.rb | 111 |
1 files changed, 0 insertions, 111 deletions
diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb deleted file mode 100644 index 43698b7955f..00000000000 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ /dev/null @@ -1,111 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class processes a batch of rows in `untracked_files_for_uploads` by - # adding each file to the `uploads` table if it does not exist. - class PopulateUntrackedUploads - def perform(start_id, end_id) - return unless migrate? - - files = Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.where(id: start_id..end_id) - processed_files = insert_uploads_if_needed(files) - processed_files.delete_all - - drop_temp_table_if_finished - end - - private - - def migrate? - Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.table_exists? && - Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::Upload.table_exists? - end - - def insert_uploads_if_needed(files) - filtered_files, error_files = filter_error_files(files) - filtered_files = filter_existing_uploads(filtered_files) - filtered_files = filter_deleted_models(filtered_files) - insert(filtered_files) - - processed_files = files.where.not(id: error_files.map(&:id)) - processed_files - end - - def filter_error_files(files) - files.partition do |file| - file.to_h - true - rescue => e - msg = <<~MSG - Error parsing path "#{file.path}": - #{e.message} - #{e.backtrace.join("\n ")} - MSG - Rails.logger.error(msg) # rubocop:disable Gitlab/RailsLogger - false - end - end - - def filter_existing_uploads(files) - paths = files.map(&:upload_path) - existing_paths = Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::Upload.where(path: paths).pluck(:path).to_set - - files.reject do |file| - existing_paths.include?(file.upload_path) - end - end - - # There are files on disk that are not in the uploads table because their - # model was deleted, and we don't delete the files on disk. - def filter_deleted_models(files) - ids = deleted_model_ids(files) - - files.reject do |file| - ids[file.model_type].include?(file.model_id) - end - end - - def deleted_model_ids(files) - ids = { - 'Appearance' => [], - 'Namespace' => [], - 'Note' => [], - 'Project' => [], - 'User' => [] - } - - # group model IDs by model type - files.each do |file| - ids[file.model_type] << file.model_id - end - - ids.each do |model_type, model_ids| - model_class = "Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::#{model_type}".constantize - found_ids = model_class.where(id: model_ids.uniq).pluck(:id) - deleted_ids = ids[model_type] - found_ids - ids[model_type] = deleted_ids - end - - ids - end - - def insert(files) - rows = files.map do |file| - file.to_h.merge(created_at: 'NOW()') - end - - Gitlab::Database.bulk_insert('uploads', # rubocop:disable Gitlab/BulkInsert - rows, - disable_quote: :created_at) - end - - def drop_temp_table_if_finished - if Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.all.empty? && !Rails.env.test? # Dropping a table intermittently breaks test cleanup - Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.connection.drop_table(:untracked_files_for_uploads, - if_exists: true) - end - end - end - end -end |