diff options
Diffstat (limited to 'lib/gitlab/database/migrations/background_migration_helpers.rb')
| -rw-r--r-- | lib/gitlab/database/migrations/background_migration_helpers.rb | 80 |
1 files changed, 65 insertions, 15 deletions
diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index 7e5c002d072..9bffed43077 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -41,6 +41,25 @@ module Gitlab # end # end def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BATCH_SIZE, other_job_arguments: [], initial_delay: 0, track_jobs: false, primary_column_name: :id) + if transaction_open? + raise 'The `#queue_background_migration_jobs_by_range_at_intervals` can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + # Background Migrations do not work well for in cases requiring to update `gitlab_shared` + # Once the decomposition is done, enqueued jobs for `gitlab_shared` tables (on CI database) + # will not be executed since the queue (which is stored in Redis) is tied to main database, not to schema. + # The batched background migrations do not have those limitations since the tracking tables + # are properly database-only. + if background_migration_restrict_gitlab_migration_schemas&.include?(:gitlab_shared) + raise 'The `#queue_background_migration_jobs_by_range_at_intervals` cannot " \ + "use `restrict_gitlab_migration:` " with `:gitlab_shared`. ' \ + 'Background migrations do encode migration worker which is tied to a given database. ' \ + 'After split this worker will not be properly duplicated into decomposed database. ' \ + 'Use batched background migrations instead that do support well working across all databases.' + end + raise "#{model_class} does not have an ID column of #{primary_column_name} to use for batch ranges" unless model_class.column_names.include?(primary_column_name.to_s) raise "#{primary_column_name} is not an integer or string column" unless [:integer, :string].include?(model_class.columns_hash[primary_column_name.to_s].type) @@ -90,6 +109,18 @@ module Gitlab # delay_interval - The duration between each job's scheduled time # batch_size - The maximum number of jobs to fetch to memory from the database. def requeue_background_migration_jobs_by_range_at_intervals(job_class_name, delay_interval, batch_size: BATCH_SIZE, initial_delay: 0) + if transaction_open? + raise 'The `#requeue_background_migration_jobs_by_range_at_intervals` can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + if background_migration_restrict_gitlab_migration_schemas&.any? + raise 'The `#requeue_background_migration_jobs_by_range_at_intervals` cannot use `restrict_gitlab_migration:`. ' \ + 'The `#requeue_background_migration_jobs_by_range_at_intervals` needs to be executed on all databases since ' \ + 'each database has its own queue of background migrations.' + end + job_coordinator = coordinator_for_tracking_database # To not overload the worker too much we enforce a minimum interval both @@ -133,23 +164,40 @@ module Gitlab # This method does not garauntee that all jobs completed successfully. # It can only be used if the previous background migration used the queue_background_migration_jobs_by_range_at_intervals helper. def finalize_background_migration(class_name, delete_tracking_jobs: ['succeeded']) + if transaction_open? + raise 'The `#finalize_background_migration` can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + if background_migration_restrict_gitlab_migration_schemas&.any? + raise 'The `#finalize_background_migration` cannot use `restrict_gitlab_migration:`. ' \ + 'The `#finalize_background_migration` needs to be executed on all databases since ' \ + 'each database has its own queue of background migrations.' + end + job_coordinator = coordinator_for_tracking_database - # Empty the sidekiq queue. - job_coordinator.steal(class_name) + with_restored_connection_stack do + # Since we are running trusted code (background migration class) allow to execute any type of finalize + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do + # Empty the sidekiq queue. + job_coordinator.steal(class_name) - # Process pending tracked jobs. - jobs = Gitlab::Database::BackgroundMigrationJob.pending.for_migration_class(class_name) + # Process pending tracked jobs. + jobs = Gitlab::Database::BackgroundMigrationJob.pending.for_migration_class(class_name) - jobs.find_each do |job| - job_coordinator.perform(job.class_name, job.arguments) - end + jobs.find_each do |job| + job_coordinator.perform(job.class_name, job.arguments) + end - # Empty the sidekiq queue. - job_coordinator.steal(class_name) + # Empty the sidekiq queue. + job_coordinator.steal(class_name) - # Delete job tracking rows. - delete_job_tracking(class_name, status: delete_tracking_jobs) if delete_tracking_jobs + # Delete job tracking rows. + delete_job_tracking(class_name, status: delete_tracking_jobs) if delete_tracking_jobs + end + end end def migrate_in(*args, coordinator: coordinator_for_tracking_database) @@ -174,6 +222,10 @@ module Gitlab private + def background_migration_restrict_gitlab_migration_schemas + self.allowed_gitlab_schemas if self.respond_to?(:allowed_gitlab_schemas) + end + def with_migration_context(&block) Gitlab::ApplicationContext.with_context(caller_id: self.class.to_s, &block) end @@ -183,11 +235,9 @@ module Gitlab end def coordinator_for_tracking_database - Gitlab::BackgroundMigration.coordinator_for_database(tracking_database) - end + tracking_database = Gitlab::Database.db_config_name(connection) - def tracking_database - Gitlab::BackgroundMigration::DEFAULT_TRACKING_DATABASE + Gitlab::BackgroundMigration.coordinator_for_database(tracking_database) end end end |