diff options
Diffstat (limited to 'lib/gitlab/database/migrations')
4 files changed, 33 insertions, 26 deletions
diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index a2a4a37ab87..0261ade0fe7 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -84,7 +84,7 @@ module Gitlab FROM #{connection.quote_table_name(batch_table_name)} SQL - migration_status = batch_max_value.nil? ? :finished : :active + status_event = batch_max_value.nil? ? :finish : :execute batch_max_value ||= batch_min_value migration = Gitlab::Database::BackgroundMigration::BatchedMigration.new( @@ -98,7 +98,7 @@ module Gitlab batch_class_name: batch_class_name, batch_size: batch_size, sub_batch_size: sub_batch_size, - status: migration_status + status_event: status_event ) # Below `BatchedMigration` attributes were introduced after the diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb index 9d28db6b886..7c21346007a 100644 --- a/lib/gitlab/database/migrations/instrumentation.rb +++ b/lib/gitlab/database/migrations/instrumentation.rb @@ -6,11 +6,8 @@ module Gitlab class Instrumentation STATS_FILENAME = 'migration-stats.json' - attr_reader :observations - def initialize(result_dir:, observer_classes: ::Gitlab::Database::Migrations::Observers.all_observers) @observer_classes = observer_classes - @observations = [] @result_dir = result_dir end @@ -38,15 +35,16 @@ module Gitlab on_each_observer(observers) { |observer| observer.after } on_each_observer(observers) { |observer| observer.record } - record_observation(observation) + record_observation(observation, destination_dir: per_migration_result_dir) end private attr_reader :observer_classes - def record_observation(observation) - @observations << observation + def record_observation(observation, destination_dir:) + stats_file_location = File.join(destination_dir, STATS_FILENAME) + File.write(stats_file_location, observation.to_json) end def on_each_observer(observers, &block) diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index 02645a0d452..3b6f52b43a8 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -6,7 +6,7 @@ module Gitlab class Runner BASE_RESULT_DIR = Rails.root.join('tmp', 'migration-testing').freeze METADATA_FILENAME = 'metadata.json' - SCHEMA_VERSION = 2 # Version of the output format produced by the runner + SCHEMA_VERSION = 3 # Version of the output format produced by the runner class << self def up @@ -17,6 +17,10 @@ module Gitlab Runner.new(direction: :down, migrations: migrations_for_down, result_dir: BASE_RESULT_DIR.join('down')) end + def background_migrations + TestBackgroundRunner.new(result_dir: BASE_RESULT_DIR.join('background_migrations')) + end + def migration_context @migration_context ||= ApplicationRecord.connection.migration_context end @@ -76,13 +80,8 @@ module Gitlab end end ensure - if instrumentation - stats_filename = File.join(result_dir, Gitlab::Database::Migrations::Instrumentation::STATS_FILENAME) - File.write(stats_filename, instrumentation.observations.to_json) - - metadata_filename = File.join(result_dir, METADATA_FILENAME) - File.write(metadata_filename, { version: SCHEMA_VERSION }.to_json) - end + metadata_filename = File.join(result_dir, METADATA_FILENAME) + File.write(metadata_filename, { version: SCHEMA_VERSION }.to_json) # We clear the cache here to mirror the cache clearing that happens at the end of `db:migrate` tasks # This clearing makes subsequent rake tasks in the same execution pick up database schema changes caused by diff --git a/lib/gitlab/database/migrations/test_background_runner.rb b/lib/gitlab/database/migrations/test_background_runner.rb index 821d68c06c9..74e54d62e05 100644 --- a/lib/gitlab/database/migrations/test_background_runner.rb +++ b/lib/gitlab/database/migrations/test_background_runner.rb @@ -4,12 +4,10 @@ module Gitlab module Database module Migrations class TestBackgroundRunner - # TODO - build a rake task to call this method, and support it in the gitlab-com-database-testing project. - # Until then, we will inject a migration with a very high timestamp during database testing - # that calls this class to run jobs - # See https://gitlab.com/gitlab-org/database-team/gitlab-com-database-testing/-/issues/41 for details + attr_reader :result_dir - def initialize + def initialize(result_dir:) + @result_dir = result_dir @job_coordinator = Gitlab::BackgroundMigration.coordinator_for_database(Gitlab::Database::MAIN_DATABASE_NAME) end @@ -24,18 +22,30 @@ module Gitlab # without .to_f, we do integer division # For example, 3.minutes / 2 == 1.minute whereas 3.minutes / 2.to_f == (1.minute + 30.seconds) duration_per_migration_type = for_duration / jobs_to_run.count.to_f - jobs_to_run.each do |_migration_name, jobs| + jobs_to_run.each do |migration_name, jobs| run_until = duration_per_migration_type.from_now - jobs.shuffle.each do |j| - break if run_until <= Time.current - run_job(j) - end + run_jobs_for_migration(migration_name: migration_name, jobs: jobs, run_until: run_until) end end private + def run_jobs_for_migration(migration_name:, jobs:, run_until:) + per_background_migration_result_dir = File.join(@result_dir, migration_name) + + instrumentation = Instrumentation.new(result_dir: per_background_migration_result_dir) + batch_names = (1..).each.lazy.map { |i| "batch_#{i}"} + + jobs.shuffle.each do |j| + break if run_until <= Time.current + + instrumentation.observe(version: nil, name: batch_names.next, connection: ActiveRecord::Migration.connection) do + run_job(j) + end + end + end + def run_job(job) Gitlab::BackgroundMigration.perform(job.args[0], job.args[1]) end |