diff options
Diffstat (limited to 'lib/tasks/gitlab/db.rake')
| -rw-r--r-- | lib/tasks/gitlab/db.rake | 122 |
1 files changed, 81 insertions, 41 deletions
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 9c92aa5eb28..c4dc7b938cc 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -283,6 +283,36 @@ namespace :gitlab do end end + namespace :execute_async_index_operations do + each_database(databases) do |database_name| + task database_name, [:pick] => :environment do |_, args| + args.with_defaults(pick: 2) + + if Feature.disabled?(:database_async_index_operations, type: :ops) + puts <<~NOTE.color(:yellow) + Note: database async index operations feature is currently disabled. + + Enable with: Feature.enable(:database_async_index_operations) + NOTE + exit + end + + Gitlab::Database::EachDatabase.each_database_connection(only: database_name) do + Gitlab::Database::AsyncIndexes.execute_pending_actions!(how_many: args[:pick].to_i) + end + end + end + + task :all, [:pick] => :environment do |_, args| + default_pick = Gitlab.dev_or_test_env? ? 1000 : 2 + args.with_defaults(pick: default_pick) + + each_database(databases) do |database_name| + Rake::Task["gitlab:db:execute_async_index_operations:#{database_name}"].invoke(args[:pick]) + end + end + end + desc 'Check if there have been user additions to the database' task active: :environment do if ActiveRecord::Base.connection.migration_context.needs_migration? @@ -306,7 +336,7 @@ namespace :gitlab do namespace :migration_testing do # Not possible to import Gitlab::Database::DATABASE_NAMES here # Specs verify that a task exists for each entry in that array. - all_databases = %i[main ci] + all_databases = %i[main ci main_clusterwide] task up: :environment do Gitlab::Database::Migrations::Runner.up(database: 'main', legacy_mode: true).run @@ -399,64 +429,73 @@ namespace :gitlab do namespace :dictionary do DB_DOCS_PATH = File.join(Rails.root, 'db', 'docs') + EE_DICTIONARY_PATH = File.join(Rails.root, 'ee', 'db', 'docs') desc 'Generate database docs yaml' task generate: :environment do - FileUtils.mkdir_p(DB_DOCS_PATH) unless Dir.exist?(DB_DOCS_PATH) + FileUtils.mkdir_p(DB_DOCS_PATH) + FileUtils.mkdir_p(EE_DICTIONARY_PATH) if Gitlab.ee? Rails.application.eager_load! - tables = Gitlab::Database.database_base_models.flat_map { |_, m| m.connection.tables } + version = Gem::Version.new(File.read('VERSION')) + milestone = version.release.segments[0..1].join('.') - views = Gitlab::Database.database_base_models.flat_map { |_, m| m.connection.views } + classes = {} - sources = tables + views + Gitlab::Database.database_base_models.each do |_, model_class| + tables = model_class.connection.tables - classes = sources.index_with { [] } + views = model_class.connection.views + + sources = tables + views + + model_classes = sources.index_with { [] } + + classes.merge!(model_classes) { |_, sources, new_sources| sources + new_sources } - Gitlab::Database.database_base_models.each do |_, model_class| model_class .descendants .reject(&:abstract_class) .reject { |c| c.name =~ /^(?:EE::)?Gitlab::(?:BackgroundMigration|DatabaseImporters)::/ } .reject { |c| c.name =~ /^HABTM_/ } .each { |c| classes[c.table_name] << c.name if classes.has_key?(c.table_name) } - end - version = Gem::Version.new(File.read('VERSION')) - milestone = version.release.segments[0..1].join('.') + sources.each do |source_name| + next if source_name.start_with?('_test_') # Ignore test tables - sources.each do |source_name| - file = dictionary_file_path(source_name, views) - key_name = "#{data_source_type(source_name, views)}_name" + database = model_class.connection_db_config.name + file = dictionary_file_path(source_name, views, database) + key_name = "#{data_source_type(source_name, views)}_name" - table_metadata = { - key_name => source_name, - 'classes' => classes[source_name]&.sort&.uniq, - 'feature_categories' => [], - 'description' => nil, - 'introduced_by_url' => nil, - 'milestone' => milestone - } + table_metadata = { + key_name => source_name, + 'classes' => classes[source_name]&.sort&.uniq, + 'feature_categories' => [], + 'description' => nil, + 'introduced_by_url' => nil, + 'milestone' => milestone + } - if File.exist?(file) - outdated = false + if File.exist?(file) + outdated = false - existing_metadata = YAML.safe_load(File.read(file)) + existing_metadata = YAML.safe_load(File.read(file)) - if existing_metadata[key_name] != table_metadata[key_name] - existing_metadata[key_name] = table_metadata[key_name] - outdated = true - end + if existing_metadata[key_name] != table_metadata[key_name] + existing_metadata[key_name] = table_metadata[key_name] + outdated = true + end - if existing_metadata['classes'].sort != table_metadata['classes'].sort - existing_metadata['classes'] = table_metadata['classes'] - outdated = true - end + if existing_metadata['classes'] && existing_metadata['classes'].sort != table_metadata['classes'].sort + existing_metadata['classes'] = table_metadata['classes'] + outdated = true + end - File.write(file, existing_metadata.to_yaml) if outdated - else - File.write(file, table_metadata.to_yaml) + File.write(file, existing_metadata.to_yaml) if outdated + else + File.write(file, table_metadata.to_yaml) + end end end end @@ -469,16 +508,17 @@ namespace :gitlab do 'table' end - def dictionary_file_path(source_name, views) + def dictionary_file_path(source_name, views, database) sub_directory = views.include?(source_name) ? 'views' : '' - File.join(DB_DOCS_PATH, sub_directory, "#{source_name}.yml") + path = database == 'geo' ? EE_DICTIONARY_PATH : DB_DOCS_PATH + + File.join(path, sub_directory, "#{source_name}.yml") end - # Temporary disable this, see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85760#note_998452069 - # Rake::Task['db:migrate'].enhance do - # Rake::Task['gitlab:db:dictionary:generate'].invoke if Rails.env.development? - # end + Rake::Task['db:migrate'].enhance do + Rake::Task['gitlab:db:dictionary:generate'].invoke if Rails.env.development? + end end end end |