diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-12-20 16:37:47 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-12-20 16:37:47 +0300 |
commit | aee0a117a889461ce8ced6fcf73207fe017f1d99 (patch) | |
tree | 891d9ef189227a8445d83f35c1b0fc99573f4380 /lib/tasks | |
parent | 8d46af3258650d305f53b819eabf7ab18d22f59e (diff) |
Add latest changes from gitlab-org/gitlab@14-6-stable-eev14.6.0-rc42
Diffstat (limited to 'lib/tasks')
-rw-r--r-- | lib/tasks/gettext.rake | 13 | ||||
-rw-r--r-- | lib/tasks/gitlab/background_migrations.rake | 19 | ||||
-rw-r--r-- | lib/tasks/gitlab/cleanup.rake | 29 | ||||
-rw-r--r-- | lib/tasks/gitlab/db.rake | 49 | ||||
-rw-r--r-- | lib/tasks/gitlab/gitaly.rake | 9 | ||||
-rw-r--r-- | lib/tasks/gitlab/seed/group_seed.rake | 2 | ||||
-rw-r--r-- | lib/tasks/gitlab/sidekiq.rake | 5 | ||||
-rw-r--r-- | lib/tasks/gitlab/update_templates.rake | 2 | ||||
-rw-r--r-- | lib/tasks/gitlab/x509/update.rake | 4 |
9 files changed, 85 insertions, 47 deletions
diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index e03c78d5a40..17f9414ad52 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -58,6 +58,7 @@ namespace :gettext do task lint: :environment do require 'simple_po_parser' require 'gitlab/utils' + require 'parallel' FastGettext.silence_errors files = Dir.glob(Rails.root.join('locale/*/gitlab.po')) @@ -70,7 +71,9 @@ namespace :gettext do linters.unshift(Gitlab::I18n::PoLinter.new(po_path: pot_file_path)) - failed_linters = linters.select { |linter| linter.errors.any? } + failed_linters = Parallel + .map(linters, progress: 'Linting po files') { |linter| linter if linter.errors.any? } + .compact if failed_linters.empty? puts 'All PO files are valid.' @@ -129,14 +132,6 @@ namespace :gettext do ) end - # Disallow HTML from translatable strings - # See: https://docs.gitlab.com/ee/development/i18n/externalization.html#html - def html_todolist - return @html_todolist if defined?(@html_todolist) - - @html_todolist = YAML.safe_load(File.read(Rails.root.join('lib/gitlab/i18n/html_todo.yml'))) - end - def report_errors_for_file(file, errors_for_file) puts "Errors in `#{file}`:" diff --git a/lib/tasks/gitlab/background_migrations.rake b/lib/tasks/gitlab/background_migrations.rake index c978a2807ca..c7f3d003f9f 100644 --- a/lib/tasks/gitlab/background_migrations.rake +++ b/lib/tasks/gitlab/background_migrations.rake @@ -2,6 +2,7 @@ namespace :gitlab do namespace :background_migrations do + desc 'Synchronously finish executing a batched background migration' task :finalize, [:job_class_name, :table_name, :column_name, :job_arguments] => :environment do |_, args| [:job_class_name, :table_name, :column_name, :job_arguments].each do |argument| unless args[argument] @@ -19,5 +20,23 @@ namespace :gitlab do puts "Done.".color(:green) end + + desc 'Display the status of batched background migrations' + task status: :environment do + statuses = Gitlab::Database::BackgroundMigration::BatchedMigration.statuses + max_status_length = statuses.keys.map(&:length).max + format_string = "%-#{max_status_length}s | %s\n" + + Gitlab::Database::BackgroundMigration::BatchedMigration.find_each(batch_size: 100) do |migration| + identification_fields = [ + migration.job_class_name, + migration.table_name, + migration.column_name, + migration.job_arguments.to_json + ].join(',') + + printf(format_string, migration.status, identification_fields) + end + end end end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 0cd4ab354c9..8f033a41e3d 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -100,13 +100,15 @@ namespace :gitlab do namespace :sessions do desc "GitLab | Cleanup | Sessions | Clean ActiveSession lookup keys" task active_sessions_lookup_keys: :gitlab_environment do - session_key_pattern = "#{Gitlab::Redis::SharedState::USER_SESSIONS_LOOKUP_NAMESPACE}:*" + use_redis_session_store = Gitlab::Utils.to_boolean(ENV['GITLAB_USE_REDIS_SESSIONS_STORE'], default: true) + redis_store_class = use_redis_session_store ? Gitlab::Redis::Sessions : Gitlab::Redis::SharedState + session_key_pattern = "#{Gitlab::Redis::Sessions::USER_SESSIONS_LOOKUP_NAMESPACE}:*" last_save_check = Time.at(0) wait_time = 10.seconds cursor = 0 total_users_scanned = 0 - Gitlab::Redis::SharedState.with do |redis| + redis_store_class.with do |redis| begin cursor, keys = redis.scan(cursor, match: session_key_pattern) total_users_scanned += keys.count @@ -119,27 +121,16 @@ namespace :gitlab do last_save_check = Time.now end + user = Struct.new(:id) + keys.each do |key| user_id = key.split(':').last - lookup_key_count = redis.scard(key) - - session_ids = ActiveSession.session_ids_for_user(user_id) - entries = ActiveSession.raw_active_session_entries(redis, session_ids, user_id) - session_ids_and_entries = session_ids.zip(entries) - - inactive_session_ids = session_ids_and_entries.map do |session_id, session| - session_id if session.nil? - end.compact - - redis.pipelined do |conn| - inactive_session_ids.each do |session_id| - conn.srem(key, session_id) - end - end + removed = [] + active = ActiveSession.cleaned_up_lookup_entries(redis, user.new(user_id), removed) - if inactive_session_ids - puts "deleted #{inactive_session_ids.count} out of #{lookup_key_count} lookup keys for User ##{user_id}" + if removed.any? + puts "deleted #{removed.count} out of #{active.count + removed.count} lookup keys for User ##{user_id}" end end end while cursor.to_i != 0 diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index e83c4cbdb39..9e733fc3a0f 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -1,5 +1,7 @@ # frozen_string_literal: true +databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml + namespace :gitlab do namespace :db do desc 'GitLab | DB | Manually insert schema migration version' @@ -49,7 +51,7 @@ namespace :gitlab do # Drop all extra schema objects GitLab owns Gitlab::Database::EXTRA_SCHEMAS.each do |schema| - connection.execute("DROP SCHEMA IF EXISTS #{connection.quote_table_name(schema)}") + connection.execute("DROP SCHEMA IF EXISTS #{connection.quote_table_name(schema)} CASCADE") end end @@ -83,7 +85,7 @@ namespace :gitlab do desc 'GitLab | DB | Sets up EE specific database functionality' if Gitlab.ee? - task setup_ee: %w[geo:db:drop geo:db:create geo:db:schema:load geo:db:migrate] + task setup_ee: %w[db:drop:geo db:create:geo db:schema:load:geo db:migrate:geo] else task :setup_ee end @@ -116,6 +118,19 @@ namespace :gitlab do Rake::Task['gitlab:db:clean_structure_sql'].invoke end + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| + # Inform Rake that custom tasks should be run every time rake db:structure:dump is run + # + # Rails 6.1 deprecates db:structure:dump in favor of db:schema:dump + Rake::Task["db:structure:dump:#{name}"].enhance do + Rake::Task['gitlab:db:clean_structure_sql'].invoke + end + + Rake::Task["db:schema:dump:#{name}"].enhance do + Rake::Task['gitlab:db:clean_structure_sql'].invoke + end + end + desc 'Create missing dynamic database partitions' task create_dynamic_partitions: :environment do Gitlab::Database::Partitioning.sync_partitions @@ -160,24 +175,30 @@ namespace :gitlab do Rake::Task['gitlab:db:create_dynamic_partitions'].invoke end - desc 'execute reindexing without downtime to eliminate bloat' + desc "Reindex database without downtime to eliminate bloat" task reindex: :environment do - unless Feature.enabled?(:database_reindexing, type: :ops, default_enabled: :yaml) + unless Gitlab::Database::Reindexing.enabled? puts "This feature (database_reindexing) is currently disabled.".color(:yellow) exit end - Gitlab::Database::EachDatabase.each_database_connection do |connection, connection_name| - Gitlab::Database::SharedModel.logger = Logger.new($stdout) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false) + Gitlab::Database::Reindexing.invoke + end + + namespace :reindex do + databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml - # Hack: Before we do actual reindexing work, create async indexes - Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops) + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database_name| + desc "Reindex #{database_name} database without downtime to eliminate bloat" + task database_name => :environment do + unless Gitlab::Database::Reindexing.enabled? + puts "This feature (database_reindexing) is currently disabled.".color(:yellow) + exit + end - Gitlab::Database::Reindexing.automatic_reindexing + Gitlab::Database::Reindexing.invoke(database_name) + end end - rescue StandardError => e - Gitlab::AppLogger.error(e) - raise end desc 'Enqueue an index for reindexing' @@ -243,7 +264,9 @@ namespace :gitlab do # Only for development environments, # we execute pending data migrations inline for convenience. Rake::Task['db:migrate'].enhance do - Rake::Task['gitlab:db:execute_batched_migrations'].invoke if Rails.env.development? + if Rails.env.development? && Gitlab::Database::BackgroundMigration::BatchedMigration.table_exists? + Rake::Task['gitlab:db:execute_batched_migrations'].invoke + end end end end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index eabbb8652f1..b01a7902bf2 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -37,8 +37,8 @@ namespace :gitlab do raise end - desc 'GitLab | Gitaly | Install or upgrade gitaly' - task :install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args| + desc 'GitLab | Gitaly | Clone and checkout gitaly' + task :clone, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args| warn_user_is_not_gitlab unless args.dir.present? && args.storage_path.present? @@ -51,6 +51,11 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]") version = Gitlab::GitalyClient.expected_server_version checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir, clone_opts: %w[--depth 1]) + end + + desc 'GitLab | Gitaly | Install or upgrade gitaly' + task :install, [:dir, :storage_path, :repo] => [:gitlab_environment, 'gitlab:gitaly:clone'] do |t, args| + warn_user_is_not_gitlab storage_paths = { 'default' => args.storage_path } Gitlab::SetupHelper::Gitaly.create_configuration(args.dir, storage_paths) diff --git a/lib/tasks/gitlab/seed/group_seed.rake b/lib/tasks/gitlab/seed/group_seed.rake index bc705c94422..a9a350fb6c3 100644 --- a/lib/tasks/gitlab/seed/group_seed.rake +++ b/lib/tasks/gitlab/seed/group_seed.rake @@ -184,7 +184,7 @@ class GroupSeeder group = Group.find(group_id) @resource_count.times do |i| - _, project_path = PROJECT_URL.split('/')[-2..-1] + _, project_path = PROJECT_URL.split('/')[-2..] project_path.gsub!('.git', '') diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake index 2e383065b64..10492e183c5 100644 --- a/lib/tasks/gitlab/sidekiq.rake +++ b/lib/tasks/gitlab/sidekiq.rake @@ -100,6 +100,10 @@ namespace :gitlab do queues_and_weights = Gitlab::SidekiqConfig.queues_for_sidekiq_queues_yml write_yaml(Gitlab::SidekiqConfig::SIDEKIQ_QUEUES_PATH, banner, queues: queues_and_weights) + + if Gitlab.jh? + write_yaml(Gitlab::SidekiqConfig::JH_SIDEKIQ_QUEUES_PATH, banner, queues: Gitlab::SidekiqConfig.jh_queues_for_sidekiq_queues_yml) + end end desc 'GitLab | Sidekiq | Validate that sidekiq_queues.yml matches worker definitions' @@ -113,6 +117,7 @@ namespace :gitlab do Then commit and push the changes from: - #{Gitlab::SidekiqConfig::SIDEKIQ_QUEUES_PATH} + #{"- " + Gitlab::SidekiqConfig::JH_SIDEKIQ_QUEUES_PATH if Gitlab.jh?} MSG end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index e3a4e7f50b8..247897bed0b 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -32,7 +32,7 @@ namespace :gitlab do tmp_namespace_path = "tmp-project-import-#{Time.now.to_i}" puts "Creating temporary namespace #{tmp_namespace_path}" - tmp_namespace = Namespace.create!(owner: admin, name: tmp_namespace_path, path: tmp_namespace_path) + tmp_namespace = Namespace.create!(owner: admin, name: tmp_namespace_path, path: tmp_namespace_path, type: Namespaces::UserNamespace.sti_name) templates = if template_names.empty? Gitlab::ProjectTemplate.all diff --git a/lib/tasks/gitlab/x509/update.rake b/lib/tasks/gitlab/x509/update.rake index d3c63fa8514..7b7d15479bf 100644 --- a/lib/tasks/gitlab/x509/update.rake +++ b/lib/tasks/gitlab/x509/update.rake @@ -12,14 +12,14 @@ namespace :gitlab do def update_certificates logger = Logger.new($stdout) - unless X509CommitSignature.exists? + unless CommitSignatures::X509CommitSignature.exists? logger.info("Unable to find any x509 commit signatures. Exiting.") return end logger.info("Start to update x509 commit signatures") - X509CommitSignature.find_each do |sig| + CommitSignatures::X509CommitSignature.find_each do |sig| sig.x509_commit&.update_signature!(sig) end |