diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-20 12:55:51 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-20 12:55:51 +0300 |
commit | e8d2c2579383897a1dd7f9debd359abe8ae8373d (patch) | |
tree | c42be41678c2586d49a75cabce89322082698334 /lib/tasks/gitlab | |
parent | fc845b37ec3a90aaa719975f607740c22ba6a113 (diff) |
Add latest changes from gitlab-org/gitlab@14-1-stable-eev14.1.0-rc42
Diffstat (limited to 'lib/tasks/gitlab')
-rw-r--r-- | lib/tasks/gitlab/assets.rake | 4 | ||||
-rw-r--r-- | lib/tasks/gitlab/background_migrations.rake | 23 | ||||
-rw-r--r-- | lib/tasks/gitlab/backup.rake | 5 | ||||
-rw-r--r-- | lib/tasks/gitlab/db.rake | 58 | ||||
-rw-r--r-- | lib/tasks/gitlab/helpers.rake | 4 | ||||
-rw-r--r-- | lib/tasks/gitlab/sidekiq.rake | 25 | ||||
-rw-r--r-- | lib/tasks/gitlab/usage_data.rake | 2 |
7 files changed, 53 insertions, 68 deletions
diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index 54e74fd9c8b..db10428e0dc 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -5,8 +5,8 @@ require 'fileutils' module Tasks module Gitlab module Assets - FOSS_ASSET_FOLDERS = %w[app/assets app/views fixtures/emojis vendor/assets/javascripts].freeze - EE_ASSET_FOLDERS = %w[ee/app/assets ee/app/views].freeze + FOSS_ASSET_FOLDERS = %w[app/assets fixtures/emojis vendor/assets/javascripts].freeze + EE_ASSET_FOLDERS = %w[ee/app/assets].freeze JS_ASSET_PATTERNS = %w[*.js config/**/*.js].freeze JS_ASSET_FILES = %w[package.json yarn.lock].freeze MASTER_MD5_HASH_FILE = 'master-assets-hash.txt' diff --git a/lib/tasks/gitlab/background_migrations.rake b/lib/tasks/gitlab/background_migrations.rake new file mode 100644 index 00000000000..c978a2807ca --- /dev/null +++ b/lib/tasks/gitlab/background_migrations.rake @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :background_migrations do + task :finalize, [:job_class_name, :table_name, :column_name, :job_arguments] => :environment do |_, args| + [:job_class_name, :table_name, :column_name, :job_arguments].each do |argument| + unless args[argument] + puts "Must specify #{argument} as an argument".color(:red) + exit 1 + end + end + + Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.finalize( + args[:job_class_name], + args[:table_name], + args[:column_name], + Gitlab::Json.parse(args[:job_arguments]) + ) + + puts "Done.".color(:green) + end + end +end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 5b17a8c185a..ed74dd472ff 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -282,6 +282,7 @@ namespace :gitlab do def puts_time(msg) progress.puts "#{Time.now} -- #{msg}" + Gitlab::BackupLogger.info(message: "#{Rainbow.uncolor(msg)}") end def progress @@ -297,7 +298,9 @@ namespace :gitlab do def repository_backup_strategy if Feature.enabled?(:gitaly_backup) - Backup::GitalyBackup.new(progress) + max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence + max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence + Backup::GitalyBackup.new(progress, parallel: max_concurrency, parallel_storage: max_storage_concurrency) else Backup::GitalyRpcBackup.new(progress) end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index ee986f4c503..2b508b341dd 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -90,73 +90,35 @@ namespace :gitlab do desc 'This adjusts and cleans db/structure.sql - it runs after db:structure:dump' task :clean_structure_sql do |task_name| - structure_file = 'db/structure.sql' - schema = File.read(structure_file) + ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env).each do |db_config| + structure_file = ActiveRecord::Tasks::DatabaseTasks.dump_filename(db_config.name) - File.open(structure_file, 'wb+') do |io| - Gitlab::Database::SchemaCleaner.new(schema).clean(io) - end - - # Allow this task to be called multiple times, as happens when running db:migrate:redo - Rake::Task[task_name].reenable - end + schema = File.read(structure_file) - desc 'This dumps GitLab specific database details - it runs after db:structure:dump' - task :dump_custom_structure do |task_name| - Gitlab::Database::CustomStructure.new.dump + File.open(structure_file, 'wb+') do |io| + Gitlab::Database::SchemaCleaner.new(schema).clean(io) + end + end # Allow this task to be called multiple times, as happens when running db:migrate:redo Rake::Task[task_name].reenable end - desc 'This loads GitLab specific database details - runs after db:structure:dump' - task :load_custom_structure do - configuration = Rails.application.config_for(:database) - - ENV['PGHOST'] = configuration['host'] if configuration['host'] - ENV['PGPORT'] = configuration['port'].to_s if configuration['port'] - ENV['PGPASSWORD'] = configuration['password'].to_s if configuration['password'] - ENV['PGUSER'] = configuration['username'].to_s if configuration['username'] - - command = 'psql' - dump_filepath = Gitlab::Database::CustomStructure.custom_dump_filepath.to_path - args = ['-v', 'ON_ERROR_STOP=1', '-q', '-X', '-f', dump_filepath, configuration['database']] - - unless Kernel.system(command, *args) - raise "failed to execute:\n#{command} #{args.join(' ')}\n\n" \ - "Please ensure `#{command}` is installed in your PATH and has proper permissions.\n\n" - end - end - # Inform Rake that custom tasks should be run every time rake db:structure:dump is run # # Rails 6.1 deprecates db:structure:dump in favor of db:schema:dump Rake::Task['db:structure:dump'].enhance do Rake::Task['gitlab:db:clean_structure_sql'].invoke - Rake::Task['gitlab:db:dump_custom_structure'].invoke end # Inform Rake that custom tasks should be run every time rake db:schema:dump is run Rake::Task['db:schema:dump'].enhance do Rake::Task['gitlab:db:clean_structure_sql'].invoke - Rake::Task['gitlab:db:dump_custom_structure'].invoke - end - - # Inform Rake that custom tasks should be run every time rake db:structure:load is run - # - # Rails 6.1 deprecates db:structure:load in favor of db:schema:load - Rake::Task['db:structure:load'].enhance do - Rake::Task['gitlab:db:load_custom_structure'].invoke - end - - # Inform Rake that custom tasks should be run every time rake db:schema:load is run - Rake::Task['db:schema:load'].enhance do - Rake::Task['gitlab:db:load_custom_structure'].invoke end desc 'Create missing dynamic database partitions' - task :create_dynamic_partitions do - Gitlab::Database::Partitioning::PartitionCreator.new.create_partitions + task create_dynamic_partitions: :environment do + Gitlab::Database::Partitioning::PartitionManager.new.sync_partitions end # This is targeted towards deploys and upgrades of GitLab. @@ -192,7 +154,7 @@ namespace :gitlab do Rake::Task['gitlab:db:create_dynamic_partitions'].invoke end - desc 'reindex a regular (non-unique) index without downtime to eliminate bloat' + desc 'reindex a regular index without downtime to eliminate bloat' task :reindex, [:index_name] => :environment do |_, args| unless Feature.enabled?(:database_reindexing, type: :ops) puts "This feature (database_reindexing) is currently disabled.".color(:yellow) diff --git a/lib/tasks/gitlab/helpers.rake b/lib/tasks/gitlab/helpers.rake index b61b1833c5a..b467aa3819d 100644 --- a/lib/tasks/gitlab/helpers.rake +++ b/lib/tasks/gitlab/helpers.rake @@ -3,6 +3,8 @@ # Prevent StateMachine warnings from outputting during a cron task StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] -task gitlab_environment: :environment do +task :gitlab_environment do + Rake::Task[:environment].invoke unless ENV['SKIP_RAILS_ENV_IN_RAKE'] + extend SystemCheck::Helpers end diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake index 6f5c3a86dd3..d3060d92e88 100644 --- a/lib/tasks/gitlab/sidekiq.rake +++ b/lib/tasks/gitlab/sidekiq.rake @@ -1,7 +1,5 @@ # frozen_string_literal: true -return if Rails.env.production? - namespace :gitlab do namespace :sidekiq do def write_yaml(path, banner, object) @@ -9,31 +7,28 @@ namespace :gitlab do end namespace :migrate_jobs do - def mappings - ::Gitlab::SidekiqConfig - .workers - .reject { |worker| worker.klass.is_a?(Gitlab::SidekiqConfig::DummyWorker) } - .to_h { |worker| [worker.klass.to_s, ::Gitlab::SidekiqConfig::WorkerRouter.global.route(worker.klass)] } - end - desc 'GitLab | Sidekiq | Migrate jobs in the scheduled set to new queue names' task schedule: :environment do ::Gitlab::SidekiqMigrateJobs .new('schedule', logger: Logger.new($stdout)) - .execute(mappings) + .execute(::Gitlab::SidekiqConfig.worker_queue_mappings) end desc 'GitLab | Sidekiq | Migrate jobs in the retry set to new queue names' task retry: :environment do ::Gitlab::SidekiqMigrateJobs .new('retry', logger: Logger.new($stdout)) - .execute(mappings) + .execute(::Gitlab::SidekiqConfig.worker_queue_mappings) end end + task :not_production do + raise 'This task cannot be run in the production environment' if Rails.env.production? + end + namespace :all_queues_yml do desc 'GitLab | Sidekiq | Generate all_queues.yml based on worker definitions' - task generate: :environment do + task generate: ['gitlab:sidekiq:not_production', :environment] do banner = <<~BANNER # This file is generated automatically by # bin/rake gitlab:sidekiq:all_queues_yml:generate @@ -51,7 +46,7 @@ namespace :gitlab do end desc 'GitLab | Sidekiq | Validate that all_queues.yml matches worker definitions' - task check: :environment do + task check: ['gitlab:sidekiq:not_production', :environment] do if Gitlab::SidekiqConfig.all_queues_yml_outdated? raise <<~MSG Changes in worker queues found, please update the metadata by running: @@ -70,7 +65,7 @@ namespace :gitlab do namespace :sidekiq_queues_yml do desc 'GitLab | Sidekiq | Generate sidekiq_queues.yml based on worker definitions' - task generate: :environment do + task generate: ['gitlab:sidekiq:not_production', :environment] do banner = <<~BANNER # This file is generated automatically by # bin/rake gitlab:sidekiq:sidekiq_queues_yml:generate @@ -104,7 +99,7 @@ namespace :gitlab do end desc 'GitLab | Sidekiq | Validate that sidekiq_queues.yml matches worker definitions' - task check: :environment do + task check: ['gitlab:sidekiq:not_production', :environment] do if Gitlab::SidekiqConfig.sidekiq_queues_yml_outdated? raise <<~MSG Changes in worker queues found, please update the metadata by running: diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake index 0ad50c0fa53..166f08ef16a 100644 --- a/lib/tasks/gitlab/usage_data.rake +++ b/lib/tasks/gitlab/usage_data.rake @@ -19,7 +19,7 @@ namespace :gitlab do desc 'GitLab | UsageData | Generate usage ping and send it to Versions Application' task generate_and_send: :environment do - result = SubmitUsagePingService.new.execute + result = ServicePing::SubmitService.new.execute puts Gitlab::Json.pretty_generate(result.attributes) end |