Add latest changes from gitlab-org/gitlab@14-1-stable-eev14.1.0-rc42

7 files changed, 53 insertions, 68 deletions

diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake
index 54e74fd9c8b..db10428e0dc 100644
--- a/lib/tasks/gitlab/assets.rake
+++ b/lib/tasks/gitlab/assets.rake
@@ -5,8 +5,8 @@ require 'fileutils'
 module Tasks
   module Gitlab
     module Assets
-      FOSS_ASSET_FOLDERS = %w[app/assets app/views fixtures/emojis vendor/assets/javascripts].freeze
-      EE_ASSET_FOLDERS = %w[ee/app/assets ee/app/views].freeze
+      FOSS_ASSET_FOLDERS = %w[app/assets fixtures/emojis vendor/assets/javascripts].freeze
+      EE_ASSET_FOLDERS = %w[ee/app/assets].freeze
       JS_ASSET_PATTERNS = %w[*.js config/**/*.js].freeze
       JS_ASSET_FILES = %w[package.json yarn.lock].freeze
       MASTER_MD5_HASH_FILE = 'master-assets-hash.txt'
diff --git a/lib/tasks/gitlab/background_migrations.rake b/lib/tasks/gitlab/background_migrations.rake
new file mode 100644
index 00000000000..c978a2807ca
--- /dev/null
+++ b/lib/tasks/gitlab/background_migrations.rake
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+namespace :gitlab do
+  namespace :background_migrations do
+    task :finalize, [:job_class_name, :table_name, :column_name, :job_arguments] => :environment do |_, args|
+      [:job_class_name, :table_name, :column_name, :job_arguments].each do |argument|
+        unless args[argument]
+          puts "Must specify #{argument} as an argument".color(:red)
+          exit 1
+        end
+      end
+
+      Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.finalize(
+        args[:job_class_name],
+        args[:table_name],
+        args[:column_name],
+        Gitlab::Json.parse(args[:job_arguments])
+      )
+
+      puts "Done.".color(:green)
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index 5b17a8c185a..ed74dd472ff 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -282,6 +282,7 @@ namespace :gitlab do
 
     def puts_time(msg)
       progress.puts "#{Time.now} -- #{msg}"
+      Gitlab::BackupLogger.info(message: "#{Rainbow.uncolor(msg)}")
     end
 
     def progress
@@ -297,7 +298,9 @@ namespace :gitlab do
 
     def repository_backup_strategy
       if Feature.enabled?(:gitaly_backup)
-        Backup::GitalyBackup.new(progress)
+        max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence
+        max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence
+        Backup::GitalyBackup.new(progress, parallel: max_concurrency, parallel_storage: max_storage_concurrency)
       else
         Backup::GitalyRpcBackup.new(progress)
       end
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index ee986f4c503..2b508b341dd 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -90,73 +90,35 @@ namespace :gitlab do
 
     desc 'This adjusts and cleans db/structure.sql - it runs after db:structure:dump'
     task :clean_structure_sql do |task_name|
-      structure_file = 'db/structure.sql'
-      schema = File.read(structure_file)
+      ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env).each do |db_config|
+        structure_file = ActiveRecord::Tasks::DatabaseTasks.dump_filename(db_config.name)
 
-      File.open(structure_file, 'wb+') do |io|
-        Gitlab::Database::SchemaCleaner.new(schema).clean(io)
-      end
-
-      # Allow this task to be called multiple times, as happens when running db:migrate:redo
-      Rake::Task[task_name].reenable
-    end
+        schema = File.read(structure_file)
 
-    desc 'This dumps GitLab specific database details - it runs after db:structure:dump'
-    task :dump_custom_structure do |task_name|
-      Gitlab::Database::CustomStructure.new.dump
+        File.open(structure_file, 'wb+') do |io|
+          Gitlab::Database::SchemaCleaner.new(schema).clean(io)
+        end
+      end
 
       # Allow this task to be called multiple times, as happens when running db:migrate:redo
       Rake::Task[task_name].reenable
     end
 
-    desc 'This loads GitLab specific database details - runs after db:structure:dump'
-    task :load_custom_structure do
-      configuration = Rails.application.config_for(:database)
-
-      ENV['PGHOST']     = configuration['host']          if configuration['host']
-      ENV['PGPORT']     = configuration['port'].to_s     if configuration['port']
-      ENV['PGPASSWORD'] = configuration['password'].to_s if configuration['password']
-      ENV['PGUSER']     = configuration['username'].to_s if configuration['username']
-
-      command = 'psql'
-      dump_filepath = Gitlab::Database::CustomStructure.custom_dump_filepath.to_path
-      args = ['-v', 'ON_ERROR_STOP=1', '-q', '-X', '-f', dump_filepath, configuration['database']]
-
-      unless Kernel.system(command, *args)
-        raise "failed to execute:\n#{command} #{args.join(' ')}\n\n" \
-          "Please ensure `#{command}` is installed in your PATH and has proper permissions.\n\n"
-      end
-    end
-
     # Inform Rake that custom tasks should be run every time rake db:structure:dump is run
     #
     # Rails 6.1 deprecates db:structure:dump in favor of db:schema:dump
     Rake::Task['db:structure:dump'].enhance do
       Rake::Task['gitlab:db:clean_structure_sql'].invoke
-      Rake::Task['gitlab:db:dump_custom_structure'].invoke
     end
 
     # Inform Rake that custom tasks should be run every time rake db:schema:dump is run
     Rake::Task['db:schema:dump'].enhance do
       Rake::Task['gitlab:db:clean_structure_sql'].invoke
-      Rake::Task['gitlab:db:dump_custom_structure'].invoke
-    end
-
-    # Inform Rake that custom tasks should be run every time rake db:structure:load is run
-    #
-    # Rails 6.1 deprecates db:structure:load in favor of db:schema:load
-    Rake::Task['db:structure:load'].enhance do
-      Rake::Task['gitlab:db:load_custom_structure'].invoke
-    end
-
-    # Inform Rake that custom tasks should be run every time rake db:schema:load is run
-    Rake::Task['db:schema:load'].enhance do
-      Rake::Task['gitlab:db:load_custom_structure'].invoke
     end
 
     desc 'Create missing dynamic database partitions'
-    task :create_dynamic_partitions do
-      Gitlab::Database::Partitioning::PartitionCreator.new.create_partitions
+    task create_dynamic_partitions: :environment do
+      Gitlab::Database::Partitioning::PartitionManager.new.sync_partitions
     end
 
     # This is targeted towards deploys and upgrades of GitLab.
@@ -192,7 +154,7 @@ namespace :gitlab do
       Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
     end
 
-    desc 'reindex a regular (non-unique) index without downtime to eliminate bloat'
+    desc 'reindex a regular index without downtime to eliminate bloat'
     task :reindex, [:index_name] => :environment do |_, args|
       unless Feature.enabled?(:database_reindexing, type: :ops)
         puts "This feature (database_reindexing) is currently disabled.".color(:yellow)
diff --git a/lib/tasks/gitlab/helpers.rake b/lib/tasks/gitlab/helpers.rake
index b61b1833c5a..b467aa3819d 100644
--- a/lib/tasks/gitlab/helpers.rake
+++ b/lib/tasks/gitlab/helpers.rake
@@ -3,6 +3,8 @@
 # Prevent StateMachine warnings from outputting during a cron task
 StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON']
 
-task gitlab_environment: :environment do
+task :gitlab_environment do
+  Rake::Task[:environment].invoke unless ENV['SKIP_RAILS_ENV_IN_RAKE']
+
   extend SystemCheck::Helpers
 end
diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake
index 6f5c3a86dd3..d3060d92e88 100644
--- a/lib/tasks/gitlab/sidekiq.rake
+++ b/lib/tasks/gitlab/sidekiq.rake
@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-return if Rails.env.production?
-
 namespace :gitlab do
   namespace :sidekiq do
     def write_yaml(path, banner, object)
@@ -9,31 +7,28 @@ namespace :gitlab do
     end
 
     namespace :migrate_jobs do
-      def mappings
-        ::Gitlab::SidekiqConfig
-          .workers
-          .reject { |worker| worker.klass.is_a?(Gitlab::SidekiqConfig::DummyWorker) }
-          .to_h { |worker| [worker.klass.to_s, ::Gitlab::SidekiqConfig::WorkerRouter.global.route(worker.klass)] }
-      end
-
       desc 'GitLab | Sidekiq | Migrate jobs in the scheduled set to new queue names'
       task schedule: :environment do
         ::Gitlab::SidekiqMigrateJobs
           .new('schedule', logger: Logger.new($stdout))
-          .execute(mappings)
+          .execute(::Gitlab::SidekiqConfig.worker_queue_mappings)
       end
 
       desc 'GitLab | Sidekiq | Migrate jobs in the retry set to new queue names'
       task retry: :environment do
         ::Gitlab::SidekiqMigrateJobs
           .new('retry', logger: Logger.new($stdout))
-          .execute(mappings)
+          .execute(::Gitlab::SidekiqConfig.worker_queue_mappings)
       end
     end
 
+    task :not_production do
+      raise 'This task cannot be run in the production environment' if Rails.env.production?
+    end
+
     namespace :all_queues_yml do
       desc 'GitLab | Sidekiq | Generate all_queues.yml based on worker definitions'
-      task generate: :environment do
+      task generate: ['gitlab:sidekiq:not_production', :environment] do
         banner = <<~BANNER
           # This file is generated automatically by
           #   bin/rake gitlab:sidekiq:all_queues_yml:generate
@@ -51,7 +46,7 @@ namespace :gitlab do
       end
 
       desc 'GitLab | Sidekiq | Validate that all_queues.yml matches worker definitions'
-      task check: :environment do
+      task check: ['gitlab:sidekiq:not_production', :environment] do
         if Gitlab::SidekiqConfig.all_queues_yml_outdated?
           raise <<~MSG
             Changes in worker queues found, please update the metadata by running:
@@ -70,7 +65,7 @@ namespace :gitlab do
 
     namespace :sidekiq_queues_yml do
       desc 'GitLab | Sidekiq | Generate sidekiq_queues.yml based on worker definitions'
-      task generate: :environment do
+      task generate: ['gitlab:sidekiq:not_production', :environment] do
         banner = <<~BANNER
           # This file is generated automatically by
           #   bin/rake gitlab:sidekiq:sidekiq_queues_yml:generate
@@ -104,7 +99,7 @@ namespace :gitlab do
       end
 
       desc 'GitLab | Sidekiq | Validate that sidekiq_queues.yml matches worker definitions'
-      task check: :environment do
+      task check: ['gitlab:sidekiq:not_production', :environment] do
         if Gitlab::SidekiqConfig.sidekiq_queues_yml_outdated?
           raise <<~MSG
             Changes in worker queues found, please update the metadata by running:
diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake
index 0ad50c0fa53..166f08ef16a 100644
--- a/lib/tasks/gitlab/usage_data.rake
+++ b/lib/tasks/gitlab/usage_data.rake
@@ -19,7 +19,7 @@ namespace :gitlab do
 
     desc 'GitLab | UsageData | Generate usage ping and send it to Versions Application'
     task generate_and_send: :environment do
-      result = SubmitUsagePingService.new.execute
+      result = ServicePing::SubmitService.new.execute
 
       puts Gitlab::Json.pretty_generate(result.attributes)
     end