Add latest changes from gitlab-org/gitlab@13-12-stable-eev13.12.0-rc42

14 files changed, 87 insertions, 40 deletions

diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake
index 94867e1a16a..4c312ea492b 100644
--- a/lib/tasks/gitlab/artifacts/migrate.rake
+++ b/lib/tasks/gitlab/artifacts/migrate.rake
@@ -8,30 +8,24 @@ namespace :gitlab do
   namespace :artifacts do
     task migrate: :environment do
       logger = Logger.new(STDOUT)
-      logger.info('Starting transfer of artifacts to remote storage')
 
-      helper = Gitlab::Artifacts::MigrationHelper.new
+      helper = Gitlab::LocalAndRemoteStorageMigration::ArtifactMigrater.new(logger)
 
       begin
-        helper.migrate_to_remote_storage do |artifact|
-          logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to object storage")
-        end
-      rescue => e
+        helper.migrate_to_remote_storage
+      rescue StandardError => e
         logger.error(e.message)
       end
     end
 
     task migrate_to_local: :environment do
       logger = Logger.new(STDOUT)
-      logger.info('Starting transfer of artifacts to local storage')
 
-      helper = Gitlab::Artifacts::MigrationHelper.new
+      helper = Gitlab::LocalAndRemoteStorageMigration::ArtifactMigrater.new(logger)
 
       begin
-        helper.migrate_to_local_storage do |artifact|
-          logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to local storage")
-        end
-      rescue => e
+        helper.migrate_to_local_storage
+      rescue StandardError => e
         logger.error(e.message)
       end
     end
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 3baf4e7b7c6..bbfdf598e42 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -191,7 +191,7 @@ namespace :gitlab do
       ActiveRecord::Base.logger = Logger.new(STDOUT) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false)
 
       Gitlab::Database::Reindexing.perform(indexes)
-    rescue => e
+    rescue StandardError => e
       Gitlab::AppLogger.error(e)
       raise
     end
@@ -217,9 +217,11 @@ namespace :gitlab do
     end
 
     desc 'Run migrations with instrumentation'
-    task :migration_testing, [:result_file] => :environment do |_, args|
-      result_file = args[:result_file] || raise("Please specify result_file argument")
-      raise "File exists already, won't overwrite: #{result_file}" if File.exist?(result_file)
+    task migration_testing: :environment do
+      result_dir = Gitlab::Database::Migrations::Instrumentation::RESULT_DIR
+      raise "Directory exists already, won't overwrite: #{result_dir}" if File.exist?(result_dir)
+
+      Dir.mkdir(result_dir)
 
       verbose_was = ActiveRecord::Migration.verbose
       ActiveRecord::Migration.verbose = true
@@ -240,7 +242,7 @@ namespace :gitlab do
       end
     ensure
       if instrumentation
-        File.open(result_file, 'wb+') do |io|
+        File.open(File.join(result_dir, Gitlab::Database::Migrations::Instrumentation::STATS_FILENAME), 'wb+') do |io|
           io << instrumentation.observations.to_json
         end
       end
@@ -248,5 +250,19 @@ namespace :gitlab do
       ActiveRecord::Base.clear_cache!
       ActiveRecord::Migration.verbose = verbose_was
     end
+
+    desc 'Run all pending batched migrations'
+    task execute_batched_migrations: :environment do
+      Gitlab::Database::BackgroundMigration::BatchedMigration.active.queue_order.each do |migration|
+        Gitlab::AppLogger.info("Executing batched migration #{migration.id} inline")
+        Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new.run_entire_migration(migration)
+      end
+    end
+
+    # Only for development environments,
+    # we execute pending data migrations inline for convenience.
+    Rake::Task['db:migrate'].enhance do
+      Rake::Task['gitlab:db:execute_batched_migrations'].invoke if Rails.env.development?
+    end
   end
 end
diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake
index 9c5549b4a54..8587ef4afdb 100644
--- a/lib/tasks/gitlab/git.rake
+++ b/lib/tasks/gitlab/git.rake
@@ -9,7 +9,7 @@ namespace :gitlab do
         begin
           project.repository.fsck
 
-        rescue => e
+        rescue StandardError => e
           failures << "#{project.full_path} on #{project.repository_storage}: #{e}"
         end
 
@@ -51,7 +51,7 @@ namespace :gitlab do
         next unless project.repo_exists?
 
         result = project.repository.checksum
-      rescue => e
+      rescue StandardError => e
         result = "Ignored error: #{e.message}".squish.truncate(255)
       ensure
         puts "#{project.id},#{result}"
diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake
index 05249a126bc..a173de7c5c7 100644
--- a/lib/tasks/gitlab/lfs/migrate.rake
+++ b/lib/tasks/gitlab/lfs/migrate.rake
@@ -14,7 +14,7 @@ namespace :gitlab do
         lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE)
 
         logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage")
-      rescue => e
+      rescue StandardError => e
         logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}")
       end
     end
@@ -28,7 +28,7 @@ namespace :gitlab do
         lfs_object.file.migrate!(LfsObjectUploader::Store::LOCAL)
 
         logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to local storage")
-      rescue => e
+      rescue StandardError => e
         logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}")
       end
     end
diff --git a/lib/tasks/gitlab/packages/events.rake b/lib/tasks/gitlab/packages/events.rake
index 4a6a014acc5..d24535d85b6 100644
--- a/lib/tasks/gitlab/packages/events.rake
+++ b/lib/tasks/gitlab/packages/events.rake
@@ -9,7 +9,7 @@ namespace :gitlab do
       task generate: :environment do
         Rake::Task["gitlab:packages:events:generate_counts"].invoke
         Rake::Task["gitlab:packages:events:generate_unique"].invoke
-      rescue => e
+      rescue StandardError => e
         logger.error("Error building events list: #{e}")
       end
 
@@ -21,7 +21,7 @@ namespace :gitlab do
         File.open(path, "w") { |file| file << counter_events_list.to_yaml }
 
         logger.info("Events file `#{path}` generated successfully")
-      rescue => e
+      rescue StandardError => e
         logger.error("Error building events list: #{e}")
       end
 
@@ -33,7 +33,7 @@ namespace :gitlab do
         File.open(path, "w") { |file| file << generate_unique_events_list.to_yaml }
 
         logger.info("Events file `#{path}` generated successfully")
-      rescue => e
+      rescue StandardError => e
         logger.error("Error building events list: #{e}")
       end
 
diff --git a/lib/tasks/gitlab/packages/migrate.rake b/lib/tasks/gitlab/packages/migrate.rake
index 20a8c51db66..febc3e7fa2d 100644
--- a/lib/tasks/gitlab/packages/migrate.rake
+++ b/lib/tasks/gitlab/packages/migrate.rake
@@ -17,7 +17,7 @@ namespace :gitlab do
         package_file.file.migrate!(::Packages::PackageFileUploader::Store::REMOTE)
 
         logger.info("Transferred package file #{package_file.id} of size #{package_file.size.to_i.bytes} to object storage")
-      rescue => e
+      rescue StandardError => e
         logger.error("Failed to transfer package file #{package_file.id} with error: #{e.message}")
       end
     end
diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake
index ee2931f0c4f..684d62d1367 100644
--- a/lib/tasks/gitlab/pages.rake
+++ b/lib/tasks/gitlab/pages.rake
@@ -61,26 +61,24 @@ namespace :gitlab do
     namespace :deployments do
       task migrate_to_object_storage: :gitlab_environment do
         logger = Logger.new(STDOUT)
-        logger.info('Starting transfer of pages deployments to remote storage')
 
-        helper = Gitlab::Pages::MigrationHelper.new(logger)
+        helper = Gitlab::LocalAndRemoteStorageMigration::PagesDeploymentMigrater.new(logger)
 
         begin
           helper.migrate_to_remote_storage
-        rescue => e
+        rescue StandardError => e
           logger.error(e.message)
         end
       end
 
       task migrate_to_local: :gitlab_environment do
         logger = Logger.new(STDOUT)
-        logger.info('Starting transfer of Pages deployments to local storage')
 
-        helper = Gitlab::Pages::MigrationHelper.new(logger)
+        helper = Gitlab::LocalAndRemoteStorageMigration::PagesDeploymentMigrater.new(logger)
 
         begin
           helper.migrate_to_local_storage
-        rescue => e
+        rescue StandardError => e
           logger.error(e.message)
         end
       end
diff --git a/lib/tasks/gitlab/praefect.rake b/lib/tasks/gitlab/praefect.rake
index 346df3e0c75..28b70f8986e 100644
--- a/lib/tasks/gitlab/praefect.rake
+++ b/lib/tasks/gitlab/praefect.rake
@@ -44,7 +44,7 @@ namespace :gitlab do
 
         row = [project.name] << replicas_resp.primary.checksum
         row.concat(sorted_replicas.map {|r| r.checksum})
-      rescue
+      rescue StandardError
         puts 'Something went wrong when getting replicas.'
         next
       end
diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake
index 6f00db42d78..6f5c3a86dd3 100644
--- a/lib/tasks/gitlab/sidekiq.rake
+++ b/lib/tasks/gitlab/sidekiq.rake
@@ -8,6 +8,29 @@ namespace :gitlab do
       File.write(path, banner + YAML.dump(object).gsub(/ *$/m, ''))
     end
 
+    namespace :migrate_jobs do
+      def mappings
+        ::Gitlab::SidekiqConfig
+          .workers
+          .reject { |worker| worker.klass.is_a?(Gitlab::SidekiqConfig::DummyWorker) }
+          .to_h { |worker| [worker.klass.to_s, ::Gitlab::SidekiqConfig::WorkerRouter.global.route(worker.klass)] }
+      end
+
+      desc 'GitLab | Sidekiq | Migrate jobs in the scheduled set to new queue names'
+      task schedule: :environment do
+        ::Gitlab::SidekiqMigrateJobs
+          .new('schedule', logger: Logger.new($stdout))
+          .execute(mappings)
+      end
+
+      desc 'GitLab | Sidekiq | Migrate jobs in the retry set to new queue names'
+      task retry: :environment do
+        ::Gitlab::SidekiqMigrateJobs
+          .new('retry', logger: Logger.new($stdout))
+          .execute(mappings)
+      end
+    end
+
     namespace :all_queues_yml do
       desc 'GitLab | Sidekiq | Generate all_queues.yml based on worker definitions'
       task generate: :environment do
diff --git a/lib/tasks/gitlab/snowplow.rake b/lib/tasks/gitlab/snowplow.rake
new file mode 100644
index 00000000000..278ba4a471c
--- /dev/null
+++ b/lib/tasks/gitlab/snowplow.rake
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+namespace :gitlab do
+  namespace :snowplow do
+    desc 'GitLab | Snowplow | Generate event dictionary'
+    task generate_event_dictionary: :environment do
+      items = Gitlab::Tracking::EventDefinition.definitions
+      Gitlab::Tracking::Docs::Renderer.new(items).write
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/terraform/migrate.rake b/lib/tasks/gitlab/terraform/migrate.rake
index a9c16049240..2bf9ec9537a 100644
--- a/lib/tasks/gitlab/terraform/migrate.rake
+++ b/lib/tasks/gitlab/terraform/migrate.rake
@@ -15,7 +15,7 @@ namespace :gitlab do
 
           logger.info(message)
         end
-      rescue => e
+      rescue StandardError => e
         logger.error("Failed to migrate: #{e.message}")
       end
     end
diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake
index 95072444fcf..0ad50c0fa53 100644
--- a/lib/tasks/gitlab/usage_data.rake
+++ b/lib/tasks/gitlab/usage_data.rake
@@ -29,5 +29,10 @@ namespace :gitlab do
       items = Gitlab::Usage::MetricDefinition.definitions
       Gitlab::Usage::Docs::Renderer.new(items).write
     end
+
+    desc 'GitLab | UsageDataMetrics | Generate usage ping from metrics definition YAML files in JSON'
+    task generate_from_yaml: :environment do
+      puts Gitlab::Json.pretty_generate(Gitlab::UsageDataMetrics.uncached_data)
+    end
   end
 end
diff --git a/lib/tasks/migrate/migrate_iids.rake b/lib/tasks/migrate/migrate_iids.rake
index e0666a87656..f5db94d5336 100644
--- a/lib/tasks/migrate/migrate_iids.rake
+++ b/lib/tasks/migrate/migrate_iids.rake
@@ -11,7 +11,7 @@ task migrate_iids: :environment do
     else
       print 'F'
     end
-  rescue
+  rescue StandardError
     print 'F'
   end
 
@@ -25,7 +25,7 @@ task migrate_iids: :environment do
     else
       print 'F'
     end
-  rescue
+  rescue StandardError
     print 'F'
   end
 
@@ -39,7 +39,7 @@ task migrate_iids: :environment do
     else
       print 'F'
     end
-  rescue
+  rescue StandardError
     print 'F'
   end
 
diff --git a/lib/tasks/tokens.rake b/lib/tasks/tokens.rake
index b0ff2cce482..74baa11c314 100644
--- a/lib/tasks/tokens.rake
+++ b/lib/tasks/tokens.rake
@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
-require_relative '../../app/models/concerns/token_authenticatable.rb'
-require_relative '../../app/models/concerns/token_authenticatable_strategies/base.rb'
-require_relative '../../app/models/concerns/token_authenticatable_strategies/insecure.rb'
-require_relative '../../app/models/concerns/token_authenticatable_strategies/digest.rb'
+require_relative '../../app/models/concerns/token_authenticatable'
+require_relative '../../app/models/concerns/token_authenticatable_strategies/base'
+require_relative '../../app/models/concerns/token_authenticatable_strategies/insecure'
+require_relative '../../app/models/concerns/token_authenticatable_strategies/digest'
 
 namespace :tokens do
   desc "Reset all GitLab incoming email tokens"