Add latest changes from gitlab-org/gitlab@14-6-stable-eev14.6.0-rc42

9 files changed, 85 insertions, 47 deletions

diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake
index e03c78d5a40..17f9414ad52 100644
--- a/lib/tasks/gettext.rake
+++ b/lib/tasks/gettext.rake
@@ -58,6 +58,7 @@ namespace :gettext do
   task lint: :environment do
     require 'simple_po_parser'
     require 'gitlab/utils'
+    require 'parallel'
 
     FastGettext.silence_errors
     files = Dir.glob(Rails.root.join('locale/*/gitlab.po'))
@@ -70,7 +71,9 @@ namespace :gettext do
 
     linters.unshift(Gitlab::I18n::PoLinter.new(po_path: pot_file_path))
 
-    failed_linters = linters.select { |linter| linter.errors.any? }
+    failed_linters = Parallel
+      .map(linters, progress: 'Linting po files') { |linter| linter if linter.errors.any? }
+      .compact
 
     if failed_linters.empty?
       puts 'All PO files are valid.'
@@ -129,14 +132,6 @@ namespace :gettext do
     )
   end
 
-  # Disallow HTML from translatable strings
-  # See: https://docs.gitlab.com/ee/development/i18n/externalization.html#html
-  def html_todolist
-    return @html_todolist if defined?(@html_todolist)
-
-    @html_todolist = YAML.safe_load(File.read(Rails.root.join('lib/gitlab/i18n/html_todo.yml')))
-  end
-
   def report_errors_for_file(file, errors_for_file)
     puts "Errors in `#{file}`:"
 
diff --git a/lib/tasks/gitlab/background_migrations.rake b/lib/tasks/gitlab/background_migrations.rake
index c978a2807ca..c7f3d003f9f 100644
--- a/lib/tasks/gitlab/background_migrations.rake
+++ b/lib/tasks/gitlab/background_migrations.rake
@@ -2,6 +2,7 @@
 
 namespace :gitlab do
   namespace :background_migrations do
+    desc 'Synchronously finish executing a batched background migration'
     task :finalize, [:job_class_name, :table_name, :column_name, :job_arguments] => :environment do |_, args|
       [:job_class_name, :table_name, :column_name, :job_arguments].each do |argument|
         unless args[argument]
@@ -19,5 +20,23 @@ namespace :gitlab do
 
       puts "Done.".color(:green)
     end
+
+    desc 'Display the status of batched background migrations'
+    task status: :environment do
+      statuses = Gitlab::Database::BackgroundMigration::BatchedMigration.statuses
+      max_status_length = statuses.keys.map(&:length).max
+      format_string = "%-#{max_status_length}s | %s\n"
+
+      Gitlab::Database::BackgroundMigration::BatchedMigration.find_each(batch_size: 100) do |migration|
+        identification_fields = [
+          migration.job_class_name,
+          migration.table_name,
+          migration.column_name,
+          migration.job_arguments.to_json
+        ].join(',')
+
+        printf(format_string, migration.status, identification_fields)
+      end
+    end
   end
 end
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index 0cd4ab354c9..8f033a41e3d 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -100,13 +100,15 @@ namespace :gitlab do
     namespace :sessions do
       desc "GitLab | Cleanup | Sessions | Clean ActiveSession lookup keys"
       task active_sessions_lookup_keys: :gitlab_environment do
-        session_key_pattern = "#{Gitlab::Redis::SharedState::USER_SESSIONS_LOOKUP_NAMESPACE}:*"
+        use_redis_session_store = Gitlab::Utils.to_boolean(ENV['GITLAB_USE_REDIS_SESSIONS_STORE'], default: true)
+        redis_store_class = use_redis_session_store ? Gitlab::Redis::Sessions : Gitlab::Redis::SharedState
+        session_key_pattern = "#{Gitlab::Redis::Sessions::USER_SESSIONS_LOOKUP_NAMESPACE}:*"
         last_save_check = Time.at(0)
         wait_time = 10.seconds
         cursor = 0
         total_users_scanned = 0
 
-        Gitlab::Redis::SharedState.with do |redis|
+        redis_store_class.with do |redis|
           begin
             cursor, keys = redis.scan(cursor, match: session_key_pattern)
             total_users_scanned += keys.count
@@ -119,27 +121,16 @@ namespace :gitlab do
               last_save_check = Time.now
             end
 
+            user = Struct.new(:id)
+
             keys.each do |key|
               user_id = key.split(':').last
 
-              lookup_key_count = redis.scard(key)
-
-              session_ids = ActiveSession.session_ids_for_user(user_id)
-              entries = ActiveSession.raw_active_session_entries(redis, session_ids, user_id)
-              session_ids_and_entries = session_ids.zip(entries)
-
-              inactive_session_ids = session_ids_and_entries.map do |session_id, session|
-                session_id if session.nil?
-              end.compact
-
-              redis.pipelined do |conn|
-                inactive_session_ids.each do |session_id|
-                  conn.srem(key, session_id)
-                end
-              end
+              removed = []
+              active = ActiveSession.cleaned_up_lookup_entries(redis, user.new(user_id), removed)
 
-              if inactive_session_ids
-                puts "deleted #{inactive_session_ids.count} out of #{lookup_key_count} lookup keys for User ##{user_id}"
+              if removed.any?
+                puts "deleted #{removed.count} out of #{active.count + removed.count} lookup keys for User ##{user_id}"
               end
             end
           end while cursor.to_i != 0
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index e83c4cbdb39..9e733fc3a0f 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml
+
 namespace :gitlab do
   namespace :db do
     desc 'GitLab | DB | Manually insert schema migration version'
@@ -49,7 +51,7 @@ namespace :gitlab do
 
       # Drop all extra schema objects GitLab owns
       Gitlab::Database::EXTRA_SCHEMAS.each do |schema|
-        connection.execute("DROP SCHEMA IF EXISTS #{connection.quote_table_name(schema)}")
+        connection.execute("DROP SCHEMA IF EXISTS #{connection.quote_table_name(schema)} CASCADE")
       end
     end
 
@@ -83,7 +85,7 @@ namespace :gitlab do
     desc 'GitLab | DB | Sets up EE specific database functionality'
 
     if Gitlab.ee?
-      task setup_ee: %w[geo:db:drop geo:db:create geo:db:schema:load geo:db:migrate]
+      task setup_ee: %w[db:drop:geo db:create:geo db:schema:load:geo db:migrate:geo]
     else
       task :setup_ee
     end
@@ -116,6 +118,19 @@ namespace :gitlab do
       Rake::Task['gitlab:db:clean_structure_sql'].invoke
     end
 
+    ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name|
+      # Inform Rake that custom tasks should be run every time rake db:structure:dump is run
+      #
+      # Rails 6.1 deprecates db:structure:dump in favor of db:schema:dump
+      Rake::Task["db:structure:dump:#{name}"].enhance do
+        Rake::Task['gitlab:db:clean_structure_sql'].invoke
+      end
+
+      Rake::Task["db:schema:dump:#{name}"].enhance do
+        Rake::Task['gitlab:db:clean_structure_sql'].invoke
+      end
+    end
+
     desc 'Create missing dynamic database partitions'
     task create_dynamic_partitions: :environment do
       Gitlab::Database::Partitioning.sync_partitions
@@ -160,24 +175,30 @@ namespace :gitlab do
       Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
     end
 
-    desc 'execute reindexing without downtime to eliminate bloat'
+    desc "Reindex database without downtime to eliminate bloat"
     task reindex: :environment do
-      unless Feature.enabled?(:database_reindexing, type: :ops, default_enabled: :yaml)
+      unless Gitlab::Database::Reindexing.enabled?
         puts "This feature (database_reindexing) is currently disabled.".color(:yellow)
         exit
       end
 
-      Gitlab::Database::EachDatabase.each_database_connection do |connection, connection_name|
-        Gitlab::Database::SharedModel.logger = Logger.new($stdout) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false)
+      Gitlab::Database::Reindexing.invoke
+    end
+
+    namespace :reindex do
+      databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml
 
-        # Hack: Before we do actual reindexing work, create async indexes
-        Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops)
+      ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database_name|
+        desc "Reindex #{database_name} database without downtime to eliminate bloat"
+        task database_name => :environment do
+          unless Gitlab::Database::Reindexing.enabled?
+            puts "This feature (database_reindexing) is currently disabled.".color(:yellow)
+            exit
+          end
 
-        Gitlab::Database::Reindexing.automatic_reindexing
+          Gitlab::Database::Reindexing.invoke(database_name)
+        end
       end
-    rescue StandardError => e
-      Gitlab::AppLogger.error(e)
-      raise
     end
 
     desc 'Enqueue an index for reindexing'
@@ -243,7 +264,9 @@ namespace :gitlab do
     # Only for development environments,
     # we execute pending data migrations inline for convenience.
     Rake::Task['db:migrate'].enhance do
-      Rake::Task['gitlab:db:execute_batched_migrations'].invoke if Rails.env.development?
+      if Rails.env.development? && Gitlab::Database::BackgroundMigration::BatchedMigration.table_exists?
+        Rake::Task['gitlab:db:execute_batched_migrations'].invoke
+      end
     end
   end
 end
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index eabbb8652f1..b01a7902bf2 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -37,8 +37,8 @@ namespace :gitlab do
       raise
     end
 
-    desc 'GitLab | Gitaly | Install or upgrade gitaly'
-    task :install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args|
+    desc 'GitLab | Gitaly | Clone and checkout gitaly'
+    task :clone, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args|
       warn_user_is_not_gitlab
 
       unless args.dir.present? && args.storage_path.present?
@@ -51,6 +51,11 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
       version = Gitlab::GitalyClient.expected_server_version
 
       checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir, clone_opts: %w[--depth 1])
+    end
+
+    desc 'GitLab | Gitaly | Install or upgrade gitaly'
+    task :install, [:dir, :storage_path, :repo] => [:gitlab_environment, 'gitlab:gitaly:clone'] do |t, args|
+      warn_user_is_not_gitlab
 
       storage_paths = { 'default' => args.storage_path }
       Gitlab::SetupHelper::Gitaly.create_configuration(args.dir, storage_paths)
diff --git a/lib/tasks/gitlab/seed/group_seed.rake b/lib/tasks/gitlab/seed/group_seed.rake
index bc705c94422..a9a350fb6c3 100644
--- a/lib/tasks/gitlab/seed/group_seed.rake
+++ b/lib/tasks/gitlab/seed/group_seed.rake
@@ -184,7 +184,7 @@ class GroupSeeder
       group = Group.find(group_id)
 
       @resource_count.times do |i|
-        _, project_path = PROJECT_URL.split('/')[-2..-1]
+        _, project_path = PROJECT_URL.split('/')[-2..]
 
         project_path.gsub!('.git', '')
 
diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake
index 2e383065b64..10492e183c5 100644
--- a/lib/tasks/gitlab/sidekiq.rake
+++ b/lib/tasks/gitlab/sidekiq.rake
@@ -100,6 +100,10 @@ namespace :gitlab do
         queues_and_weights = Gitlab::SidekiqConfig.queues_for_sidekiq_queues_yml
 
         write_yaml(Gitlab::SidekiqConfig::SIDEKIQ_QUEUES_PATH, banner, queues: queues_and_weights)
+
+        if Gitlab.jh?
+          write_yaml(Gitlab::SidekiqConfig::JH_SIDEKIQ_QUEUES_PATH, banner, queues: Gitlab::SidekiqConfig.jh_queues_for_sidekiq_queues_yml)
+        end
       end
 
       desc 'GitLab | Sidekiq | Validate that sidekiq_queues.yml matches worker definitions'
@@ -113,6 +117,7 @@ namespace :gitlab do
             Then commit and push the changes from:
 
             - #{Gitlab::SidekiqConfig::SIDEKIQ_QUEUES_PATH}
+            #{"- " + Gitlab::SidekiqConfig::JH_SIDEKIQ_QUEUES_PATH if Gitlab.jh?}
 
           MSG
         end
diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake
index e3a4e7f50b8..247897bed0b 100644
--- a/lib/tasks/gitlab/update_templates.rake
+++ b/lib/tasks/gitlab/update_templates.rake
@@ -32,7 +32,7 @@ namespace :gitlab do
 
     tmp_namespace_path = "tmp-project-import-#{Time.now.to_i}"
     puts "Creating temporary namespace #{tmp_namespace_path}"
-    tmp_namespace = Namespace.create!(owner: admin, name: tmp_namespace_path, path: tmp_namespace_path)
+    tmp_namespace = Namespace.create!(owner: admin, name: tmp_namespace_path, path: tmp_namespace_path, type: Namespaces::UserNamespace.sti_name)
 
     templates = if template_names.empty?
                   Gitlab::ProjectTemplate.all
diff --git a/lib/tasks/gitlab/x509/update.rake b/lib/tasks/gitlab/x509/update.rake
index d3c63fa8514..7b7d15479bf 100644
--- a/lib/tasks/gitlab/x509/update.rake
+++ b/lib/tasks/gitlab/x509/update.rake
@@ -12,14 +12,14 @@ namespace :gitlab do
     def update_certificates
       logger = Logger.new($stdout)
 
-      unless X509CommitSignature.exists?
+      unless CommitSignatures::X509CommitSignature.exists?
         logger.info("Unable to find any x509 commit signatures. Exiting.")
         return
       end
 
       logger.info("Start to update x509 commit signatures")
 
-      X509CommitSignature.find_each do |sig|
+      CommitSignatures::X509CommitSignature.find_each do |sig|
         sig.x509_commit&.update_signature!(sig)
       end