Add latest changes from gitlab-org/gitlab@13-2-stable-ee

5 files changed, 99 insertions, 16 deletions

diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake
index c380eb293b5..6af91d473a6 100644
--- a/lib/tasks/cache.rake
+++ b/lib/tasks/cache.rake
@@ -18,7 +18,9 @@ namespace :cache do
               count: REDIS_CLEAR_BATCH_SIZE
             )
 
-            redis.del(*keys) if keys.any?
+            Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
+              redis.del(*keys) if keys.any?
+            end
 
             break if cursor == REDIS_SCAN_START_STOP
           end
diff --git a/lib/tasks/gitlab/container_registry.rake b/lib/tasks/gitlab/container_registry.rake
index 7687cb237cc..cd18c873a5a 100644
--- a/lib/tasks/gitlab/container_registry.rake
+++ b/lib/tasks/gitlab/container_registry.rake
@@ -15,21 +15,7 @@ namespace :gitlab do
 
       warn_user_is_not_gitlab
 
-      url = registry_config.api_url
-      # registry_info will query the /v2 route of the registry API. This route
-      # requires authentication, but not authorization (the response has no body,
-      # only headers that show the version of the registry). There is no
-      # associated user when running this rake, so we need to generate a valid
-      # JWT token with no access permissions to authenticate as a trusted client.
-      token = Auth::ContainerRegistryAuthenticationService.access_token([], [])
-      client = ContainerRegistry::Client.new(url, token: token)
-      info = client.registry_info
-
-      Gitlab::CurrentSettings.update!(
-        container_registry_vendor: info[:vendor] || '',
-        container_registry_version: info[:version] || '',
-        container_registry_features: info[:features] || []
-      )
+      UpdateContainerRegistryInfoService.new.execute
     end
   end
 end
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 4917d496d07..61318570fd5 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -39,6 +39,11 @@ namespace :gitlab do
       # PG: http://www.postgresql.org/docs/current/static/ddl-depend.html
       # Add `IF EXISTS` because cascade could have already deleted a table.
       tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") }
+
+      # Drop all extra schema objects GitLab owns
+      Gitlab::Database::EXTRA_SCHEMAS.each do |schema|
+        connection.execute("DROP SCHEMA IF EXISTS #{connection.quote_table_name(schema)}")
+      end
     end
 
     desc 'GitLab | DB | Configures the database by running migrate, or by loading the schema and seeding if needed'
@@ -129,5 +134,37 @@ namespace :gitlab do
     Rake::Task['db:structure:load'].enhance do
       Rake::Task['gitlab:db:load_custom_structure'].invoke
     end
+
+    desc 'Create missing dynamic database partitions'
+    task :create_dynamic_partitions do
+      Gitlab::Database::Partitioning::PartitionCreator.new.create_partitions
+    end
+
+    # This is targeted towards deploys and upgrades of GitLab.
+    # Since we're running migrations already at this time,
+    # we also check and create partitions as needed here.
+    Rake::Task['db:migrate'].enhance do
+      Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
+    end
+
+    # When we load the database schema from db/structure.sql
+    # we don't have any dynamic partitions created. We don't really need to
+    # because application initializers/sidekiq take care of that, too.
+    # However, the presence of partitions for a table has influence on their
+    # position in db/structure.sql (which is topologically sorted).
+    #
+    # Other than that it's helpful to create partitions early when bootstrapping
+    # a new installation.
+    Rake::Task['db:structure:load'].enhance do
+      Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
+    end
+
+    # During testing, db:test:load restores the database schema from scratch
+    # which does not include dynamic partitions. We cannot rely on application
+    # initializers here as the application can continue to run while
+    # a rake task reloads the database schema.
+    Rake::Task['db:test:load'].enhance do
+      Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
+    end
   end
 end
diff --git a/lib/tasks/gitlab/external_diffs.rake b/lib/tasks/gitlab/external_diffs.rake
new file mode 100644
index 00000000000..08f25914007
--- /dev/null
+++ b/lib/tasks/gitlab/external_diffs.rake
@@ -0,0 +1,35 @@
+namespace :gitlab do
+  namespace :external_diffs do
+    desc "Override external diffs in file storage to be in object storage instead. This does not change the actual location of the data"
+    task force_object_storage: :environment do |t, args|
+      ansi = Gitlab::Utils.to_boolean(ENV.fetch('ANSI', true))
+      batch = ENV.fetch('BATCH_SIZE', 1000)
+      start_id = ENV.fetch('START_ID', nil)
+      end_id = ENV.fetch('END_ID', nil)
+      update_delay = args.fetch('UPDATE_DELAY', 1)
+
+      # Use ANSI codes to overwrite the same line repeatedly if supported
+      newline = ansi ? "\x1B8\x1B[2K" : "\n"
+
+      total = 0
+
+      # The only useful index on the table is by id, so scan through the whole
+      # table by that and filter out those we don't want on each relation
+      MergeRequestDiff.in_batches(of: batch, start: start_id, finish: end_id) do |relation| # rubocop:disable Cop/InBatches
+        count = relation
+          .except(:order)
+          .where(stored_externally: true, external_diff_store: ExternalDiffUploader::Store::LOCAL)
+          .update_all(external_diff_store: ExternalDiffUploader::Store::REMOTE)
+
+        total += count
+
+        if count > 0
+          print "#{newline}#{total} updated..."
+          sleep(update_delay) if update_delay > 0
+        end
+      end
+
+      puts "done!"
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/packages/migrate.rake b/lib/tasks/gitlab/packages/migrate.rake
new file mode 100644
index 00000000000..cd6dcf78da3
--- /dev/null
+++ b/lib/tasks/gitlab/packages/migrate.rake
@@ -0,0 +1,23 @@
+require 'logger'
+
+desc "GitLab | Packages | Migrate packages files to remote storage"
+namespace :gitlab do
+  namespace :packages do
+    task migrate: :environment do
+      logger = Logger.new(STDOUT)
+      logger.info('Starting transfer of package files to object storage')
+
+      unless ::Packages::PackageFileUploader.object_store_enabled?
+        raise 'Object store is disabled for packages feature'
+      end
+
+      ::Packages::PackageFile.with_files_stored_locally.find_each(batch_size: 10) do |package_file|
+        package_file.file.migrate!(::Packages::PackageFileUploader::Store::REMOTE)
+
+        logger.info("Transferred package file #{package_file.id} of size #{package_file.size.to_i.bytes} to object storage")
+      rescue => e
+        logger.error("Failed to transfer package file #{package_file.id} with error: #{e.message}")
+      end
+    end
+  end
+end