Add latest changes from gitlab-org/gitlab@13-4-stable-ee

8 files changed, 62 insertions, 57 deletions

diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake
index 31030d061f2..1e28d15f75e 100644
--- a/lib/tasks/gettext.rake
+++ b/lib/tasks/gettext.rake
@@ -38,7 +38,9 @@ namespace :gettext do
     Rake::Task['gettext:find'].invoke
 
     # leave only the required changes.
-    `git checkout -- locale/*/gitlab.po`
+    unless system(*%w(git checkout -- locale/*/gitlab.po))
+      raise 'failed to cleanup generated locale/*/gitlab.po files'
+    end
 
     # Remove timestamps from the pot file
     pot_content = File.read pot_file
diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake
index 69a3526b872..caa583fb3a9 100644
--- a/lib/tasks/gitlab/assets.rake
+++ b/lib/tasks/gitlab/assets.rake
@@ -55,6 +55,7 @@ namespace :gitlab do
         rake:assets:precompile
         gitlab:assets:compile_webpack_if_needed
         gitlab:assets:fix_urls
+        gitlab:assets:check_page_bundle_mixins_css_for_sideeffects
       ].each(&::Gitlab::TaskHelpers.method(:invoke_and_time_task))
     end
 
@@ -127,5 +128,10 @@ namespace :gitlab do
         abort 'Error: Unable to compile webpack DLL.'.color(:red)
       end
     end
+
+    desc 'GitLab | Assets | Check that scss mixins do not introduce any sideffects'
+    task :check_page_bundle_mixins_css_for_sideeffects do
+      system('./scripts/frontend/check_page_bundle_mixins_css_for_sideeffects.js')
+    end
   end
 end
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index b0f1ca39387..2a3713ed85c 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -47,6 +47,11 @@ namespace :gitlab do
         begin
           unless ENV['force'] == 'yes'
             warning = <<-MSG.strip_heredoc
+              Be sure to stop Puma, Sidekiq, and any other process that
+              connects to the database before proceeding. For Omnibus
+              installs, see the following link for more information:
+              https://docs.gitlab.com/ee/raketasks/backup_restore.html#restore-for-omnibus-gitlab-installations
+
               Before restoring the database, we will remove all existing
               tables to avoid future upgrade problems. Be aware that if you have
               custom tables in the GitLab database these tables and all data will be
@@ -131,7 +136,21 @@ namespace :gitlab do
 
       task restore: :gitlab_environment do
         puts_time "Restoring database ... ".color(:blue)
-        Backup::Database.new(progress).restore
+        errors = Backup::Database.new(progress).restore
+
+        if errors.present?
+          warning = <<~MSG
+            There were errors in restoring the schema. This may cause
+            issues if this results in missing indexes, constraints, or
+            columns. Please record the errors above and contact GitLab
+            Support if you have questions:
+            https://about.gitlab.com/support/
+          MSG
+
+          warn warning.color(:red)
+          ask_to_continue
+        end
+
         puts_time "done".color(:green)
       end
     end
@@ -273,5 +292,7 @@ namespace :gitlab do
         $stdout
       end
     end
-  end # namespace end: backup
-end # namespace end: gitlab
+  end
+  # namespace end: backup
+end
+# namespace end: gitlab
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 61318570fd5..425f66918b0 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -166,5 +166,12 @@ namespace :gitlab do
     Rake::Task['db:test:load'].enhance do
       Rake::Task['gitlab:db:create_dynamic_partitions'].invoke
     end
+
+    desc 'reindex a regular (non-unique) index without downtime to eliminate bloat'
+    task :reindex, [:index_name] => :environment do |_, args|
+      raise ArgumentError, 'must give the index name to reindex' unless args[:index_name]
+
+      Gitlab::Database::ConcurrentReindex.new(args[:index_name], logger: Logger.new(STDOUT)).execute
+    end
   end
 end
diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake
index 470a12c39cd..3d4c847a0f0 100644
--- a/lib/tasks/gitlab/lfs/migrate.rake
+++ b/lib/tasks/gitlab/lfs/migrate.rake
@@ -9,12 +9,12 @@ namespace :gitlab do
 
       LfsObject.with_files_stored_locally
         .find_each(batch_size: 10) do |lfs_object|
-          lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE)
+        lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE)
 
-          logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage")
-        rescue => e
-          logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}")
-        end
+        logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage")
+      rescue => e
+        logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}")
+      end
     end
 
     task migrate_to_local: :environment do
diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake
index eb3de195626..6f00db42d78 100644
--- a/lib/tasks/gitlab/sidekiq.rake
+++ b/lib/tasks/gitlab/sidekiq.rake
@@ -5,7 +5,7 @@ return if Rails.env.production?
 namespace :gitlab do
   namespace :sidekiq do
     def write_yaml(path, banner, object)
-      File.write(path, banner + YAML.dump(object))
+      File.write(path, banner + YAML.dump(object).gsub(/ *$/m, ''))
     end
 
     namespace :all_queues_yml do
diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake
new file mode 100644
index 00000000000..6f3db91c2b0
--- /dev/null
+++ b/lib/tasks/gitlab/usage_data.rake
@@ -0,0 +1,13 @@
+namespace :gitlab do
+  namespace :usage_data do
+    desc 'GitLab | UsageData | Generate raw SQLs for usage ping in YAML'
+    task dump_sql_in_yaml: :environment do
+      puts Gitlab::UsageDataQueries.uncached_data.to_yaml
+    end
+
+    desc 'GitLab | UsageData | Generate raw SQLs for usage ping in JSON'
+    task dump_sql_in_json: :environment do
+      puts Gitlab::Json.pretty_generate(Gitlab::UsageDataQueries.uncached_data)
+    end
+  end
+end
diff --git a/lib/tasks/pngquant.rake b/lib/tasks/pngquant.rake
index ceb4de55373..63bc1c7c16e 100644
--- a/lib/tasks/pngquant.rake
+++ b/lib/tasks/pngquant.rake
@@ -2,10 +2,10 @@ return if Rails.env.production?
 
 require 'png_quantizator'
 require 'parallel'
+require_relative '../../tooling/lib/tooling/images'
 
 # The amount of variance (in bytes) allowed in
 # file size when testing for compression size
-TOLERANCE = 10000
 
 namespace :pngquant do
   # Returns an array of all images eligible for compression
@@ -13,55 +13,13 @@ namespace :pngquant do
     Dir.glob('doc/**/*.png', File::FNM_CASEFOLD)
   end
 
-  # Runs pngquant on an image and optionally
-  # writes the result to disk
-  def compress_image(file, overwrite_original)
-    compressed_file = "#{file}.compressed"
-    FileUtils.copy(file, compressed_file)
-
-    pngquant_file = PngQuantizator::Image.new(compressed_file)
-
-    # Run the image repeatedly through pngquant until
-    # the change in file size is within TOLERANCE
-    loop do
-      before = File.size(compressed_file)
-      pngquant_file.quantize!
-      after = File.size(compressed_file)
-      break if before - after <= TOLERANCE
-    end
-
-    savings = File.size(file) - File.size(compressed_file)
-    is_uncompressed = savings > TOLERANCE
-
-    if is_uncompressed && overwrite_original
-      FileUtils.copy(compressed_file, file)
-    end
-
-    FileUtils.remove(compressed_file)
-
-    [is_uncompressed, savings]
-  end
-
-  # Ensures pngquant is available and prints an error if not
-  def check_executable
-    unless system('pngquant --version', out: File::NULL)
-      warn(
-        'Error: pngquant executable was not detected in the system.'.color(:red),
-        'Download pngquant at https://pngquant.org/ and place the executable in /usr/local/bin'.color(:green)
-      )
-      abort
-    end
-  end
-
   desc 'GitLab | Pngquant | Compress all documentation PNG images using pngquant'
   task :compress do
-    check_executable
-
     files = doc_images
     puts "Compressing #{files.size} PNG files in doc/**"
 
     Parallel.each(files) do |file|
-      was_uncompressed, savings = compress_image(file, true)
+      was_uncompressed, savings = Tooling::Image.compress_image(file)
 
       if was_uncompressed
         puts "#{file} was reduced by #{savings} bytes"
@@ -71,13 +29,11 @@ namespace :pngquant do
 
   desc 'GitLab | Pngquant | Checks that all documentation PNG images have been compressed with pngquant'
   task :lint do
-    check_executable
-
     files = doc_images
     puts "Checking #{files.size} PNG files in doc/**"
 
     uncompressed_files = Parallel.map(files) do |file|
-      is_uncompressed, _ = compress_image(file, false)
+      is_uncompressed, _ = Tooling::Image.compress_image(file, true)
       if is_uncompressed
         puts "Uncompressed file detected: ".color(:red) + file
         file