Add latest changes from gitlab-org/gitlab@13-4-stable-ee

7 files changed, 120 insertions, 25 deletions

diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb
index 33658ae225f..c2266f0bad6 100644
--- a/lib/backup/artifacts.rb
+++ b/lib/backup/artifacts.rb
@@ -9,7 +9,7 @@ module Backup
     def initialize(progress)
       @progress = progress
 
-      super('artifacts', JobArtifactUploader.root)
+      super('artifacts', JobArtifactUploader.root, excludes: ['tmp'])
     end
   end
 end
diff --git a/lib/backup/database.rb b/lib/backup/database.rb
index d4c1ce260e4..851445f703d 100644
--- a/lib/backup/database.rb
+++ b/lib/backup/database.rb
@@ -8,10 +8,18 @@ module Backup
     attr_reader :progress
     attr_reader :config, :db_file_name
 
-    def initialize(progress)
+    IGNORED_ERRORS = [
+      # Ignore the DROP errors; recent database dumps will use --if-exists with pg_dump
+      /does not exist$/,
+      # User may not have permissions to drop extensions or schemas
+      /must be owner of/
+    ].freeze
+    IGNORED_ERRORS_REGEXP = Regexp.union(IGNORED_ERRORS).freeze
+
+    def initialize(progress, filename: nil)
       @progress = progress
       @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env]
-      @db_file_name = File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
+      @db_file_name = filename || File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
     end
 
     def dump
@@ -27,6 +35,7 @@ module Backup
           progress.print "Dumping PostgreSQL database #{config['database']} ... "
           pg_env
           pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump.
+          pgsql_args << '--if-exists'
 
           if Gitlab.config.backup.pg_schema
             pgsql_args << '-n'
@@ -48,6 +57,8 @@ module Backup
       end
 
       report_success(success)
+      progress.flush
+
       raise Backup::Error, 'Backup failed' unless success
     end
 
@@ -56,26 +67,65 @@ module Backup
       decompress_pid = spawn(*%w(gzip -cd), out: decompress_wr, in: db_file_name)
       decompress_wr.close
 
-      restore_pid =
+      status, errors =
         case config["adapter"]
         when "postgresql" then
           progress.print "Restoring PostgreSQL database #{config['database']} ... "
           pg_env
-          spawn('psql', config['database'], in: decompress_rd)
+          execute_and_track_errors(pg_restore_cmd, decompress_rd)
         end
       decompress_rd.close
 
-      success = [decompress_pid, restore_pid].all? do |pid|
-        Process.waitpid(pid)
-        $?.success?
+      Process.waitpid(decompress_pid)
+      success = $?.success? && status.success?
+
+      if errors.present?
+        progress.print "------ BEGIN ERRORS -----\n".color(:yellow)
+        progress.print errors.join.color(:yellow)
+        progress.print "------ END ERRORS -------\n".color(:yellow)
       end
 
       report_success(success)
-      abort Backup::Error, 'Restore failed' unless success
+      raise Backup::Error, 'Restore failed' unless success
+
+      errors
     end
 
     protected
 
+    def ignore_error?(line)
+      IGNORED_ERRORS_REGEXP.match?(line)
+    end
+
+    def execute_and_track_errors(cmd, decompress_rd)
+      errors = []
+
+      Open3.popen3(ENV, *cmd) do |stdin, stdout, stderr, thread|
+        stdin.binmode
+
+        out_reader = Thread.new do
+          data = stdout.read
+          $stdout.write(data)
+        end
+
+        err_reader = Thread.new do
+          until (raw_line = stderr.gets).nil?
+            warn(raw_line)
+            errors << raw_line unless ignore_error?(raw_line)
+          end
+        end
+
+        begin
+          IO.copy_stream(decompress_rd, stdin)
+        rescue Errno::EPIPE
+        end
+
+        stdin.close
+        [thread, out_reader, err_reader].each(&:join)
+        [thread.value, errors]
+      end
+    end
+
     def pg_env
       args = {
         'username'  => 'PGUSER',
@@ -100,5 +150,11 @@ module Backup
         progress.puts '[FAILED]'.color(:red)
       end
     end
+
+    private
+
+    def pg_restore_cmd
+      ['psql', config['database']]
+    end
   end
 end
diff --git a/lib/backup/files.rb b/lib/backup/files.rb
index dae9056a47b..619a62fd6f6 100644
--- a/lib/backup/files.rb
+++ b/lib/backup/files.rb
@@ -7,14 +7,17 @@ module Backup
   class Files
     include Backup::Helper
 
-    attr_reader :name, :app_files_dir, :backup_tarball, :files_parent_dir
+    DEFAULT_EXCLUDE = 'lost+found'
 
-    def initialize(name, app_files_dir)
+    attr_reader :name, :app_files_dir, :backup_tarball, :excludes, :files_parent_dir
+
+    def initialize(name, app_files_dir, excludes: [])
       @name = name
       @app_files_dir = File.realpath(app_files_dir)
       @files_parent_dir = File.realpath(File.join(@app_files_dir, '..'))
       @backup_files_dir = File.join(Gitlab.config.backup.path, File.basename(@app_files_dir) )
       @backup_tarball = File.join(Gitlab.config.backup.path, name + '.tar.gz')
+      @excludes = [DEFAULT_EXCLUDE].concat(excludes)
     end
 
     # Copy files from public/files to backup/files
@@ -23,7 +26,7 @@ module Backup
       FileUtils.rm_f(backup_tarball)
 
       if ENV['STRATEGY'] == 'copy'
-        cmd = %W(rsync -a --exclude=lost+found #{app_files_dir} #{Gitlab.config.backup.path})
+        cmd = [%w(rsync -a), exclude_dirs(:rsync), %W(#{app_files_dir} #{Gitlab.config.backup.path})].flatten
         output, status = Gitlab::Popen.popen(cmd)
 
         unless status == 0
@@ -31,10 +34,12 @@ module Backup
           raise Backup::Error, 'Backup failed'
         end
 
-        run_pipeline!([%W(#{tar} --exclude=lost+found -C #{@backup_files_dir} -cf - .), gzip_cmd], out: [backup_tarball, 'w', 0600])
+        tar_cmd = [tar, exclude_dirs(:tar), %W(-C #{@backup_files_dir} -cf - .)].flatten
+        run_pipeline!([tar_cmd, gzip_cmd], out: [backup_tarball, 'w', 0600])
         FileUtils.rm_rf(@backup_files_dir)
       else
-        run_pipeline!([%W(#{tar} --exclude=lost+found -C #{app_files_dir} -cf - .), gzip_cmd], out: [backup_tarball, 'w', 0600])
+        tar_cmd = [tar, exclude_dirs(:tar), %W(-C #{app_files_dir} -cf - .)].flatten
+        run_pipeline!([tar_cmd, gzip_cmd], out: [backup_tarball, 'w', 0600])
       end
     end
 
@@ -81,5 +86,17 @@ module Backup
       error = err_r.read
       raise Backup::Error, "Backup failed. #{error}" unless error =~ regex
     end
+
+    def exclude_dirs(fmt)
+      excludes.map do |s|
+        if s == DEFAULT_EXCLUDE
+          '--exclude=' + s
+        elsif fmt == :rsync
+          '--exclude=/' + s
+        elsif fmt == :tar
+          '--exclude=./' + s
+        end
+      end
+    end
   end
 end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index 915567f8106..2b28b30fd74 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -47,7 +47,7 @@ module Backup
         return
       end
 
-      directory = connect_to_remote_directory(connection_settings)
+      directory = connect_to_remote_directory(Gitlab.config.backup.upload)
 
       if directory.files.create(create_attributes)
         progress.puts "done".color(:green)
@@ -88,7 +88,7 @@ module Backup
             # - 1495527097_2017_05_23_9.3.0-pre_gitlab_backup.tar
             next unless file =~ /^(\d{10})(?:_\d{4}_\d{2}_\d{2}(_\d+\.\d+\.\d+((-|\.)(pre|rc\d))?(-ee)?)?)?_gitlab_backup\.tar$/
 
-            timestamp = $1.to_i
+            timestamp = Regexp.last_match(1).to_i
 
             if Time.at(timestamp) < (Time.now - keep_time)
               begin
@@ -195,9 +195,11 @@ module Backup
       @backup_file_list.map {|item| item.gsub("#{FILE_NAME_SUFFIX}", "")}
     end
 
-    def connect_to_remote_directory(connection_settings)
-      # our settings use string keys, but Fog expects symbols
-      connection = ::Fog::Storage.new(connection_settings.symbolize_keys)
+    def connect_to_remote_directory(options)
+      config = ObjectStorage::Config.new(options)
+      config.load_provider
+
+      connection = ::Fog::Storage.new(config.credentials)
 
       # We only attempt to create the directory for local backups. For AWS
       # and other cloud providers, we cannot guarantee the user will have
diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb
index a4be728df08..d7aab33d7cb 100644
--- a/lib/backup/pages.rb
+++ b/lib/backup/pages.rb
@@ -9,7 +9,7 @@ module Backup
     def initialize(progress)
       @progress = progress
 
-      super('pages', Gitlab.config.pages.path)
+      super('pages', Gitlab.config.pages.path, excludes: [::Projects::UpdatePagesService::TMP_EXTRACT_PATH])
     end
   end
 end
diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index 51fac9e8706..eb0b230904e 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -148,20 +148,22 @@ module Backup
     private
 
     def dump_consecutive
-      Project.find_each(batch_size: 1000) do |project|
+      Project.includes(:route, :group, namespace: :owner).find_each(batch_size: 1000) do |project|
         dump_project(project)
       end
     end
 
     def dump_storage(storage, semaphore, max_storage_concurrency:)
       errors = Queue.new
-      queue = SizedQueue.new(1)
+      queue = InterlockSizedQueue.new(1)
 
       threads = Array.new(max_storage_concurrency) do
         Thread.new do
           Rails.application.executor.wrap do
             while project = queue.pop
-              semaphore.acquire
+              ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+                semaphore.acquire
+              end
 
               begin
                 dump_project(project)
@@ -176,7 +178,7 @@ module Backup
         end
       end
 
-      Project.for_repository_storage(storage).find_each(batch_size: 100) do |project|
+      Project.for_repository_storage(storage).includes(:route, :group, namespace: :owner).find_each(batch_size: 100) do |project|
         break unless errors.empty?
 
         queue.push(project)
@@ -241,5 +243,23 @@ module Backup
         pool.schedule
       end
     end
+
+    class InterlockSizedQueue < SizedQueue
+      extend ::Gitlab::Utils::Override
+
+      override :pop
+      def pop(*)
+        ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+          super
+        end
+      end
+
+      override :push
+      def push(*)
+        ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+          super
+        end
+      end
+    end
   end
 end
diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb
index 5a20b6ae0a6..b6a62bc3f29 100644
--- a/lib/backup/uploads.rb
+++ b/lib/backup/uploads.rb
@@ -9,7 +9,7 @@ module Backup
     def initialize(progress)
       @progress = progress
 
-      super('uploads', File.join(Gitlab.config.uploads.storage_path, "uploads"))
+      super('uploads', File.join(Gitlab.config.uploads.storage_path, "uploads"), excludes: ['tmp'])
     end
   end
 end