Add latest changes from gitlab-org/gitlab@master

7 files changed, 77 insertions, 38 deletions

diff --git a/lib/api/helpers/packages_manager_clients_helpers.rb b/lib/api/helpers/packages_manager_clients_helpers.rb
index a30396bf07a..e7662b03577 100644
--- a/lib/api/helpers/packages_manager_clients_helpers.rb
+++ b/lib/api/helpers/packages_manager_clients_helpers.rb
@@ -6,16 +6,6 @@ module API
       extend Grape::API::Helpers
       include ::API::Helpers::PackagesHelpers
 
-      params :workhorse_upload_params do
-        optional 'file.path', type: String, desc: 'Path to locally stored body (generated by Workhorse)'
-        optional 'file.name', type: String, desc: 'Real filename as send in Content-Disposition (generated by Workhorse)'
-        optional 'file.type', type: String, desc: 'Real content type as send in Content-Type (generated by Workhorse)'
-        optional 'file.size', type: Integer, desc: 'Real size of file (generated by Workhorse)'
-        optional 'file.md5', type: String, desc: 'MD5 checksum of the file (generated by Workhorse)'
-        optional 'file.sha1', type: String, desc: 'SHA1 checksum of the file (generated by Workhorse)'
-        optional 'file.sha256', type: String, desc: 'SHA256 checksum of the file (generated by Workhorse)'
-      end
-
       def find_job_from_http_basic_auth
         return unless request.headers
 
@@ -36,12 +26,6 @@ module API
         DeployToken.active.find_by_token(token)
       end
 
-      def uploaded_package_file(param_name = :file)
-        uploaded_file = UploadedFile.from_params(params, param_name, ::Packages::PackageFileUploader.workhorse_local_upload_path)
-        bad_request!('Missing package file!') unless uploaded_file
-        uploaded_file
-      end
-
       private
 
       def decode_token
diff --git a/lib/backup/database.rb b/lib/backup/database.rb
index ddddd57bd60..6c0029df27f 100644
--- a/lib/backup/database.rb
+++ b/lib/backup/database.rb
@@ -8,10 +8,10 @@ module Backup
     attr_reader :progress
     attr_reader :config, :db_file_name
 
-    def initialize(progress)
+    def initialize(progress, filename: nil)
       @progress = progress
       @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env]
-      @db_file_name = File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
+      @db_file_name = filename || File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
     end
 
     def dump
@@ -57,26 +57,63 @@ module Backup
       decompress_pid = spawn(*%w(gzip -cd), out: decompress_wr, in: db_file_name)
       decompress_wr.close
 
-      restore_pid =
+      status, errors =
         case config["adapter"]
         when "postgresql" then
           progress.print "Restoring PostgreSQL database #{config['database']} ... "
           pg_env
-          spawn('psql', config['database'], in: decompress_rd)
+          execute_and_track_errors(pg_restore_cmd, decompress_rd)
         end
       decompress_rd.close
 
-      success = [decompress_pid, restore_pid].all? do |pid|
-        Process.waitpid(pid)
-        $?.success?
+      Process.waitpid(decompress_pid)
+      success = $?.success? && status.success?
+
+      if errors.present?
+        progress.print "------ BEGIN ERRORS -----".color(:yellow)
+        progress.print errors.join.color(:yellow)
+        progress.print "------ END ERRORS -------".color(:yellow)
       end
 
       report_success(success)
-      abort 'Restore failed' unless success
+      raise Backup::Error, 'Restore failed' unless success
+
+      errors
     end
 
     protected
 
+    def execute_and_track_errors(cmd, decompress_rd)
+      errors = []
+
+      Open3.popen3(ENV, *cmd) do |stdin, stdout, stderr, thread|
+        stdin.binmode
+
+        Thread.new do
+          data = stdout.read
+          $stdout.write(data)
+        end
+
+        Thread.new do
+          until (raw_line = stderr.gets).nil?
+            warn(raw_line)
+            # Recent database dumps will use --if-exists with pg_dump
+            errors << raw_line unless raw_line =~ /does not exist$/
+          end
+        end
+
+        begin
+          IO.copy_stream(decompress_rd, stdin)
+        rescue Errno::EPIPE
+        end
+
+        stdin.close
+
+        thread.join
+        [thread.value, errors]
+      end
+    end
+
     def pg_env
       args = {
         'username'  => 'PGUSER',
@@ -101,5 +138,11 @@ module Backup
         progress.puts '[FAILED]'.color(:red)
       end
     end
+
+    private
+
+    def pg_restore_cmd
+      ['psql', config['database']]
+    end
   end
 end
diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb
index 9ba5df89a35..878f11eb9ce 100644
--- a/lib/gitlab/danger/helper.rb
+++ b/lib/gitlab/danger/helper.rb
@@ -206,16 +206,6 @@ module Gitlab
         usernames.map { |u| Gitlab::Danger::Teammate.new('username' => u) }
       end
 
-      def missing_database_labels(current_mr_labels)
-        labels = if has_database_scoped_labels?(current_mr_labels)
-                   ['database']
-                 else
-                   ['database', 'database::review pending']
-                 end
-
-        labels - current_mr_labels
-      end
-
       def sanitize_mr_title(title)
         title.gsub(DRAFT_REGEX, '').gsub(/`/, '\\\`')
       end
@@ -259,8 +249,6 @@ module Gitlab
         all_changed_files.grep(regex)
       end
 
-      private
-
       def has_database_scoped_labels?(current_mr_labels)
         current_mr_labels.any? { |label| label.start_with?('database::') }
       end
diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb
index 73187d8dea8..c702c6f1add 100644
--- a/lib/gitlab/task_helpers.rb
+++ b/lib/gitlab/task_helpers.rb
@@ -24,6 +24,8 @@ module Gitlab
     # Returns "yes" the user chose to continue
     # Raises Gitlab::TaskAbortedByUserError if the user chose *not* to continue
     def ask_to_continue
+      return if Gitlab::Utils.to_boolean(ENV['GITLAB_ASSUME_YES'])
+
       answer = prompt("Do you want to continue (yes/no)? ".color(:blue), %w{yes no})
       raise Gitlab::TaskAbortedByUserError unless answer == "yes"
     end
diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb
index 37688d6e0e7..2a0d902295e 100644
--- a/lib/gitlab/tracking.rb
+++ b/lib/gitlab/tracking.rb
@@ -56,12 +56,19 @@ module Gitlab
 
       def snowplow
         @snowplow ||= SnowplowTracker::Tracker.new(
-          SnowplowTracker::AsyncEmitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname, protocol: 'https'),
+          emitter,
           SnowplowTracker::Subject.new,
           SNOWPLOW_NAMESPACE,
           Gitlab::CurrentSettings.snowplow_app_id
         )
       end
+
+      def emitter
+        SnowplowTracker::AsyncEmitter.new(
+          Gitlab::CurrentSettings.snowplow_collector_hostname,
+          protocol: 'https'
+        )
+      end
     end
   end
 end
diff --git a/lib/gitlab_danger.rb b/lib/gitlab_danger.rb
index a98ac9200da..5537bd81f10 100644
--- a/lib/gitlab_danger.rb
+++ b/lib/gitlab_danger.rb
@@ -22,6 +22,7 @@ class GitlabDanger
     roulette
     ce_ee_vue_templates
     sidekiq_queues
+    specialization_labels
   ].freeze
 
   MESSAGE_PREFIX = '==>'.freeze
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index 70d22837335..2a3713ed85c 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -136,7 +136,21 @@ namespace :gitlab do
 
       task restore: :gitlab_environment do
         puts_time "Restoring database ... ".color(:blue)
-        Backup::Database.new(progress).restore
+        errors = Backup::Database.new(progress).restore
+
+        if errors.present?
+          warning = <<~MSG
+            There were errors in restoring the schema. This may cause
+            issues if this results in missing indexes, constraints, or
+            columns. Please record the errors above and contact GitLab
+            Support if you have questions:
+            https://about.gitlab.com/support/
+          MSG
+
+          warn warning.color(:red)
+          ask_to_continue
+        end
+
         puts_time "done".color(:green)
       end
     end