diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-09-03 21:08:29 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-09-03 21:08:29 +0300 |
commit | 592223823c8ebf6e32d98e4b12620ba8ff043cca (patch) | |
tree | 1c665b754b1487df374d9cb28cfc61ff40a6b317 /lib | |
parent | 0e13b2c71563d578805fc01fda9a4361f9f9d053 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/helpers/packages_manager_clients_helpers.rb | 16 | ||||
-rw-r--r-- | lib/backup/database.rb | 59 | ||||
-rw-r--r-- | lib/gitlab/danger/helper.rb | 12 | ||||
-rw-r--r-- | lib/gitlab/task_helpers.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/tracking.rb | 9 | ||||
-rw-r--r-- | lib/gitlab_danger.rb | 1 | ||||
-rw-r--r-- | lib/tasks/gitlab/backup.rake | 16 |
7 files changed, 77 insertions, 38 deletions
diff --git a/lib/api/helpers/packages_manager_clients_helpers.rb b/lib/api/helpers/packages_manager_clients_helpers.rb index a30396bf07a..e7662b03577 100644 --- a/lib/api/helpers/packages_manager_clients_helpers.rb +++ b/lib/api/helpers/packages_manager_clients_helpers.rb @@ -6,16 +6,6 @@ module API extend Grape::API::Helpers include ::API::Helpers::PackagesHelpers - params :workhorse_upload_params do - optional 'file.path', type: String, desc: 'Path to locally stored body (generated by Workhorse)' - optional 'file.name', type: String, desc: 'Real filename as send in Content-Disposition (generated by Workhorse)' - optional 'file.type', type: String, desc: 'Real content type as send in Content-Type (generated by Workhorse)' - optional 'file.size', type: Integer, desc: 'Real size of file (generated by Workhorse)' - optional 'file.md5', type: String, desc: 'MD5 checksum of the file (generated by Workhorse)' - optional 'file.sha1', type: String, desc: 'SHA1 checksum of the file (generated by Workhorse)' - optional 'file.sha256', type: String, desc: 'SHA256 checksum of the file (generated by Workhorse)' - end - def find_job_from_http_basic_auth return unless request.headers @@ -36,12 +26,6 @@ module API DeployToken.active.find_by_token(token) end - def uploaded_package_file(param_name = :file) - uploaded_file = UploadedFile.from_params(params, param_name, ::Packages::PackageFileUploader.workhorse_local_upload_path) - bad_request!('Missing package file!') unless uploaded_file - uploaded_file - end - private def decode_token diff --git a/lib/backup/database.rb b/lib/backup/database.rb index ddddd57bd60..6c0029df27f 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -8,10 +8,10 @@ module Backup attr_reader :progress attr_reader :config, :db_file_name - def initialize(progress) + def initialize(progress, filename: nil) @progress = progress @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env] - @db_file_name = File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') + @db_file_name = filename || File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') end def dump @@ -57,26 +57,63 @@ module Backup decompress_pid = spawn(*%w(gzip -cd), out: decompress_wr, in: db_file_name) decompress_wr.close - restore_pid = + status, errors = case config["adapter"] when "postgresql" then progress.print "Restoring PostgreSQL database #{config['database']} ... " pg_env - spawn('psql', config['database'], in: decompress_rd) + execute_and_track_errors(pg_restore_cmd, decompress_rd) end decompress_rd.close - success = [decompress_pid, restore_pid].all? do |pid| - Process.waitpid(pid) - $?.success? + Process.waitpid(decompress_pid) + success = $?.success? && status.success? + + if errors.present? + progress.print "------ BEGIN ERRORS -----".color(:yellow) + progress.print errors.join.color(:yellow) + progress.print "------ END ERRORS -------".color(:yellow) end report_success(success) - abort 'Restore failed' unless success + raise Backup::Error, 'Restore failed' unless success + + errors end protected + def execute_and_track_errors(cmd, decompress_rd) + errors = [] + + Open3.popen3(ENV, *cmd) do |stdin, stdout, stderr, thread| + stdin.binmode + + Thread.new do + data = stdout.read + $stdout.write(data) + end + + Thread.new do + until (raw_line = stderr.gets).nil? + warn(raw_line) + # Recent database dumps will use --if-exists with pg_dump + errors << raw_line unless raw_line =~ /does not exist$/ + end + end + + begin + IO.copy_stream(decompress_rd, stdin) + rescue Errno::EPIPE + end + + stdin.close + + thread.join + [thread.value, errors] + end + end + def pg_env args = { 'username' => 'PGUSER', @@ -101,5 +138,11 @@ module Backup progress.puts '[FAILED]'.color(:red) end end + + private + + def pg_restore_cmd + ['psql', config['database']] + end end end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 9ba5df89a35..878f11eb9ce 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -206,16 +206,6 @@ module Gitlab usernames.map { |u| Gitlab::Danger::Teammate.new('username' => u) } end - def missing_database_labels(current_mr_labels) - labels = if has_database_scoped_labels?(current_mr_labels) - ['database'] - else - ['database', 'database::review pending'] - end - - labels - current_mr_labels - end - def sanitize_mr_title(title) title.gsub(DRAFT_REGEX, '').gsub(/`/, '\\\`') end @@ -259,8 +249,6 @@ module Gitlab all_changed_files.grep(regex) end - private - def has_database_scoped_labels?(current_mr_labels) current_mr_labels.any? { |label| label.start_with?('database::') } end diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 73187d8dea8..c702c6f1add 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -24,6 +24,8 @@ module Gitlab # Returns "yes" the user chose to continue # Raises Gitlab::TaskAbortedByUserError if the user chose *not* to continue def ask_to_continue + return if Gitlab::Utils.to_boolean(ENV['GITLAB_ASSUME_YES']) + answer = prompt("Do you want to continue (yes/no)? ".color(:blue), %w{yes no}) raise Gitlab::TaskAbortedByUserError unless answer == "yes" end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 37688d6e0e7..2a0d902295e 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -56,12 +56,19 @@ module Gitlab def snowplow @snowplow ||= SnowplowTracker::Tracker.new( - SnowplowTracker::AsyncEmitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname, protocol: 'https'), + emitter, SnowplowTracker::Subject.new, SNOWPLOW_NAMESPACE, Gitlab::CurrentSettings.snowplow_app_id ) end + + def emitter + SnowplowTracker::AsyncEmitter.new( + Gitlab::CurrentSettings.snowplow_collector_hostname, + protocol: 'https' + ) + end end end end diff --git a/lib/gitlab_danger.rb b/lib/gitlab_danger.rb index a98ac9200da..5537bd81f10 100644 --- a/lib/gitlab_danger.rb +++ b/lib/gitlab_danger.rb @@ -22,6 +22,7 @@ class GitlabDanger roulette ce_ee_vue_templates sidekiq_queues + specialization_labels ].freeze MESSAGE_PREFIX = '==>'.freeze diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 70d22837335..2a3713ed85c 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -136,7 +136,21 @@ namespace :gitlab do task restore: :gitlab_environment do puts_time "Restoring database ... ".color(:blue) - Backup::Database.new(progress).restore + errors = Backup::Database.new(progress).restore + + if errors.present? + warning = <<~MSG + There were errors in restoring the schema. This may cause + issues if this results in missing indexes, constraints, or + columns. Please record the errors above and contact GitLab + Support if you have questions: + https://about.gitlab.com/support/ + MSG + + warn warning.color(:red) + ask_to_continue + end + puts_time "done".color(:green) end end |