diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-20 12:55:51 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-20 12:55:51 +0300 |
commit | e8d2c2579383897a1dd7f9debd359abe8ae8373d (patch) | |
tree | c42be41678c2586d49a75cabce89322082698334 /lib/backup | |
parent | fc845b37ec3a90aaa719975f607740c22ba6a113 (diff) |
Add latest changes from gitlab-org/gitlab@14-1-stable-eev14.1.0-rc42
Diffstat (limited to 'lib/backup')
-rw-r--r-- | lib/backup/database.rb | 40 | ||||
-rw-r--r-- | lib/backup/gitaly_backup.rb | 18 | ||||
-rw-r--r-- | lib/backup/gitaly_rpc_backup.rb | 4 | ||||
-rw-r--r-- | lib/backup/repositories.rb | 5 |
4 files changed, 44 insertions, 23 deletions
diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 0429d9496d6..f07fd786b4b 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -9,6 +9,8 @@ module Backup attr_reader :config, :db_file_name IGNORED_ERRORS = [ + # Ignore warnings + /WARNING:/, # Ignore the DROP errors; recent database dumps will use --if-exists with pg_dump /does not exist$/, # User may not have permissions to drop extensions or schemas @@ -18,7 +20,7 @@ module Backup def initialize(progress, filename: nil) @progress = progress - @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env] + @config = ActiveRecord::Base.configurations.find_db_config(Rails.env).configuration_hash @db_file_name = filename || File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') end @@ -30,9 +32,9 @@ module Backup compress_rd.close dump_pid = - case config["adapter"] + case config[:adapter] when "postgresql" then - progress.print "Dumping PostgreSQL database #{config['database']} ... " + progress.print "Dumping PostgreSQL database #{database} ... " pg_env pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump. pgsql_args << '--if-exists' @@ -47,7 +49,7 @@ module Backup end end - Process.spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr) + Process.spawn('pg_dump', *pgsql_args, database, out: compress_wr) end compress_wr.close @@ -68,9 +70,9 @@ module Backup decompress_wr.close status, errors = - case config["adapter"] + case config[:adapter] when "postgresql" then - progress.print "Restoring PostgreSQL database #{config['database']} ... " + progress.print "Restoring PostgreSQL database #{database} ... " pg_env execute_and_track_errors(pg_restore_cmd, decompress_rd) end @@ -93,6 +95,10 @@ module Backup protected + def database + @config[:database] + end + def ignore_error?(line) IGNORED_ERRORS_REGEXP.match?(line) end @@ -128,17 +134,17 @@ module Backup def pg_env args = { - 'username' => 'PGUSER', - 'host' => 'PGHOST', - 'port' => 'PGPORT', - 'password' => 'PGPASSWORD', + username: 'PGUSER', + host: 'PGHOST', + port: 'PGPORT', + password: 'PGPASSWORD', # SSL - 'sslmode' => 'PGSSLMODE', - 'sslkey' => 'PGSSLKEY', - 'sslcert' => 'PGSSLCERT', - 'sslrootcert' => 'PGSSLROOTCERT', - 'sslcrl' => 'PGSSLCRL', - 'sslcompression' => 'PGSSLCOMPRESSION' + sslmode: 'PGSSLMODE', + sslkey: 'PGSSLKEY', + sslcert: 'PGSSLCERT', + sslrootcert: 'PGSSLROOTCERT', + sslcrl: 'PGSSLCRL', + sslcompression: 'PGSSLCOMPRESSION' } args.each do |opt, arg| # This enables the use of different PostgreSQL settings in @@ -161,7 +167,7 @@ module Backup private def pg_restore_cmd - ['psql', config['database']] + ['psql', database] end end end diff --git a/lib/backup/gitaly_backup.rb b/lib/backup/gitaly_backup.rb index cfd3d463f9e..c15b0ed6a1b 100644 --- a/lib/backup/gitaly_backup.rb +++ b/lib/backup/gitaly_backup.rb @@ -3,8 +3,10 @@ module Backup # Backup and restores repositories using gitaly-backup class GitalyBackup - def initialize(progress) + def initialize(progress, parallel: nil, parallel_storage: nil) @progress = progress + @parallel = parallel + @parallel_storage = parallel_storage end def start(type) @@ -19,8 +21,12 @@ module Backup raise Error, "unknown backup type: #{type}" end + args = [] + args += ['-parallel', @parallel.to_s] if type == :create && @parallel + args += ['-parallel-storage', @parallel_storage.to_s] if type == :create && @parallel_storage + @read_io, @write_io = IO.pipe - @pid = Process.spawn(bin_path, command, '-path', backup_repos_path, in: @read_io, out: progress) + @pid = Process.spawn(bin_path, command, '-path', backup_repos_path, *args, in: @read_io, out: @progress) end def wait @@ -48,9 +54,11 @@ module Backup }.merge(Gitlab::GitalyClient.connection_data(repository.storage)).to_json) end - private + def parallel_enqueue? + false + end - attr_reader :progress + private def started? @pid.present? @@ -61,7 +69,7 @@ module Backup end def bin_path - File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-backup')) + File.absolute_path(Gitlab.config.backup.gitaly_backup_path) end end end diff --git a/lib/backup/gitaly_rpc_backup.rb b/lib/backup/gitaly_rpc_backup.rb index 53f1de40509..baac4eb26ca 100644 --- a/lib/backup/gitaly_rpc_backup.rb +++ b/lib/backup/gitaly_rpc_backup.rb @@ -44,6 +44,10 @@ module Backup end end + def parallel_enqueue? + true + end + private attr_reader :progress diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index 80d23c1eb7f..0b5a62529b4 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -12,7 +12,10 @@ module Backup def dump(max_concurrency:, max_storage_concurrency:) strategy.start(:create) - if max_concurrency <= 1 && max_storage_concurrency <= 1 + # gitaly-backup is designed to handle concurrency on its own. So we want + # to avoid entering the buggy concurrency code here when gitaly-backup + # is enabled. + if (max_concurrency <= 1 && max_storage_concurrency <= 1) || !strategy.parallel_enqueue? return enqueue_consecutive end |