diff options
Diffstat (limited to 'lib/backup/database.rb')
| -rw-r--r-- | lib/backup/database.rb | 197 |
1 files changed, 117 insertions, 80 deletions
diff --git a/lib/backup/database.rb b/lib/backup/database.rb index cf19b4fa8ff..bd3832c7327 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -6,7 +6,7 @@ module Backup class Database < Task extend ::Gitlab::Utils::Override include Backup::Helper - attr_reader :force, :config + attr_reader :force IGNORED_ERRORS = [ # Ignore warnings @@ -18,98 +18,105 @@ module Backup ].freeze IGNORED_ERRORS_REGEXP = Regexp.union(IGNORED_ERRORS).freeze - def initialize(database_name, progress, force:) + def initialize(progress, force:) super(progress) - @database_name = database_name - @config = base_model.connection_db_config.configuration_hash @force = force end override :dump - def dump(db_file_name, backup_id) - FileUtils.mkdir_p(File.dirname(db_file_name)) - FileUtils.rm_f(db_file_name) - compress_rd, compress_wr = IO.pipe - compress_pid = spawn(gzip_cmd, in: compress_rd, out: [db_file_name, 'w', 0600]) - compress_rd.close - - dump_pid = - case config[:adapter] - when "postgresql" then - progress.print "Dumping PostgreSQL database #{database} ... " - pg_env - pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump. - pgsql_args << '--if-exists' - - if Gitlab.config.backup.pg_schema - pgsql_args << '-n' - pgsql_args << Gitlab.config.backup.pg_schema + def dump(destination_dir, backup_id) + FileUtils.mkdir_p(destination_dir) - Gitlab::Database::EXTRA_SCHEMAS.each do |schema| - pgsql_args << '-n' - pgsql_args << schema.to_s - end - end + snapshot_ids.each do |database_name, snapshot_id| + base_model = base_models_for_backup[database_name] + + config = base_model.connection_db_config.configuration_hash + + db_file_name = file_name(destination_dir, database_name) + FileUtils.rm_f(db_file_name) + + pg_database = config[:database] - Process.spawn('pg_dump', *pgsql_args, database, out: compress_wr) + progress.print "Dumping PostgreSQL database #{pg_database} ... " + pg_env(config) + pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump. + pgsql_args << '--if-exists' + pgsql_args << "--snapshot=#{snapshot_ids[database_name]}" + + if Gitlab.config.backup.pg_schema + pgsql_args << '-n' + pgsql_args << Gitlab.config.backup.pg_schema + + Gitlab::Database::EXTRA_SCHEMAS.each do |schema| + pgsql_args << '-n' + pgsql_args << schema.to_s + end end - compress_wr.close - success = [compress_pid, dump_pid].all? do |pid| - Process.waitpid(pid) - $?.success? - end + success = Backup::Dump::Postgres.new.dump(pg_database, db_file_name, pgsql_args) + + base_model.connection.rollback_transaction - report_success(success) - progress.flush + raise DatabaseBackupError.new(config, db_file_name) unless success - raise DatabaseBackupError.new(config, db_file_name) unless success + report_success(success) + progress.flush + end + ensure + base_models_for_backup.each do |_database_name, base_model| + Gitlab::Database::TransactionTimeoutSettings.new(base_model.connection).restore_timeouts + end end override :restore - def restore(db_file_name) - unless File.exist?(db_file_name) - raise(Backup::Error, "Source database file does not exist #{db_file_name}") if main_database? + def restore(destination_dir) + base_models_for_backup.each do |database_name, base_model| + config = base_model.connection_db_config.configuration_hash - progress.puts "Source backup for the database #{@database_name} doesn't exist. Skipping the task" - return - end + db_file_name = file_name(destination_dir, database_name) + database = config[:database] - unless force - progress.puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) - sleep(5) - end + unless File.exist?(db_file_name) + raise(Backup::Error, "Source database file does not exist #{db_file_name}") if main_database?(database_name) - # Drop all tables Load the schema to ensure we don't have any newer tables - # hanging out from a failed upgrade - puts_time 'Cleaning the database ... '.color(:blue) - Rake::Task['gitlab:db:drop_tables'].invoke - puts_time 'done'.color(:green) - - decompress_rd, decompress_wr = IO.pipe - decompress_pid = spawn(*%w(gzip -cd), out: decompress_wr, in: db_file_name) - decompress_wr.close - - status, @errors = - case config[:adapter] - when "postgresql" then - progress.print "Restoring PostgreSQL database #{database} ... " - pg_env - execute_and_track_errors(pg_restore_cmd, decompress_rd) + progress.puts "Source backup for the database #{database_name} doesn't exist. Skipping the task" + return false end - decompress_rd.close - Process.waitpid(decompress_pid) - success = $?.success? && status.success? + unless force + progress.puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) + sleep(5) + end - if @errors.present? - progress.print "------ BEGIN ERRORS -----\n".color(:yellow) - progress.print @errors.join.color(:yellow) - progress.print "------ END ERRORS -------\n".color(:yellow) - end + # Drop all tables Load the schema to ensure we don't have any newer tables + # hanging out from a failed upgrade + drop_tables(database_name) - report_success(success) - raise Backup::Error, 'Restore failed' unless success + decompress_rd, decompress_wr = IO.pipe + decompress_pid = spawn(*%w(gzip -cd), out: decompress_wr, in: db_file_name) + decompress_wr.close + + status, @errors = + case config[:adapter] + when "postgresql" then + progress.print "Restoring PostgreSQL database #{database} ... " + pg_env(config) + execute_and_track_errors(pg_restore_cmd(database), decompress_rd) + end + decompress_rd.close + + Process.waitpid(decompress_pid) + success = $?.success? && status.success? + + if @errors.present? + progress.print "------ BEGIN ERRORS -----\n".color(:yellow) + progress.print @errors.join.color(:yellow) + progress.print "------ END ERRORS -------\n".color(:yellow) + end + + report_success(success) + raise Backup::Error, 'Restore failed' unless success + end end override :pre_restore_warning @@ -144,16 +151,22 @@ module Backup protected - def database - @config[:database] + def base_models_for_backup + @base_models_for_backup ||= Gitlab::Database.database_base_models_with_gitlab_shared end - def base_model - Gitlab::Database.database_base_models[@database_name] + def main_database?(database_name) + database_name.to_sym == :main end - def main_database? - @database_name == :main + def file_name(base_dir, database_name) + prefix = if database_name.to_sym != :main + "#{database_name}_" + else + '' + end + + File.join(base_dir, "#{prefix}database.sql.gz") end def ignore_error?(line) @@ -189,7 +202,7 @@ module Backup end end - def pg_env + def pg_env(config) args = { username: 'PGUSER', host: 'PGHOST', @@ -223,8 +236,32 @@ module Backup private - def pg_restore_cmd + def drop_tables(database_name) + if Rake::Task.task_defined? "gitlab:db:drop_tables:#{database_name}" + puts_time 'Cleaning the database ... '.color(:blue) + Rake::Task["gitlab:db:drop_tables:#{database_name}"].invoke + puts_time 'done'.color(:green) + elsif Gitlab::Database.database_base_models.one? + # In single database, we do not have rake tasks per database + puts_time 'Cleaning the database ... '.color(:blue) + Rake::Task["gitlab:db:drop_tables"].invoke + puts_time 'done'.color(:green) + end + end + + def pg_restore_cmd(database) ['psql', database] end + + def snapshot_ids + @snapshot_ids ||= base_models_for_backup.each_with_object({}) do |(database_name, base_model), snapshot_ids| + Gitlab::Database::TransactionTimeoutSettings.new(base_model.connection).disable_timeouts + + base_model.connection.begin_transaction(isolation: :repeatable_read) + + snapshot_ids[database_name] = + base_model.connection.execute("SELECT pg_export_snapshot() as snapshot_id;").first['snapshot_id'] + end + end end end |