diff options
Diffstat (limited to 'lib/tasks/gitlab/backup.rake')
| -rw-r--r-- | lib/tasks/gitlab/backup.rake | 284 |
1 files changed, 22 insertions, 262 deletions
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 0bca63a64f5..78cb7d72d4f 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -9,26 +9,7 @@ namespace :gitlab do task create: :gitlab_environment do warn_user_is_not_gitlab - %w(db repo uploads builds artifacts pages lfs terraform_state registry packages).each do |type| - Rake::Task["gitlab:backup:#{type}:create"].invoke - end - - backup = Backup::Manager.new(progress) - backup.write_info - - if ENV['SKIP'] && ENV['SKIP'].include?('tar') - backup.upload - else - backup.pack - backup.upload - backup.cleanup - backup.remove_old - end - - progress.puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ - "and are not included in this backup. You will need these files to restore a backup.\n" \ - "Please back them up manually.".color(:red) - progress.puts "Backup task is done." + Backup::Manager.new(progress).create end # Restore backup of GitLab system @@ -36,320 +17,109 @@ namespace :gitlab do task restore: :gitlab_environment do warn_user_is_not_gitlab - backup = Backup::Manager.new(progress) - cleanup_required = backup.unpack - backup.verify_backup_version - - unless backup.skipped?('db') - begin - unless ENV['force'] == 'yes' - warning = <<-MSG.strip_heredoc - Be sure to stop Puma, Sidekiq, and any other process that - connects to the database before proceeding. For Omnibus - installs, see the following link for more information: - https://docs.gitlab.com/ee/raketasks/backup_restore.html#restore-for-omnibus-gitlab-installations - - Before restoring the database, we will remove all existing - tables to avoid future upgrade problems. Be aware that if you have - custom tables in the GitLab database these tables and all data will be - removed. - MSG - puts warning.color(:red) - ask_to_continue - puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) - sleep(5) - end - - # Drop all tables Load the schema to ensure we don't have any newer tables - # hanging out from a failed upgrade - puts_time 'Cleaning the database ... '.color(:blue) - Rake::Task['gitlab:db:drop_tables'].invoke - puts_time 'done'.color(:green) - Rake::Task['gitlab:backup:db:restore'].invoke - rescue Gitlab::TaskAbortedByUserError - puts "Quitting...".color(:red) - exit 1 - end - end - - Rake::Task['gitlab:backup:repo:restore'].invoke unless backup.skipped?('repositories') - Rake::Task['gitlab:backup:uploads:restore'].invoke unless backup.skipped?('uploads') - Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds') - Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts') - Rake::Task['gitlab:backup:pages:restore'].invoke unless backup.skipped?('pages') - Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') - Rake::Task['gitlab:backup:terraform_state:restore'].invoke unless backup.skipped?('terraform_state') - Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry') - Rake::Task['gitlab:backup:packages:restore'].invoke unless backup.skipped?('packages') - Rake::Task['gitlab:shell:setup'].invoke - Rake::Task['cache:clear'].invoke - - if cleanup_required - backup.cleanup - end - - backup.remove_tmp - - puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ - "and are not included in this backup. You will need to restore these files manually.".color(:red) - puts "Restore task is done." + Backup::Manager.new(progress).restore end namespace :repo do task create: :gitlab_environment do - puts_time "Dumping repositories ...".color(:blue) - - max_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_CONCURRENCY', 1).to_i - max_storage_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY', 1).to_i - - if ENV["SKIP"] && ENV["SKIP"].include?("repositories") - puts_time "[SKIPPED]".color(:cyan) - elsif max_concurrency < 1 || max_storage_concurrency < 1 - puts "GITLAB_BACKUP_MAX_CONCURRENCY and GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY must have a value of at least 1".color(:red) - exit 1 - else - Backup::Repositories.new(progress, strategy: repository_backup_strategy).dump( - max_concurrency: max_concurrency, - max_storage_concurrency: max_storage_concurrency - ) - puts_time "done".color(:green) - end + Backup::Manager.new(progress).run_create_task('repositories') end task restore: :gitlab_environment do - puts_time "Restoring repositories ...".color(:blue) - Backup::Repositories.new(progress, strategy: repository_backup_strategy).restore - puts_time "done".color(:green) + Backup::Manager.new(progress).run_restore_task('repositories') end end namespace :db do task create: :gitlab_environment do - puts_time "Dumping database ... ".color(:blue) - - if ENV["SKIP"] && ENV["SKIP"].include?("db") - puts_time "[SKIPPED]".color(:cyan) - else - begin - Backup::Database.new(progress).dump - puts_time "done".color(:green) - rescue Backup::DatabaseBackupError => e - progress.puts "#{e.message}" - end - end + Backup::Manager.new(progress).run_create_task('db') end task restore: :gitlab_environment do - puts_time "Restoring database ... ".color(:blue) - errors = Backup::Database.new(progress).restore - - if errors.present? - warning = <<~MSG - There were errors in restoring the schema. This may cause - issues if this results in missing indexes, constraints, or - columns. Please record the errors above and contact GitLab - Support if you have questions: - https://about.gitlab.com/support/ - MSG - - warn warning.color(:red) - ask_to_continue - end - - puts_time "done".color(:green) + Backup::Manager.new(progress).run_restore_task('db') end end namespace :builds do task create: :gitlab_environment do - puts_time "Dumping builds ... ".color(:blue) - - if ENV["SKIP"] && ENV["SKIP"].include?("builds") - puts_time "[SKIPPED]".color(:cyan) - else - begin - Backup::Builds.new(progress).dump - puts_time "done".color(:green) - rescue Backup::FileBackupError => e - progress.puts "#{e.message}" - end - end + Backup::Manager.new(progress).run_create_task('builds') end task restore: :gitlab_environment do - puts_time "Restoring builds ... ".color(:blue) - Backup::Builds.new(progress).restore - puts_time "done".color(:green) + Backup::Manager.new(progress).run_restore_task('builds') end end namespace :uploads do task create: :gitlab_environment do - puts_time "Dumping uploads ... ".color(:blue) - - if ENV["SKIP"] && ENV["SKIP"].include?("uploads") - puts_time "[SKIPPED]".color(:cyan) - else - begin - Backup::Uploads.new(progress).dump - puts_time "done".color(:green) - rescue Backup::FileBackupError => e - progress.puts "#{e.message}" - end - end + Backup::Manager.new(progress).run_create_task('uploads') end task restore: :gitlab_environment do - puts_time "Restoring uploads ... ".color(:blue) - Backup::Uploads.new(progress).restore - puts_time "done".color(:green) + Backup::Manager.new(progress).run_restore_task('uploads') end end namespace :artifacts do task create: :gitlab_environment do - puts_time "Dumping artifacts ... ".color(:blue) - - if ENV["SKIP"] && ENV["SKIP"].include?("artifacts") - puts_time "[SKIPPED]".color(:cyan) - else - begin - Backup::Artifacts.new(progress).dump - puts_time "done".color(:green) - rescue Backup::FileBackupError => e - progress.puts "#{e.message}" - end - end + Backup::Manager.new(progress).run_create_task('artifacts') end task restore: :gitlab_environment do - puts_time "Restoring artifacts ... ".color(:blue) - Backup::Artifacts.new(progress).restore - puts_time "done".color(:green) + Backup::Manager.new(progress).run_restore_task('artifacts') end end namespace :pages do task create: :gitlab_environment do - puts_time "Dumping pages ... ".color(:blue) - - if ENV["SKIP"] && ENV["SKIP"].include?("pages") - puts_time "[SKIPPED]".color(:cyan) - else - begin - Backup::Pages.new(progress).dump - puts_time "done".color(:green) - rescue Backup::FileBackupError => e - progress.puts "#{e.message}" - end - end + Backup::Manager.new(progress).run_create_task('pages') end task restore: :gitlab_environment do - puts_time "Restoring pages ... ".color(:blue) - Backup::Pages.new(progress).restore - puts_time "done".color(:green) + Backup::Manager.new(progress).run_restore_task('pages') end end namespace :lfs do task create: :gitlab_environment do - puts_time "Dumping lfs objects ... ".color(:blue) - - if ENV["SKIP"] && ENV["SKIP"].include?("lfs") - puts_time "[SKIPPED]".color(:cyan) - else - begin - Backup::Lfs.new(progress).dump - puts_time "done".color(:green) - rescue Backup::FileBackupError => e - progress.puts "#{e.message}" - end - end + Backup::Manager.new(progress).run_create_task('lfs') end task restore: :gitlab_environment do - puts_time "Restoring lfs objects ... ".color(:blue) - Backup::Lfs.new(progress).restore - puts_time "done".color(:green) + Backup::Manager.new(progress).run_restore_task('lfs') end end namespace :terraform_state do task create: :gitlab_environment do - puts_time "Dumping terraform states ... ".color(:blue) - - if ENV["SKIP"] && ENV["SKIP"].include?("terraform_state") - puts_time "[SKIPPED]".color(:cyan) - else - Backup::TerraformState.new(progress).dump - puts_time "done".color(:green) - end + Backup::Manager.new(progress).run_create_task('terraform_state') end task restore: :gitlab_environment do - puts_time "Restoring terraform states ... ".color(:blue) - Backup::TerraformState.new(progress).restore - puts_time "done".color(:green) + Backup::Manager.new(progress).run_restore_task('terraform_state') end end namespace :registry do task create: :gitlab_environment do - puts_time "Dumping container registry images ... ".color(:blue) - - if Gitlab.config.registry.enabled - if ENV["SKIP"] && ENV["SKIP"].include?("registry") - puts_time "[SKIPPED]".color(:cyan) - else - begin - Backup::Registry.new(progress).dump - puts_time "done".color(:green) - rescue Backup::FileBackupError => e - progress.puts "#{e.message}" - end - end - else - puts_time "[DISABLED]".color(:cyan) - end + Backup::Manager.new(progress).run_create_task('registry') end task restore: :gitlab_environment do - puts_time "Restoring container registry images ... ".color(:blue) - - if Gitlab.config.registry.enabled - Backup::Registry.new(progress).restore - puts_time "done".color(:green) - else - puts_time "[DISABLED]".color(:cyan) - end + Backup::Manager.new(progress).run_restore_task('registry') end end namespace :packages do task create: :gitlab_environment do - puts_time "Dumping packages ... ".color(:blue) - - if ENV['SKIP'] && ENV['SKIP'].include?('packages') - puts_time "[SKIPPED]".color(:cyan) - else - Backup::Packages.new(progress).dump - puts_time "done".color(:green) - end + Backup::Manager.new(progress).run_create_task('packages') end task restore: :gitlab_environment do - puts_time "Restoring packages ...".color(:blue) - Backup::Packages.new(progress).restore - puts_time "done".color(:green) + Backup::Manager.new(progress).run_restore_task('packages') end end - def puts_time(msg) - progress.puts "#{Time.now} -- #{msg}" - Gitlab::BackupLogger.info(message: "#{Rainbow.uncolor(msg)}") - end - def progress if ENV['CRON'] # We need an object we can say 'puts' and 'print' to; let's use a @@ -360,16 +130,6 @@ namespace :gitlab do $stdout end end - - def repository_backup_strategy - if Feature.enabled?(:gitaly_backup, default_enabled: :yaml) - max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence - max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence - Backup::GitalyBackup.new(progress, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency) - else - Backup::GitalyRpcBackup.new(progress) - end - end end # namespace end: backup end |