diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-04-20 13:00:54 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-04-20 13:00:54 +0300 |
commit | 3cccd102ba543e02725d247893729e5c73b38295 (patch) | |
tree | f36a04ec38517f5deaaacb5acc7d949688d1e187 /lib/backup | |
parent | 205943281328046ef7b4528031b90fbda70c75ac (diff) |
Add latest changes from gitlab-org/gitlab@14-10-stable-eev14.10.0-rc42
Diffstat (limited to 'lib/backup')
-rw-r--r-- | lib/backup/artifacts.rb | 14 | ||||
-rw-r--r-- | lib/backup/builds.rb | 14 | ||||
-rw-r--r-- | lib/backup/database.rb | 7 | ||||
-rw-r--r-- | lib/backup/files.rb | 15 | ||||
-rw-r--r-- | lib/backup/gitaly_backup.rb | 12 | ||||
-rw-r--r-- | lib/backup/gitaly_rpc_backup.rb | 129 | ||||
-rw-r--r-- | lib/backup/lfs.rb | 14 | ||||
-rw-r--r-- | lib/backup/manager.rb | 339 | ||||
-rw-r--r-- | lib/backup/packages.rb | 14 | ||||
-rw-r--r-- | lib/backup/pages.rb | 18 | ||||
-rw-r--r-- | lib/backup/registry.rb | 19 | ||||
-rw-r--r-- | lib/backup/repositories.rb | 144 | ||||
-rw-r--r-- | lib/backup/task.rb | 15 | ||||
-rw-r--r-- | lib/backup/terraform_state.rb | 14 | ||||
-rw-r--r-- | lib/backup/uploads.rb | 14 |
15 files changed, 217 insertions, 565 deletions
diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb deleted file mode 100644 index 4ef76b0aaf3..00000000000 --- a/lib/backup/artifacts.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Backup - class Artifacts < Backup::Files - def initialize(progress) - super(progress, 'artifacts', JobArtifactUploader.root, excludes: ['tmp']) - end - - override :human_name - def human_name - _('artifacts') - end - end -end diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb deleted file mode 100644 index fbf932e3f6b..00000000000 --- a/lib/backup/builds.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Backup - class Builds < Backup::Files - def initialize(progress) - super(progress, 'builds', Settings.gitlab_ci.builds_path) - end - - override :human_name - def human_name - _('builds') - end - end -end diff --git a/lib/backup/database.rb b/lib/backup/database.rb index afc84a4b913..3cbe3cf7d88 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -25,7 +25,7 @@ module Backup end override :dump - def dump(db_file_name) + def dump(db_file_name, backup_id) FileUtils.mkdir_p(File.dirname(db_file_name)) FileUtils.rm_f(db_file_name) compress_rd, compress_wr = IO.pipe @@ -134,11 +134,6 @@ module Backup MSG end - override :human_name - def human_name - _('database') - end - protected def database diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 7fa07e40cee..55b10c008fb 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -9,19 +9,18 @@ module Backup DEFAULT_EXCLUDE = 'lost+found' - attr_reader :name, :excludes + attr_reader :excludes - def initialize(progress, name, app_files_dir, excludes: []) + def initialize(progress, app_files_dir, excludes: []) super(progress) - @name = name @app_files_dir = app_files_dir @excludes = [DEFAULT_EXCLUDE].concat(excludes) end # Copy files from public/files to backup/files override :dump - def dump(backup_tarball) + def dump(backup_tarball, backup_id) FileUtils.mkdir_p(Gitlab.config.backup.path) FileUtils.rm_f(backup_tarball) @@ -55,7 +54,7 @@ module Backup override :restore def restore(backup_tarball) - backup_existing_files_dir + backup_existing_files_dir(backup_tarball) cmd_list = [%w[gzip -cd], %W[#{tar} --unlink-first --recursive-unlink -C #{app_files_realpath} -xf -]] status_list, output = run_pipeline!(cmd_list, in: backup_tarball) @@ -73,11 +72,13 @@ module Backup end end - def backup_existing_files_dir + def backup_existing_files_dir(backup_tarball) + name = File.basename(backup_tarball, '.tar.gz') + timestamped_files_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}.#{Time.now.to_i}") if File.exist?(app_files_realpath) # Move all files in the existing repos directory except . and .. to - # repositories.old.<timestamp> directory + # repositories.<timestamp> directory FileUtils.mkdir_p(timestamped_files_path, mode: 0700) files = Dir.glob(File.join(app_files_realpath, "*"), File::FNM_DOTMATCH) - [File.join(app_files_realpath, "."), File.join(app_files_realpath, "..")] begin diff --git a/lib/backup/gitaly_backup.rb b/lib/backup/gitaly_backup.rb index b688ff7f13b..93342e789e9 100644 --- a/lib/backup/gitaly_backup.rb +++ b/lib/backup/gitaly_backup.rb @@ -9,16 +9,14 @@ module Backup # @param [StringIO] progress IO interface to output progress # @param [Integer] max_parallelism max parallelism when running backups # @param [Integer] storage_parallelism max parallelism per storage (is affected by max_parallelism) - # @param [String] backup_id unique identifier for the backup def initialize(progress, max_parallelism: nil, storage_parallelism: nil, incremental: false, backup_id: nil) @progress = progress @max_parallelism = max_parallelism @storage_parallelism = storage_parallelism @incremental = incremental - @backup_id = backup_id end - def start(type, backup_repos_path) + def start(type, backup_repos_path, backup_id: nil) raise Error, 'already started' if started? command = case type @@ -37,7 +35,7 @@ module Backup args += ['-layout', 'pointer'] if type == :create args += ['-incremental'] if @incremental - args += ['-id', @backup_id] if @backup_id + args += ['-id', backup_id] if backup_id end end @@ -68,10 +66,6 @@ module Backup schedule_backup_job(repository, always_create: repo_type.project?) end - def parallel_enqueue? - false - end - private # Schedule a new backup job through a non-blocking JSON based pipe protocol @@ -104,6 +98,8 @@ module Backup end def bin_path + raise Error, 'gitaly-backup binary not found and gitaly_backup_path is not configured' unless Gitlab.config.backup.gitaly_backup_path.present? + File.absolute_path(Gitlab.config.backup.gitaly_backup_path) end end diff --git a/lib/backup/gitaly_rpc_backup.rb b/lib/backup/gitaly_rpc_backup.rb deleted file mode 100644 index 89ed27cfa13..00000000000 --- a/lib/backup/gitaly_rpc_backup.rb +++ /dev/null @@ -1,129 +0,0 @@ -# frozen_string_literal: true - -module Backup - # Backup and restores repositories using the gitaly RPC - class GitalyRpcBackup - def initialize(progress) - @progress = progress - end - - def start(type, backup_repos_path) - raise Error, 'already started' if @type - - @type = type - @backup_repos_path = backup_repos_path - case type - when :create - FileUtils.rm_rf(backup_repos_path) - FileUtils.mkdir_p(Gitlab.config.backup.path) - FileUtils.mkdir(backup_repos_path, mode: 0700) - when :restore - # no op - else - raise Error, "unknown backup type: #{type}" - end - end - - def finish! - @type = nil - end - - def enqueue(container, repository_type) - backup_restore = BackupRestore.new( - progress, - repository_type.repository_for(container), - @backup_repos_path - ) - - case @type - when :create - backup_restore.backup - when :restore - backup_restore.restore(always_create: repository_type.project?) - else - raise Error, 'not started' - end - end - - def parallel_enqueue? - true - end - - private - - attr_reader :progress - - class BackupRestore - attr_accessor :progress, :repository, :backup_repos_path - - def initialize(progress, repository, backup_repos_path) - @progress = progress - @repository = repository - @backup_repos_path = backup_repos_path - end - - def backup - progress.puts " * #{display_repo_path} ... " - - if repository.empty? - progress.puts " * #{display_repo_path} ... " + "[EMPTY] [SKIPPED]".color(:cyan) - return - end - - FileUtils.mkdir_p(repository_backup_path) - - repository.bundle_to_disk(path_to_bundle) - repository.gitaly_repository_client.backup_custom_hooks(custom_hooks_tar) - - progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) - - rescue StandardError => e - progress.puts "[Failed] backing up #{display_repo_path}".color(:red) - progress.puts "Error #{e}".color(:red) - end - - def restore(always_create: false) - progress.puts " * #{display_repo_path} ... " - - repository.remove rescue nil - - if File.exist?(path_to_bundle) - repository.create_from_bundle(path_to_bundle) - restore_custom_hooks - elsif always_create - repository.create_repository - end - - progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) - - rescue StandardError => e - progress.puts "[Failed] restoring #{display_repo_path}".color(:red) - progress.puts "Error #{e}".color(:red) - end - - private - - def display_repo_path - "#{repository.full_path} (#{repository.disk_path})" - end - - def repository_backup_path - @repository_backup_path ||= File.join(backup_repos_path, repository.disk_path) - end - - def path_to_bundle - @path_to_bundle ||= File.join(backup_repos_path, repository.disk_path + '.bundle') - end - - def restore_custom_hooks - return unless File.exist?(custom_hooks_tar) - - repository.gitaly_repository_client.restore_custom_hooks(custom_hooks_tar) - end - - def custom_hooks_tar - File.join(repository_backup_path, "custom_hooks.tar") - end - end - end -end diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb deleted file mode 100644 index e92f235a2d7..00000000000 --- a/lib/backup/lfs.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Backup - class Lfs < Backup::Files - def initialize(progress) - super(progress, 'lfs', Settings.lfs.storage_path) - end - - override :human_name - def human_name - _('lfs objects') - end - end -end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index cb5fd959bc9..403b2d9f16c 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -5,74 +5,43 @@ module Backup FILE_NAME_SUFFIX = '_gitlab_backup.tar' MANIFEST_NAME = 'backup_information.yml' + # pages used to deploy tmp files to this path + # if some of these files are still there, we don't need them in the backup + LEGACY_PAGES_TMP_PATH = '@pages.tmp' + TaskDefinition = Struct.new( + :enabled, # `true` if the task can be used. Treated as `true` when not specified. + :human_name, # Name of the task used for logging. :destination_path, # Where the task should put its backup file/dir. :destination_optional, # `true` if the destination might not exist on a successful backup. :cleanup_path, # Path to remove after a successful backup. Uses `destination_path` when not specified. :task, keyword_init: true - ) + ) do + def enabled? + enabled.nil? || enabled + end + end attr_reader :progress def initialize(progress, definitions: nil) @progress = progress - max_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_CONCURRENCY', 1).to_i - max_storage_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY', 1).to_i - force = ENV['force'] == 'yes' - incremental = Gitlab::Utils.to_boolean(ENV['INCREMENTAL'], default: false) + @incremental = Feature.feature_flags_available? && + Feature.enabled?(:incremental_repository_backup, default_enabled: :yaml) && + Gitlab::Utils.to_boolean(ENV['INCREMENTAL'], default: false) - @definitions = definitions || { - 'db' => TaskDefinition.new( - destination_path: 'db/database.sql.gz', - cleanup_path: 'db', - task: Database.new(progress, force: force) - ), - 'repositories' => TaskDefinition.new( - destination_path: 'repositories', - destination_optional: true, - task: Repositories.new(progress, - strategy: repository_backup_strategy(incremental), - max_concurrency: max_concurrency, - max_storage_concurrency: max_storage_concurrency) - ), - 'uploads' => TaskDefinition.new( - destination_path: 'uploads.tar.gz', - task: Uploads.new(progress) - ), - 'builds' => TaskDefinition.new( - destination_path: 'builds.tar.gz', - task: Builds.new(progress) - ), - 'artifacts' => TaskDefinition.new( - destination_path: 'artifacts.tar.gz', - task: Artifacts.new(progress) - ), - 'pages' => TaskDefinition.new( - destination_path: 'pages.tar.gz', - task: Pages.new(progress) - ), - 'lfs' => TaskDefinition.new( - destination_path: 'lfs.tar.gz', - task: Lfs.new(progress) - ), - 'terraform_state' => TaskDefinition.new( - destination_path: 'terraform_state.tar.gz', - task: TerraformState.new(progress) - ), - 'registry' => TaskDefinition.new( - destination_path: 'registry.tar.gz', - task: Registry.new(progress) - ), - 'packages' => TaskDefinition.new( - destination_path: 'packages.tar.gz', - task: Packages.new(progress) - ) - }.freeze + @definitions = definitions || build_definitions end def create + if incremental? + unpack + read_backup_information + verify_backup_version + end + @definitions.keys.each do |task_name| run_create_task(task_name) end @@ -88,34 +57,33 @@ module Backup remove_old end - progress.puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ + puts_time "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ "and are not included in this backup. You will need these files to restore a backup.\n" \ "Please back them up manually.".color(:red) - progress.puts "Backup task is done." + puts_time "Backup #{backup_id} is done." end def run_create_task(task_name) definition = @definitions[task_name] build_backup_information - puts_time "Dumping #{definition.task.human_name} ... ".color(:blue) - unless definition.task.enabled - puts_time "[DISABLED]".color(:cyan) + unless definition.enabled? + puts_time "Dumping #{definition.human_name} ... ".color(:blue) + "[DISABLED]".color(:cyan) return end if skipped?(task_name) - puts_time "[SKIPPED]".color(:cyan) + puts_time "Dumping #{definition.human_name} ... ".color(:blue) + "[SKIPPED]".color(:cyan) return end - definition.task.dump(File.join(Gitlab.config.backup.path, definition.destination_path)) - - puts_time "done".color(:green) + puts_time "Dumping #{definition.human_name} ... ".color(:blue) + definition.task.dump(File.join(Gitlab.config.backup.path, definition.destination_path), backup_id) + puts_time "Dumping #{definition.human_name} ... ".color(:blue) + "done".color(:green) rescue Backup::DatabaseBackupError, Backup::FileBackupError => e - progress.puts "#{e.message}" + puts_time "Dumping #{definition.human_name} failed: #{e.message}".color(:red) end def restore @@ -136,21 +104,21 @@ module Backup remove_tmp - puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ - "and are not included in this backup. You will need to restore these files manually.".color(:red) - puts "Restore task is done." + puts_time "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ + "and are not included in this backup. You will need to restore these files manually.".color(:red) + puts_time "Restore task is done." end def run_restore_task(task_name) definition = @definitions[task_name] - puts_time "Restoring #{definition.task.human_name} ... ".color(:blue) - - unless definition.task.enabled - puts_time "[DISABLED]".color(:cyan) + unless definition.enabled? + puts_time "Restoring #{definition.human_name} ... ".color(:blue) + "[DISABLED]".color(:cyan) return end + puts_time "Restoring #{definition.human_name} ... ".color(:blue) + warning = definition.task.pre_restore_warning if warning.present? puts_time warning.color(:red) @@ -159,7 +127,7 @@ module Backup definition.task.restore(File.join(Gitlab.config.backup.path, definition.destination_path)) - puts_time "done".color(:green) + puts_time "Restoring #{definition.human_name} ... ".color(:blue) + "done".color(:green) warning = definition.task.post_restore_warning if warning.present? @@ -174,6 +142,86 @@ module Backup private + def build_definitions + { + 'db' => TaskDefinition.new( + human_name: _('database'), + destination_path: 'db/database.sql.gz', + cleanup_path: 'db', + task: build_db_task + ), + 'repositories' => TaskDefinition.new( + human_name: _('repositories'), + destination_path: 'repositories', + destination_optional: true, + task: build_repositories_task + ), + 'uploads' => TaskDefinition.new( + human_name: _('uploads'), + destination_path: 'uploads.tar.gz', + task: build_files_task(File.join(Gitlab.config.uploads.storage_path, 'uploads'), excludes: ['tmp']) + ), + 'builds' => TaskDefinition.new( + human_name: _('builds'), + destination_path: 'builds.tar.gz', + task: build_files_task(Settings.gitlab_ci.builds_path) + ), + 'artifacts' => TaskDefinition.new( + human_name: _('artifacts'), + destination_path: 'artifacts.tar.gz', + task: build_files_task(JobArtifactUploader.root, excludes: ['tmp']) + ), + 'pages' => TaskDefinition.new( + human_name: _('pages'), + destination_path: 'pages.tar.gz', + task: build_files_task(Gitlab.config.pages.path, excludes: [LEGACY_PAGES_TMP_PATH]) + ), + 'lfs' => TaskDefinition.new( + human_name: _('lfs objects'), + destination_path: 'lfs.tar.gz', + task: build_files_task(Settings.lfs.storage_path) + ), + 'terraform_state' => TaskDefinition.new( + human_name: _('terraform states'), + destination_path: 'terraform_state.tar.gz', + task: build_files_task(Settings.terraform_state.storage_path, excludes: ['tmp']) + ), + 'registry' => TaskDefinition.new( + enabled: Gitlab.config.registry.enabled, + human_name: _('container registry images'), + destination_path: 'registry.tar.gz', + task: build_files_task(Settings.registry.path) + ), + 'packages' => TaskDefinition.new( + human_name: _('packages'), + destination_path: 'packages.tar.gz', + task: build_files_task(Settings.packages.storage_path, excludes: ['tmp']) + ) + }.freeze + end + + def build_db_task + force = Gitlab::Utils.to_boolean(ENV['force'], default: false) + + Database.new(progress, force: force) + end + + def build_repositories_task + max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence + max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence + strategy = Backup::GitalyBackup.new(progress, incremental: incremental?, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency) + + Repositories.new(progress, strategy: strategy) + end + + def build_files_task(app_files_dir, excludes: []) + Files.new(progress, app_files_dir, excludes: excludes) + end + + def incremental? + @incremental + end + def read_backup_information @backup_information ||= YAML.load_file(File.join(backup_path, MANIFEST_NAME)) end @@ -209,103 +257,104 @@ module Backup def pack Dir.chdir(backup_path) do # create archive - progress.print "Creating backup archive: #{tar_file} ... " + puts_time "Creating backup archive: #{tar_file} ... ".color(:blue) # Set file permissions on open to prevent chmod races. tar_system_options = { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] } if Kernel.system('tar', '-cf', '-', *backup_contents, tar_system_options) - progress.puts "done".color(:green) + puts_time "Creating backup archive: #{tar_file} ... ".color(:blue) + 'done'.color(:green) else - puts "creating archive #{tar_file} failed".color(:red) + puts_time "Creating archive #{tar_file} failed".color(:red) raise Backup::Error, 'Backup failed' end end end def upload - progress.print "Uploading backup archive to remote storage #{remote_directory} ... " - connection_settings = Gitlab.config.backup.upload.connection - if connection_settings.blank? - progress.puts "skipped".color(:yellow) + if connection_settings.blank? || skipped?('remote') + puts_time "Uploading backup archive to remote storage #{remote_directory} ... ".color(:blue) + "[SKIPPED]".color(:cyan) return end + puts_time "Uploading backup archive to remote storage #{remote_directory} ... ".color(:blue) + directory = connect_to_remote_directory upload = directory.files.create(create_attributes) if upload if upload.respond_to?(:encryption) && upload.encryption - progress.puts "done (encrypted with #{upload.encryption})".color(:green) + puts_time "Uploading backup archive to remote storage #{remote_directory} ... ".color(:blue) + "done (encrypted with #{upload.encryption})".color(:green) else - progress.puts "done".color(:green) + puts_time "Uploading backup archive to remote storage #{remote_directory} ... ".color(:blue) + "done".color(:green) end else - puts "uploading backup to #{remote_directory} failed".color(:red) + puts_time "Uploading backup to #{remote_directory} failed".color(:red) raise Backup::Error, 'Backup failed' end end def cleanup - progress.print "Deleting tmp directories ... " + puts_time "Deleting tar staging files ... ".color(:blue) remove_backup_path(MANIFEST_NAME) @definitions.each do |_, definition| remove_backup_path(definition.cleanup_path || definition.destination_path) end + + puts_time "Deleting tar staging files ... ".color(:blue) + 'done'.color(:green) end def remove_backup_path(path) - return unless File.exist?(File.join(backup_path, path)) + absolute_path = File.join(backup_path, path) + return unless File.exist?(absolute_path) - FileUtils.rm_rf(File.join(backup_path, path)) - progress.puts "done".color(:green) + puts_time "Cleaning up #{absolute_path}" + FileUtils.rm_rf(absolute_path) end def remove_tmp # delete tmp inside backups - progress.print "Deleting backups/tmp ... " + puts_time "Deleting backups/tmp ... ".color(:blue) - if FileUtils.rm_rf(File.join(backup_path, "tmp")) - progress.puts "done".color(:green) - else - puts "deleting backups/tmp failed".color(:red) - end + FileUtils.rm_rf(File.join(backup_path, "tmp")) + puts_time "Deleting backups/tmp ... ".color(:blue) + "done".color(:green) end def remove_old # delete backups - progress.print "Deleting old backups ... " keep_time = Gitlab.config.backup.keep_time.to_i - if keep_time > 0 - removed = 0 - - Dir.chdir(backup_path) do - backup_file_list.each do |file| - # For backward compatibility, there are 3 names the backups can have: - # - 1495527122_gitlab_backup.tar - # - 1495527068_2017_05_23_gitlab_backup.tar - # - 1495527097_2017_05_23_9.3.0-pre_gitlab_backup.tar - matched = backup_file?(file) - next unless matched - - timestamp = matched[1].to_i - - if Time.at(timestamp) < (Time.now - keep_time) - begin - FileUtils.rm(file) - removed += 1 - rescue StandardError => e - progress.puts "Deleting #{file} failed: #{e.message}".color(:red) - end + if keep_time <= 0 + puts_time "Deleting old backups ... ".color(:blue) + "[SKIPPED]".color(:cyan) + return + end + + puts_time "Deleting old backups ... ".color(:blue) + removed = 0 + + Dir.chdir(backup_path) do + backup_file_list.each do |file| + # For backward compatibility, there are 3 names the backups can have: + # - 1495527122_gitlab_backup.tar + # - 1495527068_2017_05_23_gitlab_backup.tar + # - 1495527097_2017_05_23_9.3.0-pre_gitlab_backup.tar + matched = backup_file?(file) + next unless matched + + timestamp = matched[1].to_i + + if Time.at(timestamp) < (Time.now - keep_time) + begin + FileUtils.rm(file) + removed += 1 + rescue StandardError => e + puts_time "Deleting #{file} failed: #{e.message}".color(:red) end end end - - progress.puts "done. (#{removed} removed)".color(:green) - else - progress.puts "skipping".color(:yellow) end + + puts_time "Deleting old backups ... ".color(:blue) + "done. (#{removed} removed)".color(:green) end def verify_backup_version @@ -327,7 +376,7 @@ module Backup def unpack if ENV['BACKUP'].blank? && non_tarred_backup? - progress.puts "Non tarred backup found in #{backup_path}, using that" + puts_time "Non tarred backup found in #{backup_path}, using that" return false end @@ -335,15 +384,22 @@ module Backup Dir.chdir(backup_path) do # check for existing backups in the backup dir if backup_file_list.empty? - progress.puts "No backups found in #{backup_path}" - progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" + puts_time "No backups found in #{backup_path}" + puts_time "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" exit 1 elsif backup_file_list.many? && ENV["BACKUP"].nil? - progress.puts 'Found more than one backup:' + puts_time 'Found more than one backup:' # print list of available backups - progress.puts " " + available_timestamps.join("\n ") - progress.puts 'Please specify which one you want to restore:' - progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' + puts_time " " + available_timestamps.join("\n ") + + if incremental? + puts_time 'Please specify which one you want to create an incremental backup for:' + puts_time 'rake gitlab:backup:create INCREMENTAL=true BACKUP=timestamp_of_backup' + else + puts_time 'Please specify which one you want to restore:' + puts_time 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' + end + exit 1 end @@ -354,16 +410,16 @@ module Backup end unless File.exist?(tar_file) - progress.puts "The backup file #{tar_file} does not exist!" + puts_time "The backup file #{tar_file} does not exist!" exit 1 end - progress.print 'Unpacking backup ... ' + puts_time 'Unpacking backup ... '.color(:blue) if Kernel.system(*%W(tar -xf #{tar_file})) - progress.puts 'done'.color(:green) + puts_time 'Unpacking backup ... '.color(:blue) + 'done'.color(:green) else - progress.puts 'unpacking backup failed'.color(:red) + puts_time 'Unpacking backup failed'.color(:red) exit 1 end end @@ -375,11 +431,12 @@ module Backup end def skipped?(item) - backup_information[:skipped] && backup_information[:skipped].include?(item) + ENV.fetch('SKIP', '').include?(item) || + backup_information[:skipped] && backup_information[:skipped].include?(item) end def enabled_task?(task_name) - @definitions[task_name].task.enabled + @definitions[task_name].enabled? end def backup_file?(file) @@ -441,11 +498,15 @@ module Backup end def tar_file - @tar_file ||= if ENV['BACKUP'].present? - File.basename(ENV['BACKUP']) + FILE_NAME_SUFFIX - else - "#{backup_information[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{backup_information[:gitlab_version]}#{FILE_NAME_SUFFIX}" - end + @tar_file ||= "#{backup_id}#{FILE_NAME_SUFFIX}" + end + + def backup_id + @backup_id ||= if ENV['BACKUP'].present? + File.basename(ENV['BACKUP']) + else + "#{backup_information[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{backup_information[:gitlab_version]}" + end end def create_attributes @@ -481,16 +542,6 @@ module Backup Gitlab.config.backup.upload.connection&.provider&.downcase == 'google' end - def repository_backup_strategy(incremental) - if !Feature.feature_flags_available? || Feature.enabled?(:gitaly_backup, default_enabled: :yaml) - max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence - max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence - Backup::GitalyBackup.new(progress, incremental: incremental, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency) - else - Backup::GitalyRpcBackup.new(progress) - end - end - def puts_time(msg) progress.puts "#{Time.now} -- #{msg}" Gitlab::BackupLogger.info(message: "#{Rainbow.uncolor(msg)}") diff --git a/lib/backup/packages.rb b/lib/backup/packages.rb deleted file mode 100644 index 9384e007162..00000000000 --- a/lib/backup/packages.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Backup - class Packages < Backup::Files - def initialize(progress) - super(progress, 'packages', Settings.packages.storage_path, excludes: ['tmp']) - end - - override :human_name - def human_name - _('packages') - end - end -end diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb deleted file mode 100644 index ebed6820724..00000000000 --- a/lib/backup/pages.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Backup - class Pages < Backup::Files - # pages used to deploy tmp files to this path - # if some of these files are still there, we don't need them in the backup - LEGACY_PAGES_TMP_PATH = '@pages.tmp' - - def initialize(progress) - super(progress, 'pages', Gitlab.config.pages.path, excludes: [LEGACY_PAGES_TMP_PATH]) - end - - override :human_name - def human_name - _('pages') - end - end -end diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb deleted file mode 100644 index 68ea635034d..00000000000 --- a/lib/backup/registry.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module Backup - class Registry < Backup::Files - def initialize(progress) - super(progress, 'registry', Settings.registry.path) - end - - override :human_name - def human_name - _('container registry images') - end - - override :enabled - def enabled - Gitlab.config.registry.enabled - end - end -end diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index 3633ebd661e..11bed84e356 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -6,50 +6,17 @@ module Backup class Repositories < Task extend ::Gitlab::Utils::Override - def initialize(progress, strategy:, max_concurrency: 1, max_storage_concurrency: 1) + def initialize(progress, strategy:) super(progress) @strategy = strategy - @max_concurrency = max_concurrency - @max_storage_concurrency = max_storage_concurrency end override :dump - def dump(path) - strategy.start(:create, path) - - # gitaly-backup is designed to handle concurrency on its own. So we want - # to avoid entering the buggy concurrency code here when gitaly-backup - # is enabled. - if (max_concurrency <= 1 && max_storage_concurrency <= 1) || !strategy.parallel_enqueue? - return enqueue_consecutive - end - - if max_concurrency < 1 || max_storage_concurrency < 1 - puts "GITLAB_BACKUP_MAX_CONCURRENCY and GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY must have a value of at least 1".color(:red) - exit 1 - end - - check_valid_storages! - - semaphore = Concurrent::Semaphore.new(max_concurrency) - errors = Queue.new - - threads = Gitlab.config.repositories.storages.keys.map do |storage| - Thread.new do - Rails.application.executor.wrap do - enqueue_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency) - rescue StandardError => e - errors << e - end - end - end - - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - threads.each(&:join) - end + def dump(path, backup_id) + strategy.start(:create, path, backup_id: backup_id) + enqueue_consecutive - raise errors.pop unless errors.empty? ensure strategy.finish! end @@ -66,26 +33,9 @@ module Backup restore_object_pools end - override :human_name - def human_name - _('repositories') - end - private - attr_reader :strategy, :max_concurrency, :max_storage_concurrency - - def check_valid_storages! - repository_storage_klasses.each do |klass| - if klass.excluding_repository_storage(Gitlab.config.repositories.storages.keys).exists? - raise Error, "repositories.storages in gitlab.yml does not include all storages used by #{klass}" - end - end - end - - def repository_storage_klasses - [ProjectRepository, SnippetRepository] - end + attr_reader :strategy def enqueue_consecutive enqueue_consecutive_projects @@ -102,50 +52,6 @@ module Backup Snippet.find_each(batch_size: 1000) { |snippet| enqueue_snippet(snippet) } end - def enqueue_storage(storage, semaphore, max_storage_concurrency:) - errors = Queue.new - queue = InterlockSizedQueue.new(1) - - threads = Array.new(max_storage_concurrency) do - Thread.new do - Rails.application.executor.wrap do - while container = queue.pop - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - semaphore.acquire - end - - begin - enqueue_container(container) - rescue StandardError => e - errors << e - break - ensure - semaphore.release - end - end - end - end - end - - enqueue_records_for_storage(storage, queue, errors) - - raise errors.pop unless errors.empty? - ensure - queue.close - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - threads.each(&:join) - end - end - - def enqueue_container(container) - case container - when Project - enqueue_project(container) - when Snippet - enqueue_snippet(container) - end - end - def enqueue_project(project) strategy.enqueue(project, Gitlab::GlRepository::PROJECT) strategy.enqueue(project, Gitlab::GlRepository::WIKI) @@ -156,32 +62,10 @@ module Backup strategy.enqueue(snippet, Gitlab::GlRepository::SNIPPET) end - def enqueue_records_for_storage(storage, queue, errors) - records_to_enqueue(storage).each do |relation| - relation.find_each(batch_size: 100) do |project| - break unless errors.empty? - - queue.push(project) - end - end - end - - def records_to_enqueue(storage) - [projects_in_storage(storage), snippets_in_storage(storage)] - end - - def projects_in_storage(storage) - project_relation.id_in(ProjectRepository.for_repository_storage(storage).select(:project_id)) - end - def project_relation Project.includes(:route, :group, namespace: :owner) end - def snippets_in_storage(storage) - Snippet.id_in(SnippetRepository.for_repository_storage(storage).select(:snippet_id)) - end - def restore_object_pools PoolRepository.includes(:source_project).find_each do |pool| progress.puts " - Object pool #{pool.disk_path}..." @@ -216,24 +100,6 @@ module Backup Snippet.id_in(invalid_snippets).delete_all end - - class InterlockSizedQueue < SizedQueue - extend ::Gitlab::Utils::Override - - override :pop - def pop(*) - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - super - end - end - - override :push - def push(*) - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - super - end - end - end end end diff --git a/lib/backup/task.rb b/lib/backup/task.rb index 15cd2aa64d3..776c19130a7 100644 --- a/lib/backup/task.rb +++ b/lib/backup/task.rb @@ -6,13 +6,11 @@ module Backup @progress = progress end - # human readable task name used for logging - def human_name - raise NotImplementedError - end - # dump task backup to `path` - def dump(path) + # + # @param [String] path fully qualified backup task destination + # @param [String] backup_id unique identifier for the backup + def dump(path, backup_id) raise NotImplementedError end @@ -29,11 +27,6 @@ module Backup def post_restore_warning end - # returns `true` when the task should be used - def enabled - true - end - private attr_reader :progress diff --git a/lib/backup/terraform_state.rb b/lib/backup/terraform_state.rb deleted file mode 100644 index 05f61d248be..00000000000 --- a/lib/backup/terraform_state.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Backup - class TerraformState < Backup::Files - def initialize(progress) - super(progress, 'terraform_state', Settings.terraform_state.storage_path, excludes: ['tmp']) - end - - override :human_name - def human_name - _('terraform states') - end - end -end diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb deleted file mode 100644 index 700f2af4415..00000000000 --- a/lib/backup/uploads.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Backup - class Uploads < Backup::Files - def initialize(progress) - super(progress, 'uploads', File.join(Gitlab.config.uploads.storage_path, "uploads"), excludes: ['tmp']) - end - - override :human_name - def human_name - _('uploads') - end - end -end |