diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-06-20 14:10:13 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-06-20 14:10:13 +0300 |
commit | 0ea3fcec397b69815975647f5e2aa5fe944a8486 (patch) | |
tree | 7979381b89d26011bcf9bdc989a40fcc2f1ed4ff /lib/backup | |
parent | 72123183a20411a36d607d70b12d57c484394c8e (diff) |
Add latest changes from gitlab-org/gitlab@15-1-stable-eev15.1.0-rc42
Diffstat (limited to 'lib/backup')
-rw-r--r-- | lib/backup/manager.rb | 109 | ||||
-rw-r--r-- | lib/backup/repositories.rb | 33 |
2 files changed, 85 insertions, 57 deletions
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 0991177d044..16b8f21c9e9 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -11,7 +11,8 @@ module Backup LIST_ENVS = { skipped: 'SKIP', - repositories_storages: 'REPOSITORIES_STORAGES' + repositories_storages: 'REPOSITORIES_STORAGES', + repositories_paths: 'REPOSITORIES_PATHS' }.freeze TaskDefinition = Struct.new( @@ -41,29 +42,8 @@ module Backup end def create - if incremental? - unpack(ENV.fetch('PREVIOUS_BACKUP', ENV['BACKUP'])) - read_backup_information - verify_backup_version - update_backup_information - end - - build_backup_information - - definitions.keys.each do |task_name| - run_create_task(task_name) - end - - write_backup_information - - if skipped?('tar') - upload - else - pack - upload - cleanup - remove_old - end + unpack(ENV.fetch('PREVIOUS_BACKUP', ENV['BACKUP'])) if incremental? + run_all_create_tasks puts_time "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ "and are not included in this backup. You will need these files to restore a backup.\n" \ @@ -95,22 +75,8 @@ module Backup end def restore - cleanup_required = unpack(ENV['BACKUP']) - read_backup_information - verify_backup_version - - definitions.keys.each do |task_name| - run_restore_task(task_name) if !skipped?(task_name) && enabled_task?(task_name) - end - - Rake::Task['gitlab:shell:setup'].invoke - Rake::Task['cache:clear'].invoke - - if cleanup_required - cleanup - end - - remove_tmp + unpack(ENV['BACKUP']) + run_all_restore_tasks puts_time "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ "and are not included in this backup. You will need to restore these files manually.".color(:red) @@ -225,13 +191,59 @@ module Backup max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence strategy = Backup::GitalyBackup.new(progress, incremental: incremental?, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency) - Repositories.new(progress, strategy: strategy, storages: repositories_storages) + Repositories.new(progress, + strategy: strategy, + storages: list_env(:repositories_storages), + paths: list_env(:repositories_paths) + ) end def build_files_task(app_files_dir, excludes: []) Files.new(progress, app_files_dir, excludes: excludes) end + def run_all_create_tasks + if incremental? + read_backup_information + verify_backup_version + update_backup_information + end + + build_backup_information + + definitions.keys.each do |task_name| + run_create_task(task_name) + end + + write_backup_information + + unless skipped?('tar') + pack + upload + remove_old + end + + ensure + cleanup unless skipped?('tar') + remove_tmp + end + + def run_all_restore_tasks + read_backup_information + verify_backup_version + + definitions.keys.each do |task_name| + run_restore_task(task_name) if !skipped?(task_name) && enabled_task?(task_name) + end + + Rake::Task['gitlab:shell:setup'].invoke + Rake::Task['cache:clear'].invoke + + ensure + cleanup unless skipped?('tar') + remove_tmp + end + def incremental? @incremental end @@ -259,7 +271,8 @@ module Backup tar_version: tar_version, installation_type: Gitlab::INSTALLATION_TYPE, skipped: ENV['SKIP'], - repositories_storages: ENV['REPOSITORIES_STORAGES'] + repositories_storages: ENV['REPOSITORIES_STORAGES'], + repositories_paths: ENV['REPOSITORIES_PATHS'] } end @@ -272,7 +285,8 @@ module Backup tar_version: tar_version, installation_type: Gitlab::INSTALLATION_TYPE, skipped: list_env(:skipped).join(','), - repositories_storages: list_env(:repositories_storages).join(',') + repositories_storages: list_env(:repositories_storages).join(','), + repositories_paths: list_env(:repositories_paths).join(',') ) end @@ -299,7 +313,7 @@ module Backup def upload connection_settings = Gitlab.config.backup.upload.connection - if connection_settings.blank? || skipped?('remote') + if connection_settings.blank? || skipped?('remote') || skipped?('tar') puts_time "Uploading backup archive to remote storage #{remote_directory} ... ".color(:blue) + "[SKIPPED]".color(:cyan) return end @@ -405,8 +419,7 @@ module Backup def unpack(source_backup_id) if source_backup_id.blank? && non_tarred_backup? puts_time "Non tarred backup found in #{backup_path}, using that" - - return false + return end Dir.chdir(backup_path) do @@ -466,10 +479,6 @@ module Backup @skipped ||= list_env(:skipped) end - def repositories_storages - @repositories_storages ||= list_env(:repositories_storages) - end - def list_env(name) list = ENV.fetch(LIST_ENVS[name], '').split(',') list += backup_information[name].split(',') if backup_information[name] diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index 4a31e87b969..4f4a098f374 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -3,19 +3,25 @@ require 'yaml' module Backup + # Backup and restores repositories by querying the database class Repositories < Task extend ::Gitlab::Utils::Override - def initialize(progress, strategy:, storages: []) + # @param [IO] progress IO interface to output progress + # @param [Object] :strategy Fetches backups from gitaly + # @param [Array<String>] :storages Filter by specified storage names. Empty means all storages. + # @param [Array<String>] :paths Filter by specified project paths. Empty means all projects, groups and snippets. + def initialize(progress, strategy:, storages: [], paths: []) super(progress) @strategy = strategy @storages = storages + @paths = paths end override :dump - def dump(path, backup_id) - strategy.start(:create, path, backup_id: backup_id) + def dump(destination_path, backup_id) + strategy.start(:create, destination_path, backup_id: backup_id) enqueue_consecutive ensure @@ -23,8 +29,8 @@ module Backup end override :restore - def restore(path) - strategy.start(:restore, path) + def restore(destination_path) + strategy.start(:restore, destination_path) enqueue_consecutive ensure @@ -36,7 +42,7 @@ module Backup private - attr_reader :strategy, :storages + attr_reader :strategy, :storages, :paths def enqueue_consecutive enqueue_consecutive_projects @@ -66,12 +72,26 @@ module Backup def project_relation scope = Project.includes(:route, :group, namespace: :owner) scope = scope.id_in(ProjectRepository.for_repository_storage(storages).select(:project_id)) if storages.any? + if paths.any? + scope = scope.where_full_path_in(paths).or( + Project.where(namespace_id: Namespace.where_full_path_in(paths).self_and_descendants) + ) + end + scope end def snippet_relation scope = Snippet.all scope = scope.id_in(SnippetRepository.for_repository_storage(storages).select(:snippet_id)) if storages.any? + if paths.any? + scope = scope.joins(:project).merge( + Project.where_full_path_in(paths).or( + Project.where(namespace_id: Namespace.where_full_path_in(paths).self_and_descendants) + ) + ) + end + scope end @@ -79,7 +99,6 @@ module Backup PoolRepository.includes(:source_project).find_each do |pool| progress.puts " - Object pool #{pool.disk_path}..." - pool.source_project ||= pool.member_projects.first&.root_of_fork_network unless pool.source_project progress.puts " - Object pool #{pool.disk_path}... " + "[SKIPPED]".color(:cyan) next |