diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-01-20 12:16:11 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-01-20 12:16:11 +0300 |
commit | edaa33dee2ff2f7ea3fac488d41558eb5f86d68c (patch) | |
tree | 11f143effbfeba52329fb7afbd05e6e2a3790241 /lib/backup | |
parent | d8a5691316400a0f7ec4f83832698f1988eb27c1 (diff) |
Add latest changes from gitlab-org/gitlab@14-7-stable-eev14.7.0-rc42
Diffstat (limited to 'lib/backup')
-rw-r--r-- | lib/backup/database.rb | 2 | ||||
-rw-r--r-- | lib/backup/files.rb | 8 | ||||
-rw-r--r-- | lib/backup/gitaly_backup.rb | 47 | ||||
-rw-r--r-- | lib/backup/gitaly_rpc_backup.rb | 2 | ||||
-rw-r--r-- | lib/backup/manager.rb | 2 | ||||
-rw-r--r-- | lib/backup/packages.rb | 13 | ||||
-rw-r--r-- | lib/backup/repositories.rb | 4 | ||||
-rw-r--r-- | lib/backup/terraform_state.rb | 13 |
8 files changed, 70 insertions, 21 deletions
diff --git a/lib/backup/database.rb b/lib/backup/database.rb index f07fd786b4b..a4ac404d245 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -61,7 +61,7 @@ module Backup report_success(success) progress.flush - raise Backup::Error, 'Backup failed' unless success + raise DatabaseBackupError.new(config, db_file_name) unless success end def restore diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 42cfff98239..4e51dcfb79e 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -37,7 +37,7 @@ module Backup unless status == 0 puts output - raise Backup::Error, 'Backup failed' + raise_custom_error end tar_cmd = [tar, exclude_dirs(:tar), %W[-C #{@backup_files_dir} -cf - .]].flatten @@ -49,7 +49,7 @@ module Backup end unless pipeline_succeeded?(tar_status: status_list[0], gzip_status: status_list[1], output: output) - raise Backup::Error, "Backup operation failed: #{output}" + raise_custom_error end end @@ -143,5 +143,9 @@ module Backup end end end + + def raise_custom_error + raise FileBackupError.new(app_files_dir, backup_tarball) + end end end diff --git a/lib/backup/gitaly_backup.rb b/lib/backup/gitaly_backup.rb index b104beed39c..8ac09e94004 100644 --- a/lib/backup/gitaly_backup.rb +++ b/lib/backup/gitaly_backup.rb @@ -2,11 +2,17 @@ module Backup # Backup and restores repositories using gitaly-backup + # + # gitaly-backup can work in parallel and accepts a list of repositories + # through input pipe using a specific json format for both backup and restore class GitalyBackup - def initialize(progress, parallel: nil, parallel_storage: nil) + # @param [StringIO] progress IO interface to output progress + # @param [Integer] max_parallelism max parallelism when running backups + # @param [Integer] storage_parallelism max parallelism per storage (is affected by max_parallelism) + def initialize(progress, max_parallelism: nil, storage_parallelism: nil) @progress = progress - @parallel = parallel - @parallel_storage = parallel_storage + @max_parallelism = max_parallelism + @storage_parallelism = storage_parallelism end def start(type) @@ -22,20 +28,20 @@ module Backup end args = [] - args += ['-parallel', @parallel.to_s] if @parallel - args += ['-parallel-storage', @parallel_storage.to_s] if @parallel_storage + args += ['-parallel', @max_parallelism.to_s] if @max_parallelism + args += ['-parallel-storage', @storage_parallelism.to_s] if @storage_parallelism - @stdin, stdout, @thread = Open3.popen2(build_env, bin_path, command, '-path', backup_repos_path, *args) + @input_stream, stdout, @thread = Open3.popen2(build_env, bin_path, command, '-path', backup_repos_path, *args) @out_reader = Thread.new do IO.copy_stream(stdout, @progress) end end - def wait + def finish! return unless started? - @stdin.close + @input_stream.close [@thread, @out_reader].each(&:join) status = @thread.value @@ -49,12 +55,7 @@ module Backup repository = repo_type.repository_for(container) - @stdin.puts({ - storage_name: repository.storage, - relative_path: repository.relative_path, - gl_project_path: repository.gl_project_path, - always_create: repo_type.project? - }.merge(Gitlab::GitalyClient.connection_data(repository.storage)).to_json) + schedule_backup_job(repository, always_create: repo_type.project?) end def parallel_enqueue? @@ -63,6 +64,24 @@ module Backup private + # Schedule a new backup job through a non-blocking JSON based pipe protocol + # + # @see https://gitlab.com/gitlab-org/gitaly/-/blob/master/doc/gitaly-backup.md + def schedule_backup_job(repository, always_create:) + connection_params = Gitlab::GitalyClient.connection_data(repository.storage) + + json_job = { + address: connection_params['address'], + token: connection_params['token'], + storage_name: repository.storage, + relative_path: repository.relative_path, + gl_project_path: repository.gl_project_path, + always_create: always_create + }.to_json + + @input_stream.puts(json_job) + end + def build_env { 'SSL_CERT_FILE' => OpenSSL::X509::DEFAULT_CERT_FILE, diff --git a/lib/backup/gitaly_rpc_backup.rb b/lib/backup/gitaly_rpc_backup.rb index baac4eb26ca..bbd83cd2157 100644 --- a/lib/backup/gitaly_rpc_backup.rb +++ b/lib/backup/gitaly_rpc_backup.rb @@ -23,7 +23,7 @@ module Backup end end - def wait + def finish! @type = nil end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 1bdc4965e5d..ed2e001cefc 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -2,7 +2,7 @@ module Backup class Manager - ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs registry].freeze + ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs terraform_state registry packages].freeze FOLDERS_TO_BACKUP = %w[repositories db].freeze FILE_NAME_SUFFIX = '_gitlab_backup.tar' diff --git a/lib/backup/packages.rb b/lib/backup/packages.rb new file mode 100644 index 00000000000..7b6a8f086ed --- /dev/null +++ b/lib/backup/packages.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Backup + class Packages < Backup::Files + attr_reader :progress + + def initialize(progress) + @progress = progress + + super('packages', Settings.packages.storage_path, excludes: ['tmp']) + end + end +end diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index 0b5a62529b4..4c39e58c87d 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -40,7 +40,7 @@ module Backup raise errors.pop unless errors.empty? ensure - strategy.wait + strategy.finish! end def restore @@ -48,7 +48,7 @@ module Backup enqueue_consecutive ensure - strategy.wait + strategy.finish! cleanup_snippets_without_repositories restore_object_pools diff --git a/lib/backup/terraform_state.rb b/lib/backup/terraform_state.rb new file mode 100644 index 00000000000..5f71e18f1b4 --- /dev/null +++ b/lib/backup/terraform_state.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Backup + class TerraformState < Backup::Files + attr_reader :progress + + def initialize(progress) + @progress = progress + + super('terraform_state', Settings.terraform_state.storage_path, excludes: ['tmp']) + end + end +end |