diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-06-16 21:25:58 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-06-16 21:25:58 +0300 |
commit | a5f4bba440d7f9ea47046a0a561d49adf0a1e6d4 (patch) | |
tree | fb69158581673816a8cd895f9d352dcb3c678b1e /lib/backup | |
parent | d16b2e8639e99961de6ddc93909f3bb5c1445ba1 (diff) |
Add latest changes from gitlab-org/gitlab@14-0-stable-eev14.0.0-rc42
Diffstat (limited to 'lib/backup')
-rw-r--r-- | lib/backup/gitaly_backup.rb | 67 | ||||
-rw-r--r-- | lib/backup/gitaly_rpc_backup.rb | 128 | ||||
-rw-r--r-- | lib/backup/repositories.rb | 194 |
3 files changed, 236 insertions, 153 deletions
diff --git a/lib/backup/gitaly_backup.rb b/lib/backup/gitaly_backup.rb new file mode 100644 index 00000000000..cfd3d463f9e --- /dev/null +++ b/lib/backup/gitaly_backup.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module Backup + # Backup and restores repositories using gitaly-backup + class GitalyBackup + def initialize(progress) + @progress = progress + end + + def start(type) + raise Error, 'already started' if started? + + command = case type + when :create + 'create' + when :restore + 'restore' + else + raise Error, "unknown backup type: #{type}" + end + + @read_io, @write_io = IO.pipe + @pid = Process.spawn(bin_path, command, '-path', backup_repos_path, in: @read_io, out: progress) + end + + def wait + return unless started? + + @write_io.close + Process.wait(@pid) + status = $? + + @pid = nil + + raise Error, "gitaly-backup exit status #{status.exitstatus}" if status.exitstatus != 0 + end + + def enqueue(container, repo_type) + raise Error, 'not started' unless started? + + repository = repo_type.repository_for(container) + + @write_io.puts({ + storage_name: repository.storage, + relative_path: repository.relative_path, + gl_project_path: repository.gl_project_path, + always_create: repo_type.project? + }.merge(Gitlab::GitalyClient.connection_data(repository.storage)).to_json) + end + + private + + attr_reader :progress + + def started? + @pid.present? + end + + def backup_repos_path + File.absolute_path(File.join(Gitlab.config.backup.path, 'repositories')) + end + + def bin_path + File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-backup')) + end + end +end diff --git a/lib/backup/gitaly_rpc_backup.rb b/lib/backup/gitaly_rpc_backup.rb new file mode 100644 index 00000000000..53f1de40509 --- /dev/null +++ b/lib/backup/gitaly_rpc_backup.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +module Backup + # Backup and restores repositories using the gitaly RPC + class GitalyRpcBackup + def initialize(progress) + @progress = progress + end + + def start(type) + raise Error, 'already started' if @type + + @type = type + case type + when :create + FileUtils.rm_rf(backup_repos_path) + FileUtils.mkdir_p(Gitlab.config.backup.path) + FileUtils.mkdir(backup_repos_path, mode: 0700) + when :restore + # no op + else + raise Error, "unknown backup type: #{type}" + end + end + + def wait + @type = nil + end + + def enqueue(container, repository_type) + backup_restore = BackupRestore.new( + progress, + repository_type.repository_for(container), + backup_repos_path + ) + + case @type + when :create + backup_restore.backup + when :restore + backup_restore.restore(always_create: repository_type.project?) + else + raise Error, 'not started' + end + end + + private + + attr_reader :progress + + def backup_repos_path + @backup_repos_path ||= File.join(Gitlab.config.backup.path, 'repositories') + end + + class BackupRestore + attr_accessor :progress, :repository, :backup_repos_path + + def initialize(progress, repository, backup_repos_path) + @progress = progress + @repository = repository + @backup_repos_path = backup_repos_path + end + + def backup + progress.puts " * #{display_repo_path} ... " + + if repository.empty? + progress.puts " * #{display_repo_path} ... " + "[EMPTY] [SKIPPED]".color(:cyan) + return + end + + FileUtils.mkdir_p(repository_backup_path) + + repository.bundle_to_disk(path_to_bundle) + repository.gitaly_repository_client.backup_custom_hooks(custom_hooks_tar) + + progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) + + rescue StandardError => e + progress.puts "[Failed] backing up #{display_repo_path}".color(:red) + progress.puts "Error #{e}".color(:red) + end + + def restore(always_create: false) + progress.puts " * #{display_repo_path} ... " + + repository.remove rescue nil + + if File.exist?(path_to_bundle) + repository.create_from_bundle(path_to_bundle) + restore_custom_hooks + elsif always_create + repository.create_repository + end + + progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) + + rescue StandardError => e + progress.puts "[Failed] restoring #{display_repo_path}".color(:red) + progress.puts "Error #{e}".color(:red) + end + + private + + def display_repo_path + "#{repository.full_path} (#{repository.disk_path})" + end + + def repository_backup_path + @repository_backup_path ||= File.join(backup_repos_path, repository.disk_path) + end + + def path_to_bundle + @path_to_bundle ||= File.join(backup_repos_path, repository.disk_path + '.bundle') + end + + def restore_custom_hooks + return unless File.exist?(custom_hooks_tar) + + repository.gitaly_repository_client.restore_custom_hooks(custom_hooks_tar) + end + + def custom_hooks_tar + File.join(repository_backup_path, "custom_hooks.tar") + end + end + end +end diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index b1231eebfcc..80d23c1eb7f 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -4,17 +4,16 @@ require 'yaml' module Backup class Repositories - attr_reader :progress - - def initialize(progress) + def initialize(progress, strategy:) @progress = progress + @strategy = strategy end def dump(max_concurrency:, max_storage_concurrency:) - prepare + strategy.start(:create) if max_concurrency <= 1 && max_storage_concurrency <= 1 - return dump_consecutive + return enqueue_consecutive end check_valid_storages! @@ -25,7 +24,7 @@ module Backup threads = Gitlab.config.repositories.storages.keys.map do |storage| Thread.new do Rails.application.executor.wrap do - dump_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency) + enqueue_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency) rescue StandardError => e errors << e end @@ -37,32 +36,24 @@ module Backup end raise errors.pop unless errors.empty? + ensure + strategy.wait end def restore - restore_project_repositories - restore_snippets + strategy.start(:restore) + enqueue_consecutive + + ensure + strategy.wait + cleanup_snippets_without_repositories restore_object_pools end private - def restore_project_repositories - Project.find_each(batch_size: 1000) do |project| - restore_repository(project, Gitlab::GlRepository::PROJECT) - restore_repository(project, Gitlab::GlRepository::WIKI) - restore_repository(project, Gitlab::GlRepository::DESIGN) - end - end - - def restore_snippets - invalid_ids = Snippet.find_each(batch_size: 1000) - .map { |snippet| restore_snippet_repository(snippet) } - .compact - - cleanup_snippets_without_repositories(invalid_ids) - end + attr_reader :progress, :strategy def check_valid_storages! repository_storage_klasses.each do |klass| @@ -76,32 +67,22 @@ module Backup [ProjectRepository, SnippetRepository] end - def backup_repos_path - @backup_repos_path ||= File.join(Gitlab.config.backup.path, 'repositories') - end - - def prepare - FileUtils.rm_rf(backup_repos_path) - FileUtils.mkdir_p(Gitlab.config.backup.path) - FileUtils.mkdir(backup_repos_path, mode: 0700) + def enqueue_consecutive + enqueue_consecutive_projects + enqueue_consecutive_snippets end - def dump_consecutive - dump_consecutive_projects - dump_consecutive_snippets - end - - def dump_consecutive_projects + def enqueue_consecutive_projects project_relation.find_each(batch_size: 1000) do |project| - dump_project(project) + enqueue_project(project) end end - def dump_consecutive_snippets - Snippet.find_each(batch_size: 1000) { |snippet| dump_snippet(snippet) } + def enqueue_consecutive_snippets + Snippet.find_each(batch_size: 1000) { |snippet| enqueue_snippet(snippet) } end - def dump_storage(storage, semaphore, max_storage_concurrency:) + def enqueue_storage(storage, semaphore, max_storage_concurrency:) errors = Queue.new queue = InterlockSizedQueue.new(1) @@ -114,7 +95,7 @@ module Backup end begin - dump_container(container) + enqueue_container(container) rescue StandardError => e errors << e break @@ -136,23 +117,23 @@ module Backup end end - def dump_container(container) + def enqueue_container(container) case container when Project - dump_project(container) + enqueue_project(container) when Snippet - dump_snippet(container) + enqueue_snippet(container) end end - def dump_project(project) - backup_repository(project, Gitlab::GlRepository::PROJECT) - backup_repository(project, Gitlab::GlRepository::WIKI) - backup_repository(project, Gitlab::GlRepository::DESIGN) + def enqueue_project(project) + strategy.enqueue(project, Gitlab::GlRepository::PROJECT) + strategy.enqueue(project, Gitlab::GlRepository::WIKI) + strategy.enqueue(project, Gitlab::GlRepository::DESIGN) end - def dump_snippet(snippet) - backup_repository(snippet, Gitlab::GlRepository::SNIPPET) + def enqueue_snippet(snippet) + strategy.enqueue(snippet, Gitlab::GlRepository::SNIPPET) end def enqueue_records_for_storage(storage, queue, errors) @@ -181,22 +162,6 @@ module Backup Snippet.id_in(SnippetRepository.for_repository_storage(storage).select(:snippet_id)) end - def backup_repository(container, type) - BackupRestore.new( - progress, - type.repository_for(container), - backup_repos_path - ).backup - end - - def restore_repository(container, type) - BackupRestore.new( - progress, - type.repository_for(container), - backup_repos_path - ).restore(always_create: type.project?) - end - def restore_object_pools PoolRepository.includes(:source_project).find_each do |pool| progress.puts " - Object pool #{pool.disk_path}..." @@ -214,99 +179,22 @@ module Backup end end - def restore_snippet_repository(snippet) - restore_repository(snippet, Gitlab::GlRepository::SNIPPET) - - response = Snippets::RepositoryValidationService.new(nil, snippet).execute - - if response.error? - snippet.repository.remove - - progress.puts("Snippet #{snippet.full_path} can't be restored: #{response.message}") - - snippet.id - else - nil - end - end - # Snippets without a repository should be removed because they failed to import # due to having invalid repositories - def cleanup_snippets_without_repositories(ids) - Snippet.id_in(ids).delete_all - end + def cleanup_snippets_without_repositories + invalid_snippets = [] - class BackupRestore - attr_accessor :progress, :repository, :backup_repos_path + Snippet.find_each(batch_size: 1000).each do |snippet| + response = Snippets::RepositoryValidationService.new(nil, snippet).execute + next if response.success? - def initialize(progress, repository, backup_repos_path) - @progress = progress - @repository = repository - @backup_repos_path = backup_repos_path - end - - def backup - progress.puts " * #{display_repo_path} ... " - - if repository.empty? - progress.puts " * #{display_repo_path} ... " + "[EMPTY] [SKIPPED]".color(:cyan) - return - end - - FileUtils.mkdir_p(repository_backup_path) - - repository.bundle_to_disk(path_to_bundle) - repository.gitaly_repository_client.backup_custom_hooks(custom_hooks_tar) - - progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) - - rescue StandardError => e - progress.puts "[Failed] backing up #{display_repo_path}".color(:red) - progress.puts "Error #{e}".color(:red) - end - - def restore(always_create: false) - progress.puts " * #{display_repo_path} ... " - - repository.remove rescue nil - - if File.exist?(path_to_bundle) - repository.create_from_bundle(path_to_bundle) - restore_custom_hooks - elsif always_create - repository.create_repository - end - - progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) - - rescue StandardError => e - progress.puts "[Failed] restoring #{display_repo_path}".color(:red) - progress.puts "Error #{e}".color(:red) - end - - private - - def display_repo_path - "#{repository.full_path} (#{repository.disk_path})" - end - - def repository_backup_path - @repository_backup_path ||= File.join(backup_repos_path, repository.disk_path) - end - - def path_to_bundle - @path_to_bundle ||= File.join(backup_repos_path, repository.disk_path + '.bundle') - end - - def restore_custom_hooks - return unless File.exist?(custom_hooks_tar) + snippet.repository.remove + progress.puts("Snippet #{snippet.full_path} can't be restored: #{response.message}") - repository.gitaly_repository_client.restore_custom_hooks(custom_hooks_tar) + invalid_snippets << snippet.id end - def custom_hooks_tar - File.join(repository_backup_path, "custom_hooks.tar") - end + Snippet.id_in(invalid_snippets).delete_all end class InterlockSizedQueue < SizedQueue |