From d7afed34c4238d637ce71fe8dbea41a64e7a3f1f Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Thu, 26 Jul 2018 10:05:32 +0200 Subject: Remove feature flags from lib/backup Moved to OPT_OUT in 7d14b725a0da41d1ae7c0a8496b5e66832023e3b, Now, by removing the feature gates, this is an mandatory feature. Related issues: - https://gitlab.com/gitlab-org/gitaly/issues/526 - https://gitlab.com/gitlab-org/gitaly/issues/1194 Closes https://gitlab.com/gitlab-org/gitaly/issues/749 Closes https://gitlab.com/gitlab-org/gitaly/issues/1212 Closes https://gitlab.com/gitlab-org/gitaly/issues/1195 --- lib/backup/repository.rb | 171 ++++++----------------------------------------- 1 file changed, 21 insertions(+), 150 deletions(-) (limited to 'lib') diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index af762db517c..906ed498026 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -1,10 +1,7 @@ require 'yaml' -require_relative 'helper' module Backup class Repository - include Backup::Helper - attr_reader :progress def initialize(progress) @@ -42,131 +39,36 @@ module Backup end def prepare_directories - Gitlab.config.repositories.storages.each do |name, repository_storage| - delete_all_repositories(name, repository_storage) + Gitlab.config.repositories.storages.each do |name, _repository_storage| + Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories end end def backup_project(project) - gitaly_migrate(:repository_backup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - backup_project_gitaly(project) - else - backup_project_local(project) - end - end - - backup_custom_hooks(project) - rescue => e - progress_warn(project, e, 'Failed to backup repo') - end - - def backup_project_gitaly(project) path_to_project_bundle = path_to_bundle(project) Gitlab::GitalyClient::RepositoryService.new(project.repository) .create_bundle(path_to_project_bundle) - end - - def backup_project_local(project) - path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - path_to_repo(project) - end - - path_to_project_bundle = path_to_bundle(project) - - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all) - output, status = Gitlab::Popen.popen(cmd) - progress_warn(project, cmd.join(' '), output) unless status.zero? - end - - def delete_all_repositories(name, repository_storage) - gitaly_migrate(:delete_all_repositories, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories - else - local_delete_all_repositories(name, repository_storage) - end - end - end - - def local_delete_all_repositories(name, repository_storage) - path = repository_storage.legacy_disk_path - return unless File.exist?(path) - - bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) - FileUtils.mkdir_p(bk_repos_path, mode: 0700) - files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] - - begin - FileUtils.mv(files, bk_repos_path) - rescue Errno::EACCES - access_denied_error(path) - rescue Errno::EBUSY - resource_busy_error(path) - end - end - def local_restore_custom_hooks(project, dir) - path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - path_to_repo(project) - end - cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) - output, status = Gitlab::Popen.popen(cmd) - unless status.zero? - progress_warn(project, cmd.join(' '), output) - end - end - - def gitaly_restore_custom_hooks(project, dir) - custom_hooks_path = path_to_tars(project, dir) - Gitlab::GitalyClient::RepositoryService.new(project.repository) - .restore_custom_hooks(custom_hooks_path) + backup_custom_hooks(project) + rescue => e + progress_warn(project, e, 'Failed to backup repo') end - def local_backup_custom_hooks(project) - in_path(path_to_tars(project)) do |dir| - path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - path_to_repo(project) - end - break unless File.exist?(File.join(path_to_project_repo, dir)) - - FileUtils.mkdir_p(path_to_tars(project)) - cmd = %W(tar -cf #{path_to_tars(project, dir)} -c #{path_to_project_repo} #{dir}) - output, status = Gitlab::Popen.popen(cmd) - - unless status.zero? - progress_warn(project, cmd.join(' '), output) - end - end - end + def backup_custom_hooks(project) + FileUtils.mkdir_p(project_backup_path(project)) - def gitaly_backup_custom_hooks(project) - FileUtils.mkdir_p(path_to_tars(project)) - custom_hooks_path = path_to_tars(project, 'custom_hooks') + custom_hooks_path = custom_hooks_tar(project) Gitlab::GitalyClient::RepositoryService.new(project.repository) .backup_custom_hooks(custom_hooks_path) end - def backup_custom_hooks(project) - gitaly_migrate(:backup_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_backup_custom_hooks(project) - else - local_backup_custom_hooks(project) - end - end - end - def restore_custom_hooks(project) - in_path(path_to_tars(project)) do |dir| - gitaly_migrate(:restore_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_restore_custom_hooks(project, dir) - else - local_restore_custom_hooks(project, dir) - end - end - end + return unless Dir.exist?(project_backup_path(project)) + return if Dir.glob("#{project_backup_path(project)}/custom_hooks*").none? + + custom_hooks_path = custom_hooks_tar(project) + Gitlab::GitalyClient::RepositoryService.new(project.repository) + .restore_custom_hooks(custom_hooks_path) end def restore @@ -181,7 +83,8 @@ module Backup restore_repo_success = nil if File.exist?(path_to_project_bundle) begin - project.repository.create_from_bundle path_to_project_bundle + project.repository.create_from_bundle(path_to_project_bundle) + restore_custom_hooks(project) restore_repo_success = true rescue => e restore_repo_success = false @@ -197,8 +100,6 @@ module Backup progress.puts "[Failed] restoring #{project.full_path} repository".color(:red) end - restore_custom_hooks(project) - wiki = ProjectWiki.new(project) path_to_wiki_bundle = path_to_bundle(wiki) @@ -219,48 +120,28 @@ module Backup protected - def path_to_repo(project) - project.repository.path_to_repo - end - def path_to_bundle(project) File.join(backup_repos_path, project.disk_path + '.bundle') end - def path_to_tars(project, dir = nil) - path = File.join(backup_repos_path, project.disk_path) + def project_backup_path(project) + File.join(backup_repos_path, project.disk_path) + end - if dir - File.join(path, "#{dir}.tar") - else - path - end + def custom_hooks_tar(project) + File.join(project_backup_path(project), "custom_hooks.tar") end def backup_repos_path File.join(Gitlab.config.backup.path, 'repositories') end - def in_path(path) - return unless Dir.exist?(path) - - dir_entries = Dir.entries(path) - - if dir_entries.include?('custom_hooks') || dir_entries.include?('custom_hooks.tar') - yield('custom_hooks') - end - end - def prepare FileUtils.rm_rf(backup_repos_path) FileUtils.mkdir_p(Gitlab.config.backup.path) FileUtils.mkdir(backup_repos_path, mode: 0700) end - def silent - { err: '/dev/null', out: '/dev/null' } - end - private def progress_warn(project, cmd, output) @@ -273,18 +154,8 @@ module Backup project_or_wiki.repository.empty? end - def repository_storage_paths_args - Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path } - end - def display_repo_path(project) project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path end - - def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) - Gitlab::GitalyClient.migrate(method, status: status, &block) - rescue GRPC::NotFound, GRPC::BadStatus => e - raise Error, e - end end end -- cgit v1.2.3