diff options
author | Douwe Maan <douwe@gitlab.com> | 2018-05-28 12:58:14 +0300 |
---|---|---|
committer | Douwe Maan <douwe@gitlab.com> | 2018-05-28 12:58:14 +0300 |
commit | 3ab3838346ec6437bbd655417f021f8f0e7a9855 (patch) | |
tree | 3c6545b180d4d7d6dbfadab88c983a2c1d511471 | |
parent | a45b9fc96ca568b19fd9d09a8b40846c2c8b5b36 (diff) | |
parent | b529d04b69c5fd8612bfa614a536d02574cb7f07 (diff) |
Merge branch 'gitaly-backup-restore-delete-repositories' into 'master'
Use Gitaly's DeleteAllRepositories RPC during backup restore
Closes gitaly#1194
See merge request gitlab-org/gitlab-ce!19086
-rw-r--r-- | GITALY_SERVER_VERSION | 2 | ||||
-rw-r--r-- | Gemfile | 2 | ||||
-rw-r--r-- | Gemfile.lock | 4 | ||||
-rw-r--r-- | lib/backup/repository.rb | 56 | ||||
-rw-r--r-- | lib/gitlab/gitaly_client/storage_service.rb | 15 | ||||
-rw-r--r-- | spec/lib/backup/repository_spec.rb | 34 |
6 files changed, 89 insertions, 24 deletions
diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 7bb21aff834..89eba2c5b85 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -0.102.0 +0.103.0 @@ -412,7 +412,7 @@ group :ed25519 do end # Gitaly GRPC client -gem 'gitaly-proto', '~> 0.99.0', require: 'gitaly' +gem 'gitaly-proto', '~> 0.100.0', require: 'gitaly' gem 'grpc', '~> 1.11.0' # Locked until https://github.com/google/protobuf/issues/4210 is closed diff --git a/Gemfile.lock b/Gemfile.lock index 2d928b6a826..b0b7bb537a8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -281,7 +281,7 @@ GEM gettext_i18n_rails (>= 0.7.1) po_to_json (>= 1.0.0) rails (>= 3.2.0) - gitaly-proto (0.99.0) + gitaly-proto (0.100.0) google-protobuf (~> 3.1) grpc (~> 1.10) github-linguist (5.3.3) @@ -1036,7 +1036,7 @@ DEPENDENCIES gettext (~> 3.2.2) gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails_js (~> 1.3) - gitaly-proto (~> 0.99.0) + gitaly-proto (~> 0.100.0) github-linguist (~> 5.3.3) gitlab-flowdock-git-hook (~> 1.0.1) gitlab-gollum-lib (~> 4.2) diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index c3360c391af..84670d6582e 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -73,29 +73,40 @@ module Backup end def prepare_directories - # TODO: Need to find a way to do this for gitaly - # Gitaly discussion issue: https://gitlab.com/gitlab-org/gitaly/issues/1194 - Gitlab.config.repositories.storages.each do |name, repository_storage| - path = repository_storage.legacy_disk_path - next unless File.exist?(path) - - # Move all files in the existing repos directory except . and .. to - # repositories.old.<timestamp> directory - bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) - FileUtils.mkdir_p(bk_repos_path, mode: 0700) - files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] - - begin - FileUtils.mv(files, bk_repos_path) - rescue Errno::EACCES - access_denied_error(path) - rescue Errno::EBUSY - resource_busy_error(path) + delete_all_repositories(name, repository_storage) + end + end + + def delete_all_repositories(name, repository_storage) + gitaly_migrate(:delete_all_repositories) do |is_enabled| + if is_enabled + Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories + else + local_delete_all_repositories(name, repository_storage) end end end + def local_delete_all_repositories(name, repository_storage) + path = repository_storage.legacy_disk_path + return unless File.exist?(path) + + # Move all files in the existing repos directory except . and .. to + # repositories.old.<timestamp> directory + bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) + FileUtils.mkdir_p(bk_repos_path, mode: 0700) + files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] + + begin + FileUtils.mv(files, bk_repos_path) + rescue Errno::EACCES + access_denied_error(path) + rescue Errno::EBUSY + resource_busy_error(path) + end + end + def restore_custom_hooks(project) # TODO: Need to find a way to do this for gitaly # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195 @@ -113,6 +124,7 @@ module Backup def restore prepare_directories gitlab_shell = Gitlab::Shell.new + Project.find_each(batch_size: 1000) do |project| progress.print " * #{project.full_path} ... " path_to_project_bundle = path_to_bundle(project) @@ -121,7 +133,6 @@ module Backup restore_repo_success = nil if File.exist?(path_to_project_bundle) begin - gitlab_shell.remove_repository(project.repository_storage, project.disk_path) if project.repository_exists? project.repository.create_from_bundle path_to_project_bundle restore_repo_success = true rescue => e @@ -146,7 +157,6 @@ module Backup if File.exist?(path_to_wiki_bundle) progress.print " * #{wiki.full_path} ... " begin - gitlab_shell.remove_repository(wiki.repository_storage, wiki.disk_path) if wiki.repository_exists? wiki.repository.create_from_bundle(path_to_wiki_bundle) progress.puts "[DONE]".color(:green) rescue => e @@ -224,5 +234,11 @@ module Backup def display_repo_path(project) project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path end + + def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) + Gitlab::GitalyClient.migrate(method, status: status, &block) + rescue GRPC::NotFound, GRPC::BadStatus => e + raise Error, e + end end end diff --git a/lib/gitlab/gitaly_client/storage_service.rb b/lib/gitlab/gitaly_client/storage_service.rb new file mode 100644 index 00000000000..eb0e910665b --- /dev/null +++ b/lib/gitlab/gitaly_client/storage_service.rb @@ -0,0 +1,15 @@ +module Gitlab + module GitalyClient + class StorageService + def initialize(storage) + @storage = storage + end + + # Delete all repositories in the storage. This is a slow and VERY DESTRUCTIVE operation. + def delete_all_repositories + request = Gitaly::DeleteAllRepositoriesRequest.new(storage_name: @storage) + GitalyClient.call(@storage, :storage_service, :delete_all_repositories, request) + end + end + end +end diff --git a/spec/lib/backup/repository_spec.rb b/spec/lib/backup/repository_spec.rb index a44243ac82d..023bedaaebb 100644 --- a/spec/lib/backup/repository_spec.rb +++ b/spec/lib/backup/repository_spec.rb @@ -71,6 +71,40 @@ describe Backup::Repository do end end + describe '#delete_all_repositories', :seed_helper do + shared_examples('delete_all_repositories') do + before do + allow(FileUtils).to receive(:mkdir_p).and_call_original + allow(FileUtils).to receive(:mv).and_call_original + end + + after(:all) do + ensure_seeds + end + + it 'removes all repositories' do + # Sanity check: there should be something for us to delete + expect(list_repositories).to include(File.join(SEED_STORAGE_PATH, TEST_REPO_PATH)) + + subject.delete_all_repositories('default', Gitlab.config.repositories.storages['default']) + + expect(list_repositories).to be_empty + end + + def list_repositories + Dir[SEED_STORAGE_PATH + '/*.git'] + end + end + + context 'with gitaly' do + it_behaves_like 'delete_all_repositories' + end + + context 'without gitaly', :skip_gitaly_mock do + it_behaves_like 'delete_all_repositories' + end + end + describe '#empty_repo?' do context 'for a wiki' do let(:wiki) { create(:project_wiki) } |