From f6ae451a938a9bf91e6a8c3318aa44beb1a8315a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Rodr=C3=ADguez?= Date: Tue, 23 Oct 2018 19:58:28 -0300 Subject: Fix setup of repository storages for the test environment On the backup_rake_spec we require a second repository storage, which we used to accomplish through a combination of rspec-mocks and special configurations in the Gitaly installation. Since Gitaly now refuses to start with invalid storages, we modified the setup code in https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/22530, but in between tests the storage could be removed from the `tmp/` folder. These changes simplify the approach by making the secondary storage a _real_ storage and having `TestEnv` manage it the same way it does the default storage. --- config/gitlab.yml.example | 3 +++ lib/gitlab/setup_helper.rb | 9 --------- lib/tasks/gitlab/gitaly.rake | 5 +++++ spec/support/helpers/test_env.rb | 9 ++++++++- spec/tasks/gitlab/backup_rake_spec.rb | 38 +---------------------------------- 5 files changed, 17 insertions(+), 47 deletions(-) diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index a4db125f831..f67589f4361 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -772,6 +772,9 @@ test: default: path: tmp/tests/repositories/ gitaly_address: unix:tmp/tests/gitaly/gitaly.socket + secondary: + path: tmp/tests/repositories-secondary/ + gitaly_address: unix:tmp/tests/gitaly/gitaly.socket gitaly: client_path: tmp/tests/gitaly diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 2b7e12639be..7f2d08b0916 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -11,8 +11,6 @@ module Gitlab # only generate a configuration for the most common and simplest case: when # we have exactly one Gitaly process and we are sure it is running locally # because it uses a Unix socket. - # For development and testing purposes, an extra storage is added to gitaly, - # which is not known to Rails, but must be explicitly stubbed. def gitaly_configuration_toml(gitaly_dir, storage_paths, gitaly_ruby: true) storages = [] address = nil @@ -31,13 +29,6 @@ module Gitlab storages << { name: key, path: storage_paths[key] } end - if Rails.env.test? - storage_path = Rails.root.join('tmp', 'tests', 'second_storage').to_s - - FileUtils.mkdir(storage_path) unless File.exist?(storage_path) - storages << { name: 'test_second_storage', path: storage_path } - end - config = { socket_path: address.sub(/\Aunix:/, ''), storage: storages } config[:auth] = { token: 'secret' } if Rails.env.test? config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 80de3d2ef51..1ea17d7fe0c 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -27,6 +27,11 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]") end storage_paths = { 'default' => args.storage_path } + + if Rails.env.test? + storage_paths['secondary'] = 'tmp/tests/repositories-secondary/' + end + Gitlab::SetupHelper.create_gitaly_configuration(args.dir, storage_paths) Dir.chdir(args.dir) do # In CI we run scripts/gitaly-test-build instead of this command diff --git a/spec/support/helpers/test_env.rb b/spec/support/helpers/test_env.rb index 71287f28171..7c60a3b68d0 100644 --- a/spec/support/helpers/test_env.rb +++ b/spec/support/helpers/test_env.rb @@ -70,6 +70,7 @@ module TestEnv TMP_TEST_PATH = Rails.root.join('tmp', 'tests', '**') REPOS_STORAGE = 'default'.freeze + SECONDARY_REPOS_STORAGE = 'secondary'.freeze # Test environment # @@ -119,6 +120,7 @@ module TestEnv end FileUtils.mkdir_p(repos_path) + FileUtils.mkdir_p(secondary_repos_path) FileUtils.mkdir_p(backup_path) FileUtils.mkdir_p(pages_path) FileUtils.mkdir_p(artifacts_path) @@ -232,7 +234,8 @@ module TestEnv end def copy_repo(project, bare_repo:, refs:) - target_repo_path = File.expand_path(repos_path + "/#{project.disk_path}.git") + storage_path = project.repository_storage == 'secondary' ? secondary_repos_path : repos_path + target_repo_path = File.expand_path(storage_path + "/#{project.disk_path}.git") FileUtils.mkdir_p(target_repo_path) FileUtils.cp_r("#{File.expand_path(bare_repo)}/.", target_repo_path) @@ -252,6 +255,10 @@ module TestEnv @repos_path ||= Gitlab.config.repositories.storages[REPOS_STORAGE].legacy_disk_path end + def secondary_repos_path + @secondary_repos_path ||= Gitlab.config.repositories.storages[SECONDARY_REPOS_STORAGE].legacy_disk_path + end + def backup_path Gitlab.config.backup.path end diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index 3ba6caf1337..9c9189f004e 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -231,50 +231,14 @@ describe 'gitlab:app namespace rake task' do end context 'multiple repository storages' do - let(:test_second_storage) do - Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/custom_storage')) - end - let(:storages) do - { - 'default' => Gitlab.config.repositories.storages.default, - 'test_second_storage' => test_second_storage - } - end - - before(:all) do - @default_storage_hash = Gitlab.config.repositories.storages.default.to_h - end - before do # We only need a backup of the repositories for this test stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry') - - allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) - - # Avoid asking gitaly about the root ref (which will fail beacuse of the - # mocked storages) - allow_any_instance_of(Repository).to receive(:empty?).and_return(false) - end - - after do - FileUtils.rm_rf(Settings.absolute('tmp/tests/custom_storage')) end it 'includes repositories in all repository storages' do project_a = create(:project, :repository) - project_b = create(:project, :repository, repository_storage: 'test_second_storage') - - b_storage_dir = File.join(Settings.absolute('tmp/tests/custom_storage'), File.dirname(project_b.disk_path)) - - FileUtils.mkdir_p(b_storage_dir) - - # Even when overriding the storage, we have to move it there, so it exists - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - FileUtils.mv( - File.join(Settings.absolute(storages['default'].legacy_disk_path), project_b.repository.disk_path + '.git'), - Rails.root.join(storages['test_second_storage'].legacy_disk_path, project_b.repository.disk_path + '.git') - ) - end + project_b = create(:project, :repository, repository_storage: 'secondary') expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout -- cgit v1.2.3