diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-07-30 09:09:55 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-07-30 09:09:55 +0300 |
commit | 0629b103246d6c8b24e753f62ccfc8649df2c315 (patch) | |
tree | 7a68ccda8a1fbd1728bdbf97f636801843a7bdf1 /qa | |
parent | 2dfc1088178535959894aff4e80edb8936b9dedf (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'qa')
-rw-r--r-- | qa/qa/runtime/api/client.rb | 2 | ||||
-rw-r--r-- | qa/qa/runtime/api/repository_storage_moves.rb | 16 | ||||
-rw-r--r-- | qa/qa/runtime/env.rb | 4 | ||||
-rw-r--r-- | qa/qa/service/praefect_manager.rb | 50 | ||||
-rw-r--r-- | qa/qa/specs/features/api/3_create/repository/changing_repository_storage_spec.rb | 24 | ||||
-rw-r--r-- | qa/qa/support/api.rb | 24 |
6 files changed, 104 insertions, 16 deletions
diff --git a/qa/qa/runtime/api/client.rb b/qa/qa/runtime/api/client.rb index d29571df981..e4de033c309 100644 --- a/qa/qa/runtime/api/client.rb +++ b/qa/qa/runtime/api/client.rb @@ -6,6 +6,8 @@ module QA class Client attr_reader :address, :user + AuthorizationError = Class.new(RuntimeError) + def initialize(address = :gitlab, personal_access_token: nil, is_new_session: true, user: nil, ip_limits: false) @address = address @personal_access_token = personal_access_token diff --git a/qa/qa/runtime/api/repository_storage_moves.rb b/qa/qa/runtime/api/repository_storage_moves.rb index c94a693289f..d0211d3f66d 100644 --- a/qa/qa/runtime/api/repository_storage_moves.rb +++ b/qa/qa/runtime/api/repository_storage_moves.rb @@ -10,16 +10,24 @@ module QA RepositoryStorageMovesError = Class.new(RuntimeError) def has_status?(project, status, destination_storage = Env.additional_repository_storage) - all.any? do |move| - move[:project][:path_with_namespace] == project.path_with_namespace && + find_any do |move| + next unless move[:project][:path_with_namespace] == project.path_with_namespace + + QA::Runtime::Logger.debug("Move data: #{move}") + move[:state] == status && move[:destination_storage_name] == destination_storage end end - def all + def find_any Logger.debug('Getting repository storage moves') - parse_body(get(Request.new(api_client, '/project_repository_storage_moves').url)) + + Support::Waiter.wait_until do + with_paginated_response_body(Request.new(api_client, '/project_repository_storage_moves', per_page: '100').url) do |page| + break true if page.any? { |item| yield item } + end + end end private diff --git a/qa/qa/runtime/env.rb b/qa/qa/runtime/env.rb index f1c57fbfc73..cbfce95d409 100644 --- a/qa/qa/runtime/env.rb +++ b/qa/qa/runtime/env.rb @@ -61,6 +61,10 @@ module QA ENV['QA_ADDITIONAL_REPOSITORY_STORAGE'] end + def non_cluster_repository_storage + ENV['QA_GITALY_NON_CLUSTER_STORAGE'] || 'gitaly' + end + def praefect_repository_storage ENV['QA_PRAEFECT_REPOSITORY_STORAGE'] end diff --git a/qa/qa/service/praefect_manager.rb b/qa/qa/service/praefect_manager.rb index a0433689e99..00796ab1502 100644 --- a/qa/qa/service/praefect_manager.rb +++ b/qa/qa/service/praefect_manager.rb @@ -5,6 +5,8 @@ module QA class PraefectManager include Service::Shellout + attr_accessor :gitlab + def initialize @gitlab = 'gitlab-gitaly-ha' @praefect = 'praefect' @@ -100,6 +102,14 @@ module QA enable_writes end + def verify_storage_move(source_storage, destination_storage) + return if QA::Runtime::Env.dot_com? + + repo_path = verify_storage_move_from_gitaly(source_storage[:name]) + + destination_storage[:type] == :praefect ? verify_storage_move_to_praefect(repo_path, destination_storage[:name]) : verify_storage_move_to_gitaly(repo_path, destination_storage[:name]) + end + def wait_for_praefect wait_until_shell_command_matches( "docker exec #{@praefect} bash -c 'cat /var/log/gitlab/praefect/current'", @@ -163,16 +173,48 @@ module QA private - def wait_until_shell_command(cmd) - Support::Waiter.wait_until do + def verify_storage_move_from_gitaly(storage) + wait_until_shell_command("docker exec #{@gitlab} bash -c 'tail -n 50 /var/log/gitlab/gitaly/current'") do |line| + log = JSON.parse(line) + + break log['grpc.request.repoPath'] if log['grpc.method'] == 'RenameRepository' && log['grpc.request.repoStorage'] == storage && !log['grpc.request.repoPath'].include?('wiki') + rescue JSON::ParserError + # Ignore lines that can't be parsed as JSON + end + end + + def verify_storage_move_to_praefect(repo_path, virtual_storage) + wait_until_shell_command("docker exec #{@gitlab} bash -c 'tail -n 50 /var/log/gitlab/praefect/current'") do |line| + log = JSON.parse(line) + + log['grpc.method'] == 'ReplicateRepository' && log['virtual_storage'] == virtual_storage && log['relative_path'] == repo_path + rescue JSON::ParserError + # Ignore lines that can't be parsed as JSON + end + end + + def verify_storage_move_to_gitaly(repo_path, storage) + wait_until_shell_command("docker exec #{@gitlab} bash -c 'tail -n 50 /var/log/gitlab/gitaly/current'") do |line| + log = JSON.parse(line) + + log['grpc.method'] == 'ReplicateRepository' && log['grpc.request.repoStorage'] == storage && log['grpc.request.repoPath'] == repo_path + rescue JSON::ParserError + # Ignore lines that can't be parsed as JSON + end + end + + def wait_until_shell_command(cmd, **kwargs) + sleep_interval = kwargs.delete(:sleep_interval) || 1 + + Support::Waiter.wait_until(sleep_interval: sleep_interval, **kwargs) do shell cmd do |line| break true if yield line end end end - def wait_until_shell_command_matches(cmd, regex) - wait_until_shell_command(cmd) do |line| + def wait_until_shell_command_matches(cmd, regex, **kwargs) + wait_until_shell_command(cmd, kwargs) do |line| QA::Runtime::Logger.info(line.chomp) line =~ regex diff --git a/qa/qa/specs/features/api/3_create/repository/changing_repository_storage_spec.rb b/qa/qa/specs/features/api/3_create/repository/changing_repository_storage_spec.rb index 11e7db5b097..2040d340071 100644 --- a/qa/qa/specs/features/api/3_create/repository/changing_repository_storage_spec.rb +++ b/qa/qa/specs/features/api/3_create/repository/changing_repository_storage_spec.rb @@ -3,13 +3,14 @@ module QA RSpec.describe 'Create' do describe 'Changing Gitaly repository storage', :requires_admin do + praefect_manager = Service::PraefectManager.new + praefect_manager.gitlab = 'gitlab' + shared_examples 'repository storage move' do it 'confirms a `finished` status after moving project repository storage' do expect(project).to have_file('README.md') - - project.change_repository_storage(destination_storage) - - expect(Runtime::API::RepositoryStorageMoves).to have_status(project, 'finished', destination_storage) + expect { project.change_repository_storage(destination_storage[:name]) }.not_to raise_error + expect { praefect_manager.verify_storage_move(source_storage, destination_storage) }.not_to raise_error Resource::Repository::ProjectPush.fabricate! do |push| push.project = project @@ -25,28 +26,35 @@ module QA end context 'when moving from one Gitaly storage to another', :orchestrated, :repository_storage do + let(:source_storage) { { type: :gitaly, name: 'default' } } + let(:destination_storage) { { type: :gitaly, name: QA::Runtime::Env.additional_repository_storage } } + let(:project) do Resource::Project.fabricate_via_api! do |project| project.name = 'repo-storage-move-status' project.initialize_with_readme = true + project.api_client = Runtime::API::Client.as_admin end end - let(:destination_storage) { QA::Runtime::Env.additional_repository_storage } it_behaves_like 'repository storage move' end # Note: This test doesn't have the :orchestrated tag because it runs in the Test::Integration::Praefect # scenario with other tests that aren't considered orchestrated. - context 'when moving from Gitaly to Gitaly Cluster', :requires_praefect, quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/227127', type: :investigating } do + # It also runs on staging using nfs-file07 as non-cluster storage and nfs-file22 as cluster/praefect storage + context 'when moving from Gitaly to Gitaly Cluster', :requires_praefect do + let(:source_storage) { { type: :gitaly, name: QA::Runtime::Env.non_cluster_repository_storage } } + let(:destination_storage) { { type: :praefect, name: QA::Runtime::Env.praefect_repository_storage } } + let(:project) do Resource::Project.fabricate_via_api! do |project| project.name = 'repo-storage-move' project.initialize_with_readme = true - project.repository_storage = 'gitaly' + project.repository_storage = source_storage[:name] + project.api_client = Runtime::API::Client.as_admin end end - let(:destination_storage) { QA::Runtime::Env.praefect_repository_storage } it_behaves_like 'repository storage move' end diff --git a/qa/qa/support/api.rb b/qa/qa/support/api.rb index 3c46c039eae..08faacb6db3 100644 --- a/qa/qa/support/api.rb +++ b/qa/qa/support/api.rb @@ -77,6 +77,30 @@ module QA error.response end + + def with_paginated_response_body(url) + loop do + response = get(url) + + QA::Runtime::Logger.debug("Fetching page #{response.headers[:x_page]} of #{response.headers[:x_total_pages]}...") + + yield parse_body(response) + + next_link = pagination_links(response).find { |link| link[:rel] == 'next' } + break unless next_link + + url = next_link[:url] + end + end + + def pagination_links(response) + response.headers[:link].split(',').map do |link| + match = link.match(/\<(?<url>.*)\>\; rel=\"(?<rel>\w+)\"/) + break nil unless match + + { url: match[:url], rel: match[:rel] } + end.compact + end end end end |