blob: 0b4bdf550f83dd2f040b81c7b98d2bc898960acd (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
|
# frozen_string_literal: true
module QA
RSpec.describe 'Systems' do
describe 'Gitaly backend node recovery', :orchestrated, :gitaly_cluster, :skip_live_env do
let(:praefect_manager) { Service::PraefectManager.new }
let(:project) do
Resource::Project.fabricate! do |project|
project.name = "gitaly_cluster"
project.initialize_with_readme = true
end
end
before do
# Reset the cluster in case previous tests left it in a bad state
praefect_manager.start_all_nodes
end
after do
# Leave the cluster in a suitable state for subsequent tests
praefect_manager.start_all_nodes
end
it 'recovers from dataloss', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347832' do
# Create a new project with a commit and wait for it to replicate
praefect_manager.wait_for_replication(project.id)
# Stop the primary node to trigger failover, and then wait
# for Gitaly to be ready for writes again
praefect_manager.stop_primary_node
praefect_manager.wait_for_primary_node_health_check_failure
# Push a commit to the new primary
Resource::Repository::ProjectPush.fabricate! do |push|
push.project = project
push.new_branch = false
push.commit_message = 'pushed after failover'
push.file_name = 'new_file'
push.file_content = 'new file'
end
# Confirm that the commit is waiting to be replicated
expect(praefect_manager).to be_replication_pending
# Start the old primary node again
praefect_manager.start_primary_node
praefect_manager.wait_for_health_check_all_nodes
# Wait for automatic replication
praefect_manager.wait_for_replication(project.id)
# Force switch to the old primary node
# This ensures that the commit was replicated
praefect_manager.stop_secondary_node
praefect_manager.stop_tertiary_node
# Confirm that both commits are available
expect(project.commits.map { |commit| commit[:message].chomp })
.to include("Initial commit").and include("pushed after failover")
end
end
end
end
|
