blob: 1eb077fe6ca43f0aeb13160a4ed9785915153f42 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
|
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe 'Load balancer behavior with errors inside a transaction', :redis, :delete do
include StubENV
let(:model) { ActiveRecord::Base }
let(:db_host) { model.connection_pool.db_config.host }
let(:test_table_name) { '_test_foo' }
before do
# Patch in our load balancer config, simply pointing at the test database twice
allow(Gitlab::Database::LoadBalancing::Configuration).to receive(:for_model).with(model) do |base_model|
Gitlab::Database::LoadBalancing::Configuration.new(base_model, [db_host, db_host])
end
Gitlab::Database::LoadBalancing::Setup.new(model).setup
model.connection.execute(<<~SQL)
CREATE TABLE IF NOT EXISTS #{test_table_name} (id SERIAL PRIMARY KEY, value INTEGER)
SQL
# The load balancer sleeps between attempts to retry a query.
# Mocking the sleep call significantly reduces the runtime of this spec file.
allow(model.connection.load_balancer).to receive(:sleep)
end
after do
model.connection.execute(<<~SQL)
DROP TABLE IF EXISTS #{test_table_name}
SQL
# reset load balancing to original state
allow(Gitlab::Database::LoadBalancing::Configuration).to receive(:for_model).and_call_original
Gitlab::Database::LoadBalancing::Setup.new(model).setup
end
def execute(conn)
conn.execute("INSERT INTO #{test_table_name} (value) VALUES (1)")
backend_pid = conn.execute("SELECT pg_backend_pid() AS pid").to_a.first['pid']
# This will result in a PG error, which is not raised.
# Instead, we retry the statement on a fresh connection (where the pid is different and it does nothing)
# and the load balancer continues with a fresh connection and no transaction if a transaction was open previously
conn.execute(<<~SQL)
SELECT CASE
WHEN pg_backend_pid() = #{backend_pid} THEN
pg_terminate_backend(#{backend_pid})
END
SQL
# This statement will execute on a new connection, and violate transaction semantics
# if we were in a transaction before
conn.execute("INSERT INTO #{test_table_name} (value) VALUES (2)")
end
context 'with the PREVENT_LOAD_BALANCER_RETRIES_IN_TRANSACTION environment variable not set' do
it 'logs a warning when violating transaction semantics with writes' do
conn = model.connection
expect(::Gitlab::Database::LoadBalancing::Logger).to receive(:warn).with(hash_including(event: :transaction_leak))
expect(::Gitlab::Database::LoadBalancing::Logger).to receive(:warn).with(hash_including(event: :read_write_retry))
conn.transaction do
expect(conn).to be_transaction_open
execute(conn)
expect(conn).not_to be_transaction_open
end
values = conn.execute("SELECT value FROM #{test_table_name}").to_a.map { |row| row['value'] }
expect(values).to contain_exactly(2) # Does not include 1 because the transaction was aborted and leaked
end
it 'does not log a warning when no transaction is open to be leaked' do
conn = model.connection
expect(::Gitlab::Database::LoadBalancing::Logger)
.not_to receive(:warn).with(hash_including(event: :transaction_leak))
expect(::Gitlab::Database::LoadBalancing::Logger)
.to receive(:warn).with(hash_including(event: :read_write_retry))
expect(conn).not_to be_transaction_open
execute(conn)
expect(conn).not_to be_transaction_open
values = conn.execute("SELECT value FROM #{test_table_name}").to_a.map { |row| row['value'] }
expect(values).to contain_exactly(1, 2) # Includes both rows because there was no transaction to roll back
end
end
context 'with the PREVENT_LOAD_BALANCER_RETRIES_IN_TRANSACTION environment variable set' do
before do
stub_env('PREVENT_LOAD_BALANCER_RETRIES_IN_TRANSACTION' => '1')
end
it 'raises an exception when a retry would occur during a transaction' do
expect(::Gitlab::Database::LoadBalancing::Logger)
.not_to receive(:warn).with(hash_including(event: :transaction_leak))
expect do
model.transaction do
execute(model.connection)
end
end.to raise_error(ActiveRecord::StatementInvalid) { |e| expect(e.cause).to be_a(PG::ConnectionBad) }
end
it 'retries when not in a transaction' do
expect(::Gitlab::Database::LoadBalancing::Logger)
.not_to receive(:warn).with(hash_including(event: :transaction_leak))
expect(::Gitlab::Database::LoadBalancing::Logger)
.to receive(:warn).with(hash_including(event: :read_write_retry))
expect { execute(model.connection) }.not_to raise_error
end
end
end
|
