1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
|
package tracker
import (
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitaly/internal/testhelper"
)
func TestErrorTracker_IncrErrors(t *testing.T) {
ctx, cancel := testhelper.Context()
defer cancel()
writeThreshold, readThreshold := 10, 10
errors, err := newErrors(ctx, time.Second, uint32(readThreshold), uint32(writeThreshold))
require.NoError(t, err)
node := "backend-node-1"
assert.False(t, errors.WriteThresholdReached(node))
assert.False(t, errors.ReadThresholdReached(node))
for i := 0; i < writeThreshold; i++ {
errors.IncrWriteErr(node)
}
assert.True(t, errors.WriteThresholdReached(node))
for i := 0; i < readThreshold; i++ {
errors.IncrReadErr(node)
}
assert.True(t, errors.ReadThresholdReached(node))
// use negative value for window so we are ensured to clear all of the errors in the queue
errors, err = newErrors(ctx, -time.Second, uint32(readThreshold), uint32(writeThreshold))
require.NoError(t, err)
errors.clear()
assert.False(t, errors.WriteThresholdReached(node))
assert.False(t, errors.ReadThresholdReached(node))
}
func TestErrorTracker_Concurrency(t *testing.T) {
ctx, cancel := testhelper.Context()
defer cancel()
readAndWriteThreshold := 10
errors, err := newErrors(ctx, 1*time.Second, uint32(readAndWriteThreshold), uint32(readAndWriteThreshold))
require.NoError(t, err)
node := "backend-node-1"
assert.False(t, errors.WriteThresholdReached(node))
assert.False(t, errors.ReadThresholdReached(node))
var g sync.WaitGroup
for i := 0; i < readAndWriteThreshold; i++ {
g.Add(1)
go func() {
errors.IncrWriteErr(node)
errors.IncrReadErr(node)
errors.ReadThresholdReached(node)
errors.WriteThresholdReached(node)
g.Done()
}()
}
g.Wait()
}
func TestErrorTracker_ClearErrors(t *testing.T) {
ctx, cancel := testhelper.Context()
defer cancel()
writeThreshold, readThreshold := 10, 10
errors, err := newErrors(ctx, time.Second, uint32(readThreshold), uint32(writeThreshold))
require.NoError(t, err)
node := "backend-node-1"
errors.IncrWriteErr(node)
errors.IncrReadErr(node)
clearBeforeNow := time.Now()
errors.olderThan = func() time.Time {
return clearBeforeNow
}
errors.IncrWriteErr(node)
errors.IncrReadErr(node)
errors.clear()
assert.Len(t, errors.readErrors[node], 1, "clear should only have cleared the read error older than the time specifiied")
assert.Len(t, errors.writeErrors[node], 1, "clear should only have cleared the write error older than the time specifiied")
}
func TestErrorTracker_Expired(t *testing.T) {
ctx, cancel := testhelper.Context()
defer cancel()
threshold := 10
errors, err := newErrors(ctx, 10*time.Second, uint32(threshold), uint32(threshold))
require.NoError(t, err)
node := "node"
for i := 0; i < threshold; i++ {
errors.IncrWriteErr(node)
errors.IncrReadErr(node)
}
assert.True(t, errors.ReadThresholdReached(node))
assert.True(t, errors.WriteThresholdReached(node))
cancel()
assert.False(t, errors.ReadThresholdReached(node))
assert.False(t, errors.WriteThresholdReached(node))
for i := 0; i < threshold; i++ {
errors.IncrWriteErr(node)
errors.IncrReadErr(node)
}
assert.False(t, errors.ReadThresholdReached(node))
assert.False(t, errors.WriteThresholdReached(node))
}
|
