1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
|
package testhelper
import (
"bytes"
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"encoding/pem"
"errors"
"fmt"
"io"
"math/big"
mrand "math/rand"
"net"
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"reflect"
"syscall"
"testing"
"time"
"github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm"
"gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag"
"golang.org/x/exp/slices"
)
const (
// RepositoryAuthToken is the default token used to authenticate
// against other Gitaly servers. It is inject as part of the
// GitalyServers metadata.
RepositoryAuthToken = "the-secret-token"
// DefaultStorageName is the default name of the Gitaly storage.
DefaultStorageName = "default"
)
// IsPraefectEnabled returns whether this testing run is done with Praefect in front of the Gitaly.
func IsPraefectEnabled() bool {
_, enabled := os.LookupEnv("GITALY_TEST_WITH_PRAEFECT")
return enabled
}
// SkipWithPraefect skips the test if it is being executed with Praefect in front
// of the Gitaly.
func SkipWithPraefect(tb testing.TB, reason string) {
if IsPraefectEnabled() {
tb.Skipf(reason)
}
}
// MustReadFile returns the content of a file or fails at once.
func MustReadFile(tb testing.TB, filename string) []byte {
tb.Helper()
content, err := os.ReadFile(filename)
if err != nil {
tb.Fatal(err)
}
return content
}
// GitlabTestStoragePath returns the storage path to the gitlab-test repo.
func GitlabTestStoragePath() string {
if testDirectory == "" {
panic("you must call testhelper.Configure() before GitlabTestStoragePath()")
}
return filepath.Join(testDirectory, "storage")
}
// MustRunCommand runs a command with an optional standard input and returns the standard output, or fails.
func MustRunCommand(tb testing.TB, stdin io.Reader, name string, args ...string) []byte {
tb.Helper()
if filepath.Base(name) == "git" {
require.Fail(tb, "Please use gittest.Exec or gittest.ExecStream to run git commands.")
}
cmd := exec.Command(name, args...)
if stdin != nil {
cmd.Stdin = stdin
}
output, err := cmd.Output()
if err != nil {
stderr := err.(*exec.ExitError).Stderr
require.NoError(tb, err, "%s %s: %s", name, args, stderr)
}
return output
}
// MustClose calls Close() on the Closer and fails the test in case it returns
// an error. This function is useful when closing via `defer`, as a simple
// `defer require.NoError(t, closer.Close())` would cause `closer.Close()` to
// be executed early already.
func MustClose(tb testing.TB, closer io.Closer) {
require.NoError(tb, closer.Close())
}
// Server is an interface for a server that can serve requests on a specific listener. This
// interface is used by the MustServe helper function.
type Server interface {
Serve(net.Listener) error
}
// MustServe starts to serve the given server with the listener. This function asserts that the
// server was able to successfully serve and is useful in contexts where one wants to simply spawn a
// server in a Goroutine.
func MustServe(tb testing.TB, server Server, listener net.Listener) {
tb.Helper()
// `http.Server.Serve()` is expected to return `http.ErrServerClosed`, so we special-case
// this error here.
if err := server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) {
require.NoError(tb, err)
}
}
// CopyFile copies a file at the path src to a file at the path dst
func CopyFile(tb testing.TB, src, dst string) {
fsrc, err := os.Open(src)
require.NoError(tb, err)
defer MustClose(tb, fsrc)
fdst, err := os.Create(dst)
require.NoError(tb, err)
defer MustClose(tb, fdst)
_, err = io.Copy(fdst, fsrc)
require.NoError(tb, err)
}
// GetTemporaryGitalySocketFileName will return a unique, useable socket file name
func GetTemporaryGitalySocketFileName(tb testing.TB) string {
require.NotEmpty(tb, testDirectory, "you must call testhelper.Configure() before GetTemporaryGitalySocketFileName()")
tmpfile, err := os.CreateTemp(testDirectory, "gitaly.socket.")
require.NoError(tb, err)
name := tmpfile.Name()
require.NoError(tb, tmpfile.Close())
require.NoError(tb, os.Remove(name))
return name
}
// GetLocalhostListener listens on the next available TCP port and returns
// the listener and the localhost address (host:port) string.
func GetLocalhostListener(tb testing.TB) (net.Listener, string) {
l, err := net.Listen("tcp", "localhost:0")
require.NoError(tb, err)
addr := fmt.Sprintf("localhost:%d", l.Addr().(*net.TCPAddr).Port)
return l, addr
}
// ContextOpt returns a new context instance with the new additions to it.
type ContextOpt func(context.Context) context.Context
// ContextWithLogger allows to inject provided logger into the context.
func ContextWithLogger(logger *log.Entry) ContextOpt {
return func(ctx context.Context) context.Context {
return ctxlogrus.ToContext(ctx, logger)
}
}
// Context returns that gets canceled at the end of the test.
func Context(tb testing.TB, opts ...ContextOpt) context.Context {
ctx, cancel := context.WithCancel(ContextWithoutCancel(opts...))
tb.Cleanup(cancel)
return ctx
}
// ContextWithoutCancel returns a non-cancellable context.
func ContextWithoutCancel(opts ...ContextOpt) context.Context {
ctx := context.Background()
t := time.Now()
rnd := mrand.New(mrand.NewSource(t.Unix() + int64(t.Nanosecond())))
// Enable use of explicit feature flags. Each feature flag which is checked must have been
// explicitly injected into the context, or otherwise we panic. This is a sanity check to
// verify that all feature flags we introduce are tested both with the flag enabled and
// with the flag disabled.
ctx = featureflag.ContextWithExplicitFeatureFlags(ctx)
// There are some feature flags we need to enable in this function because they end up very
// deep in the call stack, so almost every test function would have to inject it into its
// context. The values of these flags should be randomized to increase the test coverage.
ctx = featureflag.ContextWithFeatureFlag(ctx, featureflag.RunCommandsInCGroup, true)
// Randomly enable the use of the catfile cache in localrepo.ReadObject.
ctx = featureflag.ContextWithFeatureFlag(ctx, featureflag.LocalrepoReadObjectCached, rnd.Int()%2 == 0)
// Randomly enable the use of the catfile cache in localrepo.ReadObject.
ctx = featureflag.ContextWithFeatureFlag(ctx, featureflag.FixRoutingWithAdditionalRepository, rnd.Int()%2 == 0)
for _, opt := range opts {
ctx = opt(ctx)
}
return ctx
}
// CreateGlobalDirectory creates a directory in the test directory that is shared across all
// between all tests.
func CreateGlobalDirectory(tb testing.TB, name string) string {
require.NotEmpty(tb, testDirectory, "global temporary directory does not exist")
path := filepath.Join(testDirectory, name)
require.NoError(tb, os.Mkdir(path, perm.PublicDir))
return path
}
// TempDir is a wrapper around os.MkdirTemp that provides a cleanup function.
func TempDir(tb testing.TB) string {
if testDirectory == "" {
panic("you must call testhelper.Configure() before TempDir()")
}
tmpDir, err := os.MkdirTemp(testDirectory, "")
require.NoError(tb, err)
tb.Cleanup(func() {
require.NoError(tb, os.RemoveAll(tmpDir))
})
return tmpDir
}
// Cleanup functions should be called in a defer statement
// immediately after they are returned from a test helper
type Cleanup func()
// WriteExecutable ensures that the parent directory exists, and writes an executable with provided
// content. The executable must not exist previous to writing it. Returns the path of the written
// executable.
func WriteExecutable(tb testing.TB, path string, content []byte) string {
dir := filepath.Dir(path)
require.NoError(tb, os.MkdirAll(dir, perm.SharedDir))
tb.Cleanup(func() {
assert.NoError(tb, os.RemoveAll(dir))
})
// Open the file descriptor and write the script into it. It may happen that any other
// Goroutine forks while we hold this writeable file descriptor, and as a consequence we
// leak it into the other process. Subsequently, even if we close the file descriptor
// ourselves this other process may still hold on to the writeable file descriptor. The
// result is that calls to execve(3P) on our just-written file will fail with ETXTBSY,
// which is raised when trying to execute a file which is still open to be written to.
//
// We thus need to perform file locking to ensure that all writeable references to this
// file have been closed before returning.
executable, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm.SharedExecutable)
require.NoError(tb, err)
_, err = io.Copy(executable, bytes.NewReader(content))
require.NoError(tb, err)
// We now lock the file descriptor for exclusive access. If there was a forked process
// holding the writeable file descriptor at this point in time, then it would refer to the
// same file descriptor and thus be locked for exclusive access, as well. If we fork after
// creating the lock and before closing the writeable file descriptor, then the dup'd file
// descriptor would automatically inherit the lock.
//
// No matter what, after this step any file descriptors referring to this writeable file
// descriptor will be exclusively locked.
require.NoError(tb, syscall.Flock(int(executable.Fd()), syscall.LOCK_EX))
// We now close this file. The file will be automatically unlocked as soon as all
// references to this file descriptor are closed.
MustClose(tb, executable)
// We now open the file again, but this time only for reading.
executable, err = os.Open(path)
require.NoError(tb, err)
// And this time, we try to acquire a shared lock on this file. This call will block until
// the exclusive file lock on the above writeable file descriptor has been dropped. So as
// soon as we're able to acquire the lock we know that there cannot be any open writeable
// file descriptors for this file anymore, and thus we won't get ETXTBSY anymore.
require.NoError(tb, syscall.Flock(int(executable.Fd()), syscall.LOCK_SH))
MustClose(tb, executable)
return path
}
// Unsetenv unsets an environment variable. The variable will be restored after the test has
// finished.
func Unsetenv(tb testing.TB, key string) {
tb.Helper()
// We're first using `tb.Setenv()` here due to two reasons: first, it will automitcally
// handle restoring the environment variable for us after the test has finished. And second,
// it performs a check whether we're running with `tb.Parallel()`.
tb.Setenv(key, "")
// And now we can unset the environment variable given that we know we're not running in a
// parallel test and where the cleanup function has been installed.
require.NoError(tb, os.Unsetenv(key))
}
// GenerateCerts creates a certificate that can be used to establish TLS protected TCP connection.
// It returns paths to the file with the certificate and its private key.
func GenerateCerts(tb testing.TB) (string, string) {
tb.Helper()
rootCert := &x509.Certificate{
SerialNumber: big.NewInt(1),
NotBefore: time.Now(),
NotAfter: time.Now().AddDate(0, 0, 1),
BasicConstraintsValid: true,
IsCA: true,
KeyUsage: x509.KeyUsageCertSign,
}
rootKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
require.NoError(tb, err)
rootBytes, err := x509.CreateCertificate(rand.Reader, rootCert, rootCert, &rootKey.PublicKey, rootKey)
require.NoError(tb, err)
entityKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
require.NoError(tb, err)
entityCert := &x509.Certificate{
SerialNumber: big.NewInt(2),
NotBefore: time.Now(),
NotAfter: time.Now().AddDate(0, 0, 1),
IPAddresses: []net.IP{net.ParseIP("0.0.0.0"), net.ParseIP("127.0.0.1"), net.ParseIP("::1"), net.ParseIP("::")},
DNSNames: []string{"localhost"},
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{
x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth,
},
}
entityBytes, err := x509.CreateCertificate(rand.Reader, entityCert, rootCert, &entityKey.PublicKey, rootKey)
require.NoError(tb, err)
certFile, err := os.CreateTemp(testDirectory, "")
require.NoError(tb, err)
defer MustClose(tb, certFile)
tb.Cleanup(func() {
require.NoError(tb, os.Remove(certFile.Name()))
})
// create chained PEM file with CA and entity cert
for _, cert := range [][]byte{entityBytes, rootBytes} {
require.NoError(tb,
pem.Encode(certFile, &pem.Block{
Type: "CERTIFICATE",
Bytes: cert,
}),
)
}
keyFile, err := os.CreateTemp(testDirectory, "")
require.NoError(tb, err)
defer MustClose(tb, keyFile)
tb.Cleanup(func() {
require.NoError(tb, os.Remove(keyFile.Name()))
})
entityKeyBytes, err := x509.MarshalECPrivateKey(entityKey)
require.NoError(tb, err)
require.NoError(tb,
pem.Encode(keyFile, &pem.Block{
Type: "ECDSA PRIVATE KEY",
Bytes: entityKeyBytes,
}),
)
return certFile.Name(), keyFile.Name()
}
// GitalyOrPraefect returns either the Gitaly- or Praefect-specific object depending on whether
// tests are running with Praefect as a proxy or not.
func GitalyOrPraefect[Type any](gitaly, praefect Type) Type {
if IsPraefectEnabled() {
return praefect
}
return gitaly
}
// SkipQuarantinedTest skips the test if the test name has been specified as
// quarantined. If no test names are provided the test is always skipped.
func SkipQuarantinedTest(t *testing.T, issue string, tests ...string) {
if issue == "" {
panic("issue not specified")
}
if len(tests) == 0 || slices.Contains(tests, t.Name()) {
t.Skipf("This test has been quarantined. Please see %s for more information.", issue)
}
}
// pkgPath is used to determine the package path using reflection.
type pkgPath struct{}
// PkgPath returns the gitaly module package path, including major version
// number. paths will be path joined to the returned package path.
func PkgPath(paths ...string) string {
internalPkgPath := path.Dir(reflect.TypeOf(pkgPath{}).PkgPath())
rootPkgPath := path.Dir(internalPkgPath)
return path.Join(append([]string{rootPkgPath}, paths...)...)
}
|