1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
|
package housekeeping
import (
"context"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"time"
"github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitaly/v14/internal/git"
"gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo"
"gitlab.com/gitlab-org/gitaly/v14/internal/helper"
"google.golang.org/grpc/codes"
)
const (
emptyRefsGracePeriod = 24 * time.Hour
deleteTempFilesOlderThanDuration = 7 * 24 * time.Hour
brokenRefsGracePeriod = 24 * time.Hour
minimumDirPerm = 0700
lockfileGracePeriod = 15 * time.Minute
referenceLockfileGracePeriod = 1 * time.Hour
packedRefsLockGracePeriod = 1 * time.Hour
packedRefsNewGracePeriod = 15 * time.Minute
)
var (
lockfiles = []string{
"config.lock",
"HEAD.lock",
"objects/info/commit-graphs/commit-graph-chain.lock",
}
optimizeEmptyDirRemovalTotals = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: "gitaly",
Subsystem: "repository",
Name: "optimizerepository_empty_dir_removal_total",
Help: "Total number of empty directories removed by OptimizeRepository RPC",
},
)
)
func init() {
prometheus.MustRegister(optimizeEmptyDirRemovalTotals)
}
type staleFileFinderFn func(context.Context, string) ([]string, error)
// Perform will perform housekeeping duties on a repository
func Perform(ctx context.Context, repo *localrepo.Repo) error {
repoPath, err := repo.Path()
if err != nil {
myLogger(ctx).WithError(err).Warn("housekeeping failed to get repo path")
if helper.GrpcCode(err) == codes.NotFound {
return nil
}
return fmt.Errorf("housekeeping failed to get repo path: %w", err)
}
logEntry := myLogger(ctx)
var filesToPrune []string
for field, staleFileFinder := range map[string]staleFileFinderFn{
"objects": findTemporaryObjects,
"locks": findStaleLockfiles,
"refs": findBrokenLooseReferences,
"reflocks": findStaleReferenceLocks,
"packedrefslock": findPackedRefsLock,
"packedrefsnew": findPackedRefsNew,
} {
staleFiles, err := staleFileFinder(ctx, repoPath)
if err != nil {
return fmt.Errorf("housekeeping failed to find %s: %w", field, err)
}
filesToPrune = append(filesToPrune, staleFiles...)
logEntry = logEntry.WithField(field, len(staleFiles))
}
unremovableFiles := 0
for _, path := range filesToPrune {
if err := os.Remove(path); err != nil {
if os.IsNotExist(err) {
continue
}
unremovableFiles++
// We cannot use `logEntry` here as it's already seeded
// with the statistics fields.
myLogger(ctx).WithError(err).WithField("path", path).Warn("unable to remove stale file")
}
}
if len(filesToPrune) > 0 {
logEntry.WithField("failures", unremovableFiles).Info("removed files")
}
if err := removeRefEmptyDirs(ctx, repo); err != nil {
return fmt.Errorf("housekeeping could not remove empty refs: %w", err)
}
// TODO: https://gitlab.com/gitlab-org/gitaly/-/issues/3138
// This is a temporary code and needs to be removed once it will be run on all repositories at least once.
if err := unsetAllConfigsByRegexp(ctx, repo, "^http\\..+\\.extraHeader$"); err != nil {
return fmt.Errorf("housekeeping could not unset extreHeaders: %w", err)
}
return nil
}
// findStaleFiles determines whether any of the given files rooted at repoPath
// are stale or not. A file is considered stale if it exists and if it has not
// been modified during the gracePeriod. A nonexistent file is not considered
// to be a stale file and will not cause an error.
func findStaleFiles(repoPath string, gracePeriod time.Duration, files ...string) ([]string, error) {
var staleFiles []string
for _, file := range files {
path := filepath.Join(repoPath, file)
fileInfo, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
continue
}
return nil, err
}
if time.Since(fileInfo.ModTime()) < gracePeriod {
continue
}
staleFiles = append(staleFiles, path)
}
return staleFiles, nil
}
// findStaleLockfiles finds a subset of lockfiles which may be created by git
// commands. We're quite conservative with what we're removing, we certaintly
// don't just scan the repo for `*.lock` files. Instead, we only remove a known
// set of lockfiles which have caused problems in the past.
func findStaleLockfiles(ctx context.Context, repoPath string) ([]string, error) {
return findStaleFiles(repoPath, lockfileGracePeriod, lockfiles...)
}
func findTemporaryObjects(ctx context.Context, repoPath string) ([]string, error) {
var temporaryObjects []string
logger := myLogger(ctx)
err := filepath.Walk(filepath.Join(repoPath, "objects"), func(path string, info os.FileInfo, err error) error {
if info == nil {
logger.WithFields(log.Fields{
"path": path,
}).WithError(err).Error("nil FileInfo in housekeeping.Perform")
return nil
}
// Git will never create temporary directories, but only
// temporary objects, packfiles and packfile indices.
if info.IsDir() {
return nil
}
if !isStaleTemporaryObject(path, info.ModTime(), info.Mode()) {
return nil
}
temporaryObjects = append(temporaryObjects, path)
return nil
})
if err != nil {
return nil, err
}
return temporaryObjects, nil
}
func isStaleTemporaryObject(path string, modTime time.Time, mode os.FileMode) bool {
base := filepath.Base(path)
// Only delete entries starting with `tmp_` and older than a week
return strings.HasPrefix(base, "tmp_") && time.Since(modTime) >= deleteTempFilesOlderThanDuration
}
func findBrokenLooseReferences(ctx context.Context, repoPath string) ([]string, error) {
logger := myLogger(ctx)
var brokenRefs []string
err := filepath.Walk(filepath.Join(repoPath, "refs"), func(path string, info os.FileInfo, err error) error {
if info == nil {
logger.WithFields(log.Fields{
"path": path,
}).WithError(err).Error("nil FileInfo in housekeeping.Perform")
return nil
}
// When git crashes or a node reboots, it may happen that it leaves behind empty
// references. These references break various assumptions made by git and cause it
// to error in various circumstances. We thus clean them up to work around the
// issue.
if info.IsDir() || info.Size() > 0 || time.Since(info.ModTime()) < brokenRefsGracePeriod {
return nil
}
brokenRefs = append(brokenRefs, path)
return nil
})
if err != nil {
return nil, err
}
return brokenRefs, nil
}
// findStaleReferenceLocks scans the refdb for stale locks for loose references.
func findStaleReferenceLocks(ctx context.Context, repoPath string) ([]string, error) {
var staleReferenceLocks []string
err := filepath.Walk(filepath.Join(repoPath, "refs"), func(path string, info os.FileInfo, err error) error {
if os.IsNotExist(err) {
// Race condition: somebody already deleted the file for us. Ignore this file.
return nil
}
if err != nil {
return err
}
if info.IsDir() {
return nil
}
if !strings.HasSuffix(info.Name(), ".lock") || time.Since(info.ModTime()) < referenceLockfileGracePeriod {
return nil
}
staleReferenceLocks = append(staleReferenceLocks, path)
return nil
})
if err != nil {
return nil, err
}
return staleReferenceLocks, nil
}
// findPackedRefsLock returns stale lockfiles for the packed-refs file.
func findPackedRefsLock(ctx context.Context, repoPath string) ([]string, error) {
return findStaleFiles(repoPath, packedRefsLockGracePeriod, "packed-refs.lock")
}
// findPackedRefsNew returns stale temporary packed-refs files.
func findPackedRefsNew(ctx context.Context, repoPath string) ([]string, error) {
return findStaleFiles(repoPath, packedRefsNewGracePeriod, "packed-refs.new")
}
// FixDirectoryPermissions does a recursive directory walk to look for
// directories that cannot be accessed by the current user, and tries to
// fix those with chmod. The motivating problem is that directories with mode
// 0 break os.RemoveAll.
func FixDirectoryPermissions(ctx context.Context, path string) error {
return fixDirectoryPermissions(ctx, path, make(map[string]struct{}))
}
func fixDirectoryPermissions(ctx context.Context, path string, retriedPaths map[string]struct{}) error {
logger := myLogger(ctx)
return filepath.Walk(path, func(path string, info os.FileInfo, errIncoming error) error {
if info == nil {
logger.WithFields(log.Fields{
"path": path,
}).WithError(errIncoming).Error("nil FileInfo in housekeeping.fixDirectoryPermissions")
return nil
}
if !info.IsDir() || info.Mode()&minimumDirPerm == minimumDirPerm {
return nil
}
if err := os.Chmod(path, info.Mode()|minimumDirPerm); err != nil {
return err
}
if _, retried := retriedPaths[path]; !retried && os.IsPermission(errIncoming) {
retriedPaths[path] = struct{}{}
return fixDirectoryPermissions(ctx, path, retriedPaths)
}
return nil
})
}
func removeRefEmptyDirs(ctx context.Context, repository *localrepo.Repo) error {
rPath, err := repository.Path()
if err != nil {
return err
}
repoRefsPath := filepath.Join(rPath, "refs")
// we never want to delete the actual "refs" directory, so we start the
// recursive functions for each subdirectory
entries, err := ioutil.ReadDir(repoRefsPath)
if err != nil {
return err
}
for _, e := range entries {
if !e.IsDir() {
continue
}
ePath := filepath.Join(repoRefsPath, e.Name())
if err := removeEmptyDirs(ctx, ePath); err != nil {
return err
}
}
return nil
}
func removeEmptyDirs(ctx context.Context, target string) error {
if err := ctx.Err(); err != nil {
return err
}
// We need to stat the directory early on in order to get its current mtime. If we
// did this after we have removed empty child directories, then its mtime would've
// changed and we wouldn't consider it for deletion.
dirStat, err := os.Stat(target)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
entries, err := ioutil.ReadDir(target)
switch {
case os.IsNotExist(err):
return nil // race condition: someone else deleted it first
case err != nil:
return err
}
for _, e := range entries {
if !e.IsDir() {
continue
}
ePath := filepath.Join(target, e.Name())
if err := removeEmptyDirs(ctx, ePath); err != nil {
return err
}
}
// If the directory is older than the grace period for empty refs, then we can
// consider it for deletion in case it's empty.
if time.Since(dirStat.ModTime()) < emptyRefsGracePeriod {
return nil
}
// recheck entries now that we have potentially removed some dirs
entries, err = ioutil.ReadDir(target)
if err != nil && !os.IsNotExist(err) {
return err
}
if len(entries) > 0 {
return nil
}
switch err := os.Remove(target); {
case os.IsNotExist(err):
return nil // race condition: someone else deleted it first
case err != nil:
return err
}
optimizeEmptyDirRemovalTotals.Inc()
return nil
}
func unsetAllConfigsByRegexp(ctx context.Context, repository *localrepo.Repo, regexp string) error {
config := repository.Config()
configPairs, err := config.GetRegexp(ctx, regexp, git.ConfigGetRegexpOpts{})
if err != nil {
return fmt.Errorf("get config keys: %w", err)
}
for _, configPair := range configPairs {
if err := config.Unset(ctx, configPair.Key, git.ConfigUnsetOpts{
All: true,
}); err != nil {
return fmt.Errorf("unset all: %w", err)
}
}
return nil
}
func myLogger(ctx context.Context) *log.Entry {
return ctxlogrus.Extract(ctx).WithField("system", "housekeeping")
}
|