housekeeping: Expose pruned stale files via Prometheus metric

Expose the number of stale files we have pruned via a Prometheus metric
so that it's easier to see what kinds of garbage we frequently need to
clean up.

Changelog: added

2 files changed, 122 insertions, 14 deletions

diff --git a/internal/git/housekeeping/clean_stale_data.go b/internal/git/housekeeping/clean_stale_data.go
index 42efc60f2..2cea9164a 100644
--- a/internal/git/housekeeping/clean_stale_data.go
+++ b/internal/git/housekeeping/clean_stale_data.go
@@ -67,6 +67,8 @@ func (m *RepositoryManager) CleanStaleData(ctx context.Context, repo *localrepo.
 
 		filesToPrune = append(filesToPrune, staleFiles...)
 		logEntry = logEntry.WithField(field, len(staleFiles))
+
+		m.prunedFilesTotal.WithLabelValues(field).Add(float64(len(staleFiles)))
 	}
 
 	unremovableFiles := 0
diff --git a/internal/git/housekeeping/clean_stale_data_test.go b/internal/git/housekeeping/clean_stale_data_test.go
index 985b9f092..28e52621c 100644
--- a/internal/git/housekeeping/clean_stale_data_test.go
+++ b/internal/git/housekeeping/clean_stale_data_test.go
@@ -6,9 +6,11 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"gitlab.com/gitlab-org/gitaly/v14/internal/backchannel"
@@ -116,10 +118,47 @@ func d(name string, mode os.FileMode, age time.Duration, finalState entryFinalSt
 	return &dirEntry{fileEntry{name, mode, age, finalState}, entries}
 }
 
+type cleanStaleDataMetrics struct {
+	objects        int
+	locks          int
+	refs           int
+	reflocks       int
+	refsEmptyDir   int
+	packedRefsLock int
+	packedRefsNew  int
+}
+
+func requireCleanStaleDataMetrics(t *testing.T, m *RepositoryManager, metrics cleanStaleDataMetrics) {
+	t.Helper()
+
+	var builder strings.Builder
+
+	_, err := builder.WriteString("# HELP gitaly_housekeeping_pruned_files_total Total number of files pruned\n")
+	require.NoError(t, err)
+	_, err = builder.WriteString("# TYPE gitaly_housekeeping_pruned_files_total counter\n")
+	require.NoError(t, err)
+
+	for metric, expectedValue := range map[string]int{
+		"objects":        metrics.objects,
+		"locks":          metrics.locks,
+		"refs":           metrics.refs,
+		"reflocks":       metrics.reflocks,
+		"packedrefslock": metrics.packedRefsLock,
+		"packedrefsnew":  metrics.packedRefsNew,
+		"refsemptydir":   metrics.refsEmptyDir,
+	} {
+		_, err := builder.WriteString(fmt.Sprintf("gitaly_housekeeping_pruned_files_total{filetype=%q} %d\n", metric, expectedValue))
+		require.NoError(t, err)
+	}
+
+	require.NoError(t, testutil.CollectAndCompare(m, strings.NewReader(builder.String()), "gitaly_housekeeping_pruned_files_total"))
+}
+
 func TestPerform(t *testing.T) {
 	testcases := []struct {
-		name    string
-		entries []entry
+		name            string
+		entries         []entry
+		expectedMetrics cleanStaleDataMetrics
 	}{
 		{
 			name: "clean",
@@ -157,6 +196,9 @@ func TestPerform(t *testing.T) {
 					f("b", 0o700, 24*time.Hour, Keep),
 				}),
 			},
+			expectedMetrics: cleanStaleDataMetrics{
+				objects: 1,
+			},
 		},
 		{
 			name: "subdir temp file",
@@ -167,6 +209,9 @@ func TestPerform(t *testing.T) {
 					}),
 				}),
 			},
+			expectedMetrics: cleanStaleDataMetrics{
+				objects: 1,
+			},
 		},
 		{
 			name: "inaccessible tmp directory",
@@ -189,6 +234,9 @@ func TestPerform(t *testing.T) {
 					}),
 				}),
 			},
+			expectedMetrics: cleanStaleDataMetrics{
+				objects: 1,
+			},
 		},
 		{
 			name: "files outside of object database",
@@ -217,11 +265,15 @@ func TestPerform(t *testing.T) {
 				e.create(t, repoPath)
 			}
 
-			require.NoError(t, NewManager(nil).CleanStaleData(ctx, repo))
+			mgr := NewManager(nil)
+
+			require.NoError(t, mgr.CleanStaleData(ctx, repo))
 
 			for _, e := range tc.entries {
 				e.validate(t, repoPath)
 			}
+
+			requireCleanStaleDataMetrics(t, mgr, tc.expectedMetrics)
 		})
 	}
 }
@@ -234,9 +286,10 @@ func TestPerform_references(t *testing.T) {
 	}
 
 	testcases := []struct {
-		desc     string
-		refs     []ref
-		expected []string
+		desc            string
+		refs            []ref
+		expected        []string
+		expectedMetrics cleanStaleDataMetrics
 	}{
 		{
 			desc: "normal reference",
@@ -262,6 +315,9 @@ func TestPerform_references(t *testing.T) {
 				{name: "refs/heads/master", age: 25 * time.Hour, size: 0},
 			},
 			expected: nil,
+			expectedMetrics: cleanStaleDataMetrics{
+				refs: 1,
+			},
 		},
 		{
 			desc: "multiple references",
@@ -284,6 +340,9 @@ func TestPerform_references(t *testing.T) {
 				"refs/heads/kept-because-recent",
 				"refs/heads/kept-because-nonempty",
 			},
+			expectedMetrics: cleanStaleDataMetrics{
+				refs: 3,
+			},
 		},
 	}
 
@@ -302,7 +361,9 @@ func TestPerform_references(t *testing.T) {
 			}
 			ctx := testhelper.Context(t)
 
-			require.NoError(t, NewManager(nil).CleanStaleData(ctx, repo))
+			mgr := NewManager(nil)
+
+			require.NoError(t, mgr.CleanStaleData(ctx, repo))
 
 			var actual []string
 			require.NoError(t, filepath.Walk(filepath.Join(repoPath, "refs"), func(path string, info os.FileInfo, _ error) error {
@@ -315,14 +376,17 @@ func TestPerform_references(t *testing.T) {
 			}))
 
 			require.ElementsMatch(t, tc.expected, actual)
+
+			requireCleanStaleDataMetrics(t, mgr, tc.expectedMetrics)
 		})
 	}
 }
 
 func TestPerform_emptyRefDirs(t *testing.T) {
 	testcases := []struct {
-		name    string
-		entries []entry
+		name            string
+		entries         []entry
+		expectedMetrics cleanStaleDataMetrics
 	}{
 		{
 			name: "unrelated empty directories",
@@ -353,6 +417,9 @@ func TestPerform_emptyRefDirs(t *testing.T) {
 					d("nested", 0o700, 240*time.Hour, Delete, []entry{}),
 				}),
 			},
+			expectedMetrics: cleanStaleDataMetrics{
+				refsEmptyDir: 1,
+			},
 		},
 		{
 			name: "hierarchy of nested stale ref dirs gets pruned",
@@ -363,6 +430,9 @@ func TestPerform_emptyRefDirs(t *testing.T) {
 					}),
 				}),
 			},
+			expectedMetrics: cleanStaleDataMetrics{
+				refsEmptyDir: 2,
+			},
 		},
 		{
 			name: "hierarchy with intermediate non-stale ref dir gets kept",
@@ -375,6 +445,9 @@ func TestPerform_emptyRefDirs(t *testing.T) {
 					}),
 				}),
 			},
+			expectedMetrics: cleanStaleDataMetrics{
+				refsEmptyDir: 1,
+			},
 		},
 		{
 			name: "stale hierrachy with refs gets partially retained",
@@ -390,6 +463,9 @@ func TestPerform_emptyRefDirs(t *testing.T) {
 					}),
 				}),
 			},
+			expectedMetrics: cleanStaleDataMetrics{
+				refsEmptyDir: 2,
+			},
 		},
 	}
 
@@ -403,11 +479,15 @@ func TestPerform_emptyRefDirs(t *testing.T) {
 				e.create(t, repoPath)
 			}
 
-			require.NoError(t, NewManager(nil).CleanStaleData(ctx, repo))
+			mgr := NewManager(nil)
+
+			require.NoError(t, mgr.CleanStaleData(ctx, repo))
 
 			for _, e := range tc.entries {
 				e.validate(t, repoPath)
 			}
+
+			requireCleanStaleDataMetrics(t, mgr, tc.expectedMetrics)
 		})
 	}
 }
@@ -416,29 +496,42 @@ func TestPerform_withSpecificFile(t *testing.T) {
 	t.Parallel()
 
 	for _, tc := range []struct {
-		desc   string
-		file   string
-		finder staleFileFinderFn
+		desc            string
+		file            string
+		finder          staleFileFinderFn
+		expectedMetrics cleanStaleDataMetrics
 	}{
 		{
 			desc:   "locked HEAD",
 			file:   "HEAD.lock",
 			finder: findStaleLockfiles,
+			expectedMetrics: cleanStaleDataMetrics{
+				locks: 1,
+			},
 		},
 		{
 			desc:   "locked config",
 			file:   "config.lock",
 			finder: findStaleLockfiles,
+			expectedMetrics: cleanStaleDataMetrics{
+				locks: 1,
+			},
 		},
 		{
 			desc:   "locked packed-refs",
 			file:   "packed-refs.lock",
 			finder: findPackedRefsLock,
+			expectedMetrics: cleanStaleDataMetrics{
+				packedRefsLock: 1,
+			},
 		},
 		{
 			desc:   "temporary packed-refs",
 			file:   "packed-refs.new",
 			finder: findPackedRefsNew,
+			expectedMetrics: cleanStaleDataMetrics{
+				packedRefsNew: 1,
+			},
 		},
 	} {
 		tc := tc
@@ -506,6 +599,8 @@ func TestPerform_withSpecificFile(t *testing.T) {
 					}
 				})
 			}
+
+			requireCleanStaleDataMetrics(t, mgr, tc.expectedMetrics)
 		})
 	}
 }
@@ -517,6 +612,7 @@ func TestPerform_referenceLocks(t *testing.T) {
 		desc                   string
 		entries                []entry
 		expectedReferenceLocks []string
+		expectedMetrics        cleanStaleDataMetrics
 	}{
 		{
 			desc: "fresh lock is kept",
@@ -538,6 +634,9 @@ func TestPerform_referenceLocks(t *testing.T) {
 			expectedReferenceLocks: []string{
 				"refs/main.lock",
 			},
+			expectedMetrics: cleanStaleDataMetrics{
+				reflocks: 1,
+			},
 		},
 		{
 			desc: "nested reference locks are deleted",
@@ -562,6 +661,9 @@ func TestPerform_referenceLocks(t *testing.T) {
 				"refs/heads/main.lock",
 				"refs/foobar/main.lock",
 			},
+			expectedMetrics: cleanStaleDataMetrics{
+				reflocks: 3,
+			},
 		},
 	} {
 		t.Run(tc.desc, func(t *testing.T) {
@@ -584,11 +686,15 @@ func TestPerform_referenceLocks(t *testing.T) {
 			require.NoError(t, err)
 			require.ElementsMatch(t, expectedReferenceLocks, staleLockfiles)
 
-			require.NoError(t, NewManager(nil).CleanStaleData(ctx, repo))
+			mgr := NewManager(nil)
+
+			require.NoError(t, mgr.CleanStaleData(ctx, repo))
 
 			for _, e := range tc.entries {
 				e.validate(t, repoPath)
 			}
+
+			requireCleanStaleDataMetrics(t, mgr, tc.expectedMetrics)
 		})
 	}
 }