housekeeping: Add metric to track time since last full repack

Add a new metric to the housekeeping manager that tracks the time since
the last full repack. This metric will gives us a better picture of how
often we're currently doing full repacks. Equipped with this information
we can eventually come up with a good estimate of how often we should be
doing full repacks in a geometric-repacking world.

2 files changed, 38 insertions, 8 deletions

diff --git a/internal/git/housekeeping/manager.go b/internal/git/housekeeping/manager.go
index 1ca976c3f..a95a396c5 100644
--- a/internal/git/housekeeping/manager.go
+++ b/internal/git/housekeeping/manager.go
@@ -3,6 +3,7 @@ package housekeeping
 import (
 	"context"
 	"sync"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/git/localrepo"
@@ -24,14 +25,15 @@ type Manager interface {
 type RepositoryManager struct {
 	txManager transaction.Manager
 
-	tasksTotal             *prometheus.CounterVec
-	tasksLatency           *prometheus.HistogramVec
-	prunedFilesTotal       *prometheus.CounterVec
-	dataStructureExistence *prometheus.CounterVec
-	dataStructureCount     *prometheus.HistogramVec
-	dataStructureSize      *prometheus.HistogramVec
-	optimizeFunc           func(context.Context, *RepositoryManager, *localrepo.Repo, OptimizationStrategy) error
-	reposInProgress        sync.Map
+	tasksTotal                             *prometheus.CounterVec
+	tasksLatency                           *prometheus.HistogramVec
+	prunedFilesTotal                       *prometheus.CounterVec
+	dataStructureExistence                 *prometheus.CounterVec
+	dataStructureCount                     *prometheus.HistogramVec
+	dataStructureSize                      *prometheus.HistogramVec
+	dataStructureTimeSinceLastOptimization *prometheus.HistogramVec
+	optimizeFunc                           func(context.Context, *RepositoryManager, *localrepo.Repo, OptimizationStrategy) error
+	reposInProgress                        sync.Map
 }
 
 // NewManager creates a new RepositoryManager.
@@ -84,6 +86,30 @@ func NewManager(promCfg gitalycfgprom.Config, txManager transaction.Manager) *Re
 			},
 			[]string{"data_structure"},
 		),
+		dataStructureTimeSinceLastOptimization: prometheus.NewHistogramVec(
+			prometheus.HistogramOpts{
+				Name: "gitaly_housekeeping_time_since_last_optimization_seconds",
+				Help: "Absolute time in seconds since a given optimization has last been performed",
+				Buckets: []float64{
+					time.Second.Seconds(),
+					time.Minute.Seconds(),
+					(5 * time.Minute).Seconds(),
+					(10 * time.Minute).Seconds(),
+					(30 * time.Minute).Seconds(),
+					(1 * time.Hour).Seconds(),
+					(3 * time.Hour).Seconds(),
+					(6 * time.Hour).Seconds(),
+					(12 * time.Hour).Seconds(),
+					(1 * 24 * time.Hour).Seconds(),
+					(2 * 24 * time.Hour).Seconds(),
+					(3 * 24 * time.Hour).Seconds(),
+					(5 * 24 * time.Hour).Seconds(),
+					(7 * 24 * time.Hour).Seconds(),
+					(10 * 24 * time.Hour).Seconds(),
+				},
+			},
+			[]string{"data_structure"},
+		),
 		optimizeFunc: optimizeRepository,
 	}
 }
diff --git a/internal/git/housekeeping/optimize_repository.go b/internal/git/housekeeping/optimize_repository.go
index bc964d8ef..1d779ef64 100644
--- a/internal/git/housekeeping/optimize_repository.go
+++ b/internal/git/housekeeping/optimize_repository.go
@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"strconv"
+	"time"
 
 	"github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
 	"github.com/prometheus/client_golang/prometheus"
@@ -110,6 +111,9 @@ func (m *RepositoryManager) reportRepositoryInfo(ctx context.Context, info stats
 	m.reportDataStructureSize("packfiles_cruft", info.Packfiles.CruftSize)
 	m.reportDataStructureSize("packfiles_keep", info.Packfiles.KeepSize)
 	m.reportDataStructureSize("packed_references", info.References.PackedReferencesSize)
+
+	now := time.Now()
+	m.dataStructureTimeSinceLastOptimization.WithLabelValues("packfiles_full_repack").Observe(now.Sub(info.Packfiles.LastFullRepack).Seconds())
 }
 
 func (m *RepositoryManager) reportDataStructureExistence(dataStructure string, exists bool) {