gitaly: Prune old cgroups in background

Currently we block on startup while pruning cgroups from old Gitaly
processes. However, stale cgroups have no impact on Gitaly's other
startup tasks as we namespace them by pid. We can safely make this a
background task so that critical startup tasks can move ahead unimpeded.

On a host with 1000 repo cgroups and teardown is ~1ms per cgroup, this
improves startup time by roughly 15%. On our hosts where cgroup teardown
is closer to 20ms this will have a much larger impact.

  Benchmark 1: ./gitaly-5b092369 serve config.toml
    Time (mean ± σ):     632.7 ms ± 150.7 ms    [User: 473.7 ms, System: 226.7 ms]
    Range (min … max):   461.9 ms … 868.1 ms    10 runs

  Benchmark 2: ./async-prune serve config.toml
    Time (mean ± σ):     549.3 ms ± 127.9 ms    [User: 464.6 ms, System: 223.4 ms]
    Range (min … max):   427.1 ms … 754.6 ms    10 runs

  Summary
    ./async-prune serve config.toml ran
      1.15 ± 0.38 times faster than ./gitaly-5b092369 serve config.toml

2 files changed, 8 insertions, 4 deletions

diff --git a/internal/cgroups/cgroups.go b/internal/cgroups/cgroups.go
index 3a69710e8..8c918399e 100644
--- a/internal/cgroups/cgroups.go
+++ b/internal/cgroups/cgroups.go
@@ -5,6 +5,7 @@ import (
 	"os/exec"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/dontpanic"
 	"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config/cgroups"
 	"gitlab.com/gitlab-org/gitaly/v16/internal/log"
 )
@@ -105,7 +106,10 @@ func NewManager(cfg cgroups.Config, logger log.Logger, pid int) Manager {
 	return &NoopManager{}
 }
 
-// PruneOldCgroups prunes old cgroups for both the memory and cpu subsystems
-func PruneOldCgroups(cfg cgroups.Config, logger log.Logger) {
-	pruneOldCgroups(cfg, logger)
+// StartPruningOldCgroups prunes old cgroups for both the memory and cpu subsystems
+// in a goroutine.
+func StartPruningOldCgroups(cfg cgroups.Config, logger log.Logger) {
+	dontpanic.Go(logger, func() {
+		pruneOldCgroups(cfg, logger)
+	})
 }
diff --git a/internal/cli/gitaly/serve.go b/internal/cli/gitaly/serve.go
index 2dce430d0..b65e63641 100644
--- a/internal/cli/gitaly/serve.go
+++ b/internal/cli/gitaly/serve.go
@@ -173,7 +173,7 @@ func run(cfg config.Cfg, logger log.Logger) error {
 	// time a gitaly process is spawned. Look through the hierarchy root
 	// to find any cgroup directories that belong to old gitaly processes
 	// and remove them.
-	cgroups.PruneOldCgroups(cfg.Cgroups, logger)
+	cgroups.StartPruningOldCgroups(cfg.Cgroups, logger)
 	cgroupMgr := cgroups.NewManager(cfg.Cgroups, logger, os.Getpid())
 
 	began := time.Now()