cgroups: Cleanup cgroups during startupjc-cgroups-use-stable-path

Currently, cgroups Cleanup up is done upon shutdown. This can be
problematic however in an upgrade scenario when one Gitaly is shutting
down whilst another is coming online. There could be a race condition
where one is trying to delete cgroups while the other is trying to
create them.

To remove this complexity, move the Cleanup call to right before the
cgroups are setup.

Changelog: changed

1 files changed, 9 insertions, 7 deletions

diff --git a/cmd/gitaly/main.go b/cmd/gitaly/main.go
index 793bb015e..e9bf095b2 100644
--- a/cmd/gitaly/main.go
+++ b/cmd/gitaly/main.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"os"
@@ -111,7 +112,14 @@ func configure(configPath string) (config.Cfg, error) {
 
 	glog.Configure(glog.Loggers, cfg.Logging.Format, cfg.Logging.Level)
 
-	if err := cgroups.NewManager(cfg.Cgroups).Setup(); err != nil {
+	cgroupMgr := cgroups.NewManager(cfg.Cgroups)
+	if err := cgroupMgr.Cleanup(); err != nil {
+		if !errors.Is(err, cgroups.ErrProcessesExist) {
+			log.WithError(err).Warn("error cleaning up cgroups")
+		}
+	}
+
+	if err := cgroupMgr.Setup(); err != nil {
 		return config.Cfg{}, fmt.Errorf("failed setting up cgroups: %w", err)
 	}
 
@@ -369,12 +377,6 @@ func run(cfg config.Cfg) error {
 	}
 	defer shutdownWorkers()
 
-	defer func() {
-		if err := cgroups.NewManager(cfg.Cgroups).Cleanup(); err != nil {
-			log.WithError(err).Warn("error cleaning up cgroups")
-		}
-	}()
-
 	gracefulStopTicker := helper.NewTimerTicker(cfg.GracefulRestartTimeout.Duration())
 	defer gracefulStopTicker.Stop()