cgroups: Add PruneOldCgroups helper

Add a helper function to specifically clean out old cgroups from both
the memory and cpu subsystems.

4 files changed, 284 insertions, 100 deletions

diff --git a/internal/cgroups/cgroups.go b/internal/cgroups/cgroups.go
index df68ef16a..53c2a47d4 100644
--- a/internal/cgroups/cgroups.go
+++ b/internal/cgroups/cgroups.go
@@ -1,9 +1,13 @@
 package cgroups
 
 import (
+	"path/filepath"
+
 	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/command"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/git/repository"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config/cgroups"
 )
 
@@ -31,3 +35,26 @@ func NewManager(cfg cgroups.Config, pid int) Manager {
 
 	return &NoopManager{}
 }
+
+// PruneOldCgroups prunes old cgroups for both the memory and cpu subsystems
+func PruneOldCgroups(cfg cgroups.Config, logger log.FieldLogger) {
+	if cfg.HierarchyRoot == "" {
+		return
+	}
+
+	if err := config.PruneOldGitalyProcessDirectories(
+		logger,
+		filepath.Join(cfg.Mountpoint, "memory",
+			cfg.HierarchyRoot),
+	); err != nil {
+		logger.WithError(err).Error("failed to clean up memory cgroups")
+	}
+
+	if err := config.PruneOldGitalyProcessDirectories(
+		logger,
+		filepath.Join(cfg.Mountpoint, "cpu",
+			cfg.HierarchyRoot),
+	); err != nil {
+		logger.WithError(err).Error("failed to clean up cpu cgroups")
+	}
+}
diff --git a/internal/cgroups/cgroups_linux_test.go b/internal/cgroups/cgroups_linux_test.go
index 9f59a9a25..a921bc778 100644
--- a/internal/cgroups/cgroups_linux_test.go
+++ b/internal/cgroups/cgroups_linux_test.go
@@ -3,8 +3,13 @@
 package cgroups
 
 import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
 	"testing"
 
+	"github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config/cgroups"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/testhelper"
@@ -20,3 +25,142 @@ func TestNewManager(t *testing.T) {
 	require.IsType(t, &CGroupV1Manager{}, &CGroupV1Manager{cfg: cfg})
 	require.IsType(t, &NoopManager{}, NewManager(cgroups.Config{}, 1))
 }
+
+func TestPruneOldCgroups(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		desc           string
+		cfg            cgroups.Config
+		expectedPruned bool
+		// setup returns a pid
+		setup func(*testing.T, cgroups.Config) int
+	}{
+		{
+			desc: "process belongs to another user",
+			cfg: cgroups.Config{
+				Mountpoint:    testhelper.TempDir(t),
+				HierarchyRoot: "gitaly",
+				Repositories: cgroups.Repositories{
+					Count:       10,
+					MemoryBytes: 10 * 1024 * 1024,
+					CPUShares:   1024,
+				},
+			},
+			setup: func(t *testing.T, cfg cgroups.Config) int {
+				pid := 1
+				cgroupManager := NewManager(cfg, pid)
+				require.NoError(t, cgroupManager.Setup())
+
+				return pid
+			},
+			expectedPruned: true,
+		},
+		{
+			desc: "no hierarchy root",
+			cfg: cgroups.Config{
+				Mountpoint:    testhelper.TempDir(t),
+				HierarchyRoot: "",
+				Repositories: cgroups.Repositories{
+					Count:       10,
+					MemoryBytes: 10 * 1024 * 1024,
+					CPUShares:   1024,
+				},
+			},
+			setup: func(t *testing.T, cfg cgroups.Config) int {
+				pid := 1
+				cgroupManager := NewManager(cfg, pid)
+				require.NoError(t, cgroupManager.Setup())
+
+				return 1
+			},
+			expectedPruned: false,
+		},
+		{
+			desc: "pid of finished process",
+			cfg: cgroups.Config{
+				Mountpoint:    testhelper.TempDir(t),
+				HierarchyRoot: "gitaly",
+				Repositories: cgroups.Repositories{
+					Count:       10,
+					MemoryBytes: 10 * 1024 * 1024,
+					CPUShares:   1024,
+				},
+			},
+			setup: func(t *testing.T, cfg cgroups.Config) int {
+				cmd := exec.Command("ls")
+				require.NoError(t, cmd.Run())
+				pid := cmd.Process.Pid
+
+				cgroupManager := NewManager(cfg, pid)
+				require.NoError(t, cgroupManager.Setup())
+
+				return pid
+			},
+			expectedPruned: true,
+		},
+		{
+			desc: "pid of running process",
+			cfg: cgroups.Config{
+				Mountpoint:    testhelper.TempDir(t),
+				HierarchyRoot: "gitaly",
+				Repositories: cgroups.Repositories{
+					Count:       10,
+					MemoryBytes: 10 * 1024 * 1024,
+					CPUShares:   1024,
+				},
+			},
+			setup: func(t *testing.T, cfg cgroups.Config) int {
+				pid := os.Getpid()
+
+				cgroupManager := NewManager(cfg, pid)
+				require.NoError(t, cgroupManager.Setup())
+
+				return pid
+			},
+			expectedPruned: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			memoryRoot := filepath.Join(
+				tc.cfg.Mountpoint,
+				"memory",
+				tc.cfg.HierarchyRoot,
+			)
+			cpuRoot := filepath.Join(
+				tc.cfg.Mountpoint,
+				"cpu",
+				tc.cfg.HierarchyRoot,
+			)
+
+			require.NoError(t, os.MkdirAll(cpuRoot, os.ModePerm))
+			require.NoError(t, os.MkdirAll(memoryRoot, os.ModePerm))
+
+			pid := tc.setup(t, tc.cfg)
+
+			logger, hook := test.NewNullLogger()
+			PruneOldCgroups(tc.cfg, logger)
+
+			// create cgroups directories with a different pid
+			oldGitalyProcessMemoryDir := filepath.Join(
+				memoryRoot,
+				fmt.Sprintf("gitaly-%d", pid),
+			)
+			oldGitalyProcesssCPUDir := filepath.Join(
+				cpuRoot,
+				fmt.Sprintf("gitaly-%d", pid),
+			)
+
+			if tc.expectedPruned {
+				require.NoDirExists(t, oldGitalyProcessMemoryDir)
+				require.NoDirExists(t, oldGitalyProcesssCPUDir)
+			} else {
+				require.DirExists(t, oldGitalyProcessMemoryDir)
+				require.DirExists(t, oldGitalyProcesssCPUDir)
+				require.Len(t, hook.Entries, 0)
+			}
+		})
+	}
+}
diff --git a/internal/gitaly/config/config_test.go b/internal/gitaly/config/config_test.go
index cd4ba8ebd..ed56ae236 100644
--- a/internal/gitaly/config/config_test.go
+++ b/internal/gitaly/config/config_test.go
@@ -6,13 +6,11 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
 	"testing"
 	"time"
 
-	"github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config/auth"
@@ -1301,101 +1299,3 @@ func TestSetupRuntimeDirectory(t *testing.T) {
 		}
 	})
 }
-
-func TestPruneOldGitalyProcessDirectories(t *testing.T) {
-	t.Run("no runtime directories", func(t *testing.T) {
-		require.NoError(t, PruneOldGitalyProcessDirectories(testhelper.NewDiscardingLogEntry(t), testhelper.TempDir(t)))
-	})
-
-	t.Run("unset runtime directory", func(t *testing.T) {
-		require.EqualError(t,
-			PruneOldGitalyProcessDirectories(testhelper.NewDiscardingLogEntry(t), ""), "list gitaly process directory: open : no such file or directory")
-	})
-
-	t.Run("non-existent runtime directory", func(t *testing.T) {
-		require.EqualError(t,
-			PruneOldGitalyProcessDirectories(testhelper.NewDiscardingLogEntry(t),
-				"/path/does/not/exist"), "list gitaly process directory: open /path/does/not/exist: no such file or directory")
-	})
-
-	t.Run("invalid, stale and active runtime directories", func(t *testing.T) {
-		baseDir := testhelper.TempDir(t)
-		cfg := Cfg{RuntimeDir: baseDir}
-
-		// Setup a runtime directory for our process, it can't be stale as long as
-		// we are running.
-		ownRuntimeDir, err := SetupRuntimeDirectory(cfg, os.Getpid())
-		require.NoError(t, err)
-
-		expectedLogs := map[string]string{}
-		expectedErrs := map[string]error{}
-
-		// Setup runtime directories for processes that have finished.
-		var prunableDirs []string
-		for i := 0; i < 2; i++ {
-			cmd := exec.Command("cat")
-			require.NoError(t, cmd.Run())
-
-			staleRuntimeDir, err := SetupRuntimeDirectory(cfg, cmd.Process.Pid)
-			require.NoError(t, err)
-
-			prunableDirs = append(prunableDirs, staleRuntimeDir)
-			expectedLogs[staleRuntimeDir] = "removed leftover gitaly process directory"
-		}
-
-		// Setup runtime directory with pid of process not owned by git user
-		rootRuntimeDir, err := SetupRuntimeDirectory(cfg, 1)
-		require.NoError(t, err)
-		expectedLogs[rootRuntimeDir] = "removed leftover gitaly process directory"
-		prunableDirs = append(prunableDirs, rootRuntimeDir)
-
-		// Create an unexpected file in the runtime directory
-		unexpectedFilePath := filepath.Join(baseDir, "unexpected-file")
-		require.NoError(t, os.WriteFile(unexpectedFilePath, []byte(""), os.ModePerm))
-		expectedLogs[unexpectedFilePath] = "could not prune entry"
-		expectedErrs[unexpectedFilePath] = errors.New("gitaly process directory contains an unexpected file")
-
-		nonPrunableDirs := []string{ownRuntimeDir}
-
-		// Setup some unexpected directories in the runtime directory
-		for _, dirName := range []string{
-			"nohyphen",
-			"too-many-hyphens",
-			"invalidprefix-3",
-			"gitaly-invalidpid",
-		} {
-			dirPath := filepath.Join(baseDir, dirName)
-			require.NoError(t, os.Mkdir(dirPath, os.ModePerm))
-			expectedLogs[dirPath] = "could not prune entry"
-			expectedErrs[dirPath] = errors.New("gitaly process directory contains an unexpected directory")
-			nonPrunableDirs = append(nonPrunableDirs, dirPath)
-		}
-
-		logger, hook := test.NewNullLogger()
-		require.NoError(t, PruneOldGitalyProcessDirectories(logger, cfg.RuntimeDir))
-
-		actualLogs := map[string]string{}
-		actualErrs := map[string]error{}
-		for _, entry := range hook.Entries {
-			actualLogs[entry.Data["path"].(string)] = entry.Message
-			if entry.Data["error"] != nil {
-				err, ok := entry.Data["error"].(error)
-				require.True(t, ok)
-				actualErrs[entry.Data["path"].(string)] = err
-			}
-		}
-
-		require.Equal(t, expectedLogs, actualLogs)
-		require.Equal(t, expectedErrs, actualErrs)
-
-		require.FileExists(t, unexpectedFilePath)
-
-		for _, nonPrunableEntry := range nonPrunableDirs {
-			require.DirExists(t, nonPrunableEntry, nonPrunableEntry)
-		}
-
-		for _, prunableEntry := range prunableDirs {
-			require.NoDirExists(t, prunableEntry, prunableEntry)
-		}
-	})
-}
diff --git a/internal/gitaly/config/temp_dir_test.go b/internal/gitaly/config/temp_dir_test.go
new file mode 100644
index 000000000..9961278ce
--- /dev/null
+++ b/internal/gitaly/config/temp_dir_test.go
@@ -0,0 +1,113 @@
+//go:build !gitaly_test_sha256
+
+package config
+
+import (
+	"errors"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/testhelper"
+)
+
+func TestPruneOldGitalyProcessDirectories(t *testing.T) {
+	t.Run("no runtime directories", func(t *testing.T) {
+		require.NoError(t, PruneOldGitalyProcessDirectories(testhelper.NewDiscardingLogEntry(t), testhelper.TempDir(t)))
+	})
+
+	t.Run("unset runtime directory", func(t *testing.T) {
+		require.EqualError(t,
+			PruneOldGitalyProcessDirectories(testhelper.NewDiscardingLogEntry(t), ""), "list gitaly process directory: open : no such file or directory")
+	})
+
+	t.Run("non-existent runtime directory", func(t *testing.T) {
+		require.EqualError(t,
+			PruneOldGitalyProcessDirectories(testhelper.NewDiscardingLogEntry(t),
+				"/path/does/not/exist"), "list gitaly process directory: open /path/does/not/exist: no such file or directory")
+	})
+
+	t.Run("invalid, stale and active runtime directories", func(t *testing.T) {
+		baseDir := testhelper.TempDir(t)
+		cfg := Cfg{RuntimeDir: baseDir}
+
+		// Setup a runtime directory for our process, it can't be stale as long as
+		// we are running.
+		ownRuntimeDir, err := SetupRuntimeDirectory(cfg, os.Getpid())
+		require.NoError(t, err)
+
+		expectedLogs := map[string]string{}
+		expectedErrs := map[string]error{}
+
+		// Setup runtime directories for processes that have finished.
+		var prunableDirs []string
+		for i := 0; i < 2; i++ {
+			cmd := exec.Command("cat")
+			require.NoError(t, cmd.Run())
+
+			staleRuntimeDir, err := SetupRuntimeDirectory(cfg, cmd.Process.Pid)
+			require.NoError(t, err)
+
+			prunableDirs = append(prunableDirs, staleRuntimeDir)
+			expectedLogs[staleRuntimeDir] = "removed leftover gitaly process directory"
+		}
+
+		// Setup runtime directory with pid of process not owned by git user
+		rootRuntimeDir, err := SetupRuntimeDirectory(cfg, 1)
+		require.NoError(t, err)
+		expectedLogs[rootRuntimeDir] = "removed leftover gitaly process directory"
+		prunableDirs = append(prunableDirs, rootRuntimeDir)
+
+		// Create an unexpected file in the runtime directory
+		unexpectedFilePath := filepath.Join(baseDir, "unexpected-file")
+		require.NoError(t, os.WriteFile(unexpectedFilePath, []byte(""), os.ModePerm))
+		expectedLogs[unexpectedFilePath] = "could not prune entry"
+		expectedErrs[unexpectedFilePath] = errors.New("gitaly process directory contains an unexpected file")
+
+		nonPrunableDirs := []string{ownRuntimeDir}
+
+		// Setup some unexpected directories in the runtime directory
+		for _, dirName := range []string{
+			"nohyphen",
+			"too-many-hyphens",
+			"invalidprefix-3",
+			"gitaly-invalidpid",
+		} {
+			dirPath := filepath.Join(baseDir, dirName)
+			require.NoError(t, os.Mkdir(dirPath, os.ModePerm))
+			expectedLogs[dirPath] = "could not prune entry"
+			expectedErrs[dirPath] = errors.New("gitaly process directory contains an unexpected directory")
+			nonPrunableDirs = append(nonPrunableDirs, dirPath)
+		}
+
+		logger, hook := test.NewNullLogger()
+		require.NoError(t, PruneOldGitalyProcessDirectories(logger, cfg.RuntimeDir))
+
+		actualLogs := map[string]string{}
+		actualErrs := map[string]error{}
+		for _, entry := range hook.Entries {
+			actualLogs[entry.Data["path"].(string)] = entry.Message
+			if entry.Data["error"] != nil {
+				err, ok := entry.Data["error"].(error)
+				require.True(t, ok)
+				actualErrs[entry.Data["path"].(string)] = err
+			}
+		}
+
+		require.Equal(t, expectedLogs, actualLogs)
+		require.Equal(t, expectedErrs, actualErrs)
+
+		require.FileExists(t, unexpectedFilePath)
+
+		for _, nonPrunableEntry := range nonPrunableDirs {
+			require.DirExists(t, nonPrunableEntry, nonPrunableEntry)
+		}
+
+		for _, prunableEntry := range prunableDirs {
+			require.NoDirExists(t, prunableEntry, prunableEntry)
+		}
+	})
+}