Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitaly.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorQuang-Minh Nguyen <qmnguyen@gitlab.com>2023-11-17 11:10:06 +0300
committerQuang-Minh Nguyen <qmnguyen@gitlab.com>2023-11-17 11:10:06 +0300
commite66f6ec5ad5b47cb43e69a4690c3e9e1a9a49b4e (patch)
tree838e6b3ddba30c7c5ea1c8d8f0f8e51fb1ec6548
parent9b36b0700fe0f1b25bd1105ce20ddf1ab66f3050 (diff)
parent04908eb5fbb468ef047d78bcadd4c8b5dd2910d9 (diff)
Merge branch 'wc/cgroup-unify-tests' into 'master'
cgroups: Unify v1 and v2 tests See merge request https://gitlab.com/gitlab-org/gitaly/-/merge_requests/6519 Merged-by: Quang-Minh Nguyen <qmnguyen@gitlab.com> Approved-by: James Liu <jliu@gitlab.com> Approved-by: Quang-Minh Nguyen <qmnguyen@gitlab.com> Co-authored-by: Will Chandler <wchandler@gitlab.com>
-rw-r--r--internal/cgroups/cgroups_linux_test.go2
-rw-r--r--internal/cgroups/handler_linux_test.go896
-rw-r--r--internal/cgroups/mock_linux_test.go76
-rw-r--r--internal/cgroups/v1_linux_test.go713
-rw-r--r--internal/cgroups/v2_linux_test.go689
5 files changed, 967 insertions, 1409 deletions
diff --git a/internal/cgroups/cgroups_linux_test.go b/internal/cgroups/cgroups_linux_test.go
index c3e7e143d..80a307a94 100644
--- a/internal/cgroups/cgroups_linux_test.go
+++ b/internal/cgroups/cgroups_linux_test.go
@@ -10,6 +10,6 @@ import (
"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper"
)
-func TestNewManager(t *testing.T) {
+func TestNewManagerNoop(t *testing.T) {
require.IsType(t, &NoopManager{}, NewManager(cgroups.Config{}, testhelper.SharedLogger(t), 1))
}
diff --git a/internal/cgroups/handler_linux_test.go b/internal/cgroups/handler_linux_test.go
new file mode 100644
index 000000000..9556b393b
--- /dev/null
+++ b/internal/cgroups/handler_linux_test.go
@@ -0,0 +1,896 @@
+//go:build linux
+
+package cgroups
+
+import (
+ "fmt"
+ "io/fs"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "testing"
+
+ cgrps "github.com/containerd/cgroups/v3"
+ "github.com/prometheus/client_golang/prometheus/testutil"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config/cgroups"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper"
+ "golang.org/x/exp/slices"
+)
+
+func defaultCgroupsConfig() cgroups.Config {
+ return cgroups.Config{
+ HierarchyRoot: "gitaly",
+ Repositories: cgroups.Repositories{
+ Count: 3,
+ MemoryBytes: 1024000,
+ CPUShares: 256,
+ CPUQuotaUs: 200,
+ },
+ }
+}
+
+func TestNewManager(t *testing.T) {
+ cfg := cgroups.Config{Repositories: cgroups.Repositories{Count: 10}}
+
+ manager := newCgroupManagerWithMode(cfg, testhelper.SharedLogger(t), 1, cgrps.Legacy)
+ require.IsType(t, &cgroupV1Handler{}, manager.handler)
+ manager = newCgroupManagerWithMode(cfg, testhelper.SharedLogger(t), 1, cgrps.Hybrid)
+ require.IsType(t, &cgroupV1Handler{}, manager.handler)
+ manager = newCgroupManagerWithMode(cfg, testhelper.SharedLogger(t), 1, cgrps.Unified)
+ require.IsType(t, &cgroupV2Handler{}, manager.handler)
+ manager = newCgroupManagerWithMode(cfg, testhelper.SharedLogger(t), 1, cgrps.Unavailable)
+ require.Nil(t, manager)
+}
+
+type expectedCgroup struct {
+ wantMemoryBytes int
+ wantCPUShares int
+ wantCPUQuotaUs int
+ wantCFSPeriod int
+ wantCPUWeight int
+ wantCPUMax string
+}
+
+func TestSetup_ParentCgroups(t *testing.T) {
+ tests := []struct {
+ name string
+ cfg cgroups.Config
+ expectedV1 expectedCgroup
+ expectedV2 expectedCgroup
+ }{
+ {
+ name: "all config specified",
+ cfg: cgroups.Config{
+ MemoryBytes: 102400,
+ CPUShares: 256,
+ CPUQuotaUs: 200,
+ },
+ expectedV1: expectedCgroup{
+ wantMemoryBytes: 102400,
+ wantCPUShares: 256,
+ wantCPUQuotaUs: 200,
+ wantCFSPeriod: int(cfsPeriodUs),
+ },
+ expectedV2: expectedCgroup{
+ wantMemoryBytes: 102400,
+ wantCPUWeight: 256,
+ wantCPUMax: "200 100000",
+ },
+ },
+ {
+ name: "only memory limit set",
+ cfg: cgroups.Config{
+ MemoryBytes: 102400,
+ },
+ expectedV1: expectedCgroup{
+ wantMemoryBytes: 102400,
+ },
+ expectedV2: expectedCgroup{
+ wantMemoryBytes: 102400,
+ },
+ },
+ {
+ name: "only cpu shares set",
+ cfg: cgroups.Config{
+ CPUShares: 512,
+ },
+ expectedV1: expectedCgroup{
+ wantCPUShares: 512,
+ },
+ expectedV2: expectedCgroup{
+ wantCPUWeight: 512,
+ },
+ },
+ {
+ name: "only cpu quota set",
+ cfg: cgroups.Config{
+ CPUQuotaUs: 200,
+ },
+ expectedV1: expectedCgroup{
+ wantCPUQuotaUs: 200,
+ wantCFSPeriod: int(cfsPeriodUs),
+ },
+ expectedV2: expectedCgroup{
+ wantCPUMax: "200 100000",
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ tt := tt
+ t.Run(tt.name, func(t *testing.T) {
+ for _, version := range []int{1, 2} {
+ version := version
+ t.Run("cgroups-v"+strconv.Itoa(version), func(t *testing.T) {
+ t.Parallel()
+
+ mock := newMock(t, version)
+
+ pid := 1
+
+ cfg := tt.cfg
+ cfg.HierarchyRoot = "gitaly"
+ cfg.Mountpoint = mock.rootPath()
+
+ manager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), pid)
+ mock.setupMockCgroupFiles(t, manager, []uint{})
+
+ require.False(t, manager.Ready())
+ require.NoError(t, manager.Setup())
+ require.True(t, manager.Ready())
+
+ cgroupPath := filepath.Join("gitaly", fmt.Sprintf("gitaly-%d", pid))
+ if version == 1 {
+ requireCgroupComponents(t, version, mock.rootPath(), cgroupPath, tt.expectedV1)
+ } else {
+ requireCgroupComponents(t, version, mock.rootPath(), cgroupPath, tt.expectedV2)
+ }
+ })
+ }
+ })
+ }
+}
+
+func TestRepoCgroups(t *testing.T) {
+ tests := []struct {
+ name string
+ cfg cgroups.Repositories
+ expectedV1 expectedCgroup
+ expectedV2 expectedCgroup
+ }{
+ {
+ name: "all config specified",
+ cfg: defaultCgroupsConfig().Repositories,
+ expectedV1: expectedCgroup{
+ wantMemoryBytes: 1024000,
+ wantCPUShares: 256,
+ wantCPUQuotaUs: 200,
+ wantCFSPeriod: int(cfsPeriodUs),
+ },
+ expectedV2: expectedCgroup{
+ wantMemoryBytes: 1024000,
+ wantCPUWeight: 256,
+ wantCPUMax: "200 100000",
+ },
+ },
+ {
+ name: "only memory limit set",
+ cfg: cgroups.Repositories{
+ MemoryBytes: 1024000,
+ },
+ expectedV1: expectedCgroup{
+ wantMemoryBytes: 1024000,
+ },
+ expectedV2: expectedCgroup{
+ wantMemoryBytes: 1024000,
+ },
+ },
+ {
+ name: "only cpu shares set",
+ cfg: cgroups.Repositories{
+ CPUShares: 512,
+ },
+ expectedV1: expectedCgroup{
+ wantCPUShares: 512,
+ },
+ expectedV2: expectedCgroup{
+ wantCPUWeight: 512,
+ },
+ },
+ {
+ name: "only cpu quota set",
+ cfg: cgroups.Repositories{
+ CPUQuotaUs: 100,
+ },
+ expectedV1: expectedCgroup{
+ wantCPUQuotaUs: 100,
+ wantCFSPeriod: int(cfsPeriodUs),
+ },
+ expectedV2: expectedCgroup{
+ wantCPUMax: "100 100000",
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ tt := tt
+ t.Run(tt.name, func(t *testing.T) {
+ for _, version := range []int{1, 2} {
+ version := version
+ t.Run("cgroups-v"+strconv.Itoa(version), func(t *testing.T) {
+ t.Parallel()
+
+ mock := newMock(t, version)
+
+ pid := 1
+ cfg := defaultCgroupsConfig()
+ cfg.Repositories = tt.cfg
+ cfg.Repositories.Count = 3
+ cfg.HierarchyRoot = "gitaly"
+ cfg.Mountpoint = mock.rootPath()
+
+ manager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), pid)
+
+ // Validate no shards have been created. We deliberately do not call
+ // `setupMockCgroupFiles()` here to confirm that the cgroup controller
+ // is creating repository directories in the correct location.
+ requireShards(t, version, mock, manager, pid)
+
+ groupID := calcGroupID(cmdArgs, cfg.Repositories.Count)
+
+ mock.setupMockCgroupFiles(t, manager, []uint{groupID})
+
+ require.False(t, manager.Ready())
+ require.NoError(t, manager.Setup())
+ require.True(t, manager.Ready())
+
+ ctx := testhelper.Context(t)
+
+ // Create a command to force Gitaly to create the repo cgroup.
+ cmd := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...)
+ require.NoError(t, cmd.Run())
+ _, err := manager.AddCommand(cmd)
+ require.NoError(t, err)
+
+ requireShards(t, version, mock, manager, pid, groupID)
+
+ var expected expectedCgroup
+ if version == 1 {
+ expected = tt.expectedV1
+ } else {
+ expected = tt.expectedV2
+ }
+
+ for shard := uint(0); shard < cfg.Repositories.Count; shard++ {
+ // The negative case where no directory should exist is asserted
+ // by `requireShards()`.
+ if shard == groupID {
+ cgRelPath := filepath.Join(
+ "gitaly", fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", shard),
+ )
+
+ requireCgroupComponents(t, version, mock.rootPath(), cgRelPath, expected)
+ }
+ }
+ })
+ }
+ })
+ }
+}
+
+func TestAddCommand(t *testing.T) {
+ for _, version := range []int{1, 2} {
+ t.Run("cgroups-v"+strconv.Itoa(version), func(t *testing.T) {
+ mock := newMock(t, version)
+
+ config := defaultCgroupsConfig()
+ config.Repositories.Count = 10
+ config.Repositories.MemoryBytes = 1024
+ config.Repositories.CPUShares = 16
+ config.HierarchyRoot = "gitaly"
+ config.Mountpoint = mock.rootPath()
+
+ pid := 1
+ manager1 := mock.newCgroupManager(config, testhelper.SharedLogger(t), pid)
+ mock.setupMockCgroupFiles(t, manager1, []uint{})
+ require.NoError(t, manager1.Setup())
+
+ ctx := testhelper.Context(t)
+
+ cmd2 := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...)
+ require.NoError(t, cmd2.Run())
+
+ groupID := calcGroupID(cmd2.Args, config.Repositories.Count)
+
+ manager2 := mock.newCgroupManager(config, testhelper.SharedLogger(t), pid)
+
+ t.Run("without overridden key", func(t *testing.T) {
+ _, err := manager2.AddCommand(cmd2)
+ require.NoError(t, err)
+ requireShards(t, version, mock, manager2, pid, groupID)
+
+ for _, path := range mock.repoPaths(pid, groupID) {
+ procsPath := filepath.Join(path, "cgroup.procs")
+ content := readCgroupFile(t, procsPath)
+
+ cmdPid, err := strconv.Atoi(string(content))
+ require.NoError(t, err)
+
+ require.Equal(t, cmd2.Process.Pid, cmdPid)
+ }
+ })
+
+ t.Run("with overridden key", func(t *testing.T) {
+ overriddenGroupID := calcGroupID([]string{"foobar"}, config.Repositories.Count)
+
+ _, err := manager2.AddCommand(cmd2, WithCgroupKey("foobar"))
+ require.NoError(t, err)
+ requireShards(t, version, mock, manager2, pid, groupID, overriddenGroupID)
+
+ for _, path := range mock.repoPaths(pid, overriddenGroupID) {
+ procsPath := filepath.Join(path, "cgroup.procs")
+ content := readCgroupFile(t, procsPath)
+
+ cmdPid, err := strconv.Atoi(string(content))
+ require.NoError(t, err)
+
+ require.Equal(t, cmd2.Process.Pid, cmdPid)
+ }
+ })
+ })
+ }
+}
+
+func TestCleanup(t *testing.T) {
+ for _, version := range []int{1, 2} {
+ t.Run("cgroups-v"+strconv.Itoa(version), func(t *testing.T) {
+ mock := newMock(t, version)
+
+ pid := 1
+ cfg := defaultCgroupsConfig()
+ cfg.Mountpoint = mock.rootPath()
+
+ manager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), pid)
+ mock.setupMockCgroupFiles(t, manager, []uint{0, 1, 2})
+
+ require.NoError(t, manager.Setup())
+ require.NoError(t, manager.Cleanup())
+
+ for i := uint(0); i < 3; i++ {
+ for _, path := range mock.repoPaths(pid, i) {
+ require.NoDirExists(t, path)
+ }
+ }
+ })
+ }
+}
+
+func TestMetrics(t *testing.T) {
+ tests := []struct {
+ name string
+ metricsEnabled bool
+ pid int
+ expectV1 string
+ expectV2 string
+ }{
+ {
+ name: "metrics enabled: true",
+ metricsEnabled: true,
+ pid: 1,
+ expectV1: `# HELP gitaly_cgroup_cpu_usage_total CPU Usage of Cgroup
+# TYPE gitaly_cgroup_cpu_usage_total gauge
+gitaly_cgroup_cpu_usage_total{path="%s",type="kernel"} 0
+gitaly_cgroup_cpu_usage_total{path="%s",type="user"} 0
+# HELP gitaly_cgroup_memory_reclaim_attempts_total Number of memory usage hits limits
+# TYPE gitaly_cgroup_memory_reclaim_attempts_total gauge
+gitaly_cgroup_memory_reclaim_attempts_total{path="%s"} 2
+# HELP gitaly_cgroup_procs_total Total number of procs
+# TYPE gitaly_cgroup_procs_total gauge
+gitaly_cgroup_procs_total{path="%s",subsystem="cpu"} 1
+gitaly_cgroup_procs_total{path="%s",subsystem="memory"} 1
+# HELP gitaly_cgroup_cpu_cfs_periods_total Number of elapsed enforcement period intervals
+# TYPE gitaly_cgroup_cpu_cfs_periods_total counter
+gitaly_cgroup_cpu_cfs_periods_total{path="%s"} 10
+# HELP gitaly_cgroup_cpu_cfs_throttled_periods_total Number of throttled period intervals
+# TYPE gitaly_cgroup_cpu_cfs_throttled_periods_total counter
+gitaly_cgroup_cpu_cfs_throttled_periods_total{path="%s"} 20
+# HELP gitaly_cgroup_cpu_cfs_throttled_seconds_total Total time duration the Cgroup has been throttled
+# TYPE gitaly_cgroup_cpu_cfs_throttled_seconds_total counter
+gitaly_cgroup_cpu_cfs_throttled_seconds_total{path="%s"} 0.001
+`,
+ expectV2: `# HELP gitaly_cgroup_cpu_cfs_periods_total Number of elapsed enforcement period intervals
+# TYPE gitaly_cgroup_cpu_cfs_periods_total counter
+gitaly_cgroup_cpu_cfs_periods_total{path="%s"} 10
+# HELP gitaly_cgroup_cpu_cfs_throttled_periods_total Number of throttled period intervals
+# TYPE gitaly_cgroup_cpu_cfs_throttled_periods_total counter
+gitaly_cgroup_cpu_cfs_throttled_periods_total{path="%s"} 20
+# HELP gitaly_cgroup_cpu_cfs_throttled_seconds_total Total time duration the Cgroup has been throttled
+# TYPE gitaly_cgroup_cpu_cfs_throttled_seconds_total counter
+gitaly_cgroup_cpu_cfs_throttled_seconds_total{path="%s"} 0.001
+# HELP gitaly_cgroup_cpu_usage_total CPU Usage of Cgroup
+# TYPE gitaly_cgroup_cpu_usage_total gauge
+gitaly_cgroup_cpu_usage_total{path="%s",type="kernel"} 0
+gitaly_cgroup_cpu_usage_total{path="%s",type="user"} 0
+# HELP gitaly_cgroup_procs_total Total number of procs
+# TYPE gitaly_cgroup_procs_total gauge
+gitaly_cgroup_procs_total{path="%s",subsystem="cpu"} 1
+gitaly_cgroup_procs_total{path="%s",subsystem="cpuset"} 1
+gitaly_cgroup_procs_total{path="%s",subsystem="memory"} 1
+`,
+ },
+ {
+ name: "metrics enabled: false",
+ metricsEnabled: false,
+ pid: 2,
+ },
+ }
+
+ for _, tt := range tests {
+ tt := tt
+ t.Run(tt.name, func(t *testing.T) {
+ for _, version := range []int{1, 2} {
+ version := version
+ t.Run("cgroups-v"+strconv.Itoa(version), func(t *testing.T) {
+ t.Parallel()
+ mock := newMock(t, version)
+
+ config := defaultCgroupsConfig()
+ config.Repositories.Count = 1
+ config.Repositories.MemoryBytes = 1048576
+ config.Repositories.CPUShares = 16
+ config.Mountpoint = mock.rootPath()
+ config.MetricsEnabled = tt.metricsEnabled
+
+ manager1 := mock.newCgroupManager(config, testhelper.SharedLogger(t), tt.pid)
+
+ var mockFiles []mockCgroupFile
+ if version == 1 {
+ mockFiles = append(mockFiles, mockCgroupFile{"memory.failcnt", "2"})
+ }
+ mock.setupMockCgroupFiles(t, manager1, []uint{0}, mockFiles...)
+ require.NoError(t, manager1.Setup())
+
+ ctx := testhelper.Context(t)
+
+ cmd := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...)
+ require.NoError(t, cmd.Start())
+ _, err := manager1.AddCommand(cmd)
+ require.NoError(t, err)
+
+ gitCmd1 := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...)
+ require.NoError(t, gitCmd1.Start())
+ _, err = manager1.AddCommand(gitCmd1)
+ require.NoError(t, err)
+
+ gitCmd2 := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...)
+ require.NoError(t, gitCmd2.Start())
+ _, err = manager1.AddCommand(gitCmd2)
+ require.NoError(t, err)
+ defer func() {
+ require.NoError(t, gitCmd2.Wait())
+ }()
+
+ require.NoError(t, cmd.Wait())
+ require.NoError(t, gitCmd1.Wait())
+
+ repoCgroupPath := filepath.Join(manager1.currentProcessCgroup(), "repos-0")
+
+ var expected *strings.Reader
+ if version == 1 {
+ expected = strings.NewReader(strings.ReplaceAll(tt.expectV1, "%s", repoCgroupPath))
+ } else {
+ expected = strings.NewReader(strings.ReplaceAll(tt.expectV2, "%s", repoCgroupPath))
+ }
+ assert.NoError(t, testutil.CollectAndCompare(manager1, expected))
+ })
+ }
+ })
+ }
+}
+
+func TestPruneOldCgroups(t *testing.T) {
+ t.Parallel()
+
+ testCases := []struct {
+ desc string
+ cfg cgroups.Config
+ expectedPruned bool
+ // setup returns a pid
+ setup func(t *testing.T, cfg cgroups.Config, mock mockCgroup) int
+ }{
+ {
+ desc: "process belongs to another user",
+ cfg: cgroups.Config{
+ HierarchyRoot: "gitaly",
+ Repositories: cgroups.Repositories{
+ Count: 10,
+ MemoryBytes: 10 * 1024 * 1024,
+ CPUShares: 1024,
+ },
+ },
+ setup: func(t *testing.T, cfg cgroups.Config, mock mockCgroup) int {
+ pid := 1
+ cgroupManager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), pid)
+ mock.setupMockCgroupFiles(t, cgroupManager, []uint{0, 1, 2})
+ require.NoError(t, cgroupManager.Setup())
+
+ return pid
+ },
+ expectedPruned: true,
+ },
+ {
+ desc: "no hierarchy root",
+ cfg: cgroups.Config{
+ HierarchyRoot: "",
+ Repositories: cgroups.Repositories{
+ Count: 10,
+ MemoryBytes: 10 * 1024 * 1024,
+ CPUShares: 1024,
+ },
+ },
+ setup: func(t *testing.T, cfg cgroups.Config, mock mockCgroup) int {
+ pid := 1
+ cgroupManager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), pid)
+ mock.setupMockCgroupFiles(t, cgroupManager, []uint{0, 1, 2})
+ require.NoError(t, cgroupManager.Setup())
+ return 1
+ },
+ expectedPruned: false,
+ },
+ {
+ desc: "pid of finished process",
+ cfg: cgroups.Config{
+ HierarchyRoot: "gitaly",
+ Repositories: cgroups.Repositories{
+ Count: 10,
+ MemoryBytes: 10 * 1024 * 1024,
+ CPUShares: 1024,
+ },
+ },
+ setup: func(t *testing.T, cfg cgroups.Config, mock mockCgroup) int {
+ cmd := exec.Command("ls")
+ require.NoError(t, cmd.Run())
+ pid := cmd.Process.Pid
+
+ cgroupManager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), pid)
+ mock.setupMockCgroupFiles(t, cgroupManager, []uint{0, 1, 2})
+
+ if mock.version() == 1 {
+ memoryRoot := filepath.Join(
+ cfg.Mountpoint,
+ "memory",
+ cfg.HierarchyRoot,
+ "memory.limit_in_bytes",
+ )
+ require.NoError(t, os.WriteFile(memoryRoot, []byte{}, fs.ModeAppend))
+ } else {
+ require.NoError(t, cgroupManager.Setup())
+
+ memoryFile := filepath.Join(
+ cfg.Mountpoint,
+ cfg.HierarchyRoot,
+ "memory.limit_in_bytes",
+ )
+ require.NoError(t, os.WriteFile(memoryFile, []byte{}, fs.ModeAppend))
+ }
+
+ return pid
+ },
+ expectedPruned: true,
+ },
+ {
+ desc: "pid of running process",
+ cfg: cgroups.Config{
+ HierarchyRoot: "gitaly",
+ Repositories: cgroups.Repositories{
+ Count: 10,
+ MemoryBytes: 10 * 1024 * 1024,
+ CPUShares: 1024,
+ },
+ },
+ setup: func(t *testing.T, cfg cgroups.Config, mock mockCgroup) int {
+ pid := os.Getpid()
+
+ cgroupManager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), pid)
+ mock.setupMockCgroupFiles(t, cgroupManager, []uint{0, 1, 2})
+ require.NoError(t, cgroupManager.Setup())
+
+ return pid
+ },
+ expectedPruned: false,
+ },
+ {
+ desc: "gitaly-0 directory is deleted",
+ cfg: cgroups.Config{
+ HierarchyRoot: "gitaly",
+ Repositories: cgroups.Repositories{
+ Count: 10,
+ MemoryBytes: 10 * 1024 * 1024,
+ CPUShares: 1024,
+ },
+ },
+ setup: func(t *testing.T, cfg cgroups.Config, mock mockCgroup) int {
+ cgroupManager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), 0)
+ mock.setupMockCgroupFiles(t, cgroupManager, []uint{0, 1, 2})
+ require.NoError(t, cgroupManager.Setup())
+
+ return 0
+ },
+ expectedPruned: true,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.desc, func(t *testing.T) {
+ t.Run("cgroups-v1", func(t *testing.T) {
+ mock := newMock(t, 1)
+ tc.cfg.Mountpoint = mock.rootPath()
+
+ memoryRoot := filepath.Join(
+ tc.cfg.Mountpoint,
+ "memory",
+ tc.cfg.HierarchyRoot,
+ )
+ cpuRoot := filepath.Join(
+ tc.cfg.Mountpoint,
+ "cpu",
+ tc.cfg.HierarchyRoot,
+ )
+
+ require.NoError(t, os.MkdirAll(cpuRoot, perm.PublicDir))
+ require.NoError(t, os.MkdirAll(memoryRoot, perm.PublicDir))
+
+ pid := tc.setup(t, tc.cfg, mock)
+
+ logger := testhelper.NewLogger(t)
+ hook := testhelper.AddLoggerHook(logger)
+
+ mock.pruneOldCgroups(tc.cfg, logger)
+
+ // create cgroups directories with a different pid
+ oldGitalyProcessMemoryDir := filepath.Join(
+ memoryRoot,
+ fmt.Sprintf("gitaly-%d", pid),
+ )
+ oldGitalyProcesssCPUDir := filepath.Join(
+ cpuRoot,
+ fmt.Sprintf("gitaly-%d", pid),
+ )
+
+ if tc.expectedPruned {
+ require.NoDirExists(t, oldGitalyProcessMemoryDir)
+ require.NoDirExists(t, oldGitalyProcesssCPUDir)
+ } else {
+ require.DirExists(t, oldGitalyProcessMemoryDir)
+ require.DirExists(t, oldGitalyProcesssCPUDir)
+ require.Empty(t, hook.AllEntries())
+ }
+ })
+
+ t.Run("cgroups-v2", func(t *testing.T) {
+ mock := newMock(t, 2)
+ tc.cfg.Mountpoint = mock.rootPath()
+
+ root := filepath.Join(
+ tc.cfg.Mountpoint,
+ tc.cfg.HierarchyRoot,
+ )
+ require.NoError(t, os.MkdirAll(root, perm.PublicDir))
+
+ pid := tc.setup(t, tc.cfg, mock)
+
+ logger := testhelper.NewLogger(t)
+ mock.pruneOldCgroups(tc.cfg, logger)
+
+ // create cgroups directories with a different pid
+ oldGitalyProcessDir := filepath.Join(
+ root,
+ fmt.Sprintf("gitaly-%d", pid),
+ )
+
+ if tc.expectedPruned {
+ require.NoDirExists(t, oldGitalyProcessDir)
+ } else {
+ require.DirExists(t, oldGitalyProcessDir)
+ }
+ })
+ })
+ }
+}
+
+func TestStats(t *testing.T) {
+ t.Parallel()
+
+ for _, tc := range []struct {
+ desc string
+ version int
+ mockFiles []mockCgroupFile
+ expectedStats Stats
+ }{
+ {
+ desc: "empty statistics",
+ version: 1,
+ mockFiles: []mockCgroupFile{
+ {"memory.limit_in_bytes", "0"},
+ {"memory.usage_in_bytes", "0"},
+ {"memory.oom_control", ""},
+ {"cpu.stat", ""},
+ },
+ expectedStats: Stats{},
+ },
+ {
+ desc: "cgroupfs recorded some stats",
+ version: 1,
+ mockFiles: []mockCgroupFile{
+ {"memory.limit_in_bytes", "2000000000"},
+ {"memory.usage_in_bytes", "1234000000"},
+ {"memory.oom_control", `oom_kill_disable 1
+under_oom 1
+oom_kill 3`},
+ {"cpu.stat", `nr_periods 10
+nr_throttled 50
+throttled_time 1000000`}, // 0.001 seconds
+ },
+ expectedStats: Stats{
+ ParentStats: CgroupStats{
+ CPUThrottledCount: 50,
+ CPUThrottledDuration: 0.001,
+ MemoryUsage: 1234000000,
+ MemoryLimit: 2000000000,
+ OOMKills: 3,
+ UnderOOM: true,
+ },
+ },
+ },
+ {
+ desc: "empty statistics",
+ version: 2,
+ mockFiles: []mockCgroupFile{
+ {"memory.current", "0"},
+ {"memory.max", "0"},
+ {"cpu.stat", ""},
+ },
+ expectedStats: Stats{},
+ },
+ {
+ desc: "cgroupfs recorded some stats",
+ version: 2,
+ mockFiles: []mockCgroupFile{
+ {"memory.max", "2000000000"},
+ {"memory.current", "1234000000"},
+ {"memory.events", `low 1
+high 2
+max 3
+oom 4
+oom_kill 5`},
+ {"nr_throttled", "50"},
+ {"throttled_usec", "1000000"},
+ {"cpu.stat", `nr_periods 10
+nr_throttled 50
+throttled_usec 1000000`}, // 0.001 seconds
+ },
+ expectedStats: Stats{
+ ParentStats: CgroupStats{
+ CPUThrottledCount: 50,
+ CPUThrottledDuration: 0.001,
+ MemoryUsage: 1234000000,
+ MemoryLimit: 2000000000,
+ OOMKills: 5,
+ },
+ },
+ },
+ } {
+ t.Run(tc.desc, func(t *testing.T) {
+ mock := newMock(t, tc.version)
+
+ config := defaultCgroupsConfig()
+ config.Repositories.Count = 1
+ config.Repositories.MemoryBytes = 2000000000
+ config.Repositories.CPUShares = 16
+ config.Mountpoint = mock.rootPath()
+
+ manager := mock.newCgroupManager(config, testhelper.SharedLogger(t), 1)
+
+ mock.setupMockCgroupFiles(t, manager, []uint{0}, tc.mockFiles...)
+ require.NoError(t, manager.Setup())
+
+ stats, err := manager.Stats()
+ require.NoError(t, err)
+ require.Equal(t, tc.expectedStats, stats)
+ })
+ }
+}
+
+func requireCgroupComponents(t *testing.T, version int, root string, cgroupPath string, expected expectedCgroup) {
+ t.Helper()
+
+ if version == 1 {
+ memoryLimitPath := filepath.Join(root, "memory", cgroupPath, "memory.limit_in_bytes")
+ requireCgroupWithInt(t, memoryLimitPath, expected.wantMemoryBytes)
+
+ cpuSharesPath := filepath.Join(root, "cpu", cgroupPath, "cpu.shares")
+ requireCgroupWithInt(t, cpuSharesPath, expected.wantCPUShares)
+
+ cpuCFSQuotaPath := filepath.Join(root, "cpu", cgroupPath, "cpu.cfs_quota_us")
+ requireCgroupWithInt(t, cpuCFSQuotaPath, expected.wantCPUQuotaUs)
+
+ cpuCFSPeriodPath := filepath.Join(root, "cpu", cgroupPath, "cpu.cfs_period_us")
+ requireCgroupWithInt(t, cpuCFSPeriodPath, expected.wantCFSPeriod)
+ } else {
+ memoryMaxPath := filepath.Join(root, cgroupPath, "memory.max")
+ requireCgroupWithInt(t, memoryMaxPath, expected.wantMemoryBytes)
+
+ cpuWeightPath := filepath.Join(root, cgroupPath, "cpu.weight")
+ requireCgroupWithInt(t, cpuWeightPath, calculateWantCPUWeight(expected.wantCPUWeight))
+
+ cpuMaxPath := filepath.Join(root, cgroupPath, "cpu.max")
+ requireCgroupWithString(t, cpuMaxPath, expected.wantCPUMax)
+ }
+}
+
+func readCgroupFile(t *testing.T, path string) []byte {
+ t.Helper()
+
+ // The cgroups package defaults to permission 0 as it expects the file to be existing (the kernel creates the file)
+ // and its testing override the permission private variable to something sensible, hence we have to chmod ourselves
+ // so we can read the file.
+ require.NoError(t, os.Chmod(path, perm.PublicFile))
+
+ return testhelper.MustReadFile(t, path)
+}
+
+func requireCgroupWithInt(t *testing.T, cgroupFile string, want int) {
+ t.Helper()
+
+ if want <= 0 {
+ return
+ }
+
+ require.Equal(t,
+ string(readCgroupFile(t, cgroupFile)),
+ strconv.Itoa(want),
+ )
+}
+
+func requireCgroupWithString(t *testing.T, cgroupFile string, want string) {
+ t.Helper()
+
+ if want == "" {
+ return
+ }
+ require.Equal(t,
+ string(readCgroupFile(t, cgroupFile)),
+ want,
+ )
+}
+
+func calculateWantCPUWeight(wantCPUWeight int) int {
+ if wantCPUWeight == 0 {
+ return 0
+ }
+ return 1 + ((wantCPUWeight-2)*9999)/262142
+}
+
+func requireShards(t *testing.T, version int, mock mockCgroup, mgr *CGroupManager, pid int, expectedShards ...uint) {
+ for shard := uint(0); shard < mgr.cfg.Repositories.Count; shard++ {
+ shouldExist := slices.Contains(expectedShards, shard)
+
+ cgroupPath := filepath.Join("gitaly",
+ fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", shard))
+ cgLock := mgr.status.getLock(cgroupPath)
+ require.Equal(t, shouldExist, cgLock.isCreated())
+
+ for _, diskPath := range mock.repoPaths(pid, shard) {
+ if shouldExist {
+ require.DirExists(t, diskPath)
+ } else {
+ require.NoDirExists(t, diskPath)
+ }
+ }
+ }
+}
diff --git a/internal/cgroups/mock_linux_test.go b/internal/cgroups/mock_linux_test.go
index ac54418a0..16f7c1900 100644
--- a/internal/cgroups/mock_linux_test.go
+++ b/internal/cgroups/mock_linux_test.go
@@ -35,12 +35,41 @@ import (
"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper"
)
-type mockCgroup struct {
+type mockCgroup interface {
+ setupMockCgroupFiles(
+ t *testing.T,
+ manager *CGroupManager,
+ shards []uint,
+ inputContent ...mockCgroupFile,
+ )
+ newCgroupManager(cfg cgroupscfg.Config, logger log.Logger, pid int) *CGroupManager
+ pruneOldCgroups(cfg cgroupscfg.Config, logger log.Logger)
+ // rootPath returns the mock's root directory.
+ rootPath() string
+ // repoPaths returns the full disk path for each subcomponent, e.g. memory, cpu, of
+ // a repository cgroup. On v2 this is a single path.
+ repoPaths(pid int, repoID uint) []string
+ // version returns the cgroup version number, 1 or 2.
+ version() int
+}
+
+func newMock(t *testing.T, version int) mockCgroup {
+ var mock mockCgroup
+ if version == 1 {
+ mock = newMockV1(t)
+ } else {
+ mock = newMockV2(t)
+ }
+
+ return mock
+}
+
+type mockCgroupV1 struct {
root string
subsystems []cgroup1.Subsystem
}
-func newMock(t *testing.T) *mockCgroup {
+func newMockV1(t *testing.T) *mockCgroupV1 {
t.Helper()
root := testhelper.TempDir(t)
@@ -52,7 +81,7 @@ func newMock(t *testing.T) *mockCgroup {
require.NoError(t, os.MkdirAll(filepath.Join(root, string(s.Name())), perm.SharedDir))
}
- return &mockCgroup{
+ return &mockCgroupV1{
root: root,
subsystems: subsystems,
}
@@ -93,7 +122,7 @@ nr_throttled 20
throttled_time 1000000`,
}
-func (m *mockCgroup) setupMockCgroupFiles(
+func (m *mockCgroupV1) setupMockCgroupFiles(
t *testing.T,
manager *CGroupManager,
shards []uint,
@@ -142,14 +171,34 @@ func (m *mockCgroup) setupMockCgroupFiles(
}
}
-func (m *mockCgroup) newCgroupManager(cfg cgroupscfg.Config, logger log.Logger, pid int) *CGroupManager {
+func (m *mockCgroupV1) newCgroupManager(cfg cgroupscfg.Config, logger log.Logger, pid int) *CGroupManager {
return newCgroupManagerWithMode(cfg, logger, pid, cgrps.Legacy)
}
-func (m *mockCgroup) pruneOldCgroups(cfg cgroupscfg.Config, logger log.Logger) {
+func (m *mockCgroupV1) pruneOldCgroups(cfg cgroupscfg.Config, logger log.Logger) {
pruneOldCgroupsWithMode(cfg, logger, cgrps.Legacy)
}
+func (m *mockCgroupV1) rootPath() string {
+ return m.root
+}
+
+func (m *mockCgroupV1) repoPaths(pid int, repoID uint) []string {
+ paths := make([]string, 0, len(m.subsystems))
+
+ for _, s := range m.subsystems {
+ path := filepath.Join(m.root, string(s.Name()), "gitaly",
+ fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", repoID))
+ paths = append(paths, path)
+ }
+
+ return paths
+}
+
+func (m *mockCgroupV1) version() int {
+ return 1
+}
+
type mockCgroupV2 struct {
root string
}
@@ -221,3 +270,18 @@ func (m *mockCgroupV2) newCgroupManager(cfg cgroupscfg.Config, logger log.Logger
func (m *mockCgroupV2) pruneOldCgroups(cfg cgroupscfg.Config, logger log.Logger) {
pruneOldCgroupsWithMode(cfg, logger, cgrps.Unified)
}
+
+func (m *mockCgroupV2) rootPath() string {
+ return m.root
+}
+
+func (m *mockCgroupV2) repoPaths(pid int, repoID uint) []string {
+ path := filepath.Join(m.root, "gitaly",
+ fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", repoID))
+
+ return []string{path}
+}
+
+func (m *mockCgroupV2) version() int {
+ return 2
+}
diff --git a/internal/cgroups/v1_linux_test.go b/internal/cgroups/v1_linux_test.go
deleted file mode 100644
index 38d81293a..000000000
--- a/internal/cgroups/v1_linux_test.go
+++ /dev/null
@@ -1,713 +0,0 @@
-//go:build linux
-
-package cgroups
-
-import (
- "fmt"
- "io/fs"
- "os"
- "os/exec"
- "path/filepath"
- "strconv"
- "strings"
- "testing"
-
- cgrps "github.com/containerd/cgroups/v3"
- "github.com/prometheus/client_golang/prometheus/testutil"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
- "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config/cgroups"
- "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm"
- "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper"
- "golang.org/x/exp/slices"
-)
-
-func defaultCgroupsConfig() cgroups.Config {
- return cgroups.Config{
- HierarchyRoot: "gitaly",
- Repositories: cgroups.Repositories{
- Count: 3,
- MemoryBytes: 1024000,
- CPUShares: 256,
- CPUQuotaUs: 200,
- },
- }
-}
-
-func TestNewManagerV1(t *testing.T) {
- cfg := cgroups.Config{Repositories: cgroups.Repositories{Count: 10}}
-
- manager := newCgroupManagerWithMode(cfg, testhelper.SharedLogger(t), 1, cgrps.Legacy)
- require.IsType(t, &cgroupV1Handler{}, manager.handler)
- manager = newCgroupManagerWithMode(cfg, testhelper.SharedLogger(t), 1, cgrps.Hybrid)
- require.IsType(t, &cgroupV1Handler{}, manager.handler)
- manager = newCgroupManagerWithMode(cfg, testhelper.SharedLogger(t), 1, cgrps.Unavailable)
- require.Nil(t, manager)
-}
-
-func TestSetup_ParentCgroups(t *testing.T) {
- tests := []struct {
- name string
- cfg cgroups.Config
- wantMemoryBytes int
- wantCPUShares int
- wantCPUQuotaUs int
- wantCFSPeriod int
- }{
- {
- name: "all config specified",
- cfg: cgroups.Config{
- MemoryBytes: 102400,
- CPUShares: 256,
- CPUQuotaUs: 200,
- },
- wantMemoryBytes: 102400,
- wantCPUShares: 256,
- wantCPUQuotaUs: 200,
- wantCFSPeriod: int(cfsPeriodUs),
- },
- {
- name: "only memory limit set",
- cfg: cgroups.Config{
- MemoryBytes: 102400,
- },
- wantMemoryBytes: 102400,
- },
- {
- name: "only cpu shares set",
- cfg: cgroups.Config{
- CPUShares: 512,
- },
- wantCPUShares: 512,
- },
- {
- name: "only cpu quota set",
- cfg: cgroups.Config{
- CPUQuotaUs: 200,
- },
- wantCPUQuotaUs: 200,
- wantCFSPeriod: int(cfsPeriodUs),
- },
- }
-
- for _, tt := range tests {
- tt := tt
- t.Run(tt.name, func(t *testing.T) {
- t.Parallel()
-
- mock := newMock(t)
- pid := 1
- tt.cfg.HierarchyRoot = "gitaly"
- tt.cfg.Mountpoint = mock.root
-
- v1Manager := mock.newCgroupManager(tt.cfg, testhelper.SharedLogger(t), pid)
- require.False(t, v1Manager.Ready())
- require.NoError(t, v1Manager.Setup())
- require.True(t, v1Manager.Ready())
-
- memoryLimitPath := filepath.Join(
- mock.root, "memory", "gitaly", fmt.Sprintf("gitaly-%d", pid), "memory.limit_in_bytes",
- )
- requireCgroup(t, memoryLimitPath, tt.wantMemoryBytes)
-
- cpuSharesPath := filepath.Join(
- mock.root, "cpu", "gitaly", fmt.Sprintf("gitaly-%d", pid), "cpu.shares",
- )
- requireCgroup(t, cpuSharesPath, tt.wantCPUShares)
-
- cpuCFSQuotaPath := filepath.Join(
- mock.root, "cpu", "gitaly", fmt.Sprintf("gitaly-%d", pid), "cpu.cfs_quota_us",
- )
- requireCgroup(t, cpuCFSQuotaPath, tt.wantCPUQuotaUs)
-
- cpuCFSPeriodPath := filepath.Join(
- mock.root, "cpu", "gitaly", fmt.Sprintf("gitaly-%d", pid), "cpu.cfs_period_us",
- )
- requireCgroup(t, cpuCFSPeriodPath, tt.wantCFSPeriod)
- })
- }
-}
-
-func TestSetup_RepoCgroups(t *testing.T) {
- tests := []struct {
- name string
- cfg cgroups.Repositories
- wantMemoryBytes int
- wantCPUShares int
- wantCPUQuotaUs int
- wantCFSPeriod int
- }{
- {
- name: "all config specified",
- cfg: defaultCgroupsConfig().Repositories,
- wantMemoryBytes: 1024000,
- wantCPUShares: 256,
- wantCPUQuotaUs: 200,
- wantCFSPeriod: int(cfsPeriodUs),
- },
- {
- name: "only memory limit set",
- cfg: cgroups.Repositories{
- MemoryBytes: 1024000,
- },
- wantMemoryBytes: 1024000,
- },
- {
- name: "only cpu shares set",
- cfg: cgroups.Repositories{
- CPUShares: 512,
- },
- wantCPUShares: 512,
- },
- {
- name: "only cpu quota set",
- cfg: cgroups.Repositories{
- CPUQuotaUs: 100,
- },
- wantCPUQuotaUs: 100,
- wantCFSPeriod: int(cfsPeriodUs),
- },
- }
-
- for _, tt := range tests {
- tt := tt
- t.Run(tt.name, func(t *testing.T) {
- t.Parallel()
-
- mock := newMock(t)
-
- pid := 1
- cfg := defaultCgroupsConfig()
- cfg.Repositories = tt.cfg
- cfg.Repositories.Count = 3
- cfg.HierarchyRoot = "gitaly"
- cfg.Mountpoint = mock.root
-
- v1Manager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), pid)
-
- require.False(t, v1Manager.Ready())
- require.NoError(t, v1Manager.Setup())
- require.True(t, v1Manager.Ready())
-
- ctx := testhelper.Context(t)
-
- // Validate no shards have been created. We deliberately do not call
- // `setupMockCgroupFiles()` here to confirm that the cgroup controller
- // is creating repository directories in the correct location.
- requireShardsV1(t, mock, v1Manager, pid)
-
- // Create a command to force Gitaly to create the repo cgroup.
- cmd := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...)
- require.NoError(t, cmd.Run())
- _, err := v1Manager.AddCommand(cmd)
- require.NoError(t, err)
-
- groupID := calcGroupID(cmd.Args, cfg.Repositories.Count)
- requireShardsV1(t, mock, v1Manager, pid, groupID)
-
- for i := 0; i < 3; i++ {
- cgroupExists := uint(i) == groupID
-
- memoryLimitPath := filepath.Join(
- mock.root, "memory", "gitaly", fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", i), "memory.limit_in_bytes",
- )
- if cgroupExists {
- requireCgroup(t, memoryLimitPath, tt.wantMemoryBytes)
- } else {
- require.NoFileExists(t, memoryLimitPath)
- }
-
- cpuSharesPath := filepath.Join(
- mock.root, "cpu", "gitaly", fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", i), "cpu.shares",
- )
- if cgroupExists {
- requireCgroup(t, cpuSharesPath, tt.wantCPUShares)
- } else {
- require.NoFileExists(t, cpuSharesPath)
- }
-
- cpuCFSQuotaPath := filepath.Join(
- mock.root, "cpu", "gitaly", fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", i), "cpu.cfs_quota_us",
- )
- if cgroupExists {
- requireCgroup(t, cpuCFSQuotaPath, tt.wantCPUQuotaUs)
- } else {
- require.NoFileExists(t, cpuCFSQuotaPath)
- }
-
- cpuCFSPeriodPath := filepath.Join(
- mock.root, "cpu", "gitaly", fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", i), "cpu.cfs_period_us",
- )
- if cgroupExists {
- requireCgroup(t, cpuCFSPeriodPath, tt.wantCFSPeriod)
- } else {
- require.NoFileExists(t, cpuCFSPeriodPath)
- }
- }
- })
- }
-}
-
-func TestAddCommand(t *testing.T) {
- mock := newMock(t)
-
- config := defaultCgroupsConfig()
- config.Repositories.Count = 10
- config.Repositories.MemoryBytes = 1024
- config.Repositories.CPUShares = 16
- config.HierarchyRoot = "gitaly"
- config.Mountpoint = mock.root
-
- pid := 1
- groupID := calcGroupID(cmdArgs, config.Repositories.Count)
-
- v1Manager1 := mock.newCgroupManager(config, testhelper.SharedLogger(t), pid)
- require.NoError(t, v1Manager1.Setup())
- ctx := testhelper.Context(t)
-
- cmd2 := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...)
- require.NoError(t, cmd2.Run())
-
- v1Manager2 := mock.newCgroupManager(config, testhelper.SharedLogger(t), pid)
-
- t.Run("without overridden key", func(t *testing.T) {
- _, err := v1Manager2.AddCommand(cmd2)
- require.NoError(t, err)
- requireShardsV1(t, mock, v1Manager2, pid, groupID)
-
- for _, s := range mock.subsystems {
- path := filepath.Join(mock.root, string(s.Name()), "gitaly",
- fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", groupID), "cgroup.procs")
- content := readCgroupFile(t, path)
-
- cmdPid, err := strconv.Atoi(string(content))
- require.NoError(t, err)
-
- require.Equal(t, cmd2.Process.Pid, cmdPid)
- }
- })
-
- t.Run("with overridden key", func(t *testing.T) {
- overridenGroupID := calcGroupID([]string{"foobar"}, config.Repositories.Count)
-
- _, err := v1Manager2.AddCommand(cmd2, WithCgroupKey("foobar"))
- require.NoError(t, err)
-
- for _, s := range mock.subsystems {
- path := filepath.Join(mock.root, string(s.Name()), "gitaly",
- fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", overridenGroupID), "cgroup.procs")
- content := readCgroupFile(t, path)
-
- cmdPid, err := strconv.Atoi(string(content))
- require.NoError(t, err)
-
- require.Equal(t, cmd2.Process.Pid, cmdPid)
- }
- })
-}
-
-func TestCleanup(t *testing.T) {
- mock := newMock(t)
-
- pid := 1
- cfg := defaultCgroupsConfig()
- cfg.Mountpoint = mock.root
-
- v1Manager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), pid)
-
- require.NoError(t, v1Manager.Setup())
- require.NoError(t, v1Manager.Cleanup())
-
- for i := 0; i < 3; i++ {
- memoryPath := filepath.Join(mock.root, "memory", "gitaly", fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", i))
- cpuPath := filepath.Join(mock.root, "cpu", "gitaly", fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", i))
-
- require.NoDirExists(t, memoryPath)
- require.NoDirExists(t, cpuPath)
- }
-}
-
-func TestMetrics(t *testing.T) {
- tests := []struct {
- name string
- metricsEnabled bool
- pid int
- expect string
- }{
- {
- name: "metrics enabled: true",
- metricsEnabled: true,
- pid: 1,
- expect: `# HELP gitaly_cgroup_cpu_usage_total CPU Usage of Cgroup
-# TYPE gitaly_cgroup_cpu_usage_total gauge
-gitaly_cgroup_cpu_usage_total{path="%s",type="kernel"} 0
-gitaly_cgroup_cpu_usage_total{path="%s",type="user"} 0
-# HELP gitaly_cgroup_memory_reclaim_attempts_total Number of memory usage hits limits
-# TYPE gitaly_cgroup_memory_reclaim_attempts_total gauge
-gitaly_cgroup_memory_reclaim_attempts_total{path="%s"} 2
-# HELP gitaly_cgroup_procs_total Total number of procs
-# TYPE gitaly_cgroup_procs_total gauge
-gitaly_cgroup_procs_total{path="%s",subsystem="cpu"} 1
-gitaly_cgroup_procs_total{path="%s",subsystem="memory"} 1
-# HELP gitaly_cgroup_cpu_cfs_periods_total Number of elapsed enforcement period intervals
-# TYPE gitaly_cgroup_cpu_cfs_periods_total counter
-gitaly_cgroup_cpu_cfs_periods_total{path="%s"} 10
-# HELP gitaly_cgroup_cpu_cfs_throttled_periods_total Number of throttled period intervals
-# TYPE gitaly_cgroup_cpu_cfs_throttled_periods_total counter
-gitaly_cgroup_cpu_cfs_throttled_periods_total{path="%s"} 20
-# HELP gitaly_cgroup_cpu_cfs_throttled_seconds_total Total time duration the Cgroup has been throttled
-# TYPE gitaly_cgroup_cpu_cfs_throttled_seconds_total counter
-gitaly_cgroup_cpu_cfs_throttled_seconds_total{path="%s"} 0.001
-`,
- },
- {
- name: "metrics enabled: false",
- metricsEnabled: false,
- pid: 2,
- },
- }
-
- for _, tt := range tests {
- tt := tt
- t.Run(tt.name, func(t *testing.T) {
- t.Parallel()
- mock := newMock(t)
-
- config := defaultCgroupsConfig()
- config.Repositories.Count = 1
- config.Repositories.MemoryBytes = 1048576
- config.Repositories.CPUShares = 16
- config.Mountpoint = mock.root
- config.MetricsEnabled = tt.metricsEnabled
-
- v1Manager1 := mock.newCgroupManager(config, testhelper.SharedLogger(t), tt.pid)
-
- groupID := calcGroupID(cmdArgs, config.Repositories.Count)
-
- mock.setupMockCgroupFiles(t, v1Manager1, []uint{groupID}, mockCgroupFile{"memory.failcnt", "2"})
- require.NoError(t, v1Manager1.Setup())
-
- ctx := testhelper.Context(t)
-
- cmd := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...)
- require.NoError(t, cmd.Start())
- _, err := v1Manager1.AddCommand(cmd)
- require.NoError(t, err)
-
- gitCmd1 := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...)
- require.NoError(t, gitCmd1.Start())
- _, err = v1Manager1.AddCommand(gitCmd1)
- require.NoError(t, err)
-
- gitCmd2 := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...)
- require.NoError(t, gitCmd2.Start())
- _, err = v1Manager1.AddCommand(gitCmd2)
- require.NoError(t, err)
-
- requireShardsV1(t, mock, v1Manager1, tt.pid, groupID)
-
- defer func() {
- require.NoError(t, gitCmd2.Wait())
- }()
-
- require.NoError(t, cmd.Wait())
- require.NoError(t, gitCmd1.Wait())
-
- repoCgroupPath := filepath.Join(v1Manager1.currentProcessCgroup(), "repos-0")
-
- expected := strings.NewReader(strings.ReplaceAll(tt.expect, "%s", repoCgroupPath))
- assert.NoError(t, testutil.CollectAndCompare(v1Manager1, expected))
- })
- }
-}
-
-func TestPruneOldCgroups(t *testing.T) {
- t.Parallel()
-
- testCases := []struct {
- desc string
- cfg cgroups.Config
- expectedPruned bool
- // setup returns a pid
- setup func(t *testing.T, cfg cgroups.Config, mock *mockCgroup) int
- }{
- {
- desc: "process belongs to another user",
- cfg: cgroups.Config{
- HierarchyRoot: "gitaly",
- Repositories: cgroups.Repositories{
- Count: 10,
- MemoryBytes: 10 * 1024 * 1024,
- CPUShares: 1024,
- },
- },
- setup: func(t *testing.T, cfg cgroups.Config, mock *mockCgroup) int {
- pid := 1
- cgroupManager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), pid)
- require.NoError(t, cgroupManager.Setup())
-
- return pid
- },
- expectedPruned: true,
- },
- {
- desc: "no hierarchy root",
- cfg: cgroups.Config{
- HierarchyRoot: "",
- Repositories: cgroups.Repositories{
- Count: 10,
- MemoryBytes: 10 * 1024 * 1024,
- CPUShares: 1024,
- },
- },
- setup: func(t *testing.T, cfg cgroups.Config, mock *mockCgroup) int {
- pid := 1
- cgroupManager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), pid)
- require.NoError(t, cgroupManager.Setup())
- return 1
- },
- expectedPruned: false,
- },
- {
- desc: "pid of finished process",
- cfg: cgroups.Config{
- HierarchyRoot: "gitaly",
- Repositories: cgroups.Repositories{
- Count: 10,
- MemoryBytes: 10 * 1024 * 1024,
- CPUShares: 1024,
- },
- },
- setup: func(t *testing.T, cfg cgroups.Config, mock *mockCgroup) int {
- cmd := exec.Command("ls")
- require.NoError(t, cmd.Run())
- pid := cmd.Process.Pid
-
- cgroupManager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), pid)
- require.NoError(t, cgroupManager.Setup())
-
- memoryRoot := filepath.Join(
- cfg.Mountpoint,
- "memory",
- cfg.HierarchyRoot,
- "memory.limit_in_bytes",
- )
- require.NoError(t, os.WriteFile(memoryRoot, []byte{}, fs.ModeAppend))
-
- return pid
- },
- expectedPruned: true,
- },
- {
- desc: "pid of running process",
- cfg: cgroups.Config{
- HierarchyRoot: "gitaly",
- Repositories: cgroups.Repositories{
- Count: 10,
- MemoryBytes: 10 * 1024 * 1024,
- CPUShares: 1024,
- },
- },
- setup: func(t *testing.T, cfg cgroups.Config, mock *mockCgroup) int {
- pid := os.Getpid()
-
- cgroupManager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), pid)
- require.NoError(t, cgroupManager.Setup())
-
- return pid
- },
- expectedPruned: false,
- },
- {
- desc: "gitaly-0 directory is deleted",
- cfg: cgroups.Config{
- HierarchyRoot: "gitaly",
- Repositories: cgroups.Repositories{
- Count: 10,
- MemoryBytes: 10 * 1024 * 1024,
- CPUShares: 1024,
- },
- },
- setup: func(t *testing.T, cfg cgroups.Config, mock *mockCgroup) int {
- cgroupManager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), 0)
- require.NoError(t, cgroupManager.Setup())
-
- return 0
- },
- expectedPruned: true,
- },
- }
-
- for _, tc := range testCases {
- t.Run(tc.desc, func(t *testing.T) {
- mock := newMock(t)
- tc.cfg.Mountpoint = mock.root
-
- memoryRoot := filepath.Join(
- tc.cfg.Mountpoint,
- "memory",
- tc.cfg.HierarchyRoot,
- )
- cpuRoot := filepath.Join(
- tc.cfg.Mountpoint,
- "cpu",
- tc.cfg.HierarchyRoot,
- )
-
- require.NoError(t, os.MkdirAll(cpuRoot, perm.PublicDir))
- require.NoError(t, os.MkdirAll(memoryRoot, perm.PublicDir))
-
- pid := tc.setup(t, tc.cfg, mock)
-
- logger := testhelper.NewLogger(t)
- hook := testhelper.AddLoggerHook(logger)
-
- mock.pruneOldCgroups(tc.cfg, logger)
-
- // create cgroups directories with a different pid
- oldGitalyProcessMemoryDir := filepath.Join(
- memoryRoot,
- fmt.Sprintf("gitaly-%d", pid),
- )
- oldGitalyProcesssCPUDir := filepath.Join(
- cpuRoot,
- fmt.Sprintf("gitaly-%d", pid),
- )
-
- if tc.expectedPruned {
- require.NoDirExists(t, oldGitalyProcessMemoryDir)
- require.NoDirExists(t, oldGitalyProcesssCPUDir)
- } else {
- require.DirExists(t, oldGitalyProcessMemoryDir)
- require.DirExists(t, oldGitalyProcesssCPUDir)
- require.Empty(t, hook.AllEntries())
- }
- })
- }
-}
-
-func TestStatsV1(t *testing.T) {
- t.Parallel()
-
- for _, tc := range []struct {
- desc string
- mockFiles []mockCgroupFile
- expectedStats Stats
- }{
- {
- desc: "empty statistics",
- mockFiles: []mockCgroupFile{
- {"memory.limit_in_bytes", "0"},
- {"memory.usage_in_bytes", "0"},
- {"memory.oom_control", ""},
- {"cpu.stat", ""},
- },
- expectedStats: Stats{},
- },
- {
- desc: "cgroupfs recorded some stats",
- mockFiles: []mockCgroupFile{
- {"memory.limit_in_bytes", "2000000000"},
- {"memory.usage_in_bytes", "1234000000"},
- {"memory.oom_control", `oom_kill_disable 1
-under_oom 1
-oom_kill 3`},
- {"cpu.stat", `nr_periods 10
-nr_throttled 50
-throttled_time 1000000`}, // 0.001 seconds
- {"memory.stat", `cache 235000000
-rss 234000000
-inactive_anon 200000000
-active_anon 34000000
-inactive_file 100000000
-active_file 135000000`},
- },
- expectedStats: Stats{
- ParentStats: CgroupStats{
- CPUThrottledCount: 50,
- CPUThrottledDuration: 0.001,
- MemoryUsage: 1234000000,
- MemoryLimit: 2000000000,
- OOMKills: 3,
- UnderOOM: true,
- Anon: 234000000,
- ActiveAnon: 34000000,
- InactiveAnon: 200000000,
- File: 235000000,
- ActiveFile: 135000000,
- InactiveFile: 100000000,
- },
- },
- },
- } {
- t.Run(tc.desc, func(t *testing.T) {
- mock := newMock(t)
-
- config := defaultCgroupsConfig()
- config.Repositories.Count = 1
- config.Repositories.MemoryBytes = 2000000000
- config.Repositories.CPUShares = 16
- config.Mountpoint = mock.root
-
- v1Manager := mock.newCgroupManager(config, testhelper.SharedLogger(t), 1)
-
- mock.setupMockCgroupFiles(t, v1Manager, []uint{}, tc.mockFiles...)
- require.NoError(t, v1Manager.Setup())
-
- stats, err := v1Manager.Stats()
- require.NoError(t, err)
- require.Equal(t, tc.expectedStats, stats)
- })
- }
-}
-
-func requireShardsV1(t *testing.T, mock *mockCgroup, mgr *CGroupManager, pid int, expectedShards ...uint) {
- t.Helper()
-
- for shard := uint(0); shard < mgr.cfg.Repositories.Count; shard++ {
- for _, s := range mock.subsystems {
- cgroupPath := filepath.Join("gitaly",
- fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", shard))
- diskPath := filepath.Join(mock.root, string(s.Name()), cgroupPath)
-
- if slices.Contains(expectedShards, shard) {
- require.DirExists(t, diskPath)
-
- cgLock := mgr.status.getLock(cgroupPath)
- require.True(t, cgLock.isCreated())
- } else {
- require.NoDirExists(t, diskPath)
-
- // Confirm we pre-populated this map entry.
- _, lockInserted := mgr.status.m[cgroupPath]
- require.True(t, lockInserted)
- }
- }
- }
-}
-
-func requireCgroup(t *testing.T, cgroupFile string, want int) {
- t.Helper()
-
- if want <= 0 {
- // If files doesn't exist kernel will create it with default values
- require.NoFileExistsf(t, cgroupFile, "cgroup file should not exist: %q", cgroupFile)
- return
- }
-
- require.Equal(t,
- string(readCgroupFile(t, cgroupFile)),
- strconv.Itoa(want),
- )
-}
-
-func readCgroupFile(t *testing.T, path string) []byte {
- t.Helper()
-
- // The cgroups package defaults to permission 0 as it expects the file to be existing (the kernel creates the file)
- // and its testing override the permission private variable to something sensible, hence we have to chmod ourselves
- // so we can read the file.
- require.NoError(t, os.Chmod(path, perm.PublicFile))
-
- return testhelper.MustReadFile(t, path)
-}
diff --git a/internal/cgroups/v2_linux_test.go b/internal/cgroups/v2_linux_test.go
deleted file mode 100644
index e6a788b4e..000000000
--- a/internal/cgroups/v2_linux_test.go
+++ /dev/null
@@ -1,689 +0,0 @@
-//go:build linux
-
-package cgroups
-
-import (
- "fmt"
- "io/fs"
- "os"
- "os/exec"
- "path/filepath"
- "strconv"
- "strings"
- "testing"
-
- cgrps "github.com/containerd/cgroups/v3"
- "github.com/prometheus/client_golang/prometheus/testutil"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
- "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config/cgroups"
- "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm"
- "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper"
- "golang.org/x/exp/slices"
-)
-
-func defaultCgroupsV2Config() cgroups.Config {
- return cgroups.Config{
- HierarchyRoot: "gitaly",
- Repositories: cgroups.Repositories{
- Count: 3,
- MemoryBytes: 1024000,
- CPUShares: 256,
- CPUQuotaUs: 2000,
- },
- }
-}
-
-func TestNewManagerV2(t *testing.T) {
- cfg := cgroups.Config{Repositories: cgroups.Repositories{Count: 10}}
-
- manager := newCgroupManagerWithMode(cfg, testhelper.SharedLogger(t), 1, cgrps.Unified)
- require.IsType(t, &cgroupV2Handler{}, manager.handler)
-}
-
-func TestSetup_ParentCgroupsV2(t *testing.T) {
- tests := []struct {
- name string
- cfg cgroups.Config
- wantMemoryBytes int
- wantCPUWeight int
- wantCPUMax string
- }{
- {
- name: "all config specified",
- cfg: cgroups.Config{
- MemoryBytes: 102400,
- CPUShares: 256,
- CPUQuotaUs: 2000,
- },
- wantMemoryBytes: 102400,
- wantCPUWeight: 256,
- wantCPUMax: "2000 100000",
- },
- {
- name: "only memory limit set",
- cfg: cgroups.Config{
- MemoryBytes: 102400,
- },
- wantMemoryBytes: 102400,
- },
- {
- name: "only cpu shares set",
- cfg: cgroups.Config{
- CPUShares: 512,
- },
- wantCPUWeight: 512,
- },
- {
- name: "only cpu quota set",
- cfg: cgroups.Config{
- CPUQuotaUs: 2000,
- },
- wantCPUMax: "2000 100000",
- },
- }
-
- for _, tt := range tests {
- tt := tt
- t.Run(tt.name, func(t *testing.T) {
- t.Parallel()
-
- mock := newMockV2(t)
-
- pid := 1
- tt.cfg.HierarchyRoot = "gitaly"
- tt.cfg.Mountpoint = mock.root
-
- v2Manager := mock.newCgroupManager(tt.cfg, testhelper.SharedLogger(t), pid)
- mock.setupMockCgroupFiles(t, v2Manager, []uint{})
-
- require.False(t, v2Manager.Ready())
- require.NoError(t, v2Manager.Setup())
- require.True(t, v2Manager.Ready())
-
- memoryMaxPath := filepath.Join(
- mock.root, "gitaly", fmt.Sprintf("gitaly-%d", pid), "memory.max",
- )
- requireCgroupWithInt(t, memoryMaxPath, tt.wantMemoryBytes)
-
- cpuWeightPath := filepath.Join(
- mock.root, "gitaly", fmt.Sprintf("gitaly-%d", pid), "cpu.weight",
- )
- requireCgroupWithInt(t, cpuWeightPath, calculateWantCPUWeight(tt.wantCPUWeight))
-
- cpuMaxPath := filepath.Join(
- mock.root, "gitaly", fmt.Sprintf("gitaly-%d", pid), "cpu.max",
- )
- requireCgroupWithString(t, cpuMaxPath, tt.wantCPUMax)
- })
- }
-}
-
-func TestSetup_RepoCgroupsV2(t *testing.T) {
- tests := []struct {
- name string
- cfg cgroups.Repositories
- wantMemoryBytes int
- wantCPUWeight int
- wantCPUMax string
- }{
- {
- name: "all config specified",
- cfg: defaultCgroupsV2Config().Repositories,
- wantMemoryBytes: 1024000,
- wantCPUWeight: 256,
- wantCPUMax: "2000 100000",
- },
- {
- name: "only memory limit set",
- cfg: cgroups.Repositories{
- Count: 3,
- MemoryBytes: 1024000,
- },
- wantMemoryBytes: 1024000,
- },
- {
- name: "only cpu shares set",
- cfg: cgroups.Repositories{
- Count: 3,
- CPUShares: 512,
- },
- wantCPUWeight: 512,
- },
- {
- name: "only cpu quota set",
- cfg: cgroups.Repositories{
- Count: 3,
- CPUQuotaUs: 1000,
- },
- wantCPUMax: "1000 100000",
- },
- }
-
- for _, tt := range tests {
- tt := tt
- t.Run(tt.name, func(t *testing.T) {
- t.Parallel()
-
- mock := newMockV2(t)
-
- pid := 1
-
- cfg := defaultCgroupsV2Config()
- cfg.Mountpoint = mock.root
- cfg.Repositories = tt.cfg
-
- groupID := calcGroupID(cmdArgs, cfg.Repositories.Count)
-
- v2Manager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), pid)
-
- // Validate no shards have been created. We deliberately do not call
- // `setupMockCgroupFiles()` here to confirm that the cgroup controller
- // is creating repository directories in the correct location.
- requireShardsV2(t, mock, v2Manager, pid)
-
- mock.setupMockCgroupFiles(t, v2Manager, []uint{groupID})
-
- require.False(t, v2Manager.Ready())
- require.NoError(t, v2Manager.Setup())
- require.True(t, v2Manager.Ready())
-
- ctx := testhelper.Context(t)
-
- // Create a command to force Gitaly to create the repo cgroup.
- cmd := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...)
- require.NoError(t, cmd.Run())
- _, err := v2Manager.AddCommand(cmd)
- require.NoError(t, err)
-
- requireShardsV2(t, mock, v2Manager, pid, groupID)
-
- for i := 0; i < 3; i++ {
- cgroupExists := uint(i) == groupID
-
- memoryMaxPath := filepath.Join(
- mock.root, "gitaly", fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", i), "memory.max",
- )
-
- if cgroupExists {
- requireCgroupWithInt(t, memoryMaxPath, tt.wantMemoryBytes)
- } else {
- require.NoFileExists(t, memoryMaxPath)
- }
-
- cpuWeightPath := filepath.Join(
- mock.root, "gitaly", fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", i), "cpu.weight",
- )
-
- if cgroupExists {
- requireCgroupWithInt(t, cpuWeightPath, calculateWantCPUWeight(tt.wantCPUWeight))
- } else {
- require.NoFileExists(t, cpuWeightPath)
- }
-
- cpuMaxPath := filepath.Join(
- mock.root, "gitaly", fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", i), "cpu.max",
- )
-
- if cgroupExists {
- requireCgroupWithString(t, cpuMaxPath, tt.wantCPUMax)
- } else {
- require.NoFileExists(t, cpuMaxPath)
- }
- }
- })
- }
-}
-
-func TestAddCommandV2(t *testing.T) {
- mock := newMockV2(t)
-
- config := defaultCgroupsV2Config()
- config.Repositories.Count = 10
- config.Repositories.MemoryBytes = 1024
- config.Repositories.CPUShares = 16
- config.Mountpoint = mock.root
-
- pid := 1
- groupID := calcGroupID(cmdArgs, config.Repositories.Count)
-
- v2Manager1 := mock.newCgroupManager(config, testhelper.SharedLogger(t), pid)
- mock.setupMockCgroupFiles(t, v2Manager1, []uint{})
-
- require.NoError(t, v2Manager1.Setup())
- ctx := testhelper.Context(t)
-
- cmd2 := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...)
- require.NoError(t, cmd2.Run())
-
- v2Manager2 := mock.newCgroupManager(config, testhelper.SharedLogger(t), pid)
-
- t.Run("without overridden key", func(t *testing.T) {
- groupID := calcGroupID(cmd2.Args, config.Repositories.Count)
-
- _, err := v2Manager2.AddCommand(cmd2)
- require.NoError(t, err)
- requireShardsV2(t, mock, v2Manager2, pid, groupID)
-
- path := filepath.Join(mock.root, "gitaly",
- fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", groupID), "cgroup.procs")
- content := readCgroupFile(t, path)
-
- cmdPid, err := strconv.Atoi(string(content))
- require.NoError(t, err)
-
- require.Equal(t, cmd2.Process.Pid, cmdPid)
- })
-
- t.Run("with overridden key", func(t *testing.T) {
- overriddenGroupID := calcGroupID([]string{"foobar"}, config.Repositories.Count)
-
- _, err := v2Manager2.AddCommand(cmd2, WithCgroupKey("foobar"))
- require.NoError(t, err)
- requireShardsV2(t, mock, v2Manager2, pid, groupID, overriddenGroupID)
-
- path := filepath.Join(mock.root, "gitaly",
- fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", overriddenGroupID), "cgroup.procs")
- content := readCgroupFile(t, path)
-
- cmdPid, err := strconv.Atoi(string(content))
- require.NoError(t, err)
-
- require.Equal(t, cmd2.Process.Pid, cmdPid)
- })
-}
-
-func TestCleanupV2(t *testing.T) {
- mock := newMockV2(t)
-
- pid := 1
- cfg := defaultCgroupsV2Config()
- cfg.Mountpoint = mock.root
-
- v2Manager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), pid)
- mock.setupMockCgroupFiles(t, v2Manager, []uint{0, 1, 2})
-
- require.NoError(t, v2Manager.Setup())
- require.NoError(t, v2Manager.Cleanup())
-
- for i := 0; i < 3; i++ {
- require.NoDirExists(t, filepath.Join(mock.root, "gitaly", fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", i)))
- }
-}
-
-func TestMetricsV2(t *testing.T) {
- tests := []struct {
- name string
- metricsEnabled bool
- pid int
- expect string
- }{
- {
- name: "metrics enabled: true",
- metricsEnabled: true,
- pid: 1,
- expect: `# HELP gitaly_cgroup_cpu_cfs_periods_total Number of elapsed enforcement period intervals
-# TYPE gitaly_cgroup_cpu_cfs_periods_total counter
-gitaly_cgroup_cpu_cfs_periods_total{path="%s"} 10
-# HELP gitaly_cgroup_cpu_cfs_throttled_periods_total Number of throttled period intervals
-# TYPE gitaly_cgroup_cpu_cfs_throttled_periods_total counter
-gitaly_cgroup_cpu_cfs_throttled_periods_total{path="%s"} 20
-# HELP gitaly_cgroup_cpu_cfs_throttled_seconds_total Total time duration the Cgroup has been throttled
-# TYPE gitaly_cgroup_cpu_cfs_throttled_seconds_total counter
-gitaly_cgroup_cpu_cfs_throttled_seconds_total{path="%s"} 0.001
-# HELP gitaly_cgroup_cpu_usage_total CPU Usage of Cgroup
-# TYPE gitaly_cgroup_cpu_usage_total gauge
-gitaly_cgroup_cpu_usage_total{path="%s",type="kernel"} 0
-gitaly_cgroup_cpu_usage_total{path="%s",type="user"} 0
-# HELP gitaly_cgroup_procs_total Total number of procs
-# TYPE gitaly_cgroup_procs_total gauge
-gitaly_cgroup_procs_total{path="%s",subsystem="cpu"} 1
-gitaly_cgroup_procs_total{path="%s",subsystem="cpuset"} 1
-gitaly_cgroup_procs_total{path="%s",subsystem="memory"} 1
-`,
- },
- {
- name: "metrics enabled: false",
- metricsEnabled: false,
- pid: 2,
- },
- }
-
- for _, tt := range tests {
- tt := tt
- t.Run(tt.name, func(t *testing.T) {
- t.Parallel()
- mock := newMockV2(t)
-
- config := defaultCgroupsV2Config()
- config.Repositories.Count = 1
- config.Repositories.MemoryBytes = 1048576
- config.Repositories.CPUShares = 16
- config.Mountpoint = mock.root
- config.MetricsEnabled = tt.metricsEnabled
-
- groupID := calcGroupID(cmdArgs, config.Repositories.Count)
- v2Manager1 := mock.newCgroupManager(config, testhelper.SharedLogger(t), tt.pid)
-
- mock.setupMockCgroupFiles(t, v2Manager1, []uint{groupID})
- require.NoError(t, v2Manager1.Setup())
-
- ctx := testhelper.Context(t)
-
- cmd := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...)
- require.NoError(t, cmd.Start())
- _, err := v2Manager1.AddCommand(cmd)
- require.NoError(t, err)
-
- gitCmd1 := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...)
- require.NoError(t, gitCmd1.Start())
- _, err = v2Manager1.AddCommand(gitCmd1)
- require.NoError(t, err)
-
- gitCmd2 := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...)
- require.NoError(t, gitCmd2.Start())
- _, err = v2Manager1.AddCommand(gitCmd2)
- require.NoError(t, err)
-
- requireShardsV2(t, mock, v2Manager1, tt.pid, groupID)
-
- defer func() {
- require.NoError(t, gitCmd2.Wait())
- }()
-
- require.NoError(t, cmd.Wait())
- require.NoError(t, gitCmd1.Wait())
-
- repoCgroupPath := filepath.Join(v2Manager1.currentProcessCgroup(), "repos-0")
-
- expected := strings.NewReader(strings.ReplaceAll(tt.expect, "%s", repoCgroupPath))
-
- assert.NoError(t, testutil.CollectAndCompare(v2Manager1, expected))
- })
- }
-}
-
-func TestPruneOldCgroupsV2(t *testing.T) {
- t.Parallel()
-
- testCases := []struct {
- desc string
- cfg cgroups.Config
- expectedPruned bool
- // setup returns a pid
- setup func(*testing.T, cgroups.Config, *mockCgroupV2) int
- }{
- {
- desc: "process belongs to another user",
- cfg: cgroups.Config{
- HierarchyRoot: "gitaly",
- Repositories: cgroups.Repositories{
- Count: 10,
- MemoryBytes: 10 * 1024 * 1024,
- CPUShares: 1024,
- },
- },
- setup: func(t *testing.T, cfg cgroups.Config, mock *mockCgroupV2) int {
- pid := 1
-
- cgroupManager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), pid)
- mock.setupMockCgroupFiles(t, cgroupManager, []uint{0, 1, 2})
- require.NoError(t, cgroupManager.Setup())
-
- return pid
- },
- expectedPruned: true,
- },
- {
- desc: "no hierarchy root",
- cfg: cgroups.Config{
- HierarchyRoot: "",
- Repositories: cgroups.Repositories{
- Count: 10,
- MemoryBytes: 10 * 1024 * 1024,
- CPUShares: 1024,
- },
- },
- setup: func(t *testing.T, cfg cgroups.Config, mock *mockCgroupV2) int {
- pid := 1
-
- cgroupManager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), pid)
- mock.setupMockCgroupFiles(t, cgroupManager, []uint{0, 1, 2})
- require.NoError(t, cgroupManager.Setup())
- return 1
- },
- expectedPruned: false,
- },
- {
- desc: "pid of finished process",
- cfg: cgroups.Config{
- HierarchyRoot: "gitaly",
- Repositories: cgroups.Repositories{
- Count: 10,
- MemoryBytes: 10 * 1024 * 1024,
- CPUShares: 1024,
- },
- },
- setup: func(t *testing.T, cfg cgroups.Config, mock *mockCgroupV2) int {
- cmd := exec.Command("ls")
- require.NoError(t, cmd.Run())
- pid := cmd.Process.Pid
-
- cgroupManager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), pid)
- mock.setupMockCgroupFiles(t, cgroupManager, []uint{0, 1, 2})
- require.NoError(t, cgroupManager.Setup())
-
- memoryFile := filepath.Join(
- cfg.Mountpoint,
- cfg.HierarchyRoot,
- "memory.limit_in_bytes",
- )
- require.NoError(t, os.WriteFile(memoryFile, []byte{}, fs.ModeAppend))
-
- return pid
- },
- expectedPruned: true,
- },
- {
- desc: "pid of running process",
- cfg: cgroups.Config{
- HierarchyRoot: "gitaly",
- Repositories: cgroups.Repositories{
- Count: 10,
- MemoryBytes: 10 * 1024 * 1024,
- CPUShares: 1024,
- },
- },
- setup: func(t *testing.T, cfg cgroups.Config, mock *mockCgroupV2) int {
- pid := os.Getpid()
-
- cgroupManager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), pid)
- mock.setupMockCgroupFiles(t, cgroupManager, []uint{0, 1, 2})
- require.NoError(t, cgroupManager.Setup())
-
- return pid
- },
- expectedPruned: false,
- },
- {
- desc: "gitaly-0 directory is deleted",
- cfg: cgroups.Config{
- HierarchyRoot: "gitaly",
- Repositories: cgroups.Repositories{
- Count: 10,
- MemoryBytes: 10 * 1024 * 1024,
- CPUShares: 1024,
- },
- },
- setup: func(t *testing.T, cfg cgroups.Config, mock *mockCgroupV2) int {
- cgroupManager := mock.newCgroupManager(cfg, testhelper.SharedLogger(t), 0)
- mock.setupMockCgroupFiles(t, cgroupManager, []uint{0, 1, 2})
- require.NoError(t, cgroupManager.Setup())
-
- return 0
- },
- expectedPruned: true,
- },
- }
-
- for _, tc := range testCases {
- t.Run(tc.desc, func(t *testing.T) {
- mock := newMockV2(t)
- tc.cfg.Mountpoint = mock.root
-
- root := filepath.Join(
- tc.cfg.Mountpoint,
- tc.cfg.HierarchyRoot,
- )
- require.NoError(t, os.MkdirAll(root, perm.PublicDir))
-
- pid := tc.setup(t, tc.cfg, mock)
-
- logger := testhelper.NewLogger(t)
- mock.pruneOldCgroups(tc.cfg, logger)
-
- // create cgroups directories with a different pid
- oldGitalyProcessDir := filepath.Join(
- root,
- fmt.Sprintf("gitaly-%d", pid),
- )
-
- if tc.expectedPruned {
- require.NoDirExists(t, oldGitalyProcessDir)
- } else {
- require.DirExists(t, oldGitalyProcessDir)
- }
- })
- }
-}
-
-func TestStatsV2(t *testing.T) {
- t.Parallel()
-
- for _, tc := range []struct {
- desc string
- mockFiles []mockCgroupFile
- expectedStats Stats
- }{
- {
- desc: "empty statistics",
- mockFiles: []mockCgroupFile{
- {"memory.current", "0"},
- {"memory.max", "0"},
- {"cpu.stat", ""},
- },
- expectedStats: Stats{},
- },
- {
- desc: "cgroupfs recorded some stats",
- mockFiles: []mockCgroupFile{
- {"memory.max", "2000000000"},
- {"memory.current", "1234000000"},
- {"memory.events", `low 1
-high 2
-max 3
-oom 4
-oom_kill 5`},
- {"nr_throttled", "50"},
- {"throttled_usec", "1000000"},
- {"cpu.stat", `nr_periods 10
-nr_throttled 50
-throttled_usec 1000000`}, // 0.001 seconds
- {"memory.stat", `anon 234000000
-file 235000000
-inactive_anon 200000000
-active_anon 34000000
-inactive_file 100000000
-active_file 135000000`},
- },
- expectedStats: Stats{
- ParentStats: CgroupStats{
- CPUThrottledCount: 50,
- CPUThrottledDuration: 0.001,
- MemoryUsage: 1234000000,
- MemoryLimit: 2000000000,
- OOMKills: 5,
- Anon: 234000000,
- ActiveAnon: 34000000,
- InactiveAnon: 200000000,
- File: 235000000,
- ActiveFile: 135000000,
- InactiveFile: 100000000,
- },
- },
- },
- } {
- t.Run(tc.desc, func(t *testing.T) {
- mock := newMockV2(t)
-
- config := defaultCgroupsConfig()
- config.Repositories.Count = 1
- config.Repositories.MemoryBytes = 2000000000
- config.Repositories.CPUShares = 16
- config.Mountpoint = mock.root
-
- v2Manager := mock.newCgroupManager(config, testhelper.SharedLogger(t), 1)
-
- mock.setupMockCgroupFiles(t, v2Manager, []uint{0}, tc.mockFiles...)
- require.NoError(t, v2Manager.Setup())
-
- stats, err := v2Manager.Stats()
- require.NoError(t, err)
- require.Equal(t, tc.expectedStats, stats)
- })
- }
-}
-
-func calculateWantCPUWeight(wantCPUWeight int) int {
- if wantCPUWeight == 0 {
- return 0
- }
- return 1 + ((wantCPUWeight-2)*9999)/262142
-}
-
-func requireShardsV2(t *testing.T, mock *mockCgroupV2, mgr *CGroupManager, pid int, expectedShards ...uint) {
- t.Helper()
-
- for shard := uint(0); shard < mgr.cfg.Repositories.Count; shard++ {
- cgroupPath := filepath.Join("gitaly", fmt.Sprintf("gitaly-%d", pid), fmt.Sprintf("repos-%d", shard))
- diskPath := filepath.Join(mock.root, cgroupPath)
-
- if slices.Contains(expectedShards, shard) {
- require.DirExists(t, diskPath)
-
- cgLock := mgr.status.getLock(cgroupPath)
- require.True(t, cgLock.isCreated())
- } else {
- require.NoDirExists(t, diskPath)
-
- // Confirm we pre-populated this map entry.
- _, lockInserted := mgr.status.m[cgroupPath]
- require.True(t, lockInserted)
- }
- }
-}
-
-func requireCgroupWithString(t *testing.T, cgroupFile string, want string) {
- t.Helper()
-
- if want == "" {
- return
- }
- require.Equal(t,
- string(readCgroupFile(t, cgroupFile)),
- want,
- )
-}
-
-func requireCgroupWithInt(t *testing.T, cgroupFile string, want int) {
- t.Helper()
-
- if want <= 0 {
- return
- }
-
- require.Equal(t,
- string(readCgroupFile(t, cgroupFile)),
- strconv.Itoa(want),
- )
-}