diff options
author | James Fargher <proglottis@gmail.com> | 2022-03-31 01:14:58 +0300 |
---|---|---|
committer | James Fargher <proglottis@gmail.com> | 2022-03-31 01:14:58 +0300 |
commit | 0b2f6dfc5b1e6d29e805180c0ebd592f62661df3 (patch) | |
tree | af048c8b47aa1f9acbf846dba1c80b0ffdbfd235 /internal/git | |
parent | 131adc858a199fec86ad128d41b10340e0e9fabc (diff) | |
parent | b0dd8088588a31bdd522e1f76144a0d232ffaa7c (diff) |
Merge branch 'pks-git-grep-limit-threading' into 'master'
git: Globally limit number of threads
Closes #4120
See merge request gitlab-org/gitaly!4443
Diffstat (limited to 'internal/git')
-rw-r--r-- | internal/git/command_description.go | 15 | ||||
-rw-r--r-- | internal/git/command_description_test.go | 22 | ||||
-rw-r--r-- | internal/git/housekeeping/objects.go | 12 |
3 files changed, 37 insertions, 12 deletions
diff --git a/internal/git/command_description.go b/internal/git/command_description.go index cb26d08d5..d68560554 100644 --- a/internal/git/command_description.go +++ b/internal/git/command_description.go @@ -3,6 +3,8 @@ package git import ( "fmt" "log" + "math" + "runtime" "strings" ) @@ -126,6 +128,12 @@ var commandDescriptions = map[string]commandDescription{ // git-grep(1) does not support disambiguating options from paths from // revisions. flags: scNoRefUpdates | scNoEndOfOptions, + opts: []GlobalOption{ + // This command by default spawns as many threads as there are CPUs. This + // easily impacts concurrently running commands by exhausting cores and + // generating excessive I/O load. + ConfigPair{Key: "grep.threads", Value: threadsConfigValue(runtime.NumCPU())}, + }, }, "hash-object": { flags: scNoRefUpdates, @@ -434,5 +442,12 @@ func packConfiguration() []GlobalOption { return []GlobalOption{ ConfigPair{Key: "pack.windowMemory", Value: "100m"}, ConfigPair{Key: "pack.writeReverseIndex", Value: "true"}, + ConfigPair{Key: "pack.threads", Value: threadsConfigValue(runtime.NumCPU())}, } } + +// threadsConfigValue returns the log-2 number of threads based on the number of provided CPUs. This +// prevents us from using excessively many threads and thus avoids exhaustion of all available CPUs. +func threadsConfigValue(numCPUs int) string { + return fmt.Sprintf("%d", int(math.Max(1, math.Floor(math.Log2(float64(numCPUs)))))) +} diff --git a/internal/git/command_description_test.go b/internal/git/command_description_test.go index 09209df15..52da034c3 100644 --- a/internal/git/command_description_test.go +++ b/internal/git/command_description_test.go @@ -45,3 +45,25 @@ func TestCommandDescriptions_revListPositionalArgs(t *testing.T) { }) } } + +func TestThreadsConfigValue(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + cpus int + threads string + }{ + {1, "1"}, + {2, "1"}, + {3, "1"}, + {4, "2"}, + {8, "3"}, + {9, "3"}, + {13, "3"}, + {16, "4"}, + {27, "4"}, + {32, "5"}, + } { + actualThreads := threadsConfigValue(tt.cpus) + require.Equal(t, tt.threads, actualThreads) + } +} diff --git a/internal/git/housekeeping/objects.go b/internal/git/housekeeping/objects.go index 7fbe33f06..7f5f13e60 100644 --- a/internal/git/housekeeping/objects.go +++ b/internal/git/housekeeping/objects.go @@ -2,9 +2,6 @@ package housekeeping import ( "context" - "fmt" - "math" - "runtime" "gitlab.com/gitlab-org/gitaly/v14/internal/git" "gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo" @@ -38,7 +35,6 @@ func RepackObjects(ctx context.Context, repo *localrepo.Repo, cfg RepackObjectsC git.Flag{Name: "-A"}, git.Flag{Name: "--pack-kept-objects"}, git.Flag{Name: "-l"}, - log2Threads(runtime.NumCPU()), ) } @@ -82,11 +78,3 @@ func GetRepackGitConfig(ctx context.Context, bitmap bool) []git.ConfigPair { return config } - -// log2Threads returns the log-2 number of threads based on the number of -// provided CPUs. This prevents repacking operations from exhausting all -// available CPUs and increasing request latency -func log2Threads(numCPUs int) git.ValueFlag { - n := math.Max(1, math.Floor(math.Log2(float64(numCPUs)))) - return git.ValueFlag{Name: "--threads", Value: fmt.Sprint(n)} -} |