diff options
author | Patrick Steinhardt <psteinhardt@gitlab.com> | 2022-03-29 08:59:55 +0300 |
---|---|---|
committer | Patrick Steinhardt <psteinhardt@gitlab.com> | 2022-03-29 09:15:01 +0300 |
commit | b2d62949e1855d4b72a39818fc09250365f230dd (patch) | |
tree | b2c30d8cdebbb95272365013c54936482d5e1760 /internal/git | |
parent | cc42cf8f28dc37bf808dabaac8a055a84b83a5db (diff) |
git: Globally limit threads used by git-pack-objects(1)
By default, git-pack-objects(1) will spawn as many threads as there are
logical CPU cores on the machine. While this typically makes sense on a
client's machine because a client will want the command to finish as
fast as possible, on the server-side this is less of a good idea because
it can easily impact other, concurrently running Git commands.
This is why we have started to limit the number of threads this command
may use when performing repository maintenance. Packing of objects is in
no way limited to maintenance though, but it's also executed e.g. when
serving packfiles.
Let's globally limit the number of threads git-pack-objects(1) uses by
injecting the `pack.threads` configuration into all commands which may
generate packfiles.
Changelog: performance
Diffstat (limited to 'internal/git')
-rw-r--r-- | internal/git/command_description.go | 9 | ||||
-rw-r--r-- | internal/git/command_description_test.go | 22 | ||||
-rw-r--r-- | internal/git/housekeeping/objects.go | 12 |
3 files changed, 31 insertions, 12 deletions
diff --git a/internal/git/command_description.go b/internal/git/command_description.go index cb26d08d5..162ab30a3 100644 --- a/internal/git/command_description.go +++ b/internal/git/command_description.go @@ -3,6 +3,8 @@ package git import ( "fmt" "log" + "math" + "runtime" "strings" ) @@ -434,5 +436,12 @@ func packConfiguration() []GlobalOption { return []GlobalOption{ ConfigPair{Key: "pack.windowMemory", Value: "100m"}, ConfigPair{Key: "pack.writeReverseIndex", Value: "true"}, + ConfigPair{Key: "pack.threads", Value: threadsConfigValue(runtime.NumCPU())}, } } + +// threadsConfigValue returns the log-2 number of threads based on the number of provided CPUs. This +// prevents us from using excessively many threads and thus avoids exhaustion of all available CPUs. +func threadsConfigValue(numCPUs int) string { + return fmt.Sprintf("%d", int(math.Max(1, math.Floor(math.Log2(float64(numCPUs)))))) +} diff --git a/internal/git/command_description_test.go b/internal/git/command_description_test.go index 09209df15..52da034c3 100644 --- a/internal/git/command_description_test.go +++ b/internal/git/command_description_test.go @@ -45,3 +45,25 @@ func TestCommandDescriptions_revListPositionalArgs(t *testing.T) { }) } } + +func TestThreadsConfigValue(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + cpus int + threads string + }{ + {1, "1"}, + {2, "1"}, + {3, "1"}, + {4, "2"}, + {8, "3"}, + {9, "3"}, + {13, "3"}, + {16, "4"}, + {27, "4"}, + {32, "5"}, + } { + actualThreads := threadsConfigValue(tt.cpus) + require.Equal(t, tt.threads, actualThreads) + } +} diff --git a/internal/git/housekeeping/objects.go b/internal/git/housekeeping/objects.go index 7fbe33f06..7f5f13e60 100644 --- a/internal/git/housekeeping/objects.go +++ b/internal/git/housekeeping/objects.go @@ -2,9 +2,6 @@ package housekeeping import ( "context" - "fmt" - "math" - "runtime" "gitlab.com/gitlab-org/gitaly/v14/internal/git" "gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo" @@ -38,7 +35,6 @@ func RepackObjects(ctx context.Context, repo *localrepo.Repo, cfg RepackObjectsC git.Flag{Name: "-A"}, git.Flag{Name: "--pack-kept-objects"}, git.Flag{Name: "-l"}, - log2Threads(runtime.NumCPU()), ) } @@ -82,11 +78,3 @@ func GetRepackGitConfig(ctx context.Context, bitmap bool) []git.ConfigPair { return config } - -// log2Threads returns the log-2 number of threads based on the number of -// provided CPUs. This prevents repacking operations from exhausting all -// available CPUs and increasing request latency -func log2Threads(numCPUs int) git.ValueFlag { - n := math.Max(1, math.Floor(math.Log2(float64(numCPUs)))) - return git.ValueFlag{Name: "--threads", Value: fmt.Sprint(n)} -} |