From 47274251a4fa15846644f3a3cb9f2fae1fcea644 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Fri, 21 Aug 2020 13:53:39 -0400 Subject: p5302: disable thread-count parameter tests by default The primary function of the perf suite is to detect regressions (or improvements) between versions of Git. The only numbers we show a direct comparison for are timings between the same test run on two different versions. However, it can sometimes be used to collect other information. For instance, p5302 runs the same index-pack operation with different thread counts. The output doesn't directly compare these, but anybody interested in working on index-pack can manually compare the results. For a normal regression run of the full perf-suite, though, this incurs a significant cost to generate numbers nobody will actually look at; about 25% of the total time of the test suite is spent in p5302. And the low-thread-count runs are the most expensive part of it, since they're (unsurprisingly) not using as many threads. Let's skip these tests by default, but make it possible for people working on index-pack to still run them by setting an environment variable. Rather than make this specific to p5302, let's introduce a generic mechanism. This makes it possible to run the full suite with every possible test if somebody really wants to burn some CPU. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- t/perf/README | 9 +++++++++ t/perf/p5302-pack-index.sh | 10 +++++----- t/perf/perf-lib.sh | 2 ++ 3 files changed, 16 insertions(+), 5 deletions(-) (limited to 't') diff --git a/t/perf/README b/t/perf/README index c7b70e2d28..bd649afa97 100644 --- a/t/perf/README +++ b/t/perf/README @@ -84,6 +84,15 @@ You can set the following variables (also in your config.mak): probably be about linux.git size for optimal results. Both default to the git.git you are running from. + GIT_PERF_EXTRA + Boolean to enable additional tests. Most test scripts are + written to detect regressions between two versions of Git, and + the output will compare timings for individual tests between + those versions. Some scripts have additional tests which are not + run by default, that show patterns within a single version of + Git (e.g., performance of index-pack as the number of threads + changes). These can be enabled with GIT_PERF_EXTRA. + You can also pass the options taken by ordinary git tests; the most useful one is: diff --git a/t/perf/p5302-pack-index.sh b/t/perf/p5302-pack-index.sh index a9b3e112d9..23011ab739 100755 --- a/t/perf/p5302-pack-index.sh +++ b/t/perf/p5302-pack-index.sh @@ -13,31 +13,31 @@ test_expect_success 'repack' ' export PACK ' -test_perf 'index-pack 0 threads' ' +test_perf PERF_EXTRA 'index-pack 0 threads' ' rm -rf repo.git && git init --bare repo.git && GIT_DIR=repo.git git index-pack --threads=1 --stdin < $PACK ' -test_perf 'index-pack 1 thread ' ' +test_perf PERF_EXTRA 'index-pack 1 thread ' ' rm -rf repo.git && git init --bare repo.git && GIT_DIR=repo.git GIT_FORCE_THREADS=1 git index-pack --threads=1 --stdin < $PACK ' -test_perf 'index-pack 2 threads' ' +test_perf PERF_EXTRA 'index-pack 2 threads' ' rm -rf repo.git && git init --bare repo.git && GIT_DIR=repo.git git index-pack --threads=2 --stdin < $PACK ' -test_perf 'index-pack 4 threads' ' +test_perf PERF_EXTRA 'index-pack 4 threads' ' rm -rf repo.git && git init --bare repo.git && GIT_DIR=repo.git git index-pack --threads=4 --stdin < $PACK ' -test_perf 'index-pack 8 threads' ' +test_perf PERF_EXTRA 'index-pack 8 threads' ' rm -rf repo.git && git init --bare repo.git && GIT_DIR=repo.git git index-pack --threads=8 --stdin < $PACK diff --git a/t/perf/perf-lib.sh b/t/perf/perf-lib.sh index 13e389367a..821581a885 100644 --- a/t/perf/perf-lib.sh +++ b/t/perf/perf-lib.sh @@ -245,3 +245,5 @@ test_at_end_hook_ () { test_export () { export "$@" } + +test_lazy_prereq PERF_EXTRA 'test_bool_env GIT_PERF_EXTRA false' -- cgit v1.2.3 From 218389b9f3d1b5c8583cc0e944692f7c66af8a03 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Fri, 21 Aug 2020 13:54:51 -0400 Subject: p5302: count up to online-cpus for thread tests When PERF_EXTRA is enabled, p5302 checks the performance of index-pack with various numbers of threads. This can be useful for deciding what the default should be (which is currently capped at 3 threads based on the results of this script). However, we only go up to 8 threads, and modern machines may have more. Let's get the number of CPUs from test-tool, and test various numbers of threads between one and that maximum. Note that the current tests aren't all identical, as we have to set GIT_FORCE_THREADS for the --threads=1 test (which measures the overhead of starting a single worker thread versus the "0" case of using the main thread). To keep the loop simple, we'll keep the "0" case out of it, and set GIT_FORCE_THREADS=1 for all of the other cases (it's a noop for all but the "1" case, since numbers higher than 1 would always need threads). Note also that we could skip running "test-tool" if PERF_EXTRA isn't set. However, there's some small value in knowing the number of threads, so that we can mark each test as skipped in the output. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- t/perf/p5302-pack-index.sh | 47 +++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) (limited to 't') diff --git a/t/perf/p5302-pack-index.sh b/t/perf/p5302-pack-index.sh index 23011ab739..228593d9ad 100755 --- a/t/perf/p5302-pack-index.sh +++ b/t/perf/p5302-pack-index.sh @@ -13,35 +13,36 @@ test_expect_success 'repack' ' export PACK ' -test_perf PERF_EXTRA 'index-pack 0 threads' ' - rm -rf repo.git && - git init --bare repo.git && - GIT_DIR=repo.git git index-pack --threads=1 --stdin < $PACK +# Rather than counting up and doubling each time, count down from the endpoint, +# halving each time. That ensures that our final test uses as many threads as +# CPUs, even if it isn't a power of 2. +test_expect_success 'set up thread-counting tests' ' + t=$(test-tool online-cpus) && + threads= && + while test $t -gt 0 + do + threads="$t $threads" + t=$((t / 2)) + done ' -test_perf PERF_EXTRA 'index-pack 1 thread ' ' - rm -rf repo.git && - git init --bare repo.git && - GIT_DIR=repo.git GIT_FORCE_THREADS=1 git index-pack --threads=1 --stdin < $PACK -' - -test_perf PERF_EXTRA 'index-pack 2 threads' ' - rm -rf repo.git && - git init --bare repo.git && - GIT_DIR=repo.git git index-pack --threads=2 --stdin < $PACK -' - -test_perf PERF_EXTRA 'index-pack 4 threads' ' +test_perf PERF_EXTRA 'index-pack 0 threads' ' rm -rf repo.git && git init --bare repo.git && - GIT_DIR=repo.git git index-pack --threads=4 --stdin < $PACK + GIT_DIR=repo.git git index-pack --threads=1 --stdin < $PACK ' -test_perf PERF_EXTRA 'index-pack 8 threads' ' - rm -rf repo.git && - git init --bare repo.git && - GIT_DIR=repo.git git index-pack --threads=8 --stdin < $PACK -' +for t in $threads +do + THREADS=$t + export THREADS + test_perf PERF_EXTRA "index-pack $t threads" ' + rm -rf repo.git && + git init --bare repo.git && + GIT_DIR=repo.git GIT_FORCE_THREADS=1 \ + git index-pack --threads=$THREADS --stdin <$PACK + ' +done test_perf 'index-pack default number of threads' ' rm -rf repo.git && -- cgit v1.2.3