Merge branch 'ac/bitmap-lookup-table'

The pack bitmap file gained a bitmap-lookup table to speed up
locating the necessary bitmap for a given commit.

* ac/bitmap-lookup-table:
  pack-bitmap-write: drop unused pack_idx_entry parameters
  bitmap-lookup-table: add performance tests for lookup table
  pack-bitmap: prepare to read lookup table extension
  pack-bitmap-write: learn pack.writeBitmapLookupTable and add tests
  pack-bitmap-write.c: write lookup table extension
  bitmap: move `get commit positions` code to `bitmap_writer_finish`
  Documentation/technical: describe bitmap lookup table extension

5 files changed, 199 insertions, 122 deletions

diff --git a/t/perf/lib-bitmap.sh b/t/perf/lib-bitmap.sh
index 63d3bc7cec..55a8feb1dc 100644
--- a/t/perf/lib-bitmap.sh
+++ b/t/perf/lib-bitmap.sh
@@ -67,3 +67,34 @@ test_partial_bitmap () {
 			--filter=tree:0 >/dev/null
 	'
 }
+
+test_pack_bitmap () {
+	test_perf "repack to disk" '
+		git repack -ad
+	'
+
+	test_full_bitmap
+
+	test_expect_success "create partial bitmap state" '
+		# pick a commit to represent the repo tip in the past
+		cutoff=$(git rev-list HEAD~100 -1) &&
+		orig_tip=$(git rev-parse HEAD) &&
+
+		# now kill off all of the refs and pretend we had
+		# just the one tip
+		rm -rf .git/logs .git/refs/* .git/packed-refs &&
+		git update-ref HEAD $cutoff &&
+
+		# and then repack, which will leave us with a nice
+		# big bitmap pack of the "old" history, and all of
+		# the new history will be loose, as if it had been pushed
+		# up incrementally and exploded via unpack-objects
+		git repack -Ad &&
+
+		# and now restore our original tip, as if the pushes
+		# had happened
+		git update-ref HEAD $orig_tip
+	'
+
+	test_partial_bitmap
+}
diff --git a/t/perf/p5310-pack-bitmaps.sh b/t/perf/p5310-pack-bitmaps.sh
index 7ad4f237bc..b1399f1007 100755
--- a/t/perf/p5310-pack-bitmaps.sh
+++ b/t/perf/p5310-pack-bitmaps.sh
@@ -4,51 +4,37 @@ test_description='Tests pack performance using bitmaps'
 . ./perf-lib.sh
 . "${TEST_DIRECTORY}/perf/lib-bitmap.sh"
 
-test_perf_large_repo
-
-# note that we do everything through config,
-# since we want to be able to compare bitmap-aware
-# git versus non-bitmap git
-#
-# We intentionally use the deprecated pack.writebitmaps
-# config so that we can test against older versions of git.
-test_expect_success 'setup bitmap config' '
-	git config pack.writebitmaps true
-'
-
-# we need to create the tag up front such that it is covered by the repack and
-# thus by generated bitmaps.
-test_expect_success 'create tags' '
-	git tag --message="tag pointing to HEAD" perf-tag HEAD
-'
-
-test_perf 'repack to disk' '
-	git repack -ad
-'
-
-test_full_bitmap
-
-test_expect_success 'create partial bitmap state' '
-	# pick a commit to represent the repo tip in the past
-	cutoff=$(git rev-list HEAD~100 -1) &&
-	orig_tip=$(git rev-parse HEAD) &&
-
-	# now kill off all of the refs and pretend we had
-	# just the one tip
-	rm -rf .git/logs .git/refs/* .git/packed-refs &&
-	git update-ref HEAD $cutoff &&
-
-	# and then repack, which will leave us with a nice
-	# big bitmap pack of the "old" history, and all of
-	# the new history will be loose, as if it had been pushed
-	# up incrementally and exploded via unpack-objects
-	git repack -Ad &&
-
-	# and now restore our original tip, as if the pushes
-	# had happened
-	git update-ref HEAD $orig_tip
-'
-
-test_partial_bitmap
+test_lookup_pack_bitmap () {
+	test_expect_success 'start the test from scratch' '
+		rm -rf * .git
+	'
+
+	test_perf_large_repo
+
+	# note that we do everything through config,
+	# since we want to be able to compare bitmap-aware
+	# git versus non-bitmap git
+	#
+	# We intentionally use the deprecated pack.writebitmaps
+	# config so that we can test against older versions of git.
+	test_expect_success 'setup bitmap config' '
+		git config pack.writebitmaps true
+	'
+
+	# we need to create the tag up front such that it is covered by the repack and
+	# thus by generated bitmaps.
+	test_expect_success 'create tags' '
+		git tag --message="tag pointing to HEAD" perf-tag HEAD
+	'
+
+	test_perf "enable lookup table: $1" '
+		git config pack.writeBitmapLookupTable '"$1"'
+	'
+
+	test_pack_bitmap
+}
+
+test_lookup_pack_bitmap false
+test_lookup_pack_bitmap true
 
 test_done
diff --git a/t/perf/p5311-pack-bitmaps-fetch.sh b/t/perf/p5311-pack-bitmaps-fetch.sh
index 47c3fd7581..426fab87e3 100755
--- a/t/perf/p5311-pack-bitmaps-fetch.sh
+++ b/t/perf/p5311-pack-bitmaps-fetch.sh
@@ -3,42 +3,52 @@
 test_description='performance of fetches from bitmapped packs'
 . ./perf-lib.sh
 
-test_perf_default_repo
-
-test_expect_success 'create bitmapped server repo' '
-	git config pack.writebitmaps true &&
-	git repack -ad
-'
-
-# simulate a fetch from a repository that last fetched N days ago, for
-# various values of N. We do so by following the first-parent chain,
-# and assume the first entry in the chain that is N days older than the current
-# HEAD is where the HEAD would have been then.
-for days in 1 2 4 8 16 32 64 128; do
-	title=$(printf '%10s' "($days days)")
-	test_expect_success "setup revs from $days days ago" '
-		now=$(git log -1 --format=%ct HEAD) &&
-		then=$(($now - ($days * 86400))) &&
-		tip=$(git rev-list -1 --first-parent --until=$then HEAD) &&
-		{
-			echo HEAD &&
-			echo ^$tip
-		} >revs
+test_fetch_bitmaps () {
+	test_expect_success 'setup test directory' '
+		rm -fr * .git
 	'
 
-	test_perf "server $title" '
-		git pack-objects --stdout --revs \
-				 --thin --delta-base-offset \
-				 <revs >tmp.pack
-	'
+	test_perf_default_repo
 
-	test_size "size   $title" '
-		wc -c <tmp.pack
+	test_expect_success 'create bitmapped server repo' '
+		git config pack.writebitmaps true &&
+		git config pack.writeBitmapLookupTable '"$1"' &&
+		git repack -ad
 	'
 
-	test_perf "client $title" '
-		git index-pack --stdin --fix-thin <tmp.pack
-	'
-done
+	# simulate a fetch from a repository that last fetched N days ago, for
+	# various values of N. We do so by following the first-parent chain,
+	# and assume the first entry in the chain that is N days older than the current
+	# HEAD is where the HEAD would have been then.
+	for days in 1 2 4 8 16 32 64 128; do
+		title=$(printf '%10s' "($days days)")
+		test_expect_success "setup revs from $days days ago" '
+			now=$(git log -1 --format=%ct HEAD) &&
+			then=$(($now - ($days * 86400))) &&
+			tip=$(git rev-list -1 --first-parent --until=$then HEAD) &&
+			{
+				echo HEAD &&
+				echo ^$tip
+			} >revs
+		'
+
+		test_perf "server $title (lookup=$1)" '
+			git pack-objects --stdout --revs \
+					--thin --delta-base-offset \
+					<revs >tmp.pack
+		'
+
+		test_size "size   $title" '
+			wc -c <tmp.pack
+		'
+
+		test_perf "client $title (lookup=$1)" '
+			git index-pack --stdin --fix-thin <tmp.pack
+		'
+	done
+}
+
+test_fetch_bitmaps true
+test_fetch_bitmaps false
 
 test_done
diff --git a/t/perf/p5312-pack-bitmaps-revs.sh b/t/perf/p5312-pack-bitmaps-revs.sh
new file mode 100755
index 0000000000..0684b690af
--- /dev/null
+++ b/t/perf/p5312-pack-bitmaps-revs.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+
+test_description='Tests pack performance using bitmaps (rev index enabled)'
+. ./perf-lib.sh
+. "${TEST_DIRECTORY}/perf/lib-bitmap.sh"
+
+test_lookup_pack_bitmap () {
+	test_expect_success 'start the test from scratch' '
+		rm -rf * .git
+	'
+
+	test_perf_large_repo
+
+	test_expect_success 'setup bitmap config' '
+		git config pack.writebitmaps true &&
+		git config pack.writeReverseIndex true
+	'
+
+	# we need to create the tag up front such that it is covered by the repack and
+	# thus by generated bitmaps.
+	test_expect_success 'create tags' '
+		git tag --message="tag pointing to HEAD" perf-tag HEAD
+	'
+
+	test_perf "enable lookup table: $1" '
+		git config pack.writeBitmapLookupTable '"$1"'
+	'
+
+	test_pack_bitmap
+}
+
+test_lookup_pack_bitmap false
+test_lookup_pack_bitmap true
+
+test_done
diff --git a/t/perf/p5326-multi-pack-bitmaps.sh b/t/perf/p5326-multi-pack-bitmaps.sh
index f2fa228f16..d082e6cacb 100755
--- a/t/perf/p5326-multi-pack-bitmaps.sh
+++ b/t/perf/p5326-multi-pack-bitmaps.sh
@@ -4,49 +4,64 @@ test_description='Tests performance using midx bitmaps'
 . ./perf-lib.sh
 . "${TEST_DIRECTORY}/perf/lib-bitmap.sh"
 
-test_perf_large_repo
-
-# we need to create the tag up front such that it is covered by the repack and
-# thus by generated bitmaps.
-test_expect_success 'create tags' '
-	git tag --message="tag pointing to HEAD" perf-tag HEAD
-'
-
-test_expect_success 'start with bitmapped pack' '
-	git repack -adb
-'
-
-test_perf 'setup multi-pack index' '
-	git multi-pack-index write --bitmap
-'
-
-test_expect_success 'drop pack bitmap' '
-	rm -f .git/objects/pack/pack-*.bitmap
-'
-
-test_full_bitmap
-
-test_expect_success 'create partial bitmap state' '
-	# pick a commit to represent the repo tip in the past
-	cutoff=$(git rev-list HEAD~100 -1) &&
-	orig_tip=$(git rev-parse HEAD) &&
-
-	# now pretend we have just one tip
-	rm -rf .git/logs .git/refs/* .git/packed-refs &&
-	git update-ref HEAD $cutoff &&
-
-	# and then repack, which will leave us with a nice
-	# big bitmap pack of the "old" history, and all of
-	# the new history will be loose, as if it had been pushed
-	# up incrementally and exploded via unpack-objects
-	git repack -Ad &&
-	git multi-pack-index write --bitmap &&
-
-	# and now restore our original tip, as if the pushes
-	# had happened
-	git update-ref HEAD $orig_tip
-'
-
-test_partial_bitmap
+test_bitmap () {
+	local enabled="$1"
+
+	test_expect_success "remove existing repo (lookup=$enabled)" '
+		rm -fr * .git
+	'
+
+	test_perf_large_repo
+
+	# we need to create the tag up front such that it is covered by the repack and
+	# thus by generated bitmaps.
+	test_expect_success 'create tags' '
+		git tag --message="tag pointing to HEAD" perf-tag HEAD
+	'
+
+	test_expect_success "use lookup table: $enabled" '
+		git config pack.writeBitmapLookupTable '"$enabled"'
+	'
+
+	test_expect_success "start with bitmapped pack (lookup=$enabled)" '
+		git repack -adb
+	'
+
+	test_perf "setup multi-pack index (lookup=$enabled)" '
+		git multi-pack-index write --bitmap
+	'
+
+	test_expect_success "drop pack bitmap (lookup=$enabled)" '
+		rm -f .git/objects/pack/pack-*.bitmap
+	'
+
+	test_full_bitmap
+
+	test_expect_success "create partial bitmap state (lookup=$enabled)" '
+		# pick a commit to represent the repo tip in the past
+		cutoff=$(git rev-list HEAD~100 -1) &&
+		orig_tip=$(git rev-parse HEAD) &&
+
+		# now pretend we have just one tip
+		rm -rf .git/logs .git/refs/* .git/packed-refs &&
+		git update-ref HEAD $cutoff &&
+
+		# and then repack, which will leave us with a nice
+		# big bitmap pack of the "old" history, and all of
+		# the new history will be loose, as if it had been pushed
+		# up incrementally and exploded via unpack-objects
+		git repack -Ad &&
+		git multi-pack-index write --bitmap &&
+
+		# and now restore our original tip, as if the pushes
+		# had happened
+		git update-ref HEAD $orig_tip
+	'
+
+	test_partial_bitmap
+}
+
+test_bitmap false
+test_bitmap true
 
 test_done