Include keep-around and other references in the checksum calculation

4 files changed, 70 insertions, 42 deletions

diff --git a/Makefile b/Makefile
index 7ec1b220b..4afaee5bc 100644
--- a/Makefile
+++ b/Makefile
@@ -116,6 +116,8 @@ govendor-tagged: $(TARGET_SETUP) $(GOVENDOR)
 
 $(TEST_REPO):
 	git clone --bare https://gitlab.com/gitlab-org/gitlab-test.git $@
+	# Git notes aren't fetched by default with git clone
+	git -C $@ fetch origin refs/notes/*:refs/notes/*
 
 .PHONY: prepare-tests
 prepare-tests: $(TARGET_SETUP) $(TEST_REPO) .ruby-bundle
diff --git a/internal/service/repository/calculate_checksum.go b/internal/service/repository/calculate_checksum.go
index 433b43de3..9b89e69e1 100644
--- a/internal/service/repository/calculate_checksum.go
+++ b/internal/service/repository/calculate_checksum.go
@@ -7,6 +7,7 @@ import (
 	"encoding/hex"
 	"math/big"
 	"os/exec"
+	"regexp"
 	"strings"
 
 	pb "gitlab.com/gitlab-org/gitaly-proto/go"
@@ -22,6 +23,8 @@ import (
 
 const blankChecksum = "0000000000000000000000000000000000000000"
 
+var refWhitelist = regexp.MustCompile(`HEAD|(refs/(heads|tags|keep-around|merge-requests|environments|notes)/)`)
+
 func (s *server) CalculateChecksum(ctx context.Context, in *pb.CalculateChecksumRequest) (*pb.CalculateChecksumResponse, error) {
 	repo := in.GetRepository()
 
@@ -32,8 +35,7 @@ func (s *server) CalculateChecksum(ctx context.Context, in *pb.CalculateChecksum
 
 	args := []string{
 		"show-ref",
-		"--heads",
-		"--tags",
+		"--head",
 	}
 
 	cmd, err := git.Command(ctx, repo, args...)
@@ -49,10 +51,14 @@ func (s *server) CalculateChecksum(ctx context.Context, in *pb.CalculateChecksum
 
 	scanner := bufio.NewScanner(cmd)
 	for scanner.Scan() {
-		ref := scanner.Text()
+		ref := scanner.Bytes()
+
+		if !refWhitelist.Match(ref) {
+			continue
+		}
 
 		h := sha1.New()
-		h.Write([]byte(ref))
+		h.Write(ref)
 
 		hash := hex.EncodeToString(h.Sum(nil))
 		hashIntBase16, _ := new(big.Int).SetString(hash, 16)
diff --git a/internal/service/repository/calculate_checksum_test.go b/internal/service/repository/calculate_checksum_test.go
index b507c6abd..c0508d4ba 100644
--- a/internal/service/repository/calculate_checksum_test.go
+++ b/internal/service/repository/calculate_checksum_test.go
@@ -24,10 +24,11 @@ func TestSuccessfulCalculateChecksum(t *testing.T) {
 
 	// Force the refs database of testRepo into a known state
 	require.NoError(t, os.RemoveAll(path.Join(testRepoPath, "refs")))
-	for _, d := range []string{"refs/heads", "refs/tags"} {
+	for _, d := range []string{"refs/heads", "refs/tags", "refs/notes"} {
 		require.NoError(t, os.MkdirAll(path.Join(testRepoPath, d), 0755))
 	}
 	require.NoError(t, exec.Command("cp", "testdata/checksum-test-packed-refs", path.Join(testRepoPath, "packed-refs")).Run())
+	require.NoError(t, exec.Command("git", "-C", testRepoPath, "symbolic-ref", "HEAD", "refs/heads/feature").Run())
 
 	request := &pb.CalculateChecksumRequest{Repository: testRepo}
 	testCtx, cancelCtx := testhelper.Context()
@@ -35,7 +36,37 @@ func TestSuccessfulCalculateChecksum(t *testing.T) {
 
 	response, err := client.CalculateChecksum(testCtx, request)
 	require.NoError(t, err)
-	require.Equal(t, "7b5dbc8bbacb2bfd4584b5e26ed363e7a1cce041", response.Checksum)
+	require.Equal(t, "0c500d7c8a9dbf65e4cf5e58914bec45bfb6e9ab", response.Checksum)
+}
+
+func TestRefWhitelist(t *testing.T) {
+	testCases := []struct {
+		ref   string
+		match bool
+	}{
+		{ref: "1450cd639e0bc6721eb02800169e464f212cde06 foo/bar", match: false},
+		{ref: "259a6fba859cc91c54cd86a2cbd4c2f720e3a19d foo/bar:baz", match: false},
+		{ref: "d0a293c0ac821fadfdc086fe528f79423004229d refs/foo/bar:baz", match: false},
+		{ref: "21751bf5cb2b556543a11018c1f13b35e44a99d7 tags/v0.0.1", match: false},
+		{ref: "498214de67004b1da3d820901307bed2a68a8ef6 keep-around/498214de67004b1da3d820901307bed2a68a8ef6", match: false},
+		{ref: "38008cb17ce1466d8fec2dfa6f6ab8dcfe5cf49e merge-requests/11/head", match: false},
+		{ref: "c347ca2e140aa667b968e51ed0ffe055501fe4f4 environments/3/head", match: false},
+		{ref: "4ed78158b5b018c43005cec917129861541e25bc notes/commits", match: false},
+		{ref: "9298d46305ee0d3f4ce288370beaa02637584ff2 HEAD", match: true},
+		{ref: "0ed8c6c6752e8c6ea63e7b92a517bf5ac1209c80 refs/heads/markdown", match: true},
+		{ref: "f4e6814c3e4e7a0de82a9e7cd20c626cc963a2f8 refs/tags/v1.0.0", match: true},
+		{ref: "78a30867c755d774340108cdad5f11254818fb0c refs/keep-around/78a30867c755d774340108cdad5f11254818fb0c", match: true},
+		{ref: "c347ca2e140aa667b968e51ed0ffe055501fe4f4 refs/merge-requests/10/head", match: true},
+		{ref: "c347ca2e140aa667b968e51ed0ffe055501fe4f4 refs/environments/2/head", match: true},
+		{ref: "4ed78158b5b018c43005cec917129861541e25bc refs/notes/commits", match: true},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.ref, func(t *testing.T) {
+			match := refWhitelist.Match([]byte(tc.ref))
+			require.Equal(t, match, tc.match)
+		})
+	}
 }
 
 func TestEmptyRepositoryCalculateChecksum(t *testing.T) {
diff --git a/internal/service/repository/testdata/checksum-test-packed-refs b/internal/service/repository/testdata/checksum-test-packed-refs
index 888d48dc1..bc4886984 100644
--- a/internal/service/repository/testdata/checksum-test-packed-refs
+++ b/internal/service/repository/testdata/checksum-test-packed-refs
@@ -1,15 +1,7 @@
-# pack-refs with: peeled fully-peeled 
+# pack-refs with: peeled fully-peeled sorted
 e56497bb5f03a90a51293fc6d516788730953899 refs/heads/'test'
 1b12f15a11fc6e62177bef08f47bc7b5ce50b141 refs/heads/100%branch
-1942eed5cc108b19c7405106e81fa96125d0be22 refs/heads/1942eed5cc108b19c7405106e81fa96125d0be22
-b89b56d795dcf0351e9d627b58fd2fc67616da05 refs/heads/add-balsamiq-file
-93ee73207dcfa56fe60ab8245e0623160ab8b212 refs/heads/add-ipython-files
-e774ebd33ca5de8e6ef1e633fd887bb52b9d0a7a refs/heads/add-pdf-file
-79faa7bfb522480533f4d60368d8f6ecc1e03d9b refs/heads/add-pdf-text-binary
-010d10642f5950c8d5041f0469593ccc5c646521 refs/heads/add_images_and_changes
-ef16b8d2b204706bd8dc211d4011a5bffb6fc0c2 refs/heads/big-files
 7b1cf4336b528e0f3d1d140ee50cafdbc703597c refs/heads/binary-encoding
-4e07b243e436fd6ca00ca421fa507daad3310839 refs/heads/blob-with-tricky-encoding
 498214de67004b1da3d820901307bed2a68a8ef6 refs/heads/branch-merged
 259a6fba859cc91c54cd86a2cbd4c2f720e3a19d refs/heads/conflict-binary-file
 78a30867c755d774340108cdad5f11254818fb0c refs/heads/conflict-contains-conflict-markers
@@ -22,7 +14,6 @@ d0a293c0ac821fadfdc086fe528f79423004229d refs/heads/conflict-non-utf8
 f0f390655872bb2772c85a0128b2fbc2d88670cb refs/heads/conflict_branch_b
 593890758a6f845c600f38ffa05be2749211caee refs/heads/crlf-diff
 3dd08961455abf80ef9115f4afdc1c6f968b503c refs/heads/csv
-1eda481c3d5499e81cbbe15d34c53282e87627d2 refs/heads/custom-encoding
 6c177980d2073c20c4e4f0f45253a5500f1b23e1 refs/heads/deleted-image-test
 7efb185dd22fd5c51ef044795d62b7847900c341 refs/heads/empty-branch
 98b0d8b3aec8e7f7b339fd40c446ad22cbc565fe refs/heads/ends-with.json
@@ -30,44 +21,42 @@ f0f390655872bb2772c85a0128b2fbc2d88670cb refs/heads/conflict_branch_b
 025db92c6c720f030b936133cb44a16de3445daa refs/heads/expand-collapse-files
 238e82dcc48eebf0036fdd23dd234ee950f0e0d3 refs/heads/expand-collapse-lines
 0b4bc9a49b562e85de7cc9e834518ea6828729b9 refs/heads/feature
-1a35b5a77cf6af7edf6703f88e82f6aff613666f refs/heads/feature-and-encoding
-8309e68585b28d61eb85b7e2834849dda6bf1733 refs/heads/feature-encoding-conflict
 5a62481d563af92b8e32d735f2fa63b94e806835 refs/heads/feature.custom-highlighting
 bb5206fee213d983da88c47f9cf4cc6caf9c66dc refs/heads/feature_conflict
-0031876facac3f2b2702a0e53a26e89939a42209 refs/heads/few-commits
 48f0be4bd10c1decee6fae52f9ae6d10f77b60f4 refs/heads/fix
-ce369011c189f62c815f5971d096b26759bab0d1 refs/heads/flat-path
-d25b6d94034242f3930dfcfeb6d8d9aac3583992 refs/heads/flat-path-2
 e56497bb5f03a90a51293fc6d516788730953899 refs/heads/flatten-dirs
-ab2c9622c02288a2bbaaf35d96088cfdff31d9d9 refs/heads/gitaly-diff-stuff
-0999bb770f8dc92ab5581cc0b474b3e31a96bf5c refs/heads/gitaly-non-utf8-commit
-94bb47ca1297b7b3731ff2a36923640991e9236f refs/heads/gitaly-rename-test
-cb19058ecc02d01f8e4290b7e79cafd16a8839b6 refs/heads/gitaly-stuff
-e63f41fe459e62e1228fcef60d7189127aeba95a refs/heads/gitaly-test-ref
-c809470461118b7bcab850f6e9a7ca97ac42f8ea refs/heads/gitaly-windows-1251
 5937ac0a7beb003549fc5fd26fc247adbce4a52e refs/heads/improve/awesome
-d23bddc916b96c98ff192e198b1adee0f6871085 refs/heads/many_files
 0ed8c6c6752e8c6ea63e7b92a517bf5ac1209c80 refs/heads/markdown
-7975be0116940bf2ad4321f79d02a55c5f7779aa refs/heads/master
+b83d6e391c22777fca1ed3012fce84f633d7fed0 refs/heads/master
 21751bf5cb2b556543a11018c1f13b35e44a99d7 refs/heads/merged-target
-46abbb087fcc0fd02c340f0f2f052bd2c7708da3 refs/heads/moar-lfs-ptrs
 b83d6e391c22777fca1ed3012fce84f633d7fed0 refs/heads/not-merged-branch
-b83d6e391c22777fca1ed3012fce84f633d7fed0 refs/heads/not-mergéd-branch
 45127a93e4fa99ee1709a3a9aed3d677d78cbf1b refs/heads/orphaned-branch
-c84ff944ff4529a70788a5e9003c2b7feae29047 refs/heads/rd-add-file-larger-than-1-mb
-ca47bfd5e930148c42ed74c3b561a8783e381f7f refs/heads/rebase-encoding-failure-trigger
-2d1096e3a0ecf1d2baf6dee036cc80775d4940ba refs/heads/signed-commits
-ba3343bc4fa403a8dfbfcab7fc1a8c29ee34bd69 refs/heads/spooky-stuff
-35236eac0ce603467ba37d2f99cb591df4cb8f63 refs/heads/squash-encoding-error-trigger
-54cec5282aa9f21856362fe321c800c236a61615 refs/heads/squash-large-files
 81e6355ce4e1544a3524b230952c12455de0777b refs/heads/symlink-expand-diff
-899d3d27b04690ac1cd9ef4d8a74fde0667c57f1 refs/heads/test-do-not-touch
-304d257dcb821665ab5110318fc58a007bd104ed refs/heads/two-commits
-b83d6e391c22777fca1ed3012fce84f633d7fed0 refs/heads/v1.1.0
 88790590ed1337ab189bccaa355f068481c90bec refs/heads/video
-b9238ee5bf1d7359dd3b8c89fd76c1c7f8b75aba refs/heads/wip
-e63f41fe459e62e1228fcef60d7189127aeba95a refs/heads/ʕ•ᴥ•ʔ
+1450cd639e0bc6721eb02800169e464f212cde06 refs/keep-around/1450cd639e0bc6721eb02800169e464f212cde06
+1a0b36b3cdad1d2ee32457c102a8c0b7056fa863 refs/keep-around/1a0b36b3cdad1d2ee32457c102a8c0b7056fa863
+1b12f15a11fc6e62177bef08f47bc7b5ce50b141 refs/keep-around/1b12f15a11fc6e62177bef08f47bc7b5ce50b141
+38008cb17ce1466d8fec2dfa6f6ab8dcfe5cf49e refs/keep-around/38008cb17ce1466d8fec2dfa6f6ab8dcfe5cf49e
+4842455ecb8ec1428d0e83ee6389885bf98919d3 refs/keep-around/4842455ecb8ec1428d0e83ee6389885bf98919d3
+498214de67004b1da3d820901307bed2a68a8ef6 refs/keep-around/498214de67004b1da3d820901307bed2a68a8ef6
+6907208d755b60ebeacb2e9dfea74c92c3449a1f refs/keep-around/6907208d755b60ebeacb2e9dfea74c92c3449a1f
+78a30867c755d774340108cdad5f11254818fb0c refs/keep-around/78a30867c755d774340108cdad5f11254818fb0c
+824be604a34828eb682305f0d963056cfac87b2d refs/keep-around/824be604a34828eb682305f0d963056cfac87b2d
+b83d6e391c22777fca1ed3012fce84f633d7fed0 refs/keep-around/b83d6e391c22777fca1ed3012fce84f633d7fed0
+c347ca2e140aa667b968e51ed0ffe055501fe4f4 refs/merge-requests/10/head
+38008cb17ce1466d8fec2dfa6f6ab8dcfe5cf49e refs/merge-requests/11/head
+498214de67004b1da3d820901307bed2a68a8ef6 refs/merge-requests/12/head
+78a30867c755d774340108cdad5f11254818fb0c refs/merge-requests/13/head
+1450cd639e0bc6721eb02800169e464f212cde06 refs/merge-requests/14/head
+4cd80ccab63c82b4bad16faa5193fbd2aa06df40 refs/merge-requests/2/head
+2ea1f3dec713d940208fb5ce4a38765ecb5d3f73 refs/merge-requests/5/head
+048721d90c449b244b7b4c53a9186b04330174ec refs/merge-requests/6/head
+3dd08961455abf80ef9115f4afdc1c6f968b503c refs/merge-requests/7/head
+5a62481d563af92b8e32d735f2fa63b94e806835 refs/merge-requests/9/head
+4ed78158b5b018c43005cec917129861541e25bc refs/notes/commits
 f4e6814c3e4e7a0de82a9e7cd20c626cc963a2f8 refs/tags/v1.0.0
 ^6f6d7e7ed97bb5f0054f2b1df789b39ca89b6ff9
 8a2a6eb295bb170b34c24c76c49ed0e9b2eaf34b refs/tags/v1.1.0
 ^5937ac0a7beb003549fc5fd26fc247adbce4a52e
+66eceea0db202bb39c4e445e8ca28689645366c5 refs/tmp/22d4aebeed55cbb2d0e1dd995d9205b0/head
+e56497bb5f03a90a51293fc6d516788730953899 refs/tmp/268965be3d178a5c26dae90ca181b715/head