From cbe566a0710b39691ca8f6cb6a764f6c553cd97b Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Mon, 17 Aug 2020 21:01:31 -0700 Subject: negotiator/noop: add noop fetch negotiator Add a noop fetch negotiator. This is introduced to allow partial clones to skip the unneeded negotiation step when fetching missing objects using a "git fetch" subprocess. (The implementation of spawning a "git fetch" subprocess will be done in a subsequent patch.) But this can also be useful for end users, e.g. as a blunt fix for object corruption. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano --- t/t5554-noop-fetch-negotiator.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100755 t/t5554-noop-fetch-negotiator.sh (limited to 't') diff --git a/t/t5554-noop-fetch-negotiator.sh b/t/t5554-noop-fetch-negotiator.sh new file mode 100755 index 0000000000..2ac7b5859e --- /dev/null +++ b/t/t5554-noop-fetch-negotiator.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +test_description='test noop fetch negotiator' +. ./test-lib.sh + +test_expect_success 'noop negotiator does not emit any "have"' ' + rm -f trace && + + test_create_repo server && + test_commit -C server to_fetch && + + test_create_repo client && + test_commit -C client we_have && + + test_config -C client fetch.negotiationalgorithm noop && + GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch "$(pwd)/server" && + + ! grep "fetch> have" trace && + grep "fetch> done" trace +' + +test_done -- cgit v1.2.3 From 5c3b801dab9d29c63c6c929405f808f064c11b77 Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Mon, 17 Aug 2020 21:01:35 -0700 Subject: fetch-pack: do not lazy-fetch during ref iteration In order to determine negotiation tips, "fetch-pack" iterates over all refs and dereferences all annotated tags found. This causes the existence of targets of refs and annotated tags to be checked. Avoiding this is especially important when we use "git fetch" (which invokes "fetch-pack") to perform lazy fetches in a partial clone because a target of such a ref or annotated tag may need to be itself lazy-fetched (and otherwise causing an infinite loop). Therefore, teach "fetch-pack" not to lazy fetch whenever iterating over refs. This is done by using the raw form of ref iteration and by dereferencing tags ourselves. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano --- t/t5616-partial-clone.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 't') diff --git a/t/t5616-partial-clone.sh b/t/t5616-partial-clone.sh index 8827c2ed18..b1a7bd15ca 100755 --- a/t/t5616-partial-clone.sh +++ b/t/t5616-partial-clone.sh @@ -417,6 +417,26 @@ test_expect_success 'fetch lazy-fetches only to resolve deltas, protocol v2' ' grep "want $(cat hash)" trace ' +test_expect_success 'fetch does not lazy-fetch missing targets of its refs' ' + rm -rf server client trace && + + test_create_repo server && + test_config -C server uploadpack.allowfilter 1 && + test_config -C server uploadpack.allowanysha1inwant 1 && + test_commit -C server foo && + + git clone --filter=blob:none "file://$(pwd)/server" client && + # Make all refs point to nothing by deleting all objects. + rm client/.git/objects/pack/* && + + test_commit -C server bar && + GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \ + --no-tags --recurse-submodules=no \ + origin refs/tags/bar && + FOO_HASH=$(git -C server rev-parse foo) && + ! grep "want $FOO_HASH" trace +' + # The following two tests must be in this order. It is important that # the srv.bare repository did not have tags during clone, but has tags # in the fetch. -- cgit v1.2.3 From 7ca3c0ac37bfb803f11db14b7223f6d8f0cd142e Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Mon, 17 Aug 2020 21:01:36 -0700 Subject: promisor-remote: lazy-fetch objects in subprocess Teach Git to lazy-fetch missing objects in a subprocess instead of doing it in-process. This allows any fatal errors that occur during the fetch to be isolated and converted into an error return value, instead of causing the current command being run to terminate. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano --- t/t0410-partial-clone.sh | 2 +- t/t4067-diff-partial-clone.sh | 8 ++++---- t/t5300-pack-object.sh | 2 +- t/t5601-clone.sh | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) (limited to 't') diff --git a/t/t0410-partial-clone.sh b/t/t0410-partial-clone.sh index a5ebdf9ff3..d681e90640 100755 --- a/t/t0410-partial-clone.sh +++ b/t/t0410-partial-clone.sh @@ -214,7 +214,7 @@ test_expect_success 'fetching of missing objects works with ref-in-want enabled' rm -rf repo/.git/objects/* && rm -f trace && GIT_TRACE_PACKET="$(pwd)/trace" git -C repo cat-file -p "$HASH" && - grep "git< fetch=.*ref-in-want" trace + grep "fetch< fetch=.*ref-in-want" trace ' test_expect_success 'fetching of missing objects from another promisor remote' ' diff --git a/t/t4067-diff-partial-clone.sh b/t/t4067-diff-partial-clone.sh index ef8e0e9cb0..804f2a82e8 100755 --- a/t/t4067-diff-partial-clone.sh +++ b/t/t4067-diff-partial-clone.sh @@ -20,7 +20,7 @@ test_expect_success 'git show batches blobs' ' # Ensure that there is exactly 1 negotiation by checking that there is # only 1 "done" line sent. ("done" marks the end of negotiation.) GIT_TRACE_PACKET="$(pwd)/trace" git -C client show HEAD && - grep "git> done" trace >done_lines && + grep "fetch> done" trace >done_lines && test_line_count = 1 done_lines ' @@ -44,7 +44,7 @@ test_expect_success 'diff batches blobs' ' # Ensure that there is exactly 1 negotiation by checking that there is # only 1 "done" line sent. ("done" marks the end of negotiation.) GIT_TRACE_PACKET="$(pwd)/trace" git -C client diff HEAD^ HEAD && - grep "git> done" trace >done_lines && + grep "fetch> done" trace >done_lines && test_line_count = 1 done_lines ' @@ -127,7 +127,7 @@ test_expect_success 'diff with rename detection batches blobs' ' # only 1 "done" line sent. ("done" marks the end of negotiation.) GIT_TRACE_PACKET="$(pwd)/trace" git -C client diff --raw -M HEAD^ HEAD >out && grep ":100644 100644.*R[0-9][0-9][0-9].*b.*c" out && - grep "git> done" trace >done_lines && + grep "fetch> done" trace >done_lines && test_line_count = 1 done_lines ' @@ -175,7 +175,7 @@ test_expect_success 'diff --break-rewrites fetches only if necessary, and batche # by checking that there is only 1 "done" line sent. ("done" marks the # end of negotiation.) GIT_TRACE_PACKET="$(pwd)/trace" git -C client diff --break-rewrites --raw -M HEAD^ HEAD && - grep "git> done" trace >done_lines && + grep "fetch> done" trace >done_lines && test_line_count = 1 done_lines ' diff --git a/t/t5300-pack-object.sh b/t/t5300-pack-object.sh index 3d6a93343a..392201cabd 100755 --- a/t/t5300-pack-object.sh +++ b/t/t5300-pack-object.sh @@ -528,7 +528,7 @@ test_expect_success 'prefetch objects' ' TWO=$(git -C server rev-parse three_branch^) && git -C client fetch --filter=blob:none origin "$TWO" && GIT_TRACE_PACKET=$(pwd)/trace git -C client push origin "$TWO":refs/heads/two_branch && - grep "git> done" trace >donelines && + grep "fetch> done" trace >donelines && test_line_count = 1 donelines ' diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh index eb9a093e25..15fb64c18d 100755 --- a/t/t5601-clone.sh +++ b/t/t5601-clone.sh @@ -704,7 +704,7 @@ test_expect_success 'batch missing blob request during checkout' ' # Ensure that there is only one negotiation by checking that there is # only "done" line sent. ("done" marks the end of negotiation.) GIT_TRACE_PACKET="$(pwd)/trace" git -C client checkout HEAD^ && - grep "git> done" trace >done_lines && + grep "fetch> done" trace >done_lines && test_line_count = 1 done_lines ' -- cgit v1.2.3 From db3c293ecded67128b74a03f01e65c6799ff1116 Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Wed, 2 Sep 2020 14:05:39 -0700 Subject: fetch: no FETCH_HEAD display if --no-write-fetch-head 887952b8c6 ("fetch: optionally allow disabling FETCH_HEAD update", 2020-08-18) introduced the ability to disable writing to FETCH_HEAD during fetch, but did not suppress the " -> FETCH_HEAD" message when this ability is used. This message is misleading in this case, because FETCH_HEAD is not written. Also, because "fetch" is used to lazy-fetch missing objects in a partial clone, this significantly clutters up the output in that case since the objects to be fetched are potentially numerous. Therefore, suppress this message when --no-write-fetch-head is passed (but not when --dry-run is set). Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano --- t/t0410-partial-clone.sh | 7 +++++-- t/t5510-fetch.sh | 18 ++++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) (limited to 't') diff --git a/t/t0410-partial-clone.sh b/t/t0410-partial-clone.sh index d681e90640..584a039b85 100755 --- a/t/t0410-partial-clone.sh +++ b/t/t0410-partial-clone.sh @@ -183,7 +183,7 @@ test_expect_success 'missing CLI object, but promised, passes fsck' ' ' test_expect_success 'fetching of missing objects' ' - rm -rf repo && + rm -rf repo err && test_create_repo server && test_commit -C server foo && git -C server repack -a -d --write-bitmap-index && @@ -194,7 +194,10 @@ test_expect_success 'fetching of missing objects' ' git -C repo config core.repositoryformatversion 1 && git -C repo config extensions.partialclone "origin" && - git -C repo cat-file -p "$HASH" && + git -C repo cat-file -p "$HASH" 2>err && + + # Ensure that no spurious FETCH_HEAD messages are written + ! grep FETCH_HEAD err && # Ensure that the .promisor file is written, and check that its # associated packfile contains the object diff --git a/t/t5510-fetch.sh b/t/t5510-fetch.sh index 2a1abe91f0..759aec9305 100755 --- a/t/t5510-fetch.sh +++ b/t/t5510-fetch.sh @@ -543,16 +543,18 @@ test_expect_success 'fetch into the current branch with --update-head-ok' ' ' -test_expect_success 'fetch --dry-run does not touch FETCH_HEAD' ' - rm -f .git/FETCH_HEAD && - git fetch --dry-run . && - ! test -f .git/FETCH_HEAD +test_expect_success 'fetch --dry-run does not touch FETCH_HEAD, but still prints what would be written' ' + rm -f .git/FETCH_HEAD err && + git fetch --dry-run . 2>err && + ! test -f .git/FETCH_HEAD && + grep FETCH_HEAD err ' -test_expect_success '--no-write-fetch-head does not touch FETCH_HEAD' ' - rm -f .git/FETCH_HEAD && - git fetch --no-write-fetch-head . && - ! test -f .git/FETCH_HEAD +test_expect_success '--no-write-fetch-head does not touch FETCH_HEAD, and does not print what would be written' ' + rm -f .git/FETCH_HEAD err && + git fetch --no-write-fetch-head . 2>err && + ! test -f .git/FETCH_HEAD && + ! grep FETCH_HEAD err ' test_expect_success '--write-fetch-head gets defeated by --dry-run' ' -- cgit v1.2.3