diff options
| -rw-r--r-- | bundle-uri.c | 156 | ||||
| -rwxr-xr-x | t/t5558-clone-bundle-uri.sh | 40 | ||||
| -rwxr-xr-x | t/t5601-clone.sh | 46 |
3 files changed, 233 insertions, 9 deletions
diff --git a/bundle-uri.c b/bundle-uri.c index d4277b2e3a..af48938d24 100644 --- a/bundle-uri.c +++ b/bundle-uri.c @@ -447,6 +447,139 @@ static int download_bundle_to_file(struct remote_bundle_info *bundle, void *data return 0; } +struct bundles_for_sorting { + struct remote_bundle_info **items; + size_t alloc; + size_t nr; +}; + +static int append_bundle(struct remote_bundle_info *bundle, void *data) +{ + struct bundles_for_sorting *list = data; + list->items[list->nr++] = bundle; + return 0; +} + +/** + * For use in QSORT() to get a list sorted by creationToken + * in decreasing order. + */ +static int compare_creation_token_decreasing(const void *va, const void *vb) +{ + const struct remote_bundle_info * const *a = va; + const struct remote_bundle_info * const *b = vb; + + if ((*a)->creationToken > (*b)->creationToken) + return -1; + if ((*a)->creationToken < (*b)->creationToken) + return 1; + return 0; +} + +static int fetch_bundles_by_token(struct repository *r, + struct bundle_list *list) +{ + int cur; + int move_direction = 0; + struct bundle_list_context ctx = { + .r = r, + .list = list, + .mode = list->mode, + }; + struct bundles_for_sorting bundles = { + .alloc = hashmap_get_size(&list->bundles), + }; + + ALLOC_ARRAY(bundles.items, bundles.alloc); + + for_all_bundles_in_list(list, append_bundle, &bundles); + + QSORT(bundles.items, bundles.nr, compare_creation_token_decreasing); + + /* + * Attempt to download and unbundle the minimum number of bundles by + * creationToken in decreasing order. If we fail to unbundle (after + * a successful download) then move to the next non-downloaded bundle + * and attempt downloading. Once we succeed in applying a bundle, + * move to the previous unapplied bundle and attempt to unbundle it + * again. + * + * In the case of a fresh clone, we will likely download all of the + * bundles before successfully unbundling the oldest one, then the + * rest of the bundles unbundle successfully in increasing order + * of creationToken. + * + * If there are existing objects, then this process may terminate + * early when all required commits from "new" bundles exist in the + * repo's object store. + */ + cur = 0; + while (cur >= 0 && cur < bundles.nr) { + struct remote_bundle_info *bundle = bundles.items[cur]; + if (!bundle->file) { + /* + * Not downloaded yet. Try downloading. + * + * Note that bundle->file is non-NULL if a download + * was attempted, even if it failed to download. + */ + if (fetch_bundle_uri_internal(ctx.r, bundle, ctx.depth + 1, ctx.list)) { + /* Mark as unbundled so we do not retry. */ + bundle->unbundled = 1; + + /* Try looking deeper in the list. */ + move_direction = 1; + goto move; + } + + /* We expect bundles when using creationTokens. */ + if (!is_bundle(bundle->file, 1)) { + warning(_("file downloaded from '%s' is not a bundle"), + bundle->uri); + break; + } + } + + if (bundle->file && !bundle->unbundled) { + /* + * This was downloaded, but not successfully + * unbundled. Try unbundling again. + */ + if (unbundle_from_file(ctx.r, bundle->file)) { + /* Try looking deeper in the list. */ + move_direction = 1; + } else { + /* + * Succeeded in unbundle. Retry bundles + * that previously failed to unbundle. + */ + move_direction = -1; + bundle->unbundled = 1; + } + } + + /* + * Else case: downloaded and unbundled successfully. + * Skip this by moving in the same direction as the + * previous step. + */ + +move: + /* Move in the specified direction and repeat. */ + cur += move_direction; + } + + free(bundles.items); + + /* + * We succeed if the loop terminates because 'cur' drops below + * zero. The other case is that we terminate because 'cur' + * reaches the end of the list, so we have a failure no matter + * which bundles we apply from the list. + */ + return cur >= 0; +} + static int download_bundle_list(struct repository *r, struct bundle_list *local_list, struct bundle_list *global_list, @@ -484,7 +617,15 @@ static int fetch_bundle_list_in_config_format(struct repository *r, goto cleanup; } - if ((result = download_bundle_list(r, &list_from_bundle, + /* + * If this list uses the creationToken heuristic, then the URIs + * it advertises are expected to be bundles, not nested lists. + * We can drop 'global_list' and 'depth'. + */ + if (list_from_bundle.heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN) { + result = fetch_bundles_by_token(r, &list_from_bundle); + global_list->heuristic = BUNDLE_HEURISTIC_CREATIONTOKEN; + } else if ((result = download_bundle_list(r, &list_from_bundle, global_list, depth))) goto cleanup; @@ -626,6 +767,14 @@ int fetch_bundle_list(struct repository *r, struct bundle_list *list) int result; struct bundle_list global_list; + /* + * If the creationToken heuristic is used, then the URIs + * advertised by 'list' are not nested lists and instead + * direct bundles. We do not need to use global_list. + */ + if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN) + return fetch_bundles_by_token(r, list); + init_bundle_list(&global_list); /* If a bundle is added to this global list, then it is required. */ @@ -634,7 +783,10 @@ int fetch_bundle_list(struct repository *r, struct bundle_list *list) if ((result = download_bundle_list(r, list, &global_list, 0))) goto cleanup; - result = unbundle_all_bundles(r, &global_list); + if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN) + result = fetch_bundles_by_token(r, list); + else + result = unbundle_all_bundles(r, &global_list); cleanup: for_all_bundles_in_list(&global_list, unlink_bundle, NULL); diff --git a/t/t5558-clone-bundle-uri.sh b/t/t5558-clone-bundle-uri.sh index 474432c8ac..6f9417a0af 100755 --- a/t/t5558-clone-bundle-uri.sh +++ b/t/t5558-clone-bundle-uri.sh @@ -401,17 +401,43 @@ test_expect_success 'clone bundle list (http, creationToken)' ' git -C clone-list-http-2 cat-file --batch-check <oids && cat >expect <<-EOF && - $HTTPD_URL/bundle-1.bundle - $HTTPD_URL/bundle-2.bundle - $HTTPD_URL/bundle-3.bundle + $HTTPD_URL/bundle-list $HTTPD_URL/bundle-4.bundle + $HTTPD_URL/bundle-3.bundle + $HTTPD_URL/bundle-2.bundle + $HTTPD_URL/bundle-1.bundle + EOF + + test_remote_https_urls <trace-clone.txt >actual && + test_cmp expect actual +' + +test_expect_success 'clone incomplete bundle list (http, creationToken)' ' + test_when_finished rm -f trace*.txt && + + cp clone-from/bundle-*.bundle "$HTTPD_DOCUMENT_ROOT_PATH/" && + cat >"$HTTPD_DOCUMENT_ROOT_PATH/bundle-list" <<-EOF && + [bundle] + version = 1 + mode = all + heuristic = creationToken + + [bundle "bundle-1"] + uri = bundle-1.bundle + creationToken = 1 + EOF + + GIT_TRACE2_EVENT=$(pwd)/trace-clone.txt \ + git clone --bundle-uri="$HTTPD_URL/bundle-list" \ + --single-branch --branch=base --no-tags \ + "$HTTPD_URL/smart/fetch.git" clone-token-http && + + cat >expect <<-EOF && $HTTPD_URL/bundle-list + $HTTPD_URL/bundle-1.bundle EOF - # Since the creationToken heuristic is not yet understood by the - # client, the order cannot be verified at this moment. Sort the - # list for consistent results. - test_remote_https_urls <trace-clone.txt | sort >actual && + test_remote_https_urls <trace-clone.txt >actual && test_cmp expect actual ' diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh index 1928ea1dd7..b7d5551262 100755 --- a/t/t5601-clone.sh +++ b/t/t5601-clone.sh @@ -831,6 +831,52 @@ test_expect_success 'auto-discover multiple bundles from HTTP clone' ' grep -f pattern trace.txt ' +test_expect_success 'auto-discover multiple bundles from HTTP clone: creationToken heuristic' ' + test_when_finished rm -rf "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" && + test_when_finished rm -rf clone-heuristic trace*.txt && + + test_commit -C src newest && + git -C src bundle create "$HTTPD_DOCUMENT_ROOT_PATH/newest.bundle" HEAD~1..HEAD && + git clone --bare --no-local src "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" && + + cat >>"$HTTPD_DOCUMENT_ROOT_PATH/repo4.git/config" <<-EOF && + [uploadPack] + advertiseBundleURIs = true + + [bundle] + version = 1 + mode = all + heuristic = creationToken + + [bundle "everything"] + uri = $HTTPD_URL/everything.bundle + creationtoken = 1 + + [bundle "new"] + uri = $HTTPD_URL/new.bundle + creationtoken = 2 + + [bundle "newest"] + uri = $HTTPD_URL/newest.bundle + creationtoken = 3 + EOF + + GIT_TRACE2_EVENT="$(pwd)/trace-clone.txt" \ + git -c protocol.version=2 \ + -c transfer.bundleURI=true clone \ + "$HTTPD_URL/smart/repo4.git" clone-heuristic && + + cat >expect <<-EOF && + $HTTPD_URL/newest.bundle + $HTTPD_URL/new.bundle + $HTTPD_URL/everything.bundle + EOF + + # We should fetch all bundles in the expected order. + test_remote_https_urls <trace-clone.txt >actual && + test_cmp expect actual +' + # DO NOT add non-httpd-specific tests here, because the last part of this # test script is only executed when httpd is available and enabled. |