From e8e44de7874f6ea7cdf35b526f790cc7fa8f34e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Sun, 12 Jun 2016 17:53:45 +0700 Subject: upload-pack: move shallow deepen code out of receive_needs() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a prep step for further refactoring. Besides reindentation and s/shallows\./shallows->/g, no other changes are expected. Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- upload-pack.c | 99 +++++++++++++++++++++++++++++++---------------------------- 1 file changed, 52 insertions(+), 47 deletions(-) (limited to 'upload-pack.c') diff --git a/upload-pack.c b/upload-pack.c index b3f6653ffd..97ed62002e 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -538,6 +538,55 @@ error: } } +static void deepen(int depth, const struct object_array *shallows) +{ + struct commit_list *result = NULL, *backup = NULL; + int i; + if (depth == INFINITE_DEPTH && !is_repository_shallow()) + for (i = 0; i < shallows->nr; i++) { + struct object *object = shallows->objects[i].item; + object->flags |= NOT_SHALLOW; + } + else + backup = result = + get_shallow_commits(&want_obj, depth, + SHALLOW, NOT_SHALLOW); + while (result) { + struct object *object = &result->item->object; + if (!(object->flags & (CLIENT_SHALLOW|NOT_SHALLOW))) { + packet_write(1, "shallow %s", + oid_to_hex(&object->oid)); + register_shallow(object->oid.hash); + shallow_nr++; + } + result = result->next; + } + free_commit_list(backup); + for (i = 0; i < shallows->nr; i++) { + struct object *object = shallows->objects[i].item; + if (object->flags & NOT_SHALLOW) { + struct commit_list *parents; + packet_write(1, "unshallow %s", + oid_to_hex(&object->oid)); + object->flags &= ~CLIENT_SHALLOW; + /* make sure the real parents are parsed */ + unregister_shallow(object->oid.hash); + object->parsed = 0; + parse_commit_or_die((struct commit *)object); + parents = ((struct commit *)object)->parents; + while (parents) { + add_object_array(&parents->item->object, + NULL, &want_obj); + parents = parents->next; + } + add_object_array(object, NULL, &extra_edge_obj); + } + /* make sure commit traversal conforms to client */ + register_shallow(object->oid.hash); + } + packet_flush(1); +} + static void receive_needs(void) { struct object_array shallows = OBJECT_ARRAY_INIT; @@ -630,53 +679,9 @@ static void receive_needs(void) if (depth == 0 && shallows.nr == 0) return; - if (depth > 0) { - struct commit_list *result = NULL, *backup = NULL; - int i; - if (depth == INFINITE_DEPTH && !is_repository_shallow()) - for (i = 0; i < shallows.nr; i++) { - struct object *object = shallows.objects[i].item; - object->flags |= NOT_SHALLOW; - } - else - backup = result = - get_shallow_commits(&want_obj, depth, - SHALLOW, NOT_SHALLOW); - while (result) { - struct object *object = &result->item->object; - if (!(object->flags & (CLIENT_SHALLOW|NOT_SHALLOW))) { - packet_write(1, "shallow %s", - oid_to_hex(&object->oid)); - register_shallow(object->oid.hash); - shallow_nr++; - } - result = result->next; - } - free_commit_list(backup); - for (i = 0; i < shallows.nr; i++) { - struct object *object = shallows.objects[i].item; - if (object->flags & NOT_SHALLOW) { - struct commit_list *parents; - packet_write(1, "unshallow %s", - oid_to_hex(&object->oid)); - object->flags &= ~CLIENT_SHALLOW; - /* make sure the real parents are parsed */ - unregister_shallow(object->oid.hash); - object->parsed = 0; - parse_commit_or_die((struct commit *)object); - parents = ((struct commit *)object)->parents; - while (parents) { - add_object_array(&parents->item->object, - NULL, &want_obj); - parents = parents->next; - } - add_object_array(object, NULL, &extra_edge_obj); - } - /* make sure commit traversal conforms to client */ - register_shallow(object->oid.hash); - } - packet_flush(1); - } else + if (depth > 0) + deepen(depth, &shallows); + else if (shallows.nr > 0) { int i; for (i = 0; i < shallows.nr; i++) -- cgit v1.2.3 From 5c24cdea1e33c72bfed4af25a363eb5ceae11199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Sun, 12 Jun 2016 17:53:46 +0700 Subject: upload-pack: move "shallow" sending code out of deepen() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- upload-pack.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) (limited to 'upload-pack.c') diff --git a/upload-pack.c b/upload-pack.c index 97ed62002e..0eb9a0b8fa 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -538,6 +538,20 @@ error: } } +static void send_shallow(struct commit_list *result) +{ + while (result) { + struct object *object = &result->item->object; + if (!(object->flags & (CLIENT_SHALLOW|NOT_SHALLOW))) { + packet_write(1, "shallow %s", + oid_to_hex(&object->oid)); + register_shallow(object->oid.hash); + shallow_nr++; + } + result = result->next; + } +} + static void deepen(int depth, const struct object_array *shallows) { struct commit_list *result = NULL, *backup = NULL; @@ -551,16 +565,7 @@ static void deepen(int depth, const struct object_array *shallows) backup = result = get_shallow_commits(&want_obj, depth, SHALLOW, NOT_SHALLOW); - while (result) { - struct object *object = &result->item->object; - if (!(object->flags & (CLIENT_SHALLOW|NOT_SHALLOW))) { - packet_write(1, "shallow %s", - oid_to_hex(&object->oid)); - register_shallow(object->oid.hash); - shallow_nr++; - } - result = result->next; - } + send_shallow(result); free_commit_list(backup); for (i = 0; i < shallows->nr; i++) { struct object *object = shallows->objects[i].item; -- cgit v1.2.3 From ef635b9056e23c06b4b5bcb4de294c1f6d9dda24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Sun, 12 Jun 2016 17:53:47 +0700 Subject: upload-pack: remove unused variable "backup" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the last patch, "result" and "backup" are the same. "result" used to move, but the movement is now contained in send_shallow(). Delete this redundant variable. Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- upload-pack.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'upload-pack.c') diff --git a/upload-pack.c b/upload-pack.c index 0eb9a0b8fa..ee5d20b801 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -554,7 +554,7 @@ static void send_shallow(struct commit_list *result) static void deepen(int depth, const struct object_array *shallows) { - struct commit_list *result = NULL, *backup = NULL; + struct commit_list *result = NULL; int i; if (depth == INFINITE_DEPTH && !is_repository_shallow()) for (i = 0; i < shallows->nr; i++) { @@ -562,11 +562,10 @@ static void deepen(int depth, const struct object_array *shallows) object->flags |= NOT_SHALLOW; } else - backup = result = - get_shallow_commits(&want_obj, depth, - SHALLOW, NOT_SHALLOW); + result = get_shallow_commits(&want_obj, depth, + SHALLOW, NOT_SHALLOW); send_shallow(result); - free_commit_list(backup); + free_commit_list(result); for (i = 0; i < shallows->nr; i++) { struct object *object = shallows->objects[i].item; if (object->flags & NOT_SHALLOW) { -- cgit v1.2.3 From 873700c92e01dbd5ccbe5d36878bb96d1bef3c36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Sun, 12 Jun 2016 17:53:48 +0700 Subject: upload-pack: move "unshallow" sending code out of deepen() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also add some more comments in this code because it takes too long to understand what it does (to me, who should be familiar enough to understand this code well!) Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- upload-pack.c | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) (limited to 'upload-pack.c') diff --git a/upload-pack.c b/upload-pack.c index ee5d20b801..73a8b28f57 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -552,20 +552,10 @@ static void send_shallow(struct commit_list *result) } } -static void deepen(int depth, const struct object_array *shallows) +static void send_unshallow(const struct object_array *shallows) { - struct commit_list *result = NULL; int i; - if (depth == INFINITE_DEPTH && !is_repository_shallow()) - for (i = 0; i < shallows->nr; i++) { - struct object *object = shallows->objects[i].item; - object->flags |= NOT_SHALLOW; - } - else - result = get_shallow_commits(&want_obj, depth, - SHALLOW, NOT_SHALLOW); - send_shallow(result); - free_commit_list(result); + for (i = 0; i < shallows->nr; i++) { struct object *object = shallows->objects[i].item; if (object->flags & NOT_SHALLOW) { @@ -573,7 +563,13 @@ static void deepen(int depth, const struct object_array *shallows) packet_write(1, "unshallow %s", oid_to_hex(&object->oid)); object->flags &= ~CLIENT_SHALLOW; - /* make sure the real parents are parsed */ + /* + * We want to _register_ "object" as shallow, but we + * also need to traverse object's parents to deepen a + * shallow clone. Unregister it for now so we can + * parse and add the parents to the want list, then + * re-register it. + */ unregister_shallow(object->oid.hash); object->parsed = 0; parse_commit_or_die((struct commit *)object); @@ -588,6 +584,27 @@ static void deepen(int depth, const struct object_array *shallows) /* make sure commit traversal conforms to client */ register_shallow(object->oid.hash); } +} + +static void deepen(int depth, const struct object_array *shallows) +{ + if (depth == INFINITE_DEPTH && !is_repository_shallow()) { + int i; + + for (i = 0; i < shallows->nr; i++) { + struct object *object = shallows->objects[i].item; + object->flags |= NOT_SHALLOW; + } + } else { + struct commit_list *result; + + result = get_shallow_commits(&want_obj, depth, + SHALLOW, NOT_SHALLOW); + send_shallow(result); + free_commit_list(result); + } + + send_unshallow(shallows); packet_flush(1); } -- cgit v1.2.3 From 8bf3b75841ead5a796b62a54534b5a203adffae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Sun, 12 Jun 2016 17:53:49 +0700 Subject: upload-pack: use skip_prefix() instead of starts_with() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- upload-pack.c | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) (limited to 'upload-pack.c') diff --git a/upload-pack.c b/upload-pack.c index 73a8b28f57..fa7ce092e7 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -276,7 +276,7 @@ static void create_pack_file(void) die("git upload-pack: %s", abort_msg); } -static int got_sha1(char *hex, unsigned char *sha1) +static int got_sha1(const char *hex, unsigned char *sha1) { struct object *o; int we_knew_they_have = 0; @@ -382,6 +382,8 @@ static int get_common_commits(void) for (;;) { char *line = packet_read_line(0, NULL); + const char *arg; + reset_timeout(); if (!line) { @@ -403,8 +405,8 @@ static int get_common_commits(void) got_other = 0; continue; } - if (starts_with(line, "have ")) { - switch (got_sha1(line+5, sha1)) { + if (skip_prefix(line, "have ", &arg)) { + switch (got_sha1(arg, sha1)) { case -1: /* they have what we do not */ got_other = 1; if (multi_ack && ok_to_give_up()) { @@ -620,14 +622,16 @@ static void receive_needs(void) const char *features; unsigned char sha1_buf[20]; char *line = packet_read_line(0, NULL); + const char *arg; + reset_timeout(); if (!line) break; - if (starts_with(line, "shallow ")) { + if (skip_prefix(line, "shallow ", &arg)) { unsigned char sha1[20]; struct object *object; - if (get_sha1_hex(line + 8, sha1)) + if (get_sha1_hex(arg, sha1)) die("invalid shallow line: %s", line); object = parse_object(sha1); if (!object) @@ -640,19 +644,19 @@ static void receive_needs(void) } continue; } - if (starts_with(line, "deepen ")) { + if (skip_prefix(line, "deepen ", &arg)) { char *end; - depth = strtol(line + 7, &end, 0); - if (end == line + 7 || depth <= 0) + depth = strtol(arg, &end, 0); + if (end == arg || depth <= 0) die("Invalid deepen: %s", line); continue; } - if (!starts_with(line, "want ") || - get_sha1_hex(line+5, sha1_buf)) + if (!skip_prefix(line, "want ", &arg) || + get_sha1_hex(arg, sha1_buf)) die("git upload-pack: protocol error, " "expected to get sha, not '%s'", line); - features = line + 45; + features = arg + 40; if (parse_feature_request(features, "multi_ack_detailed")) multi_ack = 2; @@ -859,7 +863,7 @@ int main(int argc, char **argv) check_replace_refs = 0; for (i = 1; i < argc; i++) { - char *arg = argv[i]; + const char *arg = argv[i]; if (arg[0] != '-') break; @@ -875,8 +879,8 @@ int main(int argc, char **argv) strict = 1; continue; } - if (starts_with(arg, "--timeout=")) { - timeout = atoi(arg+10); + if (skip_prefix(arg, "--timeout=", &arg)) { + timeout = atoi(arg); daemon_mode = 1; continue; } -- cgit v1.2.3 From 6e414e30fd12460d7a7ba3ce67d20aa82a41a737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Sun, 12 Jun 2016 17:53:50 +0700 Subject: upload-pack: tighten number parsing at "deepen" lines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- upload-pack.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'upload-pack.c') diff --git a/upload-pack.c b/upload-pack.c index fa7ce092e7..8f4d7f46cc 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -645,9 +645,9 @@ static void receive_needs(void) continue; } if (skip_prefix(line, "deepen ", &arg)) { - char *end; + char *end = NULL; depth = strtol(arg, &end, 0); - if (end == arg || depth <= 0) + if (!end || *end || depth <= 0) die("Invalid deepen: %s", line); continue; } -- cgit v1.2.3 From 7fcbd37f9c1b7413b408d0223344d070613777ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Sun, 12 Jun 2016 17:53:51 +0700 Subject: upload-pack: make check_non_tip() clean things up on error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On error check_non_tip() will die and not closing file descriptors is no big deal. The next patch will split the majority of this function out for reuse in other cases, where die() may not be the only outcome. Same story for popping SIGPIPE out of the signal chain. So let's make sure we clean things up properly first. Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- upload-pack.c | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) (limited to 'upload-pack.c') diff --git a/upload-pack.c b/upload-pack.c index 8f4d7f46cc..7ce97ecd90 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -475,16 +475,16 @@ static void check_non_tip(void) cmd.in = -1; cmd.out = -1; - if (start_command(&cmd)) - goto error; - /* - * If rev-list --stdin encounters an unknown commit, it - * terminates, which will cause SIGPIPE in the write loop + * If the next rev-list --stdin encounters an unknown commit, + * it terminates, which will cause SIGPIPE in the write loop * below. */ sigchain_push(SIGPIPE, SIG_IGN); + if (start_command(&cmd)) + goto error; + namebuf[0] = '^'; namebuf[41] = '\n'; for (i = get_max_object_index(); 0 < i; ) { @@ -507,8 +507,7 @@ static void check_non_tip(void) goto error; } close(cmd.in); - - sigchain_pop(SIGPIPE); + cmd.in = -1; /* * The commits out of the rev-list are not ancestors of @@ -518,6 +517,7 @@ static void check_non_tip(void) if (i) goto error; close(cmd.out); + cmd.out = -1; /* * rev-list may have died by encountering a bad commit @@ -527,10 +527,19 @@ static void check_non_tip(void) if (finish_command(&cmd)) goto error; + sigchain_pop(SIGPIPE); + /* All the non-tip ones are ancestors of what we advertised */ return; error: + sigchain_pop(SIGPIPE); + + if (cmd.in >= 0) + close(cmd.in); + if (cmd.out >= 0) + close(cmd.out); + /* Pick one of them (we know there at least is one) */ for (i = 0; i < want_obj.nr; i++) { o = want_obj.objects[i].item; -- cgit v1.2.3 From 3f0f6624f554a663ca0c83fd80f769dc9cef93e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Sun, 12 Jun 2016 17:53:52 +0700 Subject: upload-pack: move rev-list code out of check_non_tip() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- upload-pack.c | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) (limited to 'upload-pack.c') diff --git a/upload-pack.c b/upload-pack.c index 7ce97ecd90..93c05229cb 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -451,7 +451,7 @@ static int is_our_ref(struct object *o) return o->flags & ((allow_hidden_ref ? HIDDEN_REF : 0) | OUR_REF); } -static void check_non_tip(void) +static int has_unreachable(struct object_array *src) { static const char *argv[] = { "rev-list", "--stdin", NULL, @@ -461,14 +461,6 @@ static void check_non_tip(void) char namebuf[42]; /* ^ + SHA-1 + LF */ int i; - /* - * In the normal in-process case without - * uploadpack.allowReachableSHA1InWant, - * non-tip requests can never happen. - */ - if (!stateless_rpc && !(allow_unadvertised_object_request & ALLOW_REACHABLE_SHA1)) - goto error; - cmd.argv = argv; cmd.git_cmd = 1; cmd.no_stderr = 1; @@ -498,8 +490,8 @@ static void check_non_tip(void) goto error; } namebuf[40] = '\n'; - for (i = 0; i < want_obj.nr; i++) { - o = want_obj.objects[i].item; + for (i = 0; i < src->nr; i++) { + o = src->objects[i].item; if (is_our_ref(o)) continue; memcpy(namebuf, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ); @@ -530,7 +522,7 @@ static void check_non_tip(void) sigchain_pop(SIGPIPE); /* All the non-tip ones are ancestors of what we advertised */ - return; + return 0; error: sigchain_pop(SIGPIPE); @@ -539,10 +531,28 @@ error: close(cmd.in); if (cmd.out >= 0) close(cmd.out); + return 1; +} +static void check_non_tip(void) +{ + int i; + + /* + * In the normal in-process case without + * uploadpack.allowReachableSHA1InWant, + * non-tip requests can never happen. + */ + if (!stateless_rpc && !(allow_unadvertised_object_request & ALLOW_REACHABLE_SHA1)) + goto error; + if (!has_unreachable(&want_obj)) + /* All the non-tip ones are ancestors of what we advertised */ + return; + +error: /* Pick one of them (we know there at least is one) */ for (i = 0; i < want_obj.nr; i++) { - o = want_obj.objects[i].item; + struct object *o = want_obj.objects[i].item; if (!is_our_ref(o)) die("git upload-pack: not our ref %s", oid_to_hex(&o->oid)); -- cgit v1.2.3 From 569e554be9cb88047d9f2752750e0c260241f446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Sun, 12 Jun 2016 17:53:58 +0700 Subject: upload-pack: add deepen-since to cut shallow repos based on time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This should allow the user to say "create a shallow clone containing the work from last year" (once the client side is fixed up, of course). In theory deepen-since and deepen (aka --depth) can be used together to draw the shallow boundary (whether it's intersection or union is up to discussion, but if rev-list is used, it's likely intersection). However, because deepen goes with a custom commit walker, we can't mix the two yet. Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- upload-pack.c | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) (limited to 'upload-pack.c') diff --git a/upload-pack.c b/upload-pack.c index 93c05229cb..5269461cf1 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -14,6 +14,7 @@ #include "sigchain.h" #include "version.h" #include "string-list.h" +#include "argv-array.h" static const char upload_pack_usage[] = "git upload-pack [--strict] [--timeout=] "; @@ -629,11 +630,25 @@ static void deepen(int depth, const struct object_array *shallows) packet_flush(1); } +static void deepen_by_rev_list(int ac, const char **av, + struct object_array *shallows) +{ + struct commit_list *result; + + result = get_shallow_commits_by_rev_list(ac, av, SHALLOW, NOT_SHALLOW); + send_shallow(result); + free_commit_list(result); + send_unshallow(shallows); + packet_flush(1); +} + static void receive_needs(void) { struct object_array shallows = OBJECT_ARRAY_INIT; int depth = 0; int has_non_tip = 0; + unsigned long deepen_since = 0; + int deepen_rev_list = 0; shallow_nr = 0; for (;;) { @@ -670,6 +685,16 @@ static void receive_needs(void) die("Invalid deepen: %s", line); continue; } + if (skip_prefix(line, "deepen-since ", &arg)) { + char *end = NULL; + deepen_since = strtoul(arg, &end, 0); + if (!end || *end || !deepen_since || + /* revisions.c's max_age -1 is special */ + deepen_since == -1) + die("Invalid deepen-since: %s", line); + deepen_rev_list = 1; + continue; + } if (!skip_prefix(line, "want ", &arg) || get_sha1_hex(arg, sha1_buf)) die("git upload-pack: protocol error, " @@ -721,10 +746,26 @@ static void receive_needs(void) if (!use_sideband && daemon_mode) no_progress = 1; - if (depth == 0 && shallows.nr == 0) + if (depth == 0 && !deepen_rev_list && shallows.nr == 0) return; + if (depth > 0 && deepen_rev_list) + die("git upload-pack: deepen and deepen-since cannot be used together"); if (depth > 0) deepen(depth, &shallows); + else if (deepen_rev_list) { + struct argv_array av = ARGV_ARRAY_INIT; + int i; + + argv_array_push(&av, "rev-list"); + if (deepen_since) + argv_array_pushf(&av, "--max-age=%lu", deepen_since); + for (i = 0; i < want_obj.nr; i++) { + struct object *o = want_obj.objects[i].item; + argv_array_push(&av, oid_to_hex(&o->oid)); + } + deepen_by_rev_list(av.argc, av.argv, &shallows); + argv_array_clear(&av); + } else if (shallows.nr > 0) { int i; @@ -773,7 +814,7 @@ static int send_ref(const char *refname, const struct object_id *oid, int flag, void *cb_data) { static const char *capabilities = "multi_ack thin-pack side-band" - " side-band-64k ofs-delta shallow no-progress" + " side-band-64k ofs-delta shallow deepen-since no-progress" " include-tag multi_ack_detailed"; const char *refname_nons = strip_namespace(refname); struct object_id peeled; -- cgit v1.2.3 From 269a7a831636b7c7a453f6621fc8b440ff28a408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Sun, 12 Jun 2016 17:54:03 +0700 Subject: upload-pack: support define shallow boundary by excluding revisions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This should allow the user to say "create a shallow clone of this branch after version ". Short refs are accepted and expanded at the server side with expand_ref() because we cannot expand (unknown) refs from the client side. Like deepen-since, deepen-not cannot be used with deepen. But deepen-not can be mixed with deepen-since. The result is exactly how you do the command "git rev-list --since=... --not ref". Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- upload-pack.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) (limited to 'upload-pack.c') diff --git a/upload-pack.c b/upload-pack.c index 5269461cf1..acc6d9767b 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -645,6 +645,7 @@ static void deepen_by_rev_list(int ac, const char **av, static void receive_needs(void) { struct object_array shallows = OBJECT_ARRAY_INIT; + struct string_list deepen_not = STRING_LIST_INIT_DUP; int depth = 0; int has_non_tip = 0; unsigned long deepen_since = 0; @@ -695,6 +696,16 @@ static void receive_needs(void) deepen_rev_list = 1; continue; } + if (skip_prefix(line, "deepen-not ", &arg)) { + char *ref = NULL; + unsigned char sha1[20]; + if (expand_ref(arg, strlen(arg), sha1, &ref) != 1) + die("git upload-pack: ambiguous deepen-not: %s", line); + string_list_append(&deepen_not, ref); + free(ref); + deepen_rev_list = 1; + continue; + } if (!skip_prefix(line, "want ", &arg) || get_sha1_hex(arg, sha1_buf)) die("git upload-pack: protocol error, " @@ -749,7 +760,7 @@ static void receive_needs(void) if (depth == 0 && !deepen_rev_list && shallows.nr == 0) return; if (depth > 0 && deepen_rev_list) - die("git upload-pack: deepen and deepen-since cannot be used together"); + die("git upload-pack: deepen and deepen-since (or deepen-not) cannot be used together"); if (depth > 0) deepen(depth, &shallows); else if (deepen_rev_list) { @@ -759,6 +770,14 @@ static void receive_needs(void) argv_array_push(&av, "rev-list"); if (deepen_since) argv_array_pushf(&av, "--max-age=%lu", deepen_since); + if (deepen_not.nr) { + argv_array_push(&av, "--not"); + for (i = 0; i < deepen_not.nr; i++) { + struct string_list_item *s = deepen_not.items + i; + argv_array_push(&av, s->string); + } + argv_array_push(&av, "--not"); + } for (i = 0; i < want_obj.nr; i++) { struct object *o = want_obj.objects[i].item; argv_array_push(&av, oid_to_hex(&o->oid)); @@ -814,7 +833,7 @@ static int send_ref(const char *refname, const struct object_id *oid, int flag, void *cb_data) { static const char *capabilities = "multi_ack thin-pack side-band" - " side-band-64k ofs-delta shallow deepen-since no-progress" + " side-band-64k ofs-delta shallow deepen-since deepen-not no-progress" " include-tag multi_ack_detailed"; const char *refname_nons = strip_namespace(refname); struct object_id peeled; -- cgit v1.2.3 From 2997178ee63c76a4c449f35f299e20b32956795a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Sun, 12 Jun 2016 17:54:07 +0700 Subject: upload-pack: split check_unreachable() in two, prep for get_reachable_list() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- upload-pack.c | 56 ++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 18 deletions(-) (limited to 'upload-pack.c') diff --git a/upload-pack.c b/upload-pack.c index acc6d9767b..adb8e33757 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -452,21 +452,24 @@ static int is_our_ref(struct object *o) return o->flags & ((allow_hidden_ref ? HIDDEN_REF : 0) | OUR_REF); } -static int has_unreachable(struct object_array *src) +/* + * on successful case, it's up to the caller to close cmd->out + */ +static int do_reachable_revlist(struct child_process *cmd, + struct object_array *src) { static const char *argv[] = { "rev-list", "--stdin", NULL, }; - static struct child_process cmd = CHILD_PROCESS_INIT; struct object *o; char namebuf[42]; /* ^ + SHA-1 + LF */ int i; - cmd.argv = argv; - cmd.git_cmd = 1; - cmd.no_stderr = 1; - cmd.in = -1; - cmd.out = -1; + cmd->argv = argv; + cmd->git_cmd = 1; + cmd->no_stderr = 1; + cmd->in = -1; + cmd->out = -1; /* * If the next rev-list --stdin encounters an unknown commit, @@ -475,7 +478,7 @@ static int has_unreachable(struct object_array *src) */ sigchain_push(SIGPIPE, SIG_IGN); - if (start_command(&cmd)) + if (start_command(cmd)) goto error; namebuf[0] = '^'; @@ -487,7 +490,7 @@ static int has_unreachable(struct object_array *src) if (!is_our_ref(o)) continue; memcpy(namebuf + 1, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ); - if (write_in_full(cmd.in, namebuf, 42) < 0) + if (write_in_full(cmd->in, namebuf, 42) < 0) goto error; } namebuf[40] = '\n'; @@ -496,17 +499,39 @@ static int has_unreachable(struct object_array *src) if (is_our_ref(o)) continue; memcpy(namebuf, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ); - if (write_in_full(cmd.in, namebuf, 41) < 0) + if (write_in_full(cmd->in, namebuf, 41) < 0) goto error; } - close(cmd.in); - cmd.in = -1; + close(cmd->in); + cmd->in = -1; + sigchain_pop(SIGPIPE); + + return 0; + +error: + sigchain_pop(SIGPIPE); + + if (cmd->in >= 0) + close(cmd->in); + if (cmd->out >= 0) + close(cmd->out); + return -1; +} + +static int has_unreachable(struct object_array *src) +{ + struct child_process cmd = CHILD_PROCESS_INIT; + char buf[1]; + int i; + + if (do_reachable_revlist(&cmd, src) < 0) + return 1; /* * The commits out of the rev-list are not ancestors of * our ref. */ - i = read_in_full(cmd.out, namebuf, 1); + i = read_in_full(cmd.out, buf, 1); if (i) goto error; close(cmd.out); @@ -520,16 +545,11 @@ static int has_unreachable(struct object_array *src) if (finish_command(&cmd)) goto error; - sigchain_pop(SIGPIPE); - /* All the non-tip ones are ancestors of what we advertised */ return 0; error: sigchain_pop(SIGPIPE); - - if (cmd.in >= 0) - close(cmd.in); if (cmd.out >= 0) close(cmd.out); return 1; -- cgit v1.2.3 From 079aa97e24daff1329f041e00ba621b0afd59163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Sun, 12 Jun 2016 17:54:08 +0700 Subject: upload-pack: add get_reachable_list() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- upload-pack.c | 52 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) (limited to 'upload-pack.c') diff --git a/upload-pack.c b/upload-pack.c index adb8e33757..3227df872c 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -456,7 +456,8 @@ static int is_our_ref(struct object *o) * on successful case, it's up to the caller to close cmd->out */ static int do_reachable_revlist(struct child_process *cmd, - struct object_array *src) + struct object_array *src, + struct object_array *reachable) { static const char *argv[] = { "rev-list", "--stdin", NULL, @@ -487,6 +488,8 @@ static int do_reachable_revlist(struct child_process *cmd, o = get_indexed_object(--i); if (!o) continue; + if (reachable && o->type == OBJ_COMMIT) + o->flags &= ~TMP_MARK; if (!is_our_ref(o)) continue; memcpy(namebuf + 1, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ); @@ -496,8 +499,13 @@ static int do_reachable_revlist(struct child_process *cmd, namebuf[40] = '\n'; for (i = 0; i < src->nr; i++) { o = src->objects[i].item; - if (is_our_ref(o)) + if (is_our_ref(o)) { + if (reachable) + add_object_array(o, NULL, reachable); continue; + } + if (reachable && o->type == OBJ_COMMIT) + o->flags |= TMP_MARK; memcpy(namebuf, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ); if (write_in_full(cmd->in, namebuf, 41) < 0) goto error; @@ -518,13 +526,51 @@ error: return -1; } +static int get_reachable_list(struct object_array *src, + struct object_array *reachable) +{ + struct child_process cmd = CHILD_PROCESS_INIT; + int i; + struct object *o; + char namebuf[42]; /* ^ + SHA-1 + LF */ + + if (do_reachable_revlist(&cmd, src, reachable) < 0) + return -1; + + while ((i = read_in_full(cmd.out, namebuf, 41)) == 41) { + struct object_id sha1; + + if (namebuf[40] != '\n' || get_oid_hex(namebuf, &sha1)) + break; + + o = lookup_object(sha1.hash); + if (o && o->type == OBJ_COMMIT) { + o->flags &= ~TMP_MARK; + } + } + for (i = get_max_object_index(); 0 < i; i--) { + o = get_indexed_object(i - 1); + if (o && o->type == OBJ_COMMIT && + (o->flags & TMP_MARK)) { + add_object_array(o, NULL, reachable); + o->flags &= ~TMP_MARK; + } + } + close(cmd.out); + + if (finish_command(&cmd)) + return -1; + + return 0; +} + static int has_unreachable(struct object_array *src) { struct child_process cmd = CHILD_PROCESS_INIT; char buf[1]; int i; - if (do_reachable_revlist(&cmd, src) < 0) + if (do_reachable_revlist(&cmd, src, NULL) < 0) return 1; /* -- cgit v1.2.3 From cccf74e2da85808478c784e403a69bbfe2b9f518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Sun, 12 Jun 2016 17:54:09 +0700 Subject: fetch, upload-pack: --deepen=N extends shallow boundary by N commits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In git-fetch, --depth argument is always relative with the latest remote refs. This makes it a bit difficult to cover this use case, where the user wants to make the shallow history, say 3 levels deeper. It would work if remote refs have not moved yet, but nobody can guarantee that, especially when that use case is performed a couple months after the last clone or "git fetch --depth". Also, modifying shallow boundary using --depth does not work well with clones created by --since or --not. This patch fixes that. A new argument --deepen= will add more (*) parent commits to the current history regardless of where remote refs are. Have/Want negotiation is still respected. So if remote refs move, the server will send two chunks: one between "have" and "want" and another to extend shallow history. In theory, the client could send no "want"s in order to get the second chunk only. But the protocol does not allow that. Either you send no want lines, which means ls-remote; or you have to send at least one want line that carries deep-relative to the server.. The main work was done by Dongcan Jiang. I fixed it up here and there. And of course all the bugs belong to me. (*) We could even support --deepen= where is negative. In that case we can cut some history from the shallow clone. This operation (and --depth=) does not require interaction with remote side (and more complicated to implement as a result). Helped-by: Duy Nguyen Helped-by: Eric Sunshine Helped-by: Junio C Hamano Signed-off-by: Dongcan Jiang Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- upload-pack.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) (limited to 'upload-pack.c') diff --git a/upload-pack.c b/upload-pack.c index 3227df872c..e40d15adaf 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -32,6 +32,7 @@ static const char upload_pack_usage[] = "git upload-pack [--strict] [--timeout=< static unsigned long oldest_have; +static int deepen_relative; static int multi_ack; static int no_done; static int use_thin_pack, use_ofs_delta, use_include_tag; @@ -674,7 +675,8 @@ static void send_unshallow(const struct object_array *shallows) } } -static void deepen(int depth, const struct object_array *shallows) +static void deepen(int depth, int deepen_relative, + struct object_array *shallows) { if (depth == INFINITE_DEPTH && !is_repository_shallow()) { int i; @@ -683,6 +685,17 @@ static void deepen(int depth, const struct object_array *shallows) struct object *object = shallows->objects[i].item; object->flags |= NOT_SHALLOW; } + } else if (deepen_relative) { + struct object_array reachable_shallows = OBJECT_ARRAY_INIT; + struct commit_list *result; + + get_reachable_list(shallows, &reachable_shallows); + result = get_shallow_commits(&reachable_shallows, + depth + 1, + SHALLOW, NOT_SHALLOW); + send_shallow(result); + free_commit_list(result); + object_array_clear(&reachable_shallows); } else { struct commit_list *result; @@ -779,6 +792,8 @@ static void receive_needs(void) features = arg + 40; + if (parse_feature_request(features, "deepen-relative")) + deepen_relative = 1; if (parse_feature_request(features, "multi_ack_detailed")) multi_ack = 2; else if (parse_feature_request(features, "multi_ack")) @@ -828,7 +843,7 @@ static void receive_needs(void) if (depth > 0 && deepen_rev_list) die("git upload-pack: deepen and deepen-since (or deepen-not) cannot be used together"); if (depth > 0) - deepen(depth, &shallows); + deepen(depth, deepen_relative, &shallows); else if (deepen_rev_list) { struct argv_array av = ARGV_ARRAY_INIT; int i; @@ -899,8 +914,8 @@ static int send_ref(const char *refname, const struct object_id *oid, int flag, void *cb_data) { static const char *capabilities = "multi_ack thin-pack side-band" - " side-band-64k ofs-delta shallow deepen-since deepen-not no-progress" - " include-tag multi_ack_detailed"; + " side-band-64k ofs-delta shallow deepen-since deepen-not" + " deepen-relative no-progress include-tag multi_ack_detailed"; const char *refname_nons = strip_namespace(refname); struct object_id peeled; -- cgit v1.2.3