From 1a30f5a2f2a3d5d9b3cf6e126ac19deb40324515 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Thu, 5 Dec 2013 20:02:34 +0700 Subject: shallow.c: extend setup_*_shallow() to accept extra shallow commits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- fetch-pack.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'fetch-pack.c') diff --git a/fetch-pack.c b/fetch-pack.c index 1042448fa0..0e7483e1fe 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -311,7 +311,7 @@ static int find_common(struct fetch_pack_args *args, } if (is_repository_shallow()) - write_shallow_commits(&req_buf, 1); + write_shallow_commits(&req_buf, 1, NULL); if (args->depth > 0) packet_buf_write(&req_buf, "deepen %d", args->depth); packet_buf_flush(&req_buf); @@ -850,7 +850,8 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, if (args->stateless_rpc) packet_flush(fd[1]); if (args->depth > 0) - setup_alternate_shallow(&shallow_lock, &alternate_shallow_file); + setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, + NULL); else alternate_shallow_file = NULL; if (get_pack(args, fd, pack_lockfile)) -- cgit v1.2.3 From a796ccee5198c6ae11dd73c837f3ec46aaa1e8bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Thu, 5 Dec 2013 20:02:37 +0700 Subject: fetch-pack.c: move shallow update code out of fetch_pack() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- fetch-pack.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) (limited to 'fetch-pack.c') diff --git a/fetch-pack.c b/fetch-pack.c index 0e7483e1fe..35d097e1b1 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -925,6 +925,18 @@ static int remove_duplicates_in_refs(struct ref **ref, int nr) return dst; } +static void update_shallow(struct fetch_pack_args *args) +{ + if (args->depth > 0 && alternate_shallow_file) { + if (*alternate_shallow_file == '\0') { /* --unshallow */ + unlink_or_warn(git_path("shallow")); + rollback_lock_file(&shallow_lock); + } else + commit_lock_file(&shallow_lock); + return; + } +} + struct ref *fetch_pack(struct fetch_pack_args *args, int fd[], struct child_process *conn, const struct ref *ref, @@ -943,15 +955,7 @@ struct ref *fetch_pack(struct fetch_pack_args *args, die("no matching remote head"); } ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, pack_lockfile); - - if (args->depth > 0 && alternate_shallow_file) { - if (*alternate_shallow_file == '\0') { /* --unshallow */ - unlink_or_warn(git_path("shallow")); - rollback_lock_file(&shallow_lock); - } else - commit_lock_file(&shallow_lock); - } - + update_shallow(args); reprepare_packed_git(); return ref_cpy; } -- cgit v1.2.3 From beea4152d94cf7c77eeb6b226805b315d22b3a2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Thu, 5 Dec 2013 20:02:39 +0700 Subject: clone: support remote shallow repository MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cloning from a shallow repository does not follow the "8 steps for new .git/shallow" because if it does we need to get through step 6 for all refs. That means commit walking down to the bottom. Instead the rule to create .git/shallow is simpler and, more importantly, cheap: if a shallow commit is found in the pack, it's probably used (i.e. reachable from some refs), so we add it. Others are dropped. One may notice this method seems flawed by the word "probably". A shallow commit may not be reachable from any refs at all if it's attached to an object island (a group of objects that are not reachable by any refs). If that object island is not complete, a new fetch request may send more objects to connect it to some ref. At that time, because we incorrectly installed the shallow commit in this island, the user will not see anything after that commit (fsck is still ok). This is not desired. Given that object islands are rare (C Git never sends such islands for security reasons) and do not really harm the repository integrity, a tradeoff is made to surprise the user occasionally but work faster everyday. A new option --strict could be added later that follows exactly the 8 steps. "git prune" can also learn to remove dangling objects _and_ the shallow commits that are attached to them from .git/shallow. Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- fetch-pack.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) (limited to 'fetch-pack.c') diff --git a/fetch-pack.c b/fetch-pack.c index 35d097e1b1..6c980cd39f 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -13,6 +13,7 @@ #include "transport.h" #include "version.h" #include "prio-queue.h" +#include "sha1-array.h" static int transfer_unpack_limit = -1; static int fetch_unpack_limit = -1; @@ -774,6 +775,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, int fd[2], const struct ref *orig_ref, struct ref **sought, int nr_sought, + struct shallow_info *si, char **pack_lockfile) { struct ref *ref = copy_ref_list(orig_ref); @@ -852,6 +854,8 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, if (args->depth > 0) setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL); + else if (args->cloning && si->shallow && si->shallow->nr) + alternate_shallow_file = setup_temporary_shallow(si->shallow); else alternate_shallow_file = NULL; if (get_pack(args, fd, pack_lockfile)) @@ -925,8 +929,11 @@ static int remove_duplicates_in_refs(struct ref **ref, int nr) return dst; } -static void update_shallow(struct fetch_pack_args *args) +static void update_shallow(struct fetch_pack_args *args, + struct shallow_info *si) { + int i; + if (args->depth > 0 && alternate_shallow_file) { if (*alternate_shallow_file == '\0') { /* --unshallow */ unlink_or_warn(git_path("shallow")); @@ -935,6 +942,42 @@ static void update_shallow(struct fetch_pack_args *args) commit_lock_file(&shallow_lock); return; } + + if (!si->shallow || !si->shallow->nr) + return; + + if (alternate_shallow_file) { + /* + * The temporary shallow file is only useful for + * index-pack and unpack-objects because it may + * contain more roots than we want. Delete it. + */ + if (*alternate_shallow_file) + unlink(alternate_shallow_file); + free((char *)alternate_shallow_file); + } + + if (args->cloning) { + /* + * remote is shallow, but this is a clone, there are + * no objects in repo to worry about. Accept any + * shallow points that exist in the pack (iow in repo + * after get_pack() and reprepare_packed_git()) + */ + struct sha1_array extra = SHA1_ARRAY_INIT; + unsigned char (*sha1)[20] = si->shallow->sha1; + for (i = 0; i < si->shallow->nr; i++) + if (has_sha1_file(sha1[i])) + sha1_array_append(&extra, sha1[i]); + if (extra.nr) { + setup_alternate_shallow(&shallow_lock, + &alternate_shallow_file, + &extra); + commit_lock_file(&shallow_lock); + } + sha1_array_clear(&extra); + return; + } } struct ref *fetch_pack(struct fetch_pack_args *args, @@ -942,9 +985,11 @@ struct ref *fetch_pack(struct fetch_pack_args *args, const struct ref *ref, const char *dest, struct ref **sought, int nr_sought, + struct sha1_array *shallow, char **pack_lockfile) { struct ref *ref_cpy; + struct shallow_info si; fetch_pack_setup(); if (nr_sought) @@ -954,8 +999,11 @@ struct ref *fetch_pack(struct fetch_pack_args *args, packet_flush(fd[1]); die("no matching remote head"); } - ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, pack_lockfile); - update_shallow(args); + prepare_shallow_info(&si, shallow); + ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, + &si, pack_lockfile); reprepare_packed_git(); + update_shallow(args, &si); + clear_shallow_info(&si); return ref_cpy; } -- cgit v1.2.3 From 4820a33baa963c4559736d7a1c4c35f8dcb37293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Thu, 5 Dec 2013 20:02:40 +0700 Subject: fetch: support fetching from a shallow repository MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch just put together pieces from the 8 steps patch. We stop at step 7 and reject refs that require new shallow commits. Note that, by rejecting refs that require new shallow commits, we leave dangling objects in the repo, which become "object islands" by the next "git fetch" of the same source. If the first fetch our "ours" set is zero and we do practically nothing at step 7, "ours" is full at the next fetch and we may need to walk through commits for reachability test. Room for improvement. Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- fetch-pack.c | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) (limited to 'fetch-pack.c') diff --git a/fetch-pack.c b/fetch-pack.c index 6c980cd39f..34c544d0ca 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -854,7 +854,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, if (args->depth > 0) setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL); - else if (args->cloning && si->shallow && si->shallow->nr) + else if (si->nr_ours || si->nr_theirs) alternate_shallow_file = setup_temporary_shallow(si->shallow); else alternate_shallow_file = NULL; @@ -930,8 +930,11 @@ static int remove_duplicates_in_refs(struct ref **ref, int nr) } static void update_shallow(struct fetch_pack_args *args, + struct ref **sought, int nr_sought, struct shallow_info *si) { + struct sha1_array ref = SHA1_ARRAY_INIT; + int *status; int i; if (args->depth > 0 && alternate_shallow_file) { @@ -978,6 +981,31 @@ static void update_shallow(struct fetch_pack_args *args, sha1_array_clear(&extra); return; } + + if (!si->nr_ours && !si->nr_theirs) + return; + + remove_nonexistent_theirs_shallow(si); + /* XXX remove_nonexistent_ours_in_pack() */ + if (!si->nr_ours && !si->nr_theirs) + return; + for (i = 0; i < nr_sought; i++) + sha1_array_append(&ref, sought[i]->old_sha1); + si->ref = &ref; + + /* + * remote is also shallow, check what ref is safe to update + * without updating .git/shallow + */ + status = xcalloc(nr_sought, sizeof(*status)); + assign_shallow_commits_to_refs(si, NULL, status); + if (si->nr_ours || si->nr_theirs) { + for (i = 0; i < nr_sought; i++) + if (status[i]) + sought[i]->status = REF_STATUS_REJECT_SHALLOW; + } + free(status); + sha1_array_clear(&ref); } struct ref *fetch_pack(struct fetch_pack_args *args, @@ -1003,7 +1031,7 @@ struct ref *fetch_pack(struct fetch_pack_args *args, ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, &si, pack_lockfile); reprepare_packed_git(); - update_shallow(args, &si); + update_shallow(args, sought, nr_sought, &si); clear_shallow_info(&si); return ref_cpy; } -- cgit v1.2.3 From 48d25cae22667dfc2c31ad620172c0f0a3ac1490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Thu, 5 Dec 2013 20:02:42 +0700 Subject: fetch: add --update-shallow to accept refs that update .git/shallow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The same steps are done as in when --update-shallow is not given. The only difference is we now add all shallow commits in "ours" and "theirs" to .git/shallow (aka "step 8"). Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- fetch-pack.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'fetch-pack.c') diff --git a/fetch-pack.c b/fetch-pack.c index 34c544d0ca..a2d1b4ab28 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -993,6 +993,33 @@ static void update_shallow(struct fetch_pack_args *args, sha1_array_append(&ref, sought[i]->old_sha1); si->ref = &ref; + if (args->update_shallow) { + /* + * remote is also shallow, .git/shallow may be updated + * so all refs can be accepted. Make sure we only add + * shallow roots that are actually reachable from new + * refs. + */ + struct sha1_array extra = SHA1_ARRAY_INIT; + unsigned char (*sha1)[20] = si->shallow->sha1; + assign_shallow_commits_to_refs(si, NULL, NULL); + if (!si->nr_ours && !si->nr_theirs) { + sha1_array_clear(&ref); + return; + } + for (i = 0; i < si->nr_ours; i++) + sha1_array_append(&extra, sha1[si->ours[i]]); + for (i = 0; i < si->nr_theirs; i++) + sha1_array_append(&extra, sha1[si->theirs[i]]); + setup_alternate_shallow(&shallow_lock, + &alternate_shallow_file, + &extra); + commit_lock_file(&shallow_lock); + sha1_array_clear(&extra); + sha1_array_clear(&ref); + return; + } + /* * remote is also shallow, check what ref is safe to update * without updating .git/shallow -- cgit v1.2.3 From feefdf62c107fd63056becf547ea324dbb730e30 Mon Sep 17 00:00:00 2001 From: Ramsay Jones Date: Mon, 6 Jan 2014 00:00:58 +0000 Subject: shallow: remove unused code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 58babfff ("shallow.c: the 8 steps to select new commits for .git/shallow", 05-12-2013) added a function to implement step 5 of the quoted eight steps, namely 'remove_nonexistent_ours_in_pack()'. This function implements an optional optimization step in the new shallow commit selection algorithm. However, this function has no callers. (The commented out call sites would need to change, in order to provide information required by the function.) Signed-off-by: Ramsay Jones Acked-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- fetch-pack.c | 1 - 1 file changed, 1 deletion(-) (limited to 'fetch-pack.c') diff --git a/fetch-pack.c b/fetch-pack.c index a2d1b4ab28..9bc29cfce7 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -986,7 +986,6 @@ static void update_shallow(struct fetch_pack_args *args, return; remove_nonexistent_theirs_shallow(si); - /* XXX remove_nonexistent_ours_in_pack() */ if (!si->nr_ours && !si->nr_theirs) return; for (i = 0; i < nr_sought; i++) -- cgit v1.2.3