From ad491366de6c883cd04539cb86db31049201dfbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Thu, 5 Dec 2013 20:02:32 +0700 Subject: make the sender advertise shallow commits to the receiver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If either receive-pack or upload-pack is called on a shallow repository, shallow commits (*) will be sent after the ref advertisement (but before the packet flush), so that the receiver has the full "shape" of the sender's commit graph. This will be needed for the receiver to update its .git/shallow if necessary. This breaks the protocol for all clients trying to push to a shallow repo, or fetch from one. Which is basically the same end result as today's "is_repository_shallow() && die()" in receive-pack and upload-pack. New clients will be made aware of shallow upstream and can make use of this information. The sender must send all shallow commits that are sent in the following pack. It may send more shallow commits than necessary. upload-pack for example may choose to advertise no shallow commits if it knows in advance that the pack it's going to send contains no shallow commits. But upload-pack is the server, so we choose the cheaper way, send full .git/shallow and let the client deal with it. Smart HTTP is not affected by this patch. Shallow support on smart-http comes later separately. (*) A shallow commit is a commit that terminates the revision walker. It is usually put in .git/shallow in order to keep the revision walker from going out of bound because there is no guarantee that objects behind this commit is available. Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- shallow.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'shallow.c') diff --git a/shallow.c b/shallow.c index cdf37d694d..f2c04b28dc 100644 --- a/shallow.c +++ b/shallow.c @@ -220,3 +220,18 @@ void setup_alternate_shallow(struct lock_file *shallow_lock, *alternate_shallow_file = ""; strbuf_release(&sb); } + +static int advertise_shallow_grafts_cb(const struct commit_graft *graft, void *cb) +{ + int fd = *(int *)cb; + if (graft->nr_parent == -1) + packet_write(fd, "shallow %s\n", sha1_to_hex(graft->sha1)); + return 0; +} + +void advertise_shallow_grafts(int fd) +{ + if (!is_repository_shallow()) + return; + for_each_commit_graft(advertise_shallow_grafts_cb, &fd); +} -- cgit v1.2.3 From 1a30f5a2f2a3d5d9b3cf6e126ac19deb40324515 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Thu, 5 Dec 2013 20:02:34 +0700 Subject: shallow.c: extend setup_*_shallow() to accept extra shallow commits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- shallow.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'shallow.c') diff --git a/shallow.c b/shallow.c index f2c04b28dc..822c626600 100644 --- a/shallow.c +++ b/shallow.c @@ -165,22 +165,31 @@ static int write_one_shallow(const struct commit_graft *graft, void *cb_data) return 0; } -int write_shallow_commits(struct strbuf *out, int use_pack_protocol) +int write_shallow_commits(struct strbuf *out, int use_pack_protocol, + const struct sha1_array *extra) { struct write_shallow_data data; + int i; data.out = out; data.use_pack_protocol = use_pack_protocol; data.count = 0; for_each_commit_graft(write_one_shallow, &data); + if (!extra) + return data.count; + for (i = 0; i < extra->nr; i++) { + strbuf_addstr(out, sha1_to_hex(extra->sha1[i])); + strbuf_addch(out, '\n'); + data.count++; + } return data.count; } -char *setup_temporary_shallow(void) +char *setup_temporary_shallow(const struct sha1_array *extra) { struct strbuf sb = STRBUF_INIT; int fd; - if (write_shallow_commits(&sb, 0)) { + if (write_shallow_commits(&sb, 0, extra)) { struct strbuf path = STRBUF_INIT; strbuf_addstr(&path, git_path("shallow_XXXXXX")); fd = xmkstemp(path.buf); @@ -199,7 +208,8 @@ char *setup_temporary_shallow(void) } void setup_alternate_shallow(struct lock_file *shallow_lock, - const char **alternate_shallow_file) + const char **alternate_shallow_file, + const struct sha1_array *extra) { struct strbuf sb = STRBUF_INIT; int fd; @@ -207,7 +217,7 @@ void setup_alternate_shallow(struct lock_file *shallow_lock, check_shallow_file_for_update(); fd = hold_lock_file_for_update(shallow_lock, git_path("shallow"), LOCK_DIE_ON_ERROR); - if (write_shallow_commits(&sb, 0)) { + if (write_shallow_commits(&sb, 0, extra)) { if (write_in_full(fd, sb.buf, sb.len) != sb.len) die_errno("failed to write to %s", shallow_lock->filename); -- cgit v1.2.3 From 58babfffdeeecaa4d6edecaac1fb0c595218b801 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Thu, 5 Dec 2013 20:02:35 +0700 Subject: shallow.c: the 8 steps to select new commits for .git/shallow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Suppose a fetch or push is requested between two shallow repositories (with no history deepening or shortening). A pack that contains necessary objects is transferred over together with .git/shallow of the sender. The receiver has to determine whether it needs to update .git/shallow if new refs needs new shallow comits. The rule here is avoid updating .git/shallow by default. But we don't want to waste the received pack. If the pack contains two refs, one needs new shallow commits installed in .git/shallow and one does not, we keep the latter and reject/warn about the former. Even if .git/shallow update is allowed, we only add shallow commits strictly necessary for the former ref (remember the sender can send more shallow commits than necessary) and pay attention not to accidentally cut the receiver history short (no history shortening is asked for) So the steps to figure out what ref need what new shallow commits are: 1. Split the sender shallow commit list into "ours" and "theirs" list by has_sha1_file. Those that exist in current repo in "ours", the remaining in "theirs". 2. Check the receiver .git/shallow, remove from "ours" the ones that also exist in .git/shallow. 3. Fetch the new pack. Either install or unpack it. 4. Do has_sha1_file on "theirs" list again. Drop the ones that fail has_sha1_file. Obviously the new pack does not need them. 5. If the pack is kept, remove from "ours" the ones that do not exist in the new pack. 6. Walk the new refs to answer the question "what shallow commits, both ours and theirs, are required in .git/shallow in order to add this ref?". Shallow commits not associated to any refs are removed from their respective list. 7. (*) Check reachability (from the current refs) of all remaining commits in "ours". Those reachable are removed. We do not want to cut any part of our (reachable) history. We only check up commits. True reachability test is done by check_everything_connected() at the end as usual. 8. Combine the final "ours" and "theirs" and add them all to .git/shallow. Install new refs. The case where some hook rejects some refs on a push is explained in more detail in the push patches. Of these steps, #6 and #7 are expensive. Both require walking through some commits, or in the worst case all commits. And we rather avoid them in at least common case, where the transferred pack does not contain any shallow commits that the sender advertises. Let's look at each scenario: 1) the sender has longer history than the receiver All shallow commits from the sender will be put into "theirs" list at step 1 because none of them exists in current repo. In the common case, "theirs" becomes empty at step 4 and exit early. 2) the sender has shorter history than the receiver All shallow commits from the sender are likely in "ours" list at step 1. In the common case, if the new pack is kept, we could empty "ours" and exit early at step 5. If the pack is not kept, we hit the expensive step 6 then exit after "ours" is emptied. There'll be only a handful of objects to walk in fast-forward case. If it's forced update, we may need to walk to the bottom. 3) the sender has same .git/shallow as the receiver This is similar to case 2 except that "ours" should be emptied at step 2 and exit early. A fetch after "clone --depth=X" is case 1. A fetch after "clone" (from a shallow repo) is case 3. Luckily they're cheap for the common case. A push from "clone --depth=X" falls into case 2, which is expensive. Some more work may be done at the sender/client side to avoid more work on the server side: if the transferred pack does not contain any shallow commits, send-pack should not send any shallow commits to the receive-pack, effectively turning it into a normal push and avoid all steps. This patch implements all steps except #3, already handled by fetch-pack and receive-pack, #6 and #7, which has their own patch due to their size. (*) in previous versions step 7 was put before step 3. I reorder it so that the common case that keeps the pack does not need to walk commits at all. In future if we implement faster commit reachability check (maybe with the help of pack bitmaps or commit cache), step 7 could become cheap and be moved up before 6 again. Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- shallow.c | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) (limited to 'shallow.c') diff --git a/shallow.c b/shallow.c index 822c626600..ecd950fd04 100644 --- a/shallow.c +++ b/shallow.c @@ -2,6 +2,12 @@ #include "commit.h" #include "tag.h" #include "pkt-line.h" +#include "remote.h" +#include "refs.h" +#include "sha1-array.h" +#include "diff.h" +#include "revision.h" +#include "commit-slab.h" static int is_shallow = -1; static struct stat shallow_stat; @@ -245,3 +251,69 @@ void advertise_shallow_grafts(int fd) return; for_each_commit_graft(advertise_shallow_grafts_cb, &fd); } + +#define TRACE_KEY "GIT_TRACE_SHALLOW" + +/* + * Step 1, split sender shallow commits into "ours" and "theirs" + * Step 2, clean "ours" based on .git/shallow + */ +void prepare_shallow_info(struct shallow_info *info, struct sha1_array *sa) +{ + int i; + trace_printf_key(TRACE_KEY, "shallow: prepare_shallow_info\n"); + memset(info, 0, sizeof(*info)); + info->shallow = sa; + if (!sa) + return; + info->ours = xmalloc(sizeof(*info->ours) * sa->nr); + info->theirs = xmalloc(sizeof(*info->theirs) * sa->nr); + for (i = 0; i < sa->nr; i++) { + if (has_sha1_file(sa->sha1[i])) { + struct commit_graft *graft; + graft = lookup_commit_graft(sa->sha1[i]); + if (graft && graft->nr_parent < 0) + continue; + info->ours[info->nr_ours++] = i; + } else + info->theirs[info->nr_theirs++] = i; + } +} + +void clear_shallow_info(struct shallow_info *info) +{ + free(info->ours); + free(info->theirs); +} + +/* Step 4, remove non-existent ones in "theirs" after getting the pack */ + +void remove_nonexistent_theirs_shallow(struct shallow_info *info) +{ + unsigned char (*sha1)[20] = info->shallow->sha1; + int i, dst; + trace_printf_key(TRACE_KEY, "shallow: remove_nonexistent_theirs_shallow\n"); + for (i = dst = 0; i < info->nr_theirs; i++) { + if (i != dst) + info->theirs[dst] = info->theirs[i]; + if (has_sha1_file(sha1[info->theirs[i]])) + dst++; + } + info->nr_theirs = dst; +} + +/* Step 5, remove non-existent ones in "ours" in the pack */ +void remove_nonexistent_ours_in_pack(struct shallow_info *info, + struct packed_git *p) +{ + unsigned char (*sha1)[20] = info->shallow->sha1; + int i, dst; + trace_printf_key(TRACE_KEY, "shallow: remove_nonexistent_ours_in_pack\n"); + for (i = dst = 0; i < info->nr_ours; i++) { + if (i != dst) + info->ours[dst] = info->ours[i]; + if (find_pack_entry_one(sha1[info->ours[i]], p)) + dst++; + } + info->nr_ours = dst; +} -- cgit v1.2.3 From 8e277383e0902551c0d5d3ef9591196de5a7078c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Thu, 5 Dec 2013 20:02:36 +0700 Subject: shallow.c: steps 6 and 7 to select new commits for .git/shallow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- shallow.c | 294 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 294 insertions(+) (limited to 'shallow.c') diff --git a/shallow.c b/shallow.c index ecd950fd04..fb6069ba0c 100644 --- a/shallow.c +++ b/shallow.c @@ -317,3 +317,297 @@ void remove_nonexistent_ours_in_pack(struct shallow_info *info, } info->nr_ours = dst; } + +define_commit_slab(ref_bitmap, uint32_t *); + +struct paint_info { + struct ref_bitmap ref_bitmap; + unsigned nr_bits; + char **slab; + char *free, *end; + unsigned slab_count; +}; + +static uint32_t *paint_alloc(struct paint_info *info) +{ + unsigned nr = (info->nr_bits + 31) / 32; + unsigned size = nr * sizeof(uint32_t); + void *p; + if (!info->slab_count || info->free + size > info->end) { + info->slab_count++; + info->slab = xrealloc(info->slab, + info->slab_count * sizeof(*info->slab)); + info->free = xmalloc(COMMIT_SLAB_SIZE); + info->slab[info->slab_count - 1] = info->free; + info->end = info->free + COMMIT_SLAB_SIZE; + } + p = info->free; + info->free += size; + return p; +} + +/* + * Given a commit SHA-1, walk down to parents until either SEEN, + * UNINTERESTING or BOTTOM is hit. Set the id-th bit in ref_bitmap for + * all walked commits. + */ +static void paint_down(struct paint_info *info, const unsigned char *sha1, + int id) +{ + unsigned int i, nr; + struct commit_list *head = NULL; + int bitmap_nr = (info->nr_bits + 31) / 32; + int bitmap_size = bitmap_nr * sizeof(uint32_t); + uint32_t *tmp = xmalloc(bitmap_size); /* to be freed before return */ + uint32_t *bitmap = paint_alloc(info); + struct commit *c = lookup_commit_reference_gently(sha1, 1); + if (!c) + return; + memset(bitmap, 0, bitmap_size); + bitmap[id / 32] |= (1 << (id % 32)); + commit_list_insert(c, &head); + while (head) { + struct commit_list *p; + struct commit *c = head->item; + uint32_t **refs = ref_bitmap_at(&info->ref_bitmap, c); + + p = head; + head = head->next; + free(p); + + /* XXX check "UNINTERESTING" from pack bitmaps if available */ + if (c->object.flags & (SEEN | UNINTERESTING)) + continue; + else + c->object.flags |= SEEN; + + if (*refs == NULL) + *refs = bitmap; + else { + memcpy(tmp, *refs, bitmap_size); + for (i = 0; i < bitmap_nr; i++) + tmp[i] |= bitmap[i]; + if (memcmp(tmp, *refs, bitmap_size)) { + *refs = paint_alloc(info); + memcpy(*refs, tmp, bitmap_size); + } + } + + if (c->object.flags & BOTTOM) + continue; + + if (parse_commit(c)) + die("unable to parse commit %s", + sha1_to_hex(c->object.sha1)); + + for (p = c->parents; p; p = p->next) { + uint32_t **p_refs = ref_bitmap_at(&info->ref_bitmap, + p->item); + if (p->item->object.flags & SEEN) + continue; + if (*p_refs == NULL || *p_refs == *refs) + *p_refs = *refs; + commit_list_insert(p->item, &head); + } + } + + nr = get_max_object_index(); + for (i = 0; i < nr; i++) { + struct object *o = get_indexed_object(i); + if (o && o->type == OBJ_COMMIT) + o->flags &= ~SEEN; + } + + free(tmp); +} + +static int mark_uninteresting(const char *refname, + const unsigned char *sha1, + int flags, void *cb_data) +{ + struct commit *commit = lookup_commit_reference_gently(sha1, 1); + if (!commit) + return 0; + commit->object.flags |= UNINTERESTING; + mark_parents_uninteresting(commit); + return 0; +} + +static void post_assign_shallow(struct shallow_info *info, + struct ref_bitmap *ref_bitmap, + int *ref_status); +/* + * Step 6(+7), associate shallow commits with new refs + * + * info->ref must be initialized before calling this function. + * + * If used is not NULL, it's an array of info->shallow->nr + * bitmaps. The n-th bit set in the m-th bitmap if ref[n] needs the + * m-th shallow commit from info->shallow. + * + * If used is NULL, "ours" and "theirs" are updated. And if ref_status + * is not NULL it's an array of ref->nr ints. ref_status[i] is true if + * the ref needs some shallow commits from either info->ours or + * info->theirs. + */ +void assign_shallow_commits_to_refs(struct shallow_info *info, + uint32_t **used, int *ref_status) +{ + unsigned char (*sha1)[20] = info->shallow->sha1; + struct sha1_array *ref = info->ref; + unsigned int i, nr; + int *shallow, nr_shallow = 0; + struct paint_info pi; + + trace_printf_key(TRACE_KEY, "shallow: assign_shallow_commits_to_refs\n"); + shallow = xmalloc(sizeof(*shallow) * (info->nr_ours + info->nr_theirs)); + for (i = 0; i < info->nr_ours; i++) + shallow[nr_shallow++] = info->ours[i]; + for (i = 0; i < info->nr_theirs; i++) + shallow[nr_shallow++] = info->theirs[i]; + + /* + * Prepare the commit graph to track what refs can reach what + * (new) shallow commits. + */ + nr = get_max_object_index(); + for (i = 0; i < nr; i++) { + struct object *o = get_indexed_object(i); + if (!o || o->type != OBJ_COMMIT) + continue; + + o->flags &= ~(UNINTERESTING | BOTTOM | SEEN); + } + + memset(&pi, 0, sizeof(pi)); + init_ref_bitmap(&pi.ref_bitmap); + pi.nr_bits = ref->nr; + + /* + * "--not --all" to cut short the traversal if new refs + * connect to old refs. If not (e.g. force ref updates) it'll + * have to go down to the current shallow commits. + */ + head_ref(mark_uninteresting, NULL); + for_each_ref(mark_uninteresting, NULL); + + /* Mark potential bottoms so we won't go out of bound */ + for (i = 0; i < nr_shallow; i++) { + struct commit *c = lookup_commit(sha1[shallow[i]]); + c->object.flags |= BOTTOM; + } + + for (i = 0; i < ref->nr; i++) + paint_down(&pi, ref->sha1[i], i); + + if (used) { + int bitmap_size = ((pi.nr_bits + 31) / 32) * sizeof(uint32_t); + memset(used, 0, sizeof(*used) * info->shallow->nr); + for (i = 0; i < nr_shallow; i++) { + const struct commit *c = lookup_commit(sha1[shallow[i]]); + uint32_t **map = ref_bitmap_at(&pi.ref_bitmap, c); + if (*map) + used[shallow[i]] = xmemdupz(*map, bitmap_size); + } + /* + * unreachable shallow commits are not removed from + * "ours" and "theirs". The user is supposed to run + * step 7 on every ref separately and not trust "ours" + * and "theirs" any more. + */ + } else + post_assign_shallow(info, &pi.ref_bitmap, ref_status); + + clear_ref_bitmap(&pi.ref_bitmap); + for (i = 0; i < pi.slab_count; i++) + free(pi.slab[i]); + free(pi.slab); + free(shallow); +} + +struct commit_array { + struct commit **commits; + int nr, alloc; +}; + +static int add_ref(const char *refname, + const unsigned char *sha1, int flags, void *cb_data) +{ + struct commit_array *ca = cb_data; + ALLOC_GROW(ca->commits, ca->nr + 1, ca->alloc); + ca->commits[ca->nr] = lookup_commit_reference_gently(sha1, 1); + if (ca->commits[ca->nr]) + ca->nr++; + return 0; +} + +static void update_refstatus(int *ref_status, int nr, uint32_t *bitmap) +{ + int i; + if (!ref_status) + return; + for (i = 0; i < nr; i++) + if (bitmap[i / 32] & (1 << (i % 32))) + ref_status[i]++; +} + +/* + * Step 7, reachability test on "ours" at commit level + */ +static void post_assign_shallow(struct shallow_info *info, + struct ref_bitmap *ref_bitmap, + int *ref_status) +{ + unsigned char (*sha1)[20] = info->shallow->sha1; + struct commit *c; + uint32_t **bitmap; + int dst, i, j; + int bitmap_nr = (info->ref->nr + 31) / 32; + struct commit_array ca; + + trace_printf_key(TRACE_KEY, "shallow: post_assign_shallow\n"); + if (ref_status) + memset(ref_status, 0, sizeof(*ref_status) * info->ref->nr); + + /* Remove unreachable shallow commits from "theirs" */ + for (i = dst = 0; i < info->nr_theirs; i++) { + if (i != dst) + info->theirs[dst] = info->theirs[i]; + c = lookup_commit(sha1[info->theirs[i]]); + bitmap = ref_bitmap_at(ref_bitmap, c); + if (!*bitmap) + continue; + for (j = 0; j < bitmap_nr; j++) + if (bitmap[0][j]) { + update_refstatus(ref_status, info->ref->nr, *bitmap); + dst++; + break; + } + } + info->nr_theirs = dst; + + memset(&ca, 0, sizeof(ca)); + head_ref(add_ref, &ca); + for_each_ref(add_ref, &ca); + + /* Remove unreachable shallow commits from "ours" */ + for (i = dst = 0; i < info->nr_ours; i++) { + if (i != dst) + info->ours[dst] = info->ours[i]; + c = lookup_commit(sha1[info->ours[i]]); + bitmap = ref_bitmap_at(ref_bitmap, c); + if (!*bitmap) + continue; + for (j = 0; j < bitmap_nr; j++) + if (bitmap[0][j] && + /* Step 7, reachability test at commit level */ + !in_merge_bases_many(c, ca.nr, ca.commits)) { + update_refstatus(ref_status, info->ref->nr, *bitmap); + dst++; + break; + } + } + info->nr_ours = dst; + + free(ca.commits); +} -- cgit v1.2.3 From 79d3a236c551ad59719a6835bee03a1446296e65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Thu, 5 Dec 2013 20:02:41 +0700 Subject: upload-pack: make sure deepening preserves shallow roots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When "fetch --depth=N" where N exceeds the longest chain of history in the source repo, usually we just send an "unshallow" line to the client so full history is obtained. When the source repo is shallow we need to make sure to "unshallow" the current shallow point _and_ "shallow" again when the commit reaches its shallow bottom in the source repo. This should fix both cases: large and --unshallow. Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- shallow.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'shallow.c') diff --git a/shallow.c b/shallow.c index fb6069ba0c..52268544fd 100644 --- a/shallow.c +++ b/shallow.c @@ -75,6 +75,7 @@ struct commit_list *get_shallow_commits(struct object_array *heads, int depth, struct commit_list *result = NULL; struct object_array stack = OBJECT_ARRAY_INIT; struct commit *commit = NULL; + struct commit_graft *graft; while (commit || i < heads->nr || stack.nr) { struct commit_list *p; @@ -99,7 +100,10 @@ struct commit_list *get_shallow_commits(struct object_array *heads, int depth, if (parse_commit(commit)) die("invalid commit"); cur_depth++; - if (cur_depth >= depth) { + if ((depth != INFINITE_DEPTH && cur_depth >= depth) || + (is_repository_shallow() && !commit->parents && + (graft = lookup_commit_graft(commit->object.sha1)) != NULL && + graft->nr_parent < 0)) { commit_list_insert(commit, &result); commit->object.flags |= shallow_flag; commit = NULL; -- cgit v1.2.3 From 069c053222bfc62a6522430a137e9b2c7ff36e4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Thu, 5 Dec 2013 20:02:45 +0700 Subject: add GIT_SHALLOW_FILE to propagate --shallow-file to subprocesses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This may be needed when a hook is run after a new shallow pack is received, but .git/shallow is not settled yet. A temporary shallow file to plug all loose ends should be used instead. GIT_SHALLOW_FILE is overriden by --shallow-file. --shallow-file does not work in this case because the hook may spawn many git subprocesses and the launch commands do not have --shallow-file as it's a recent addition. Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- shallow.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'shallow.c') diff --git a/shallow.c b/shallow.c index 52268544fd..ec9179480f 100644 --- a/shallow.c +++ b/shallow.c @@ -13,10 +13,12 @@ static int is_shallow = -1; static struct stat shallow_stat; static char *alternate_shallow_file; -void set_alternate_shallow_file(const char *path) +void set_alternate_shallow_file(const char *path, int override) { if (is_shallow != -1) die("BUG: is_repository_shallow must not be called before set_alternate_shallow_file"); + if (alternate_shallow_file && !override) + return; free(alternate_shallow_file); alternate_shallow_file = path ? xstrdup(path) : NULL; } -- cgit v1.2.3 From 0a1bc12b6e401825f009ac8bb14fc438f77e2d9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Thu, 5 Dec 2013 20:02:47 +0700 Subject: receive-pack: allow pushes that update .git/shallow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The basic 8 steps to update .git/shallow does not fully apply here because the user may choose to accept just a few refs (while fetch always accepts all refs). The steps are modified a bit. 1-6. same as before. After calling assign_shallow_commits_to_refs at step 6, each shallow commit has a bitmap that marks all refs that require it. 7. mark all "ours" shallow commits that are reachable from any refs. We will need to do the original step 7 on them later. 8. go over all shallow commit bitmaps, mark refs that require new shallow commits. 9. setup a strict temporary shallow file to plug all the holes, even if it may cut some of our history short. This file is used by all hooks. The hooks could use --shallow-file=$GIT_DIR/shallow to overcome this and reach everything in current repo. 10. go over the new refs one by one. For each ref, do the reachability test if it needs a shallow commit on the list from step 7. Remove it if it's reachable from our refs. Gather all required shallow commits, run check_everything_connected() with the new ref, then install them to .git/shallow. This mode is disabled by default and can be turned on with receive.shallowupdate Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- shallow.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'shallow.c') diff --git a/shallow.c b/shallow.c index ec9179480f..3c36dd82bc 100644 --- a/shallow.c +++ b/shallow.c @@ -617,3 +617,26 @@ static void post_assign_shallow(struct shallow_info *info, free(ca.commits); } + +/* (Delayed) step 7, reachability test at commit level */ +int delayed_reachability_test(struct shallow_info *si, int c) +{ + if (si->need_reachability_test[c]) { + struct commit *commit = lookup_commit(si->shallow->sha1[c]); + + if (!si->commits) { + struct commit_array ca; + memset(&ca, 0, sizeof(ca)); + head_ref(add_ref, &ca); + for_each_ref(add_ref, &ca); + si->commits = ca.commits; + si->nr_commits = ca.nr; + } + + si->reachable[c] = in_merge_bases_many(commit, + si->nr_commits, + si->commits); + si->need_reachability_test[c] = 0; + } + return si->reachable[c]; +} -- cgit v1.2.3 From eab3296c7e5c99f559818357e70eeae09c24ac99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Th=C3=A1i=20Ng=E1=BB=8Dc=20Duy?= Date: Thu, 5 Dec 2013 20:02:54 +0700 Subject: prune: clean .git/shallow after pruning objects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch teaches "prune" to remove shallow roots that are no longer reachable from any refs (e.g. when the relevant refs are removed). Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- shallow.c | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) (limited to 'shallow.c') diff --git a/shallow.c b/shallow.c index 3c36dd82bc..c766fc3012 100644 --- a/shallow.c +++ b/shallow.c @@ -155,10 +155,14 @@ void check_shallow_file_for_update(void) die("shallow file was changed during fetch"); } +#define SEEN_ONLY 1 +#define VERBOSE 2 + struct write_shallow_data { struct strbuf *out; int use_pack_protocol; int count; + unsigned flags; }; static int write_one_shallow(const struct commit_graft *graft, void *cb_data) @@ -167,6 +171,15 @@ static int write_one_shallow(const struct commit_graft *graft, void *cb_data) const char *hex = sha1_to_hex(graft->sha1); if (graft->nr_parent != -1) return 0; + if (data->flags & SEEN_ONLY) { + struct commit *c = lookup_commit(graft->sha1); + if (!c || !(c->object.flags & SEEN)) { + if (data->flags & VERBOSE) + printf("Removing %s from .git/shallow\n", + sha1_to_hex(c->object.sha1)); + return 0; + } + } data->count++; if (data->use_pack_protocol) packet_buf_write(data->out, "shallow %s", hex); @@ -177,14 +190,16 @@ static int write_one_shallow(const struct commit_graft *graft, void *cb_data) return 0; } -int write_shallow_commits(struct strbuf *out, int use_pack_protocol, - const struct sha1_array *extra) +static int write_shallow_commits_1(struct strbuf *out, int use_pack_protocol, + const struct sha1_array *extra, + unsigned flags) { struct write_shallow_data data; int i; data.out = out; data.use_pack_protocol = use_pack_protocol; data.count = 0; + data.flags = flags; for_each_commit_graft(write_one_shallow, &data); if (!extra) return data.count; @@ -196,6 +211,12 @@ int write_shallow_commits(struct strbuf *out, int use_pack_protocol, return data.count; } +int write_shallow_commits(struct strbuf *out, int use_pack_protocol, + const struct sha1_array *extra) +{ + return write_shallow_commits_1(out, use_pack_protocol, extra, 0); +} + char *setup_temporary_shallow(const struct sha1_array *extra) { struct strbuf sb = STRBUF_INIT; @@ -258,6 +279,36 @@ void advertise_shallow_grafts(int fd) for_each_commit_graft(advertise_shallow_grafts_cb, &fd); } +/* + * mark_reachable_objects() should have been run prior to this and all + * reachable commits marked as "SEEN". + */ +void prune_shallow(int show_only) +{ + static struct lock_file shallow_lock; + struct strbuf sb = STRBUF_INIT; + int fd; + + if (show_only) { + write_shallow_commits_1(&sb, 0, NULL, SEEN_ONLY | VERBOSE); + strbuf_release(&sb); + return; + } + check_shallow_file_for_update(); + fd = hold_lock_file_for_update(&shallow_lock, git_path("shallow"), + LOCK_DIE_ON_ERROR); + if (write_shallow_commits_1(&sb, 0, NULL, SEEN_ONLY)) { + if (write_in_full(fd, sb.buf, sb.len) != sb.len) + die_errno("failed to write to %s", + shallow_lock.filename); + commit_lock_file(&shallow_lock); + } else { + unlink(git_path("shallow")); + rollback_lock_file(&shallow_lock); + } + strbuf_release(&sb); +} + #define TRACE_KEY "GIT_TRACE_SHALLOW" /* -- cgit v1.2.3 From feefdf62c107fd63056becf547ea324dbb730e30 Mon Sep 17 00:00:00 2001 From: Ramsay Jones Date: Mon, 6 Jan 2014 00:00:58 +0000 Subject: shallow: remove unused code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 58babfff ("shallow.c: the 8 steps to select new commits for .git/shallow", 05-12-2013) added a function to implement step 5 of the quoted eight steps, namely 'remove_nonexistent_ours_in_pack()'. This function implements an optional optimization step in the new shallow commit selection algorithm. However, this function has no callers. (The commented out call sites would need to change, in order to provide information required by the function.) Signed-off-by: Ramsay Jones Acked-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- shallow.c | 16 ---------------- 1 file changed, 16 deletions(-) (limited to 'shallow.c') diff --git a/shallow.c b/shallow.c index c766fc3012..e483780d49 100644 --- a/shallow.c +++ b/shallow.c @@ -359,22 +359,6 @@ void remove_nonexistent_theirs_shallow(struct shallow_info *info) info->nr_theirs = dst; } -/* Step 5, remove non-existent ones in "ours" in the pack */ -void remove_nonexistent_ours_in_pack(struct shallow_info *info, - struct packed_git *p) -{ - unsigned char (*sha1)[20] = info->shallow->sha1; - int i, dst; - trace_printf_key(TRACE_KEY, "shallow: remove_nonexistent_ours_in_pack\n"); - for (i = dst = 0; i < info->nr_ours; i++) { - if (i != dst) - info->ours[dst] = info->ours[i]; - if (find_pack_entry_one(sha1[info->ours[i]], p)) - dst++; - } - info->nr_ours = dst; -} - define_commit_slab(ref_bitmap, uint32_t *); struct paint_info { -- cgit v1.2.3