From 6fb22ca463077a07f42675be52e68891f319b5c2 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 28 Sep 2021 21:55:04 -0400 Subject: builtin/multi-pack-index.c: support `--stdin-packs` mode To power a new `--write-midx` mode, `git repack` will want to write a multi-pack index containing a certain set of packs in the repository. This new option will be used by `git repack` to write a MIDX which contains only the packs which will survive after the repack (that is, it will exclude any packs which are about to be deleted). This patch effectively exposes the function implemented in the previous commit via the `git multi-pack-index` builtin. An alternative approach would have been to call that function from the `git repack` builtin directly, but this introduces awkward problems around closing and reopening the object store, so the MIDX will be written out-of-process. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- builtin/multi-pack-index.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'builtin/multi-pack-index.c') diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index 73c0113b48..047647b5f2 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -47,6 +47,7 @@ static struct opts_multi_pack_index { const char *preferred_pack; unsigned long batch_size; unsigned flags; + int stdin_packs; } opts; static struct option common_opts[] = { @@ -61,6 +62,16 @@ static struct option *add_common_options(struct option *prev) return parse_options_concat(common_opts, prev); } +static void read_packs_from_stdin(struct string_list *to) +{ + struct strbuf buf = STRBUF_INIT; + while (strbuf_getline(&buf, stdin) != EOF) + string_list_append(to, buf.buf); + string_list_sort(to); + + strbuf_release(&buf); +} + static int cmd_multi_pack_index_write(int argc, const char **argv) { struct option *options; @@ -70,6 +81,8 @@ static int cmd_multi_pack_index_write(int argc, const char **argv) N_("pack for reuse when computing a multi-pack bitmap")), OPT_BIT(0, "bitmap", &opts.flags, N_("write multi-pack bitmap"), MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX), + OPT_BOOL(0, "stdin-packs", &opts.stdin_packs, + N_("write multi-pack index containing only given indexes")), OPT_END(), }; @@ -86,6 +99,20 @@ static int cmd_multi_pack_index_write(int argc, const char **argv) FREE_AND_NULL(options); + if (opts.stdin_packs) { + struct string_list packs = STRING_LIST_INIT_DUP; + int ret; + + read_packs_from_stdin(&packs); + + ret = write_midx_file_only(opts.object_dir, &packs, + opts.preferred_pack, opts.flags); + + string_list_clear(&packs, 0); + + return ret; + + } return write_midx_file(opts.object_dir, opts.preferred_pack, opts.flags); } -- cgit v1.2.3 From 08944d1c221a7f4fe42a50c0f11f129769edc9b1 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 28 Sep 2021 21:55:07 -0400 Subject: midx: preliminary support for `--refs-snapshot` To figure out which commits we can write a bitmap for, the multi-pack index/bitmap code does a reachability traversal, marking any commit which can be found in the MIDX as eligible to receive a bitmap. This approach will cause a problem when multi-pack bitmaps are able to be generated from `git repack`, since the reference tips can change during the repack. Even though we ignore commits that don't exist in the MIDX (when doing a scan of the ref tips), it's possible that a commit in the MIDX reaches something that isn't. This can happen when a multi-pack index contains some pack which refers to loose objects (e.g., if a pack was pushed after starting the repack but before generating the MIDX which depends on an object which is stored as loose in the repository, and by definition isn't included in the multi-pack index). By taking a snapshot of the references before we start repacking, we can close that race window. In the above scenario (where we have a packed object pointing at a loose one), we'll either (a) take a snapshot of the references before seeing the packed one, or (b) take it after, at which point we can guarantee that the loose object will be packed and included in the MIDX. This patch does just that. It writes a temporary "reference snapshot", which is a list of OIDs that are at the ref tips before writing a multi-pack bitmap. References that are "preferred" (i.e,. are a suffix of at least one value of the 'pack.preferBitmapTips' configuration) are marked with a special '+'. The format is simple: one line per commit at each tip, with an optional '+' at the beginning (for preferred references, as described above). When provided, the reference snapshot is used to drive bitmap selection instead of the MIDX code doing its own traversal. When it isn't provided, the usual traversal takes place instead. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- builtin/multi-pack-index.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'builtin/multi-pack-index.c') diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index 047647b5f2..4b827a07c0 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -7,7 +7,8 @@ #include "object-store.h" #define BUILTIN_MIDX_WRITE_USAGE \ - N_("git multi-pack-index [] write [--preferred-pack=]") + N_("git multi-pack-index [] write [--preferred-pack=]" \ + "[--refs-snapshot=]") #define BUILTIN_MIDX_VERIFY_USAGE \ N_("git multi-pack-index [] verify") @@ -45,6 +46,7 @@ static char const * const builtin_multi_pack_index_usage[] = { static struct opts_multi_pack_index { const char *object_dir; const char *preferred_pack; + const char *refs_snapshot; unsigned long batch_size; unsigned flags; int stdin_packs; @@ -83,6 +85,8 @@ static int cmd_multi_pack_index_write(int argc, const char **argv) MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX), OPT_BOOL(0, "stdin-packs", &opts.stdin_packs, N_("write multi-pack index containing only given indexes")), + OPT_FILENAME(0, "refs-snapshot", &opts.refs_snapshot, + N_("refs snapshot for selecting bitmap commits")), OPT_END(), }; @@ -106,7 +110,8 @@ static int cmd_multi_pack_index_write(int argc, const char **argv) read_packs_from_stdin(&packs); ret = write_midx_file_only(opts.object_dir, &packs, - opts.preferred_pack, opts.flags); + opts.preferred_pack, + opts.refs_snapshot, opts.flags); string_list_clear(&packs, 0); @@ -114,7 +119,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv) } return write_midx_file(opts.object_dir, opts.preferred_pack, - opts.flags); + opts.refs_snapshot, opts.flags); } static int cmd_multi_pack_index_verify(int argc, const char **argv) -- cgit v1.2.3