diff options
| -rw-r--r-- | builtin/sparse-checkout.c | 8 | ||||
| -rw-r--r-- | cache-tree.c | 27 | ||||
| -rw-r--r-- | cache-tree.h | 2 | ||||
| -rw-r--r-- | cache.h | 33 | ||||
| -rw-r--r-- | read-cache.c | 6 | ||||
| -rw-r--r-- | sparse-index.c | 132 | ||||
| -rw-r--r-- | sparse-index.h | 13 | ||||
| -rwxr-xr-x | t/perf/p2000-sparse-operations.sh | 1 | ||||
| -rwxr-xr-x | t/t1092-sparse-checkout-compatibility.sh | 95 | ||||
| -rw-r--r-- | unpack-trees.c | 4 |
10 files changed, 275 insertions, 46 deletions
diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index e27407c3bb..f91e29b56a 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -128,7 +128,7 @@ static void clean_tracked_sparse_directories(struct repository *r) * sparse index will not delete directories that contain * conflicted entries or submodules. */ - if (!r->index->sparse_index) { + if (r->index->sparse_index == INDEX_EXPANDED) { /* * If something, such as a merge conflict or other concern, * prevents us from converting to a sparse index, then do @@ -413,6 +413,9 @@ static int update_modes(int *cone_mode, int *sparse_index) /* force an index rewrite */ repo_read_index(the_repository); the_repository->index->updated_workdir = 1; + + if (!*sparse_index) + ensure_full_index(the_repository->index); } return 0; @@ -934,6 +937,9 @@ int cmd_sparse_checkout(int argc, const char **argv, const char *prefix) git_config(git_default_config, NULL); + prepare_repo_settings(the_repository); + the_repository->settings.command_requires_full_index = 0; + if (argc > 0) { if (!strcmp(argv[0], "list")) return sparse_checkout_list(argc, argv); diff --git a/cache-tree.c b/cache-tree.c index 8c5e882271..a92c23eec3 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -101,6 +101,33 @@ struct cache_tree_sub *cache_tree_sub(struct cache_tree *it, const char *path) return find_subtree(it, path, pathlen, 1); } +struct cache_tree *cache_tree_find_path(struct cache_tree *it, const char *path) +{ + const char *slash; + int namelen; + struct cache_tree_sub it_sub = { + .cache_tree = it, + }; + struct cache_tree_sub *down = &it_sub; + + while (down) { + slash = strchrnul(path, '/'); + namelen = slash - path; + down->cache_tree->entry_count = -1; + if (!*slash) { + int pos; + pos = cache_tree_subtree_pos(down->cache_tree, path, namelen); + if (0 <= pos) + return down->cache_tree->down[pos]->cache_tree; + return NULL; + } + down = find_subtree(it, path, namelen, 0); + path = slash + 1; + } + + return NULL; +} + static int do_invalidate_path(struct cache_tree *it, const char *path) { /* a/b/c diff --git a/cache-tree.h b/cache-tree.h index 8efeccebfc..f75f8e74dc 100644 --- a/cache-tree.h +++ b/cache-tree.h @@ -29,6 +29,8 @@ struct cache_tree_sub *cache_tree_sub(struct cache_tree *, const char *); int cache_tree_subtree_pos(struct cache_tree *it, const char *path, int pathlen); +struct cache_tree *cache_tree_find_path(struct cache_tree *it, const char *path); + void cache_tree_write(struct strbuf *, struct cache_tree *root); struct cache_tree *cache_tree_read(const char *buffer, unsigned long size); @@ -310,6 +310,29 @@ struct untracked_cache; struct progress; struct pattern_list; +enum sparse_index_mode { + /* + * There are no sparse directories in the index at all. + * + * Repositories that don't use cone-mode sparse-checkout will + * always have their indexes in this mode. + */ + INDEX_EXPANDED = 0, + + /* + * The index has already been collapsed to sparse directories + * whereever possible. + */ + INDEX_COLLAPSED, + + /* + * The sparse directories that exist are outside the + * sparse-checkout boundary, but it is possible that some file + * entries could collapse to sparse directory entries. + */ + INDEX_PARTIALLY_SPARSE, +}; + struct index_state { struct cache_entry **cache; unsigned int version; @@ -323,14 +346,8 @@ struct index_state { drop_cache_tree : 1, updated_workdir : 1, updated_skipworktree : 1, - fsmonitor_has_run_once : 1, - - /* - * sparse_index == 1 when sparse-directory - * entries exist. Requires sparse-checkout - * in cone mode. - */ - sparse_index : 1; + fsmonitor_has_run_once : 1; + enum sparse_index_mode sparse_index; struct hashmap name_hash; struct hashmap dir_hash; struct object_id oid; diff --git a/read-cache.c b/read-cache.c index 60355f5ad6..96ce489c7c 100644 --- a/read-cache.c +++ b/read-cache.c @@ -112,7 +112,7 @@ static const char *alternate_index_output; static void set_index_entry(struct index_state *istate, int nr, struct cache_entry *ce) { if (S_ISSPARSEDIR(ce->ce_mode)) - istate->sparse_index = 1; + istate->sparse_index = INDEX_COLLAPSED; istate->cache[nr] = ce; add_name_hash(istate, ce); @@ -1856,7 +1856,7 @@ static int read_index_extension(struct index_state *istate, break; case CACHE_EXT_SPARSE_DIRECTORIES: /* no content, only an indicator */ - istate->sparse_index = 1; + istate->sparse_index = INDEX_COLLAPSED; break; default: if (*ext < 'A' || 'Z' < *ext) @@ -3165,7 +3165,7 @@ static int do_write_locked_index(struct index_state *istate, struct lock_file *l unsigned flags) { int ret; - int was_full = !istate->sparse_index; + int was_full = istate->sparse_index == INDEX_EXPANDED; ret = convert_to_sparse(istate, 0); diff --git a/sparse-index.c b/sparse-index.c index ffbab7d35f..e4a54ce194 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -9,6 +9,11 @@ #include "dir.h" #include "fsmonitor.h" +struct modify_index_context { + struct index_state *write; + struct pattern_list *pl; +}; + static struct cache_entry *construct_sparse_dir_entry( struct index_state *istate, const char *sparse_dir, @@ -173,7 +178,7 @@ int convert_to_sparse(struct index_state *istate, int flags) * If the index is already sparse, empty, or otherwise * cannot be converted to sparse, do not convert. */ - if (istate->sparse_index || !istate->cache_nr || + if (istate->sparse_index == INDEX_COLLAPSED || !istate->cache_nr || !is_sparse_index_allowed(istate, flags)) return 0; @@ -214,7 +219,7 @@ int convert_to_sparse(struct index_state *istate, int flags) FREE_AND_NULL(istate->fsmonitor_dirty); FREE_AND_NULL(istate->fsmonitor_last_update); - istate->sparse_index = 1; + istate->sparse_index = INDEX_COLLAPSED; trace2_region_leave("index", "convert_to_sparse", istate->repo); return 0; } @@ -231,56 +236,148 @@ static int add_path_to_index(const struct object_id *oid, struct strbuf *base, const char *path, unsigned int mode, void *context) { - struct index_state *istate = (struct index_state *)context; + struct modify_index_context *ctx = (struct modify_index_context *)context; struct cache_entry *ce; size_t len = base->len; - if (S_ISDIR(mode)) - return READ_TREE_RECURSIVE; + if (S_ISDIR(mode)) { + int dtype; + size_t baselen = base->len; + if (!ctx->pl) + return READ_TREE_RECURSIVE; + + /* + * Have we expanded to a point outside of the sparse-checkout? + * + * Artificially pad the path name with a slash "/" to + * indicate it as a directory, and add an arbitrary file + * name ("-") so we can consider base->buf as a file name + * to match against the cone-mode patterns. + * + * If we compared just "path", then we would expand more + * than we should. Since every file at root is always + * included, we would expand every directory at root at + * least one level deep instead of using sparse directory + * entries. + */ + strbuf_addstr(base, path); + strbuf_add(base, "/-", 2); + + if (path_matches_pattern_list(base->buf, base->len, + NULL, &dtype, + ctx->pl, ctx->write)) { + strbuf_setlen(base, baselen); + return READ_TREE_RECURSIVE; + } - strbuf_addstr(base, path); + /* + * The path "{base}{path}/" is a sparse directory. Create the correct + * name for inserting the entry into the index. + */ + strbuf_setlen(base, base->len - 1); + } else { + strbuf_addstr(base, path); + } - ce = make_cache_entry(istate, mode, oid, base->buf, 0, 0); + ce = make_cache_entry(ctx->write, mode, oid, base->buf, 0, 0); ce->ce_flags |= CE_SKIP_WORKTREE | CE_EXTENDED; - set_index_entry(istate, istate->cache_nr++, ce); + set_index_entry(ctx->write, ctx->write->cache_nr++, ce); strbuf_setlen(base, len); return 0; } -void ensure_full_index(struct index_state *istate) +void expand_index(struct index_state *istate, struct pattern_list *pl) { int i; struct index_state *full; struct strbuf base = STRBUF_INIT; + const char *tr_region; + struct modify_index_context ctx; - if (!istate || !istate->sparse_index) + /* + * If the index is already full, then keep it full. We will convert + * it to a sparse index on write, if possible. + */ + if (!istate || istate->sparse_index == INDEX_EXPANDED) return; + /* + * If our index is sparse, but our new pattern set does not use + * cone mode patterns, then we need to expand the index before we + * continue. A NULL pattern set indicates a full expansion to a + * full index. + */ + if (pl && !pl->use_cone_patterns) { + pl = NULL; + } else { + /* + * We might contract file entries into sparse-directory + * entries, and for that we will need the cache tree to + * be recomputed. + */ + cache_tree_free(&istate->cache_tree); + + /* + * If there is a problem creating the cache tree, then we + * need to expand to a full index since we cannot satisfy + * the current request as a sparse index. + */ + if (cache_tree_update(istate, 0)) + pl = NULL; + } + if (!istate->repo) istate->repo = the_repository; - trace2_region_enter("index", "ensure_full_index", istate->repo); + /* + * A NULL pattern set indicates we are expanding a full index, so + * we use a special region name that indicates the full expansion. + * This is used by test cases, but also helps to differentiate the + * two cases. + */ + tr_region = pl ? "expand_index" : "ensure_full_index"; + trace2_region_enter("index", tr_region, istate->repo); /* initialize basics of new index */ full = xcalloc(1, sizeof(struct index_state)); memcpy(full, istate, sizeof(struct index_state)); + /* + * This slightly-misnamed 'full' index might still be sparse if we + * are only modifying the list of sparse directories. This hinges + * on whether we have a non-NULL pattern list. + */ + full->sparse_index = pl ? INDEX_PARTIALLY_SPARSE : INDEX_EXPANDED; + /* then change the necessary things */ - full->sparse_index = 0; full->cache_alloc = (3 * istate->cache_alloc) / 2; full->cache_nr = 0; ALLOC_ARRAY(full->cache, full->cache_alloc); + ctx.write = full; + ctx.pl = pl; + for (i = 0; i < istate->cache_nr; i++) { struct cache_entry *ce = istate->cache[i]; struct tree *tree; struct pathspec ps; + int dtype; if (!S_ISSPARSEDIR(ce->ce_mode)) { set_index_entry(full, full->cache_nr++, ce); continue; } + + /* We now have a sparse directory entry. Should we expand? */ + if (pl && + path_matches_pattern_list(ce->name, ce->ce_namelen, + NULL, &dtype, + pl, istate) == NOT_MATCHED) { + set_index_entry(full, full->cache_nr++, ce); + continue; + } + if (!(ce->ce_flags & CE_SKIP_WORKTREE)) warning(_("index entry is a directory, but not sparse (%08x)"), ce->ce_flags); @@ -297,7 +394,7 @@ void ensure_full_index(struct index_state *istate) strbuf_add(&base, ce->name, strlen(ce->name)); read_tree_at(istate->repo, tree, &base, &ps, - add_path_to_index, full); + add_path_to_index, &ctx); /* free directory entries. full entries are re-used */ discard_cache_entry(ce); @@ -306,7 +403,7 @@ void ensure_full_index(struct index_state *istate) /* Copy back into original index. */ memcpy(&istate->name_hash, &full->name_hash, sizeof(full->name_hash)); memcpy(&istate->dir_hash, &full->dir_hash, sizeof(full->dir_hash)); - istate->sparse_index = 0; + istate->sparse_index = pl ? INDEX_PARTIALLY_SPARSE : INDEX_EXPANDED; free(istate->cache); istate->cache = full->cache; istate->cache_nr = full->cache_nr; @@ -322,7 +419,12 @@ void ensure_full_index(struct index_state *istate) cache_tree_free(&istate->cache_tree); cache_tree_update(istate, 0); - trace2_region_leave("index", "ensure_full_index", istate->repo); + trace2_region_leave("index", tr_region, istate->repo); +} + +void ensure_full_index(struct index_state *istate) +{ + expand_index(istate, NULL); } void ensure_correct_sparsity(struct index_state *istate) diff --git a/sparse-index.h b/sparse-index.h index f57c65d972..59a92d819e 100644 --- a/sparse-index.h +++ b/sparse-index.h @@ -24,4 +24,17 @@ void expand_to_path(struct index_state *istate, struct repository; int set_sparse_index_config(struct repository *repo, int enable); +struct pattern_list; + +/** + * Scan the given index and compare its entries to the given pattern list. + * If the index is sparse and the pattern list uses cone mode patterns, + * then modify the index to contain the all of the file entries within that + * new pattern list. This expands sparse directories only as far as needed. + * + * If the pattern list is NULL or does not use cone mode patterns, then the + * index is expanded to a full index. + */ +void expand_index(struct index_state *istate, struct pattern_list *pl); + #endif diff --git a/t/perf/p2000-sparse-operations.sh b/t/perf/p2000-sparse-operations.sh index 76710cbef3..c181110a43 100755 --- a/t/perf/p2000-sparse-operations.sh +++ b/t/perf/p2000-sparse-operations.sh @@ -112,6 +112,7 @@ test_perf_on_all git add -A test_perf_on_all git add . test_perf_on_all git commit -a -m A test_perf_on_all git checkout -f - +test_perf_on_all "git sparse-checkout add f2/f3/f1 && git sparse-checkout set $SPARSE_CONE" test_perf_on_all git reset test_perf_on_all git reset --hard test_perf_on_all git reset -- does-not-exist diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index 6f778cf28c..f9f8c988bb 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -205,36 +205,68 @@ test_sparse_unstaged () { done } -test_expect_success 'sparse-index contents' ' - init_repos && - +# Usage: test_sprase_checkout_set "<c1> ... <cN>" "<s1> ... <sM>" +# Verifies that "git sparse-checkout set <c1> ... <cN>" succeeds and +# leaves the sparse index in a state where <s1> ... <sM> are sparse +# directories (and <c1> ... <cN> are not). +test_sparse_checkout_set () { + CONE_DIRS=$1 && + SPARSE_DIRS=$2 && + git -C sparse-index sparse-checkout set --skip-checks $CONE_DIRS && git -C sparse-index ls-files --sparse --stage >cache && - for dir in folder1 folder2 x + + # Check that the directories outside of the sparse-checkout cone + # have sparse directory entries. + for dir in $SPARSE_DIRS do TREE=$(git -C sparse-index rev-parse HEAD:$dir) && grep "040000 $TREE 0 $dir/" cache \ || return 1 done && - git -C sparse-index sparse-checkout set folder1 && - - git -C sparse-index ls-files --sparse --stage >cache && - for dir in deep folder2 x + # Check that the directories in the sparse-checkout cone + # are not sparse directory entries. + for dir in $CONE_DIRS do - TREE=$(git -C sparse-index rev-parse HEAD:$dir) && - grep "040000 $TREE 0 $dir/" cache \ + # Allow TREE to not exist because + # $dir does not exist at HEAD. + TREE=$(git -C sparse-index rev-parse HEAD:$dir) || + ! grep "040000 $TREE 0 $dir/" cache \ || return 1 - done && + done +} - git -C sparse-index sparse-checkout set deep/deeper1 && +test_expect_success 'sparse-index contents' ' + init_repos && - git -C sparse-index ls-files --sparse --stage >cache && - for dir in deep/deeper2 folder1 folder2 x - do - TREE=$(git -C sparse-index rev-parse HEAD:$dir) && - grep "040000 $TREE 0 $dir/" cache \ - || return 1 - done && + # Remove deep, add three other directories. + test_sparse_checkout_set \ + "folder1 folder2 x" \ + "before deep" && + + # Remove folder1, add deep + test_sparse_checkout_set \ + "deep folder2 x" \ + "before folder1" && + + # Replace deep with deep/deeper2 (dropping deep/deeper1) + # Add folder1 + test_sparse_checkout_set \ + "deep/deeper2 folder1 folder2 x" \ + "before deep/deeper1" && + + # Replace deep/deeper2 with deep/deeper1 + # Replace folder1 with folder1/0/0 + # Replace folder2 with non-existent folder2/2/3 + # Add non-existent "bogus" + test_sparse_checkout_set \ + "bogus deep/deeper1 folder1/0/0 folder2/2/3 x" \ + "before deep/deeper2 folder2/0" && + + # Drop down to only files at root + test_sparse_checkout_set \ + "" \ + "before deep folder1 folder2 x" && # Disabling the sparse-index replaces tree entries with full ones git -C sparse-index sparse-checkout init --no-sparse-index && @@ -1628,6 +1660,31 @@ test_expect_success 'ls-files' ' ensure_not_expanded ls-files --sparse ' +test_expect_success 'sparse index is not expanded: sparse-checkout' ' + init_repos && + + ensure_not_expanded sparse-checkout set deep/deeper2 && + ensure_not_expanded sparse-checkout set deep/deeper1 && + ensure_not_expanded sparse-checkout set deep && + ensure_not_expanded sparse-checkout add folder1 && + ensure_not_expanded sparse-checkout set deep/deeper1 && + ensure_not_expanded sparse-checkout set folder2 && + + # Demonstrate that the checks that "folder1/a" is a file + # do not cause a sparse-index expansion (since it is in the + # sparse-checkout cone). + echo >>sparse-index/folder2/a && + git -C sparse-index add folder2/a && + + ensure_not_expanded sparse-checkout add folder1 && + + # Skip checks here, since deep/deeper1 is inside a sparse directory + # that must be expanded to check whether `deep/deeper1` is a file + # or not. + ensure_not_expanded sparse-checkout set --skip-checks deep/deeper1 && + ensure_not_expanded sparse-checkout set +' + # NEEDSWORK: a sparse-checkout behaves differently from a full checkout # in this scenario, but it shouldn't. test_expect_success 'reset mixed and checkout orphan' ' diff --git a/unpack-trees.c b/unpack-trees.c index a1d0ff3a4d..7d73f62aee 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -19,6 +19,7 @@ #include "promisor-remote.h" #include "entry.h" #include "parallel-checkout.h" +#include "sparse-index.h" /* * Error messages expected by scripts out of plumbing commands such as @@ -2024,6 +2025,9 @@ enum update_sparsity_result update_sparsity(struct unpack_trees_options *o) goto skip_sparse_checkout; } + /* Expand sparse directories as needed */ + expand_index(o->src_index, o->pl); + /* Set NEW_SKIP_WORKTREE on existing entries. */ mark_all_ce_unused(o->src_index); mark_new_skip_worktree(o->pl, o->src_index, 0, |