From e27eab45c758bf194157b819fae4fd0fcce498fb Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 8 Sep 2021 01:42:26 +0000 Subject: sparse-index: silently return when not using cone-mode patterns While the sparse-index is only enabled when core.sparseCheckoutCone is also enabled, it is possible for the user to modify the sparse-checkout file manually in a way that does not match cone-mode patterns. In this case, we should refuse to convert an index into a sparse index, since the sparse_checkout_patterns will not be initialized with recursive and parent path hashsets. Also silently return if there are no cache entries, which is a simple case: there are no paths to make sparse! Signed-off-by: Derrick Stolee Reviewed-by: Elijah Newren Signed-off-by: Junio C Hamano --- sparse-index.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'sparse-index.c') diff --git a/sparse-index.c b/sparse-index.c index c6b4feec41..cd6e0d5f40 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -130,7 +130,7 @@ static int index_has_unmerged_entries(struct index_state *istate) int convert_to_sparse(struct index_state *istate) { int test_env; - if (istate->split_index || istate->sparse_index || + if (istate->split_index || istate->sparse_index || !istate->cache_nr || !core_apply_sparse_checkout || !core_sparse_checkout_cone) return 0; @@ -158,10 +158,16 @@ int convert_to_sparse(struct index_state *istate) return 0; } - if (!istate->sparse_checkout_patterns->use_cone_patterns) { - warning(_("attempting to use sparse-index without cone mode")); - return -1; - } + /* + * We need cone-mode patterns to use sparse-index. If a user edits + * their sparse-checkout file manually, then we can detect during + * parsing that they are not actually using cone-mode patterns and + * hence we need to abort this conversion _without error_. Warnings + * already exist in the pattern parsing to inform the user of their + * bad patterns. + */ + if (!istate->sparse_checkout_patterns->use_cone_patterns) + return 0; /* * NEEDSWORK: If we have unmerged entries, then stay full. -- cgit v1.2.3 From 5dc16756b220429ca91cbf263b936764f83cd4bb Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 8 Sep 2021 01:42:28 +0000 Subject: sparse-index: silently return when cache tree fails If cache_tree_update() returns a non-zero value, then it could not create the cache tree. This is likely due to a path having a merge conflict. Since we are already returning early, let's return silently to avoid making it seem like we failed to write the index at all. If we remove our dependence on the cache tree within convert_to_sparse(), then we could still recover from this scenario and have a sparse index. Signed-off-by: Derrick Stolee Reviewed-by: Elijah Newren Signed-off-by: Junio C Hamano --- sparse-index.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'sparse-index.c') diff --git a/sparse-index.c b/sparse-index.c index cd6e0d5f40..d9b0769595 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -178,10 +178,12 @@ int convert_to_sparse(struct index_state *istate) /* Clear and recompute the cache-tree */ cache_tree_free(&istate->cache_tree); - if (cache_tree_update(istate, 0)) { - warning(_("unable to update cache-tree, staying full")); - return -1; - } + /* + * Silently return if there is a problem with the cache tree update, + * which might just be due to a conflict state in some entry. + */ + if (cache_tree_update(istate, 0)) + return 0; remove_fsmonitor(istate); -- cgit v1.2.3 From 8a96b9d0a7c85a25e3550a14e5403eb95aca6d37 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 8 Sep 2021 01:42:29 +0000 Subject: sparse-index: use WRITE_TREE_MISSING_OK When updating the cache tree in convert_to_sparse(), the WRITE_TREE_MISSING_OK flag indicates that trees might be computed that do not already exist within the object database. This happens in cases such as 'git add' creating new trees that it wants to store in anticipation of a following 'git commit'. If this flag is not specified, then it might trigger a promisor fetch or a failure due to the object not existing locally. Use WRITE_TREE_MISSING_OK during convert_to_sparse() to avoid these possible reasons for the cache_tree_update() to fail. Signed-off-by: Derrick Stolee Reviewed-by: Elijah Newren Signed-off-by: Junio C Hamano --- sparse-index.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'sparse-index.c') diff --git a/sparse-index.c b/sparse-index.c index d9b0769595..880c5f7233 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -181,8 +181,11 @@ int convert_to_sparse(struct index_state *istate) /* * Silently return if there is a problem with the cache tree update, * which might just be due to a conflict state in some entry. + * + * This might create new tree objects, so be sure to use + * WRITE_TREE_MISSING_OK. */ - if (cache_tree_update(istate, 0)) + if (cache_tree_update(istate, WRITE_TREE_MISSING_OK)) return 0; remove_fsmonitor(istate); -- cgit v1.2.3 From 02155c8c005d0f8ee20a38245e9a065e8b5cc7dc Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 8 Sep 2021 01:42:30 +0000 Subject: sparse-checkout: create helper methods As we integrate the sparse index into more builtins, we occasionally need to check the sparse-checkout patterns to see if a path is within the sparse-checkout cone. Create some helper methods that help initialize the patterns and check for pattern matching to make this easier. The existing callers of commands like get_sparse_checkout_patterns() use a custom 'struct pattern_list' that is not necessarily the one in the 'struct index_state', so there are not many previous uses that could adopt these helpers. There are just two in builtin/add.c and sparse-index.c that can use path_in_sparse_checkout(). We add a path_in_cone_mode_sparse_checkout() as well that will only return false if the path is outside of the sparse-checkout definition _and_ the sparse-checkout patterns are in cone mode. Signed-off-by: Derrick Stolee Reviewed-by: Elijah Newren Signed-off-by: Junio C Hamano --- sparse-index.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) (limited to 'sparse-index.c') diff --git a/sparse-index.c b/sparse-index.c index 880c5f7233..23f7c3bd36 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -33,19 +33,14 @@ static int convert_to_sparse_rec(struct index_state *istate, { int i, can_convert = 1; int start_converted = num_converted; - enum pattern_match_result match; - int dtype = DT_UNKNOWN; struct strbuf child_path = STRBUF_INIT; - struct pattern_list *pl = istate->sparse_checkout_patterns; /* * Is the current path outside of the sparse cone? * Then check if the region can be replaced by a sparse * directory entry (everything is sparse and merged). */ - match = path_matches_pattern_list(ct_path, ct_pathlen, - NULL, &dtype, pl, istate); - if (match != NOT_MATCHED) + if (path_in_sparse_checkout(ct_path, istate)) can_convert = 0; for (i = start; can_convert && i < end; i++) { @@ -152,11 +147,8 @@ int convert_to_sparse(struct index_state *istate) if (!istate->repo->settings.sparse_index) return 0; - if (!istate->sparse_checkout_patterns) { - istate->sparse_checkout_patterns = xcalloc(1, sizeof(struct pattern_list)); - if (get_sparse_checkout_patterns(istate->sparse_checkout_patterns) < 0) - return 0; - } + if (init_sparse_checkout_patterns(istate)) + return 0; /* * We need cone-mode patterns to use sparse-index. If a user edits -- cgit v1.2.3 From ce7a9f014167ccf137eb09d5546eb1e8b550f11b Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 8 Sep 2021 01:42:32 +0000 Subject: sparse-index: add SPARSE_INDEX_MEMORY_ONLY flag The convert_to_sparse() method checks for the GIT_TEST_SPARSE_INDEX environment variable or the "index.sparse" config setting before converting the index to a sparse one. This is for ease of use since all current consumers are preparing to compress the index before writing it to disk. If these settings are not enabled, then convert_to_sparse() silently returns without doing anything. We will add a consumer in the next change that wants to use the sparse index as an in-memory data structure, regardless of whether the on-disk format should be sparse. To that end, create the SPARSE_INDEX_MEMORY_ONLY flag that will skip these config checks when enabled. All current consumers are modified to pass '0' in the new 'flags' parameter. Signed-off-by: Derrick Stolee Reviewed-by: Elijah Newren Signed-off-by: Junio C Hamano --- sparse-index.c | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) (limited to 'sparse-index.c') diff --git a/sparse-index.c b/sparse-index.c index 23f7c3bd36..0bc45f60ac 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -122,30 +122,37 @@ static int index_has_unmerged_entries(struct index_state *istate) return 0; } -int convert_to_sparse(struct index_state *istate) +int convert_to_sparse(struct index_state *istate, int flags) { int test_env; - if (istate->split_index || istate->sparse_index || !istate->cache_nr || + if (istate->sparse_index || !istate->cache_nr || !core_apply_sparse_checkout || !core_sparse_checkout_cone) return 0; if (!istate->repo) istate->repo = the_repository; - /* - * The GIT_TEST_SPARSE_INDEX environment variable triggers the - * index.sparse config variable to be on. - */ - test_env = git_env_bool("GIT_TEST_SPARSE_INDEX", -1); - if (test_env >= 0) - set_sparse_index_config(istate->repo, test_env); + if (!(flags & SPARSE_INDEX_MEMORY_ONLY)) { + /* + * The sparse index is not (yet) integrated with a split index. + */ + if (istate->split_index) + return 0; + /* + * The GIT_TEST_SPARSE_INDEX environment variable triggers the + * index.sparse config variable to be on. + */ + test_env = git_env_bool("GIT_TEST_SPARSE_INDEX", -1); + if (test_env >= 0) + set_sparse_index_config(istate->repo, test_env); - /* - * Only convert to sparse if index.sparse is set. - */ - prepare_repo_settings(istate->repo); - if (!istate->repo->settings.sparse_index) - return 0; + /* + * Only convert to sparse if index.sparse is set. + */ + prepare_repo_settings(istate->repo); + if (!istate->repo->settings.sparse_index) + return 0; + } if (init_sparse_checkout_patterns(istate)) return 0; -- cgit v1.2.3