From 4300f8442a276aa48d327b8371af9c2917bd3d5a Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 30 Mar 2021 13:10:48 +0000 Subject: sparse-index: implement ensure_full_index() We will mark an in-memory index_state as having sparse directory entries with the sparse_index bit. These currently cannot exist, but we will add a mechanism for collapsing a full index to a sparse one in a later change. That will happen at write time, so we must first allow parsing the format before writing it. Commands or methods that require a full index in order to operate can call ensure_full_index() to expand that index in-memory. This requires parsing trees using that index's repository. Sparse directory entries have a specific 'ce_mode' value. The macro S_ISSPARSEDIR(ce->ce_mode) can check if a cache_entry 'ce' has this type. This ce_mode is not possible with the existing index formats, so we don't also verify all properties of a sparse-directory entry, which are: 1. ce->ce_mode == 0040000 2. ce->flags & CE_SKIP_WORKTREE is true 3. ce->name[ce->namelen - 1] == '/' (ends in dir separator) 4. ce->oid references a tree object. These are all semi-enforced in ensure_full_index() to some extent. Any deviation will cause a warning at minimum or a failure in the worst case. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- read-cache.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'read-cache.c') diff --git a/read-cache.c b/read-cache.c index 1e9a50c6c7..dd3980c12b 100644 --- a/read-cache.c +++ b/read-cache.c @@ -101,6 +101,9 @@ static const char *alternate_index_output; static void set_index_entry(struct index_state *istate, int nr, struct cache_entry *ce) { + if (S_ISSPARSEDIR(ce->ce_mode)) + istate->sparse_index = 1; + istate->cache[nr] = ce; add_name_hash(istate, ce); } @@ -2273,6 +2276,12 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) trace2_data_intmax("index", the_repository, "read/cache_nr", istate->cache_nr); + if (!istate->repo) + istate->repo = the_repository; + prepare_repo_settings(istate->repo); + if (istate->repo->settings.command_requires_full_index) + ensure_full_index(istate); + return istate->cache_nr; unmap: -- cgit v1.3 From cd42415fb4c9680f308ebf8f9f85ba90ae6014be Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 30 Mar 2021 13:10:54 +0000 Subject: sparse-index: add 'sdir' index extension The index format does not currently allow for sparse directory entries. This violates some expectations that older versions of Git or third-party tools might not understand. We need an indicator inside the index file to warn these tools to not interact with a sparse index unless they are aware of sparse directory entries. Add a new _required_ index extension, 'sdir', that indicates that the index may contain sparse directory entries. This allows us to continue to use the differences in index formats 2, 3, and 4 before we create a new index version 5 in a later change. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- Documentation/technical/index-format.txt | 12 ++++++++++++ read-cache.c | 9 +++++++++ 2 files changed, 21 insertions(+) (limited to 'read-cache.c') diff --git a/Documentation/technical/index-format.txt b/Documentation/technical/index-format.txt index 3b74c05647..65da0daaa5 100644 --- a/Documentation/technical/index-format.txt +++ b/Documentation/technical/index-format.txt @@ -392,3 +392,15 @@ The remaining data of each directory block is grouped by type: in this block of entries. - 32-bit count of cache entries in this block + +== Sparse Directory Entries + + When using sparse-checkout in cone mode, some entire directories within + the index can be summarized by pointing to a tree object instead of the + entire expanded list of paths within that tree. An index containing such + entries is a "sparse index". Index format versions 4 and less were not + implemented with such entries in mind. Thus, for these versions, an + index containing sparse directory entries will include this extension + with signature { 's', 'd', 'i', 'r' }. Like the split-index extension, + tools should avoid interacting with a sparse index unless they understand + this extension. diff --git a/read-cache.c b/read-cache.c index dd3980c12b..b8f092d1b7 100644 --- a/read-cache.c +++ b/read-cache.c @@ -47,6 +47,7 @@ #define CACHE_EXT_FSMONITOR 0x46534D4E /* "FSMN" */ #define CACHE_EXT_ENDOFINDEXENTRIES 0x454F4945 /* "EOIE" */ #define CACHE_EXT_INDEXENTRYOFFSETTABLE 0x49454F54 /* "IEOT" */ +#define CACHE_EXT_SPARSE_DIRECTORIES 0x73646972 /* "sdir" */ /* changes that can be kept in $GIT_DIR/index (basically all extensions) */ #define EXTMASK (RESOLVE_UNDO_CHANGED | CACHE_TREE_CHANGED | \ @@ -1763,6 +1764,10 @@ static int read_index_extension(struct index_state *istate, case CACHE_EXT_INDEXENTRYOFFSETTABLE: /* already handled in do_read_index() */ break; + case CACHE_EXT_SPARSE_DIRECTORIES: + /* no content, only an indicator */ + istate->sparse_index = 1; + break; default: if (*ext < 'A' || 'Z' < *ext) return error(_("index uses %.4s extension, which we do not understand"), @@ -3020,6 +3025,10 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, if (err) return -1; } + if (istate->sparse_index) { + if (write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_SPARSE_DIRECTORIES, 0) < 0) + return -1; + } /* * CACHE_EXT_ENDOFINDEXENTRIES must be written as the last entry before the SHA1 -- cgit v1.3 From 6e773527b6b03976cefbb0f9571bd40dd5995e6c Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 30 Mar 2021 13:10:55 +0000 Subject: sparse-index: convert from full to sparse If we have a full index, then we can convert it to a sparse index by replacing directories outside of the sparse cone with sparse directory entries. The convert_to_sparse() method does this, when the situation is appropriate. For now, we avoid converting the index to a sparse index if: 1. the index is split. 2. the index is already sparse. 3. sparse-checkout is disabled. 4. sparse-checkout does not use cone mode. Finally, we currently limit the conversion to when the GIT_TEST_SPARSE_INDEX environment variable is enabled. A mode using Git config will be added in a later change. The trickiest thing about this conversion is that we might not be able to mark a directory as a sparse directory just because it is outside the sparse cone. There might be unmerged files within that directory, so we need to look for those. Also, if there is some strange reason why a file is not marked with CE_SKIP_WORKTREE, then we should give up on converting that directory. There is still hope that some of its subdirectories might be able to convert to sparse, so we keep looking deeper. The conversion process is assisted by the cache-tree extension. This is calculated from the full index if it does not already exist. We then abandon the cache-tree as it no longer applies to the newly-sparse index. Thus, this cache-tree will be recalculated in every sparse-full-sparse round-trip until we integrate the cache-tree extension with the sparse index. Some Git commands use the index after writing it. For example, 'git add' will update the index, then write it to disk, then read its entries to report information. To keep the in-memory index in a full state after writing, we re-expand it to a full one after the write. This is wasteful for commands that only write the index and do not read from it again, but that is only the case until we make those commands "sparse aware." We can compare the behavior of the sparse-index in t1092-sparse-checkout-compability.sh by using GIT_TEST_SPARSE_INDEX=1 when operating on the 'sparse-index' repo. We can also compare the two sparse repos directly, such as comparing their indexes (when expanded to full in the case of the 'sparse-index' repo). We also verify that the index is actually populated with sparse directory entries. The 'checkout and reset (mixed)' test is marked for failure when comparing a sparse repo to a full repo, but we can compare the two sparse-checkout cases directly to ensure that we are not changing the behavior when using a sparse index. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- cache-tree.c | 3 + cache.h | 2 + read-cache.c | 26 +++++- sparse-index.c | 139 +++++++++++++++++++++++++++++++ sparse-index.h | 1 + t/t1092-sparse-checkout-compatibility.sh | 61 +++++++++++++- 6 files changed, 228 insertions(+), 4 deletions(-) (limited to 'read-cache.c') diff --git a/cache-tree.c b/cache-tree.c index 2fb483d3c0..5f07a39e50 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -6,6 +6,7 @@ #include "object-store.h" #include "replace-object.h" #include "promisor-remote.h" +#include "sparse-index.h" #ifndef DEBUG_CACHE_TREE #define DEBUG_CACHE_TREE 0 @@ -442,6 +443,8 @@ int cache_tree_update(struct index_state *istate, int flags) if (i) return i; + ensure_full_index(istate); + if (!istate->cache_tree) istate->cache_tree = cache_tree(); diff --git a/cache.h b/cache.h index 8c4464420d..74b43aaa2b 100644 --- a/cache.h +++ b/cache.h @@ -251,6 +251,8 @@ static inline unsigned int create_ce_mode(unsigned int mode) { if (S_ISLNK(mode)) return S_IFLNK; + if (S_ISSPARSEDIR(mode)) + return S_IFDIR; if (S_ISDIR(mode) || S_ISGITLINK(mode)) return S_IFGITLINK; return S_IFREG | ce_permissions(mode); diff --git a/read-cache.c b/read-cache.c index b8f092d1b7..2410e6e0df 100644 --- a/read-cache.c +++ b/read-cache.c @@ -25,6 +25,7 @@ #include "fsmonitor.h" #include "thread-utils.h" #include "progress.h" +#include "sparse-index.h" /* Mask for the name length in ce_flags in the on-disk index */ @@ -1003,8 +1004,14 @@ inside: c = *path++; if ((c == '.' && !verify_dotfile(path, mode)) || - is_dir_sep(c) || c == '\0') + is_dir_sep(c)) return 0; + /* + * allow terminating directory separators for + * sparse directory entries. + */ + if (c == '\0') + return S_ISDIR(mode); } else if (c == '\\' && protect_ntfs) { if (is_ntfs_dotgit(path)) return 0; @@ -3088,6 +3095,14 @@ static int do_write_locked_index(struct index_state *istate, struct lock_file *l unsigned flags) { int ret; + int was_full = !istate->sparse_index; + + ret = convert_to_sparse(istate); + + if (ret) { + warning(_("failed to convert to a sparse-index")); + return ret; + } /* * TODO trace2: replace "the_repository" with the actual repo instance @@ -3099,6 +3114,9 @@ static int do_write_locked_index(struct index_state *istate, struct lock_file *l trace2_region_leave_printf("index", "do_write_index", the_repository, "%s", get_lock_file_path(lock)); + if (was_full) + ensure_full_index(istate); + if (ret) return ret; if (flags & COMMIT_LOCK) @@ -3189,9 +3207,10 @@ static int write_shared_index(struct index_state *istate, struct tempfile **temp) { struct split_index *si = istate->split_index; - int ret; + int ret, was_full = !istate->sparse_index; move_cache_to_base_index(istate); + convert_to_sparse(istate); trace2_region_enter_printf("index", "shared/do_write_index", the_repository, "%s", get_tempfile_path(*temp)); @@ -3199,6 +3218,9 @@ static int write_shared_index(struct index_state *istate, trace2_region_leave_printf("index", "shared/do_write_index", the_repository, "%s", get_tempfile_path(*temp)); + if (was_full) + ensure_full_index(istate); + if (ret) return ret; ret = adjust_shared_perm(get_tempfile_path(*temp)); diff --git a/sparse-index.c b/sparse-index.c index 7095378a1b..619ff7c2e2 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -4,6 +4,145 @@ #include "tree.h" #include "pathspec.h" #include "trace2.h" +#include "cache-tree.h" +#include "config.h" +#include "dir.h" +#include "fsmonitor.h" + +static struct cache_entry *construct_sparse_dir_entry( + struct index_state *istate, + const char *sparse_dir, + struct cache_tree *tree) +{ + struct cache_entry *de; + + de = make_cache_entry(istate, S_IFDIR, &tree->oid, sparse_dir, 0, 0); + + de->ce_flags |= CE_SKIP_WORKTREE; + return de; +} + +/* + * Returns the number of entries "inserted" into the index. + */ +static int convert_to_sparse_rec(struct index_state *istate, + int num_converted, + int start, int end, + const char *ct_path, size_t ct_pathlen, + struct cache_tree *ct) +{ + int i, can_convert = 1; + int start_converted = num_converted; + enum pattern_match_result match; + int dtype; + struct strbuf child_path = STRBUF_INIT; + struct pattern_list *pl = istate->sparse_checkout_patterns; + + /* + * Is the current path outside of the sparse cone? + * Then check if the region can be replaced by a sparse + * directory entry (everything is sparse and merged). + */ + match = path_matches_pattern_list(ct_path, ct_pathlen, + NULL, &dtype, pl, istate); + if (match != NOT_MATCHED) + can_convert = 0; + + for (i = start; can_convert && i < end; i++) { + struct cache_entry *ce = istate->cache[i]; + + if (ce_stage(ce) || + !(ce->ce_flags & CE_SKIP_WORKTREE)) + can_convert = 0; + } + + if (can_convert) { + struct cache_entry *se; + se = construct_sparse_dir_entry(istate, ct_path, ct); + + istate->cache[num_converted++] = se; + return 1; + } + + for (i = start; i < end; ) { + int count, span, pos = -1; + const char *base, *slash; + struct cache_entry *ce = istate->cache[i]; + + /* + * Detect if this is a normal entry outside of any subtree + * entry. + */ + base = ce->name + ct_pathlen; + slash = strchr(base, '/'); + + if (slash) + pos = cache_tree_subtree_pos(ct, base, slash - base); + + if (pos < 0) { + istate->cache[num_converted++] = ce; + i++; + continue; + } + + strbuf_setlen(&child_path, 0); + strbuf_add(&child_path, ce->name, slash - ce->name + 1); + + span = ct->down[pos]->cache_tree->entry_count; + count = convert_to_sparse_rec(istate, + num_converted, i, i + span, + child_path.buf, child_path.len, + ct->down[pos]->cache_tree); + num_converted += count; + i += span; + } + + strbuf_release(&child_path); + return num_converted - start_converted; +} + +int convert_to_sparse(struct index_state *istate) +{ + if (istate->split_index || istate->sparse_index || + !core_apply_sparse_checkout || !core_sparse_checkout_cone) + return 0; + + /* + * For now, only create a sparse index with the + * GIT_TEST_SPARSE_INDEX environment variable. We will relax + * this once we have a proper way to opt-in (and later still, + * opt-out). + */ + if (!git_env_bool("GIT_TEST_SPARSE_INDEX", 0)) + return 0; + + if (!istate->sparse_checkout_patterns) { + istate->sparse_checkout_patterns = xcalloc(1, sizeof(struct pattern_list)); + if (get_sparse_checkout_patterns(istate->sparse_checkout_patterns) < 0) + return 0; + } + + if (!istate->sparse_checkout_patterns->use_cone_patterns) { + warning(_("attempting to use sparse-index without cone mode")); + return -1; + } + + if (cache_tree_update(istate, 0)) { + warning(_("unable to update cache-tree, staying full")); + return -1; + } + + remove_fsmonitor(istate); + + trace2_region_enter("index", "convert_to_sparse", istate->repo); + istate->cache_nr = convert_to_sparse_rec(istate, + 0, 0, istate->cache_nr, + "", 0, istate->cache_tree); + istate->drop_cache_tree = 1; + istate->sparse_index = 1; + trace2_region_leave("index", "convert_to_sparse", istate->repo); + return 0; +} static void set_index_entry(struct index_state *istate, int nr, struct cache_entry *ce) { diff --git a/sparse-index.h b/sparse-index.h index 09a20d036c..64380e121d 100644 --- a/sparse-index.h +++ b/sparse-index.h @@ -3,5 +3,6 @@ struct index_state; void ensure_full_index(struct index_state *istate); +int convert_to_sparse(struct index_state *istate); #endif diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index a1aea141c6..1e888d1951 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -2,6 +2,11 @@ test_description='compare full workdir to sparse workdir' +# The verify_cache_tree() check is not sparse-aware (yet). +# So, disable the check until that integration is complete. +GIT_TEST_CHECK_CACHE_TREE=0 +GIT_TEST_SPLIT_INDEX=0 + . ./test-lib.sh test_expect_success 'setup' ' @@ -121,7 +126,9 @@ run_on_all () { test_all_match () { run_on_all "$@" && test_cmp full-checkout-out sparse-checkout-out && - test_cmp full-checkout-err sparse-checkout-err + test_cmp full-checkout-out sparse-index-out && + test_cmp full-checkout-err sparse-checkout-err && + test_cmp full-checkout-err sparse-index-err } test_sparse_match () { @@ -130,6 +137,38 @@ test_sparse_match () { test_cmp sparse-checkout-err sparse-index-err } +test_expect_success 'sparse-index contents' ' + init_repos && + + test-tool -C sparse-index read-cache --table >cache && + for dir in folder1 folder2 x + do + TREE=$(git -C sparse-index rev-parse HEAD:$dir) && + grep "040000 tree $TREE $dir/" cache \ + || return 1 + done && + + GIT_TEST_SPARSE_INDEX=1 git -C sparse-index sparse-checkout set folder1 && + + test-tool -C sparse-index read-cache --table >cache && + for dir in deep folder2 x + do + TREE=$(git -C sparse-index rev-parse HEAD:$dir) && + grep "040000 tree $TREE $dir/" cache \ + || return 1 + done && + + GIT_TEST_SPARSE_INDEX=1 git -C sparse-index sparse-checkout set deep/deeper1 && + + test-tool -C sparse-index read-cache --table >cache && + for dir in deep/deeper2 folder1 folder2 x + do + TREE=$(git -C sparse-index rev-parse HEAD:$dir) && + grep "040000 tree $TREE $dir/" cache \ + || return 1 + done +' + test_expect_success 'expanded in-memory index matches full index' ' init_repos && test_sparse_match test-tool read-cache --expand --table @@ -137,6 +176,7 @@ test_expect_success 'expanded in-memory index matches full index' ' test_expect_success 'status with options' ' init_repos && + test_sparse_match ls && test_all_match git status --porcelain=v2 && test_all_match git status --porcelain=v2 -z -u && test_all_match git status --porcelain=v2 -uno && @@ -273,6 +313,17 @@ test_expect_failure 'checkout and reset (mixed)' ' test_all_match git reset update-folder2 ' +# Ensure that sparse-index behaves identically to +# sparse-checkout with a full index. +test_expect_success 'checkout and reset (mixed) [sparse]' ' + init_repos && + + test_sparse_match git checkout -b reset-test update-deep && + test_sparse_match git reset deepest && + test_sparse_match git reset update-folder1 && + test_sparse_match git reset update-folder2 +' + test_expect_success 'merge' ' init_repos && @@ -309,14 +360,20 @@ test_expect_success 'clean' ' test_all_match git status --porcelain=v2 && test_all_match git clean -f && test_all_match git status --porcelain=v2 && + test_sparse_match ls && + test_sparse_match ls folder1 && test_all_match git clean -xf && test_all_match git status --porcelain=v2 && + test_sparse_match ls && + test_sparse_match ls folder1 && test_all_match git clean -xdf && test_all_match git status --porcelain=v2 && + test_sparse_match ls && + test_sparse_match ls folder1 && - test_path_is_dir sparse-checkout/folder1 + test_sparse_match test_path_is_dir folder1 ' test_done -- cgit v1.3 From 847a9e5d4f876646e128c89f0818b1a8ce792509 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 1 Apr 2021 01:49:39 +0000 Subject: *: remove 'const' qualifier for struct index_state Several methods specify that they take a 'struct index_state' pointer with the 'const' qualifier because they intend to only query the data, not change it. However, we will be introducing a step very low in the method stack that might modify a sparse-index to become a full index in the case that our queries venture inside a sparse-directory entry. This change only removes the 'const' qualifiers that are necessary for the following change which will actually modify the implementation of index_name_stage_pos(). Signed-off-by: Derrick Stolee Reviewed-by: Elijah Newren Signed-off-by: Junio C Hamano --- attr.c | 14 +++++++------- attr.h | 4 ++-- builtin/ls-files.c | 10 +++++----- cache.h | 6 +++--- convert.c | 26 +++++++++++++------------- convert.h | 20 ++++++++++---------- dir.c | 12 ++++++------ dir.h | 8 ++++---- merge-recursive.c | 2 +- pathspec.c | 6 +++--- pathspec.h | 6 +++--- read-cache.c | 10 +++++----- submodule.c | 6 +++--- submodule.h | 6 +++--- 14 files changed, 68 insertions(+), 68 deletions(-) (limited to 'read-cache.c') diff --git a/attr.c b/attr.c index 4ef85d668b..8de553293e 100644 --- a/attr.c +++ b/attr.c @@ -718,7 +718,7 @@ static struct attr_stack *read_attr_from_file(const char *path, int macro_ok) return res; } -static struct attr_stack *read_attr_from_index(const struct index_state *istate, +static struct attr_stack *read_attr_from_index(struct index_state *istate, const char *path, int macro_ok) { @@ -748,7 +748,7 @@ static struct attr_stack *read_attr_from_index(const struct index_state *istate, return res; } -static struct attr_stack *read_attr(const struct index_state *istate, +static struct attr_stack *read_attr(struct index_state *istate, const char *path, int macro_ok) { struct attr_stack *res = NULL; @@ -840,7 +840,7 @@ static void push_stack(struct attr_stack **attr_stack_p, } } -static void bootstrap_attr_stack(const struct index_state *istate, +static void bootstrap_attr_stack(struct index_state *istate, struct attr_stack **stack) { struct attr_stack *e; @@ -878,7 +878,7 @@ static void bootstrap_attr_stack(const struct index_state *istate, push_stack(stack, e, NULL, 0); } -static void prepare_attr_stack(const struct index_state *istate, +static void prepare_attr_stack(struct index_state *istate, const char *path, int dirlen, struct attr_stack **stack) { @@ -1078,7 +1078,7 @@ static void determine_macros(struct all_attrs_item *all_attrs, * If check->check_nr is non-zero, only attributes in check[] are collected. * Otherwise all attributes are collected. */ -static void collect_some_attrs(const struct index_state *istate, +static void collect_some_attrs(struct index_state *istate, const char *path, struct attr_check *check) { @@ -1107,7 +1107,7 @@ static void collect_some_attrs(const struct index_state *istate, fill(path, pathlen, basename_offset, check->stack, check->all_attrs, rem); } -void git_check_attr(const struct index_state *istate, +void git_check_attr(struct index_state *istate, const char *path, struct attr_check *check) { @@ -1124,7 +1124,7 @@ void git_check_attr(const struct index_state *istate, } } -void git_all_attrs(const struct index_state *istate, +void git_all_attrs(struct index_state *istate, const char *path, struct attr_check *check) { int i; diff --git a/attr.h b/attr.h index 404548f028..3732505eda 100644 --- a/attr.h +++ b/attr.h @@ -190,14 +190,14 @@ void attr_check_free(struct attr_check *check); */ const char *git_attr_name(const struct git_attr *); -void git_check_attr(const struct index_state *istate, +void git_check_attr(struct index_state *istate, const char *path, struct attr_check *check); /* * Retrieve all attributes that apply to the specified path. * check holds the attributes and their values. */ -void git_all_attrs(const struct index_state *istate, +void git_all_attrs(struct index_state *istate, const char *path, struct attr_check *check); enum git_attr_direction { diff --git a/builtin/ls-files.c b/builtin/ls-files.c index 60a2913a01..4f9ed1fb29 100644 --- a/builtin/ls-files.c +++ b/builtin/ls-files.c @@ -57,7 +57,7 @@ static const char *tag_modified = ""; static const char *tag_skip_worktree = ""; static const char *tag_resolve_undo = ""; -static void write_eolinfo(const struct index_state *istate, +static void write_eolinfo(struct index_state *istate, const struct cache_entry *ce, const char *path) { if (show_eol) { @@ -122,7 +122,7 @@ static void print_debug(const struct cache_entry *ce) } } -static void show_dir_entry(const struct index_state *istate, +static void show_dir_entry(struct index_state *istate, const char *tag, struct dir_entry *ent) { int len = max_prefix_len; @@ -139,7 +139,7 @@ static void show_dir_entry(const struct index_state *istate, write_name(ent->name); } -static void show_other_files(const struct index_state *istate, +static void show_other_files(struct index_state *istate, const struct dir_struct *dir) { int i; @@ -152,7 +152,7 @@ static void show_other_files(const struct index_state *istate, } } -static void show_killed_files(const struct index_state *istate, +static void show_killed_files(struct index_state *istate, const struct dir_struct *dir) { int i; @@ -254,7 +254,7 @@ static void show_ce(struct repository *repo, struct dir_struct *dir, } } -static void show_ru_info(const struct index_state *istate) +static void show_ru_info(struct index_state *istate) { struct string_list_item *item; diff --git a/cache.h b/cache.h index 8aede373ae..5006278c13 100644 --- a/cache.h +++ b/cache.h @@ -800,7 +800,7 @@ struct cache_entry *index_file_exists(struct index_state *istate, const char *na * index_name_pos(&index, "f", 1) -> -3 * index_name_pos(&index, "g", 1) -> -5 */ -int index_name_pos(const struct index_state *, const char *name, int namelen); +int index_name_pos(struct index_state *, const char *name, int namelen); /* * Some functions return the negative complement of an insert position when a @@ -850,8 +850,8 @@ int add_file_to_index(struct index_state *, const char *path, int flags); int chmod_index_entry(struct index_state *, struct cache_entry *ce, char flip); int ce_same_name(const struct cache_entry *a, const struct cache_entry *b); void set_object_name_for_intent_to_add_entry(struct cache_entry *ce); -int index_name_is_other(const struct index_state *, const char *, int); -void *read_blob_data_from_index(const struct index_state *, const char *, unsigned long *); +int index_name_is_other(struct index_state *, const char *, int); +void *read_blob_data_from_index(struct index_state *, const char *, unsigned long *); /* do stat comparison even if CE_VALID is true */ #define CE_MATCH_IGNORE_VALID 01 diff --git a/convert.c b/convert.c index ee360c2f07..20b19abffa 100644 --- a/convert.c +++ b/convert.c @@ -138,7 +138,7 @@ static const char *gather_convert_stats_ascii(const char *data, unsigned long si } } -const char *get_cached_convert_stats_ascii(const struct index_state *istate, +const char *get_cached_convert_stats_ascii(struct index_state *istate, const char *path) { const char *ret; @@ -222,7 +222,7 @@ static void check_global_conv_flags_eol(const char *path, } } -static int has_crlf_in_index(const struct index_state *istate, const char *path) +static int has_crlf_in_index(struct index_state *istate, const char *path) { unsigned long sz; void *data; @@ -496,7 +496,7 @@ static int encode_to_worktree(const char *path, const char *src, size_t src_len, return 1; } -static int crlf_to_git(const struct index_state *istate, +static int crlf_to_git(struct index_state *istate, const char *path, const char *src, size_t len, struct strbuf *buf, enum crlf_action crlf_action, int conv_flags) @@ -1307,7 +1307,7 @@ struct conv_attrs { static struct attr_check *check; -static void convert_attrs(const struct index_state *istate, +static void convert_attrs(struct index_state *istate, struct conv_attrs *ca, const char *path) { struct attr_check_item *ccheck = NULL; @@ -1369,7 +1369,7 @@ void reset_parsed_attributes(void) user_convert_tail = NULL; } -int would_convert_to_git_filter_fd(const struct index_state *istate, const char *path) +int would_convert_to_git_filter_fd(struct index_state *istate, const char *path) { struct conv_attrs ca; @@ -1388,7 +1388,7 @@ int would_convert_to_git_filter_fd(const struct index_state *istate, const char return apply_filter(path, NULL, 0, -1, NULL, ca.drv, CAP_CLEAN, NULL, NULL); } -const char *get_convert_attr_ascii(const struct index_state *istate, const char *path) +const char *get_convert_attr_ascii(struct index_state *istate, const char *path) { struct conv_attrs ca; @@ -1414,7 +1414,7 @@ const char *get_convert_attr_ascii(const struct index_state *istate, const char return ""; } -int convert_to_git(const struct index_state *istate, +int convert_to_git(struct index_state *istate, const char *path, const char *src, size_t len, struct strbuf *dst, int conv_flags) { @@ -1448,7 +1448,7 @@ int convert_to_git(const struct index_state *istate, return ret | ident_to_git(src, len, dst, ca.ident); } -void convert_to_git_filter_fd(const struct index_state *istate, +void convert_to_git_filter_fd(struct index_state *istate, const char *path, int fd, struct strbuf *dst, int conv_flags) { @@ -1466,7 +1466,7 @@ void convert_to_git_filter_fd(const struct index_state *istate, ident_to_git(dst->buf, dst->len, dst, ca.ident); } -static int convert_to_working_tree_internal(const struct index_state *istate, +static int convert_to_working_tree_internal(struct index_state *istate, const char *path, const char *src, size_t len, struct strbuf *dst, int normalizing, @@ -1510,7 +1510,7 @@ static int convert_to_working_tree_internal(const struct index_state *istate, return ret | ret_filter; } -int async_convert_to_working_tree(const struct index_state *istate, +int async_convert_to_working_tree(struct index_state *istate, const char *path, const char *src, size_t len, struct strbuf *dst, const struct checkout_metadata *meta, @@ -1519,7 +1519,7 @@ int async_convert_to_working_tree(const struct index_state *istate, return convert_to_working_tree_internal(istate, path, src, len, dst, 0, meta, dco); } -int convert_to_working_tree(const struct index_state *istate, +int convert_to_working_tree(struct index_state *istate, const char *path, const char *src, size_t len, struct strbuf *dst, const struct checkout_metadata *meta) @@ -1527,7 +1527,7 @@ int convert_to_working_tree(const struct index_state *istate, return convert_to_working_tree_internal(istate, path, src, len, dst, 0, meta, NULL); } -int renormalize_buffer(const struct index_state *istate, const char *path, +int renormalize_buffer(struct index_state *istate, const char *path, const char *src, size_t len, struct strbuf *dst) { int ret = convert_to_working_tree_internal(istate, path, src, len, dst, 1, NULL, NULL); @@ -1964,7 +1964,7 @@ static struct stream_filter *ident_filter(const struct object_id *oid) * Note that you would be crazy to set CRLF, smudge/clean or ident to a * large binary blob you would want us not to slurp into the memory! */ -struct stream_filter *get_stream_filter(const struct index_state *istate, +struct stream_filter *get_stream_filter(struct index_state *istate, const char *path, const struct object_id *oid) { diff --git a/convert.h b/convert.h index e29d1026a6..0f7c8a1f04 100644 --- a/convert.h +++ b/convert.h @@ -65,41 +65,41 @@ struct checkout_metadata { extern enum eol core_eol; extern char *check_roundtrip_encoding; -const char *get_cached_convert_stats_ascii(const struct index_state *istate, +const char *get_cached_convert_stats_ascii(struct index_state *istate, const char *path); const char *get_wt_convert_stats_ascii(const char *path); -const char *get_convert_attr_ascii(const struct index_state *istate, +const char *get_convert_attr_ascii(struct index_state *istate, const char *path); /* returns 1 if *dst was used */ -int convert_to_git(const struct index_state *istate, +int convert_to_git(struct index_state *istate, const char *path, const char *src, size_t len, struct strbuf *dst, int conv_flags); -int convert_to_working_tree(const struct index_state *istate, +int convert_to_working_tree(struct index_state *istate, const char *path, const char *src, size_t len, struct strbuf *dst, const struct checkout_metadata *meta); -int async_convert_to_working_tree(const struct index_state *istate, +int async_convert_to_working_tree(struct index_state *istate, const char *path, const char *src, size_t len, struct strbuf *dst, const struct checkout_metadata *meta, void *dco); int async_query_available_blobs(const char *cmd, struct string_list *available_paths); -int renormalize_buffer(const struct index_state *istate, +int renormalize_buffer(struct index_state *istate, const char *path, const char *src, size_t len, struct strbuf *dst); -static inline int would_convert_to_git(const struct index_state *istate, +static inline int would_convert_to_git(struct index_state *istate, const char *path) { return convert_to_git(istate, path, NULL, 0, NULL, 0); } /* Precondition: would_convert_to_git_filter_fd(path) == true */ -void convert_to_git_filter_fd(const struct index_state *istate, +void convert_to_git_filter_fd(struct index_state *istate, const char *path, int fd, struct strbuf *dst, int conv_flags); -int would_convert_to_git_filter_fd(const struct index_state *istate, +int would_convert_to_git_filter_fd(struct index_state *istate, const char *path); /* @@ -133,7 +133,7 @@ void reset_parsed_attributes(void); struct stream_filter; /* opaque */ -struct stream_filter *get_stream_filter(const struct index_state *istate, +struct stream_filter *get_stream_filter(struct index_state *istate, const char *path, const struct object_id *); void free_stream_filter(struct stream_filter *); diff --git a/dir.c b/dir.c index fd8aa7c40f..5b00dfb5b1 100644 --- a/dir.c +++ b/dir.c @@ -306,7 +306,7 @@ static int do_read_blob(const struct object_id *oid, struct oid_stat *oid_stat, * [1] Only if DO_MATCH_DIRECTORY is passed; otherwise, this is NOT a match. * [2] Only if DO_MATCH_LEADING_PATHSPEC is passed; otherwise, not a match. */ -static int match_pathspec_item(const struct index_state *istate, +static int match_pathspec_item(struct index_state *istate, const struct pathspec_item *item, int prefix, const char *name, int namelen, unsigned flags) { @@ -429,7 +429,7 @@ static int match_pathspec_item(const struct index_state *istate, * pathspec did not match any names, which could indicate that the * user mistyped the nth pathspec. */ -static int do_match_pathspec(const struct index_state *istate, +static int do_match_pathspec(struct index_state *istate, const struct pathspec *ps, const char *name, int namelen, int prefix, char *seen, @@ -500,7 +500,7 @@ static int do_match_pathspec(const struct index_state *istate, return retval; } -static int match_pathspec_with_flags(const struct index_state *istate, +static int match_pathspec_with_flags(struct index_state *istate, const struct pathspec *ps, const char *name, int namelen, int prefix, char *seen, unsigned flags) @@ -516,7 +516,7 @@ static int match_pathspec_with_flags(const struct index_state *istate, return negative ? 0 : positive; } -int match_pathspec(const struct index_state *istate, +int match_pathspec(struct index_state *istate, const struct pathspec *ps, const char *name, int namelen, int prefix, char *seen, int is_dir) @@ -529,7 +529,7 @@ int match_pathspec(const struct index_state *istate, /** * Check if a submodule is a superset of the pathspec */ -int submodule_path_match(const struct index_state *istate, +int submodule_path_match(struct index_state *istate, const struct pathspec *ps, const char *submodule_name, char *seen) @@ -892,7 +892,7 @@ void add_pattern(const char *string, const char *base, add_pattern_to_hashsets(pl, pattern); } -static int read_skip_worktree_file_from_index(const struct index_state *istate, +static int read_skip_worktree_file_from_index(struct index_state *istate, const char *path, size_t *size_out, char **data_out, struct oid_stat *oid_stat) diff --git a/dir.h b/dir.h index facfae4740..51cb0e2172 100644 --- a/dir.h +++ b/dir.h @@ -354,7 +354,7 @@ int count_slashes(const char *s); int simple_length(const char *match); int no_wildcard(const char *string); char *common_prefix(const struct pathspec *pathspec); -int match_pathspec(const struct index_state *istate, +int match_pathspec(struct index_state *istate, const struct pathspec *pathspec, const char *name, int namelen, int prefix, char *seen, int is_dir); @@ -492,12 +492,12 @@ int git_fnmatch(const struct pathspec_item *item, const char *pattern, const char *string, int prefix); -int submodule_path_match(const struct index_state *istate, +int submodule_path_match(struct index_state *istate, const struct pathspec *ps, const char *submodule_name, char *seen); -static inline int ce_path_match(const struct index_state *istate, +static inline int ce_path_match(struct index_state *istate, const struct cache_entry *ce, const struct pathspec *pathspec, char *seen) @@ -506,7 +506,7 @@ static inline int ce_path_match(const struct index_state *istate, S_ISDIR(ce->ce_mode) || S_ISGITLINK(ce->ce_mode)); } -static inline int dir_path_match(const struct index_state *istate, +static inline int dir_path_match(struct index_state *istate, const struct dir_entry *ent, const struct pathspec *pathspec, int prefix, char *seen) diff --git a/merge-recursive.c b/merge-recursive.c index 3d9207455b..b8de7a704e 100644 --- a/merge-recursive.c +++ b/merge-recursive.c @@ -2987,7 +2987,7 @@ static int blob_unchanged(struct merge_options *opt, struct strbuf obuf = STRBUF_INIT; struct strbuf abuf = STRBUF_INIT; int ret = 0; /* assume changed for safety */ - const struct index_state *idx = opt->repo->index; + struct index_state *idx = opt->repo->index; if (a->mode != o->mode) return 0; diff --git a/pathspec.c b/pathspec.c index 7a229d8d22..b6e333965c 100644 --- a/pathspec.c +++ b/pathspec.c @@ -20,7 +20,7 @@ * to use find_pathspecs_matching_against_index() instead. */ void add_pathspec_matches_against_index(const struct pathspec *pathspec, - const struct index_state *istate, + struct index_state *istate, char *seen) { int num_unmatched = 0, i; @@ -51,7 +51,7 @@ void add_pathspec_matches_against_index(const struct pathspec *pathspec, * given pathspecs achieves against all items in the index. */ char *find_pathspecs_matching_against_index(const struct pathspec *pathspec, - const struct index_state *istate) + struct index_state *istate) { char *seen = xcalloc(pathspec->nr, 1); add_pathspec_matches_against_index(pathspec, istate, seen); @@ -702,7 +702,7 @@ void clear_pathspec(struct pathspec *pathspec) pathspec->nr = 0; } -int match_pathspec_attrs(const struct index_state *istate, +int match_pathspec_attrs(struct index_state *istate, const char *name, int namelen, const struct pathspec_item *item) { diff --git a/pathspec.h b/pathspec.h index 454ce364fa..2ccc8080b6 100644 --- a/pathspec.h +++ b/pathspec.h @@ -150,11 +150,11 @@ static inline int ps_strcmp(const struct pathspec_item *item, } void add_pathspec_matches_against_index(const struct pathspec *pathspec, - const struct index_state *istate, + struct index_state *istate, char *seen); char *find_pathspecs_matching_against_index(const struct pathspec *pathspec, - const struct index_state *istate); -int match_pathspec_attrs(const struct index_state *istate, + struct index_state *istate); +int match_pathspec_attrs(struct index_state *istate, const char *name, int namelen, const struct pathspec_item *item); diff --git a/read-cache.c b/read-cache.c index 2410e6e0df..3ad9457809 100644 --- a/read-cache.c +++ b/read-cache.c @@ -549,7 +549,7 @@ int cache_name_stage_compare(const char *name1, int len1, int stage1, const char return 0; } -static int index_name_stage_pos(const struct index_state *istate, const char *name, int namelen, int stage) +static int index_name_stage_pos(struct index_state *istate, const char *name, int namelen, int stage) { int first, last; @@ -570,7 +570,7 @@ static int index_name_stage_pos(const struct index_state *istate, const char *na return -first-1; } -int index_name_pos(const struct index_state *istate, const char *name, int namelen) +int index_name_pos(struct index_state *istate, const char *name, int namelen) { return index_name_stage_pos(istate, name, namelen, 0); } @@ -3389,8 +3389,8 @@ int repo_read_index_unmerged(struct repository *repo) * We helpfully remove a trailing "/" from directories so that * the output of read_directory can be used as-is. */ -int index_name_is_other(const struct index_state *istate, const char *name, - int namelen) +int index_name_is_other(struct index_state *istate, const char *name, + int namelen) { int pos; if (namelen && name[namelen - 1] == '/') @@ -3408,7 +3408,7 @@ int index_name_is_other(const struct index_state *istate, const char *name, return 1; } -void *read_blob_data_from_index(const struct index_state *istate, +void *read_blob_data_from_index(struct index_state *istate, const char *path, unsigned long *size) { int pos, len; diff --git a/submodule.c b/submodule.c index 9767ba9893..83809a4f7b 100644 --- a/submodule.c +++ b/submodule.c @@ -33,7 +33,7 @@ static struct oid_array ref_tips_after_fetch; * will be disabled because we can't guess what might be configured in * .gitmodules unless the user resolves the conflict. */ -int is_gitmodules_unmerged(const struct index_state *istate) +int is_gitmodules_unmerged(struct index_state *istate) { int pos = index_name_pos(istate, GITMODULES_FILE, strlen(GITMODULES_FILE)); if (pos < 0) { /* .gitmodules not found or isn't merged */ @@ -301,7 +301,7 @@ int is_submodule_populated_gently(const char *path, int *return_error_code) /* * Dies if the provided 'prefix' corresponds to an unpopulated submodule */ -void die_in_unpopulated_submodule(const struct index_state *istate, +void die_in_unpopulated_submodule(struct index_state *istate, const char *prefix) { int i, prefixlen; @@ -331,7 +331,7 @@ void die_in_unpopulated_submodule(const struct index_state *istate, /* * Dies if any paths in the provided pathspec descends into a submodule */ -void die_path_inside_submodule(const struct index_state *istate, +void die_path_inside_submodule(struct index_state *istate, const struct pathspec *ps) { int i, j; diff --git a/submodule.h b/submodule.h index 4ac6e31cf1..84640c49c1 100644 --- a/submodule.h +++ b/submodule.h @@ -39,7 +39,7 @@ struct submodule_update_strategy { }; #define SUBMODULE_UPDATE_STRATEGY_INIT {SM_UPDATE_UNSPECIFIED, NULL} -int is_gitmodules_unmerged(const struct index_state *istate); +int is_gitmodules_unmerged(struct index_state *istate); int is_writing_gitmodules_ok(void); int is_staging_gitmodules_ok(struct index_state *istate); int update_path_in_gitmodules(const char *oldpath, const char *newpath); @@ -60,9 +60,9 @@ int is_submodule_active(struct repository *repo, const char *path); * Otherwise the return error code is the same as of resolve_gitdir_gently. */ int is_submodule_populated_gently(const char *path, int *return_error_code); -void die_in_unpopulated_submodule(const struct index_state *istate, +void die_in_unpopulated_submodule(struct index_state *istate, const char *prefix); -void die_path_inside_submodule(const struct index_state *istate, +void die_path_inside_submodule(struct index_state *istate, const struct pathspec *ps); enum submodule_update_type parse_submodule_update_type(const char *value); int parse_submodule_update_strategy(const char *value, -- cgit v1.3 From 95e0321c4dbb81eca5dc1c6f96b176b00a0368d7 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 1 Apr 2021 01:49:40 +0000 Subject: read-cache: expand on query into sparse-directory entry Callers to index_name_pos() or index_name_stage_pos() have a specific path in mind. If that happens to be a path with an ancestor being a sparse-directory entry, it can lead to unexpected results. In the case that we did not find the requested path, check to see if the position _before_ the inserted position is a sparse directory entry that matches the initial segment of the input path (including the directory separator at the end of the directory name). If so, then expand the index to be a full index and search again. This expansion will only happen once per index read. Future enhancements could be more careful to expand only the necessary sparse directory entry, but then we would have a special "not fully sparse, but also not fully expanded" mode that could affect writing the index to file. Since this only occurs if a specific file is requested outside of the sparse checkout definition, this is unlikely to be a common situation. Signed-off-by: Derrick Stolee Reviewed-by: Elijah Newren Signed-off-by: Junio C Hamano --- read-cache.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'read-cache.c') diff --git a/read-cache.c b/read-cache.c index 3ad9457809..3698bc7bf7 100644 --- a/read-cache.c +++ b/read-cache.c @@ -567,6 +567,27 @@ static int index_name_stage_pos(struct index_state *istate, const char *name, in } first = next+1; } + + if (istate->sparse_index && + first > 0) { + /* Note: first <= istate->cache_nr */ + struct cache_entry *ce = istate->cache[first - 1]; + + /* + * If we are in a sparse-index _and_ the entry before the + * insertion position is a sparse-directory entry that is + * an ancestor of 'name', then we need to expand the index + * and search again. This will only trigger once, because + * thereafter the index is fully expanded. + */ + if (S_ISSPARSEDIR(ce->ce_mode) && + ce_namelen(ce) < namelen && + !strncmp(name, ce->name, ce_namelen(ce))) { + ensure_full_index(istate); + return index_name_stage_pos(istate, name, namelen, stage); + } + } + return -first-1; } -- cgit v1.3 From 0c18c059a152058e30e23c4edcd24b3992599503 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 1 Apr 2021 01:49:58 +0000 Subject: read-cache: ensure full index Before iterating over all cache entries, ensure that a sparse index is expanded to a full index to avoid unexpected behavior. Signed-off-by: Derrick Stolee Reviewed-by: Elijah Newren Signed-off-by: Junio C Hamano --- read-cache.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'read-cache.c') diff --git a/read-cache.c b/read-cache.c index 3698bc7bf7..a9dcf0ab4f 100644 --- a/read-cache.c +++ b/read-cache.c @@ -1577,6 +1577,8 @@ int refresh_index(struct index_state *istate, unsigned int flags, */ preload_index(istate, pathspec, 0); trace2_region_enter("index", "refresh", NULL); + /* TODO: audit for interaction with sparse-index. */ + ensure_full_index(istate); for (i = 0; i < istate->cache_nr; i++) { struct cache_entry *ce, *new_entry; int cache_errno = 0; @@ -2498,6 +2500,8 @@ int repo_index_has_changes(struct repository *repo, diff_flush(&opt); return opt.flags.has_changes != 0; } else { + /* TODO: audit for interaction with sparse-index. */ + ensure_full_index(istate); for (i = 0; sb && i < istate->cache_nr; i++) { if (i) strbuf_addch(sb, ' '); -- cgit v1.3