From 83c094ad0dd2104adbbec034f802dceb1d052981 Mon Sep 17 00:00:00 2001 From: Nguyễn Thái Ngọc Duy Date: Sun, 8 Mar 2015 17:12:33 +0700 Subject: untracked cache: save to an index extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Helped-by: Stefan Beller Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- read-cache.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'read-cache.c') diff --git a/read-cache.c b/read-cache.c index 8d71860f69..3a058d008a 100644 --- a/read-cache.c +++ b/read-cache.c @@ -39,6 +39,7 @@ static struct cache_entry *refresh_cache_entry(struct cache_entry *ce, #define CACHE_EXT_TREE 0x54524545 /* "TREE" */ #define CACHE_EXT_RESOLVE_UNDO 0x52455543 /* "REUC" */ #define CACHE_EXT_LINK 0x6c696e6b /* "link" */ +#define CACHE_EXT_UNTRACKED 0x554E5452 /* "UNTR" */ /* changes that can be kept in $GIT_DIR/index (basically all extensions) */ #define EXTMASK (RESOLVE_UNDO_CHANGED | CACHE_TREE_CHANGED | \ @@ -2047,6 +2048,17 @@ static int do_write_index(struct index_state *istate, int newfd, if (err) return -1; } + if (!strip_extensions && istate->untracked) { + struct strbuf sb = STRBUF_INIT; + + write_untracked_extension(&sb, istate->untracked); + err = write_index_ext_header(&c, newfd, CACHE_EXT_UNTRACKED, + sb.len) < 0 || + ce_write(&c, newfd, sb.buf, sb.len) < 0; + strbuf_release(&sb); + if (err) + return -1; + } if (ce_flush(&c, newfd, istate->sha1) || fstat(newfd, &st)) return -1; -- cgit v1.3 From f9e6c649589e0940ccb82821107fb658277ed86b Mon Sep 17 00:00:00 2001 From: Nguyễn Thái Ngọc Duy Date: Sun, 8 Mar 2015 17:12:34 +0700 Subject: untracked cache: load from UNTR index extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- dir.c | 219 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ dir.h | 2 + read-cache.c | 5 ++ 3 files changed, 226 insertions(+) (limited to 'read-cache.c') diff --git a/dir.c b/dir.c index f695e0c685..b39a034ae5 100644 --- a/dir.c +++ b/dir.c @@ -2283,3 +2283,222 @@ void write_untracked_extension(struct strbuf *out, struct untracked_cache *untra strbuf_release(&wd.sb_stat); strbuf_release(&wd.sb_sha1); } + +static void free_untracked(struct untracked_cache_dir *ucd) +{ + int i; + if (!ucd) + return; + for (i = 0; i < ucd->dirs_nr; i++) + free_untracked(ucd->dirs[i]); + for (i = 0; i < ucd->untracked_nr; i++) + free(ucd->untracked[i]); + free(ucd->untracked); + free(ucd->dirs); + free(ucd); +} + +void free_untracked_cache(struct untracked_cache *uc) +{ + if (uc) + free_untracked(uc->root); + free(uc); +} + +struct read_data { + int index; + struct untracked_cache_dir **ucd; + struct ewah_bitmap *check_only; + struct ewah_bitmap *valid; + struct ewah_bitmap *sha1_valid; + const unsigned char *data; + const unsigned char *end; +}; + +static void stat_data_from_disk(struct stat_data *to, const struct stat_data *from) +{ + to->sd_ctime.sec = get_be32(&from->sd_ctime.sec); + to->sd_ctime.nsec = get_be32(&from->sd_ctime.nsec); + to->sd_mtime.sec = get_be32(&from->sd_mtime.sec); + to->sd_mtime.nsec = get_be32(&from->sd_mtime.nsec); + to->sd_dev = get_be32(&from->sd_dev); + to->sd_ino = get_be32(&from->sd_ino); + to->sd_uid = get_be32(&from->sd_uid); + to->sd_gid = get_be32(&from->sd_gid); + to->sd_size = get_be32(&from->sd_size); +} + +static int read_one_dir(struct untracked_cache_dir **untracked_, + struct read_data *rd) +{ + struct untracked_cache_dir ud, *untracked; + const unsigned char *next, *data = rd->data, *end = rd->end; + unsigned int value; + int i, len; + + memset(&ud, 0, sizeof(ud)); + + next = data; + value = decode_varint(&next); + if (next > end) + return -1; + ud.recurse = 1; + ud.untracked_alloc = value; + ud.untracked_nr = value; + if (ud.untracked_nr) + ud.untracked = xmalloc(sizeof(*ud.untracked) * ud.untracked_nr); + data = next; + + next = data; + ud.dirs_alloc = ud.dirs_nr = decode_varint(&next); + if (next > end) + return -1; + ud.dirs = xmalloc(sizeof(*ud.dirs) * ud.dirs_nr); + data = next; + + len = strlen((const char *)data); + next = data + len + 1; + if (next > rd->end) + return -1; + *untracked_ = untracked = xmalloc(sizeof(*untracked) + len); + memcpy(untracked, &ud, sizeof(ud)); + memcpy(untracked->name, data, len + 1); + data = next; + + for (i = 0; i < untracked->untracked_nr; i++) { + len = strlen((const char *)data); + next = data + len + 1; + if (next > rd->end) + return -1; + untracked->untracked[i] = xstrdup((const char*)data); + data = next; + } + + rd->ucd[rd->index++] = untracked; + rd->data = data; + + for (i = 0; i < untracked->dirs_nr; i++) { + len = read_one_dir(untracked->dirs + i, rd); + if (len < 0) + return -1; + } + return 0; +} + +static void set_check_only(size_t pos, void *cb) +{ + struct read_data *rd = cb; + struct untracked_cache_dir *ud = rd->ucd[pos]; + ud->check_only = 1; +} + +static void read_stat(size_t pos, void *cb) +{ + struct read_data *rd = cb; + struct untracked_cache_dir *ud = rd->ucd[pos]; + if (rd->data + sizeof(struct stat_data) > rd->end) { + rd->data = rd->end + 1; + return; + } + stat_data_from_disk(&ud->stat_data, (struct stat_data *)rd->data); + rd->data += sizeof(struct stat_data); + ud->valid = 1; +} + +static void read_sha1(size_t pos, void *cb) +{ + struct read_data *rd = cb; + struct untracked_cache_dir *ud = rd->ucd[pos]; + if (rd->data + 20 > rd->end) { + rd->data = rd->end + 1; + return; + } + hashcpy(ud->exclude_sha1, rd->data); + rd->data += 20; +} + +static void load_sha1_stat(struct sha1_stat *sha1_stat, + const struct stat_data *stat, + const unsigned char *sha1) +{ + stat_data_from_disk(&sha1_stat->stat, stat); + hashcpy(sha1_stat->sha1, sha1); + sha1_stat->valid = 1; +} + +struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz) +{ + const struct ondisk_untracked_cache *ouc; + struct untracked_cache *uc; + struct read_data rd; + const unsigned char *next = data, *end = (const unsigned char *)data + sz; + int len; + + if (sz <= 1 || end[-1] != '\0') + return NULL; + end--; + + ouc = (const struct ondisk_untracked_cache *)next; + if (next + ouc_size(0) > end) + return NULL; + + uc = xcalloc(1, sizeof(*uc)); + load_sha1_stat(&uc->ss_info_exclude, &ouc->info_exclude_stat, + ouc->info_exclude_sha1); + load_sha1_stat(&uc->ss_excludes_file, &ouc->excludes_file_stat, + ouc->excludes_file_sha1); + uc->dir_flags = get_be32(&ouc->dir_flags); + uc->exclude_per_dir = xstrdup(ouc->exclude_per_dir); + /* NUL after exclude_per_dir is covered by sizeof(*ouc) */ + next += ouc_size(strlen(ouc->exclude_per_dir)); + if (next >= end) + goto done2; + + len = decode_varint(&next); + if (next > end || len == 0) + goto done2; + + rd.valid = ewah_new(); + rd.check_only = ewah_new(); + rd.sha1_valid = ewah_new(); + rd.data = next; + rd.end = end; + rd.index = 0; + rd.ucd = xmalloc(sizeof(*rd.ucd) * len); + + if (read_one_dir(&uc->root, &rd) || rd.index != len) + goto done; + + next = rd.data; + len = ewah_read_mmap(rd.valid, next, end - next); + if (len < 0) + goto done; + + next += len; + len = ewah_read_mmap(rd.check_only, next, end - next); + if (len < 0) + goto done; + + next += len; + len = ewah_read_mmap(rd.sha1_valid, next, end - next); + if (len < 0) + goto done; + + ewah_each_bit(rd.check_only, set_check_only, &rd); + rd.data = next + len; + ewah_each_bit(rd.valid, read_stat, &rd); + ewah_each_bit(rd.sha1_valid, read_sha1, &rd); + next = rd.data; + +done: + free(rd.ucd); + ewah_free(rd.valid); + ewah_free(rd.check_only); + ewah_free(rd.sha1_valid); +done2: + if (next != end) { + free_untracked_cache(uc); + uc = NULL; + } + return uc; +} diff --git a/dir.h b/dir.h index dc3ee0b2e5..40a679a802 100644 --- a/dir.h +++ b/dir.h @@ -298,5 +298,7 @@ static inline int dir_path_match(const struct dir_entry *ent, has_trailing_dir); } +void free_untracked_cache(struct untracked_cache *); +struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz); void write_untracked_extension(struct strbuf *out, struct untracked_cache *untracked); #endif diff --git a/read-cache.c b/read-cache.c index 3a058d008a..ee0ef049b8 100644 --- a/read-cache.c +++ b/read-cache.c @@ -1371,6 +1371,9 @@ static int read_index_extension(struct index_state *istate, if (read_link_extension(istate, data, sz)) return -1; break; + case CACHE_EXT_UNTRACKED: + istate->untracked = read_untracked_extension(data, sz); + break; default: if (*ext < 'A' || 'Z' < *ext) return error("index uses %.4s extension, which we do not understand", @@ -1662,6 +1665,8 @@ int discard_index(struct index_state *istate) istate->cache = NULL; istate->cache_alloc = 0; discard_split_index(istate); + free_untracked_cache(istate->untracked); + istate->untracked = NULL; return 0; } -- cgit v1.3 From e931371a8f1164185486a1f5fdaaa708b4a6217c Mon Sep 17 00:00:00 2001 From: Nguyễn Thái Ngọc Duy Date: Sun, 8 Mar 2015 17:12:35 +0700 Subject: untracked cache: invalidate at index addition or removal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ideally we should implement untracked_cache_remove_from_index() and untracked_cache_add_to_index() so that they update untracked cache right away instead of invalidating it and wait for read_directory() next time to deal with it. But that may need some more work in unpack-trees.c. So stay simple as the first step. The new call in add_index_entry_with_check() may look strange because new calls usually stay close to cache_tree_invalidate_path(). We do it a bit later than c_t_i_p() in this function because if it's about replacing the entry with the same name, we don't care (but cache-tree does). Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- dir.c | 31 +++++++++++++++++++++++++++++++ dir.h | 4 ++++ read-cache.c | 4 ++++ unpack-trees.c | 7 +++++-- 4 files changed, 44 insertions(+), 2 deletions(-) (limited to 'read-cache.c') diff --git a/dir.c b/dir.c index b39a034ae5..68b46d0acb 100644 --- a/dir.c +++ b/dir.c @@ -2502,3 +2502,34 @@ done2: } return uc; } + +void untracked_cache_invalidate_path(struct index_state *istate, + const char *path) +{ + const char *sep; + struct untracked_cache_dir *d; + if (!istate->untracked || !istate->untracked->root) + return; + sep = strrchr(path, '/'); + if (sep) + d = lookup_untracked(istate->untracked, + istate->untracked->root, + path, sep - path); + else + d = istate->untracked->root; + istate->untracked->dir_invalidated++; + d->valid = 0; + d->untracked_nr = 0; +} + +void untracked_cache_remove_from_index(struct index_state *istate, + const char *path) +{ + untracked_cache_invalidate_path(istate, path); +} + +void untracked_cache_add_to_index(struct index_state *istate, + const char *path) +{ + untracked_cache_invalidate_path(istate, path); +} diff --git a/dir.h b/dir.h index 40a679a802..2ce7dd3d27 100644 --- a/dir.h +++ b/dir.h @@ -298,6 +298,10 @@ static inline int dir_path_match(const struct dir_entry *ent, has_trailing_dir); } +void untracked_cache_invalidate_path(struct index_state *, const char *); +void untracked_cache_remove_from_index(struct index_state *, const char *); +void untracked_cache_add_to_index(struct index_state *, const char *); + void free_untracked_cache(struct untracked_cache *); struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz); void write_untracked_extension(struct strbuf *out, struct untracked_cache *untracked); diff --git a/read-cache.c b/read-cache.c index ee0ef049b8..0d96c753b7 100644 --- a/read-cache.c +++ b/read-cache.c @@ -80,6 +80,7 @@ void rename_index_entry_at(struct index_state *istate, int nr, const char *new_n memcpy(new->name, new_name, namelen + 1); cache_tree_invalidate_path(istate, old->name); + untracked_cache_remove_from_index(istate, old->name); remove_index_entry_at(istate, nr); add_index_entry(istate, new, ADD_CACHE_OK_TO_ADD|ADD_CACHE_OK_TO_REPLACE); } @@ -539,6 +540,7 @@ int remove_file_from_index(struct index_state *istate, const char *path) if (pos < 0) pos = -pos-1; cache_tree_invalidate_path(istate, path); + untracked_cache_remove_from_index(istate, path); while (pos < istate->cache_nr && !strcmp(istate->cache[pos]->name, path)) remove_index_entry_at(istate, pos); return 0; @@ -981,6 +983,8 @@ static int add_index_entry_with_check(struct index_state *istate, struct cache_e } pos = -pos-1; + untracked_cache_add_to_index(istate, ce->name); + /* * Inserting a merged entry ("stage 0") into the index * will always replace all non-merged entries.. diff --git a/unpack-trees.c b/unpack-trees.c index be84ba2607..2927660d92 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -9,6 +9,7 @@ #include "refs.h" #include "attr.h" #include "split-index.h" +#include "dir.h" /* * Error messages expected by scripts out of plumbing commands such as @@ -1259,8 +1260,10 @@ static int verify_uptodate_sparse(const struct cache_entry *ce, static void invalidate_ce_path(const struct cache_entry *ce, struct unpack_trees_options *o) { - if (ce) - cache_tree_invalidate_path(o->src_index, ce->name); + if (!ce) + return; + cache_tree_invalidate_path(o->src_index, ce->name); + untracked_cache_invalidate_path(o->src_index, ce->name); } /* -- cgit v1.3 From 2bb4cda1987afe6911a1c193283213babda328d2 Mon Sep 17 00:00:00 2001 From: Nguyễn Thái Ngọc Duy Date: Sun, 8 Mar 2015 17:12:36 +0700 Subject: read-cache.c: split racy stat test to a separate function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- read-cache.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) (limited to 'read-cache.c') diff --git a/read-cache.c b/read-cache.c index 0d96c753b7..b5e9c3f8ac 100644 --- a/read-cache.c +++ b/read-cache.c @@ -272,20 +272,26 @@ static int ce_match_stat_basic(const struct cache_entry *ce, struct stat *st) return changed; } -static int is_racy_timestamp(const struct index_state *istate, - const struct cache_entry *ce) +static int is_racy_stat(const struct index_state *istate, + const struct stat_data *sd) { - return (!S_ISGITLINK(ce->ce_mode) && - istate->timestamp.sec && + return (istate->timestamp.sec && #ifdef USE_NSEC /* nanosecond timestamped files can also be racy! */ - (istate->timestamp.sec < ce->ce_stat_data.sd_mtime.sec || - (istate->timestamp.sec == ce->ce_stat_data.sd_mtime.sec && - istate->timestamp.nsec <= ce->ce_stat_data.sd_mtime.nsec)) + (istate->timestamp.sec < sd->sd_mtime.sec || + (istate->timestamp.sec == sd->sd_mtime.sec && + istate->timestamp.nsec <= sd->sd_mtime.nsec)) #else - istate->timestamp.sec <= ce->ce_stat_data.sd_mtime.sec + istate->timestamp.sec <= sd->sd_mtime.sec #endif - ); + ); +} + +static int is_racy_timestamp(const struct index_state *istate, + const struct cache_entry *ce) +{ + return (!S_ISGITLINK(ce->ce_mode) && + is_racy_stat(istate, &ce->ce_stat_data)); } int ie_match_stat(const struct index_state *istate, -- cgit v1.3 From ed4efab1b17e883b761b4482c40c04a4529be8f9 Mon Sep 17 00:00:00 2001 From: Nguyễn Thái Ngọc Duy Date: Sun, 8 Mar 2015 17:12:37 +0700 Subject: untracked cache: avoid racy timestamps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a directory is updated within the same second that its timestamp is last saved, we cannot realize the directory has been updated by checking timestamps. Assume the worst (something is update). See 29e4d36 (Racy GIT - 2005-12-20) for more information. Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- cache.h | 2 ++ dir.c | 4 ++-- read-cache.c | 8 ++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) (limited to 'read-cache.c') diff --git a/cache.h b/cache.h index 811cc36547..120d337bd4 100644 --- a/cache.h +++ b/cache.h @@ -555,6 +555,8 @@ extern void fill_stat_data(struct stat_data *sd, struct stat *st); * INODE_CHANGED, and DATA_CHANGED. */ extern int match_stat_data(const struct stat_data *sd, struct stat *st); +extern int match_stat_data_racy(const struct index_state *istate, + const struct stat_data *sd, struct stat *st); extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st); diff --git a/dir.c b/dir.c index 68b46d0acb..741484aa97 100644 --- a/dir.c +++ b/dir.c @@ -682,7 +682,7 @@ static int add_excludes(const char *fname, const char *base, int baselen, if (sha1_stat) { int pos; if (sha1_stat->valid && - !match_stat_data(&sha1_stat->stat, &st)) + !match_stat_data_racy(&the_index, &sha1_stat->stat, &st)) ; /* no content change, ss->sha1 still good */ else if (check_index && (pos = cache_name_pos(fname, strlen(fname))) >= 0 && @@ -1539,7 +1539,7 @@ static int valid_cached_dir(struct dir_struct *dir, return 0; } if (!untracked->valid || - match_stat_data(&untracked->stat_data, &st)) { + match_stat_data_racy(&the_index, &untracked->stat_data, &st)) { if (untracked->valid) invalidate_directory(dir->untracked, untracked); fill_stat_data(&untracked->stat_data, &st); diff --git a/read-cache.c b/read-cache.c index b5e9c3f8ac..57828bb3f3 100644 --- a/read-cache.c +++ b/read-cache.c @@ -294,6 +294,14 @@ static int is_racy_timestamp(const struct index_state *istate, is_racy_stat(istate, &ce->ce_stat_data)); } +int match_stat_data_racy(const struct index_state *istate, + const struct stat_data *sd, struct stat *st) +{ + if (is_racy_stat(istate, sd)) + return MTIME_CHANGED; + return match_stat_data(sd, st); +} + int ie_match_stat(const struct index_state *istate, const struct cache_entry *ce, struct stat *st, unsigned int options) -- cgit v1.3 From 1bbb3dba3fbf733db45f073ddafe89f5972c516a Mon Sep 17 00:00:00 2001 From: Nguyễn Thái Ngọc Duy Date: Sun, 8 Mar 2015 17:12:39 +0700 Subject: untracked cache: mark index dirty if untracked cache is updated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- cache.h | 1 + dir.c | 9 +++++++++ read-cache.c | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) (limited to 'read-cache.c') diff --git a/cache.h b/cache.h index 120d337bd4..1392be1030 100644 --- a/cache.h +++ b/cache.h @@ -289,6 +289,7 @@ static inline unsigned int canon_mode(unsigned int mode) #define RESOLVE_UNDO_CHANGED (1 << 4) #define CACHE_TREE_CHANGED (1 << 5) #define SPLIT_INDEX_ORDERED (1 << 6) +#define UNTRACKED_CHANGED (1 << 7) struct split_index; struct untracked_cache; diff --git a/dir.c b/dir.c index 1cf1e3002e..592b5fa795 100644 --- a/dir.c +++ b/dir.c @@ -1934,6 +1934,15 @@ int read_directory(struct dir_struct *dir, const char *path, int len, const stru dir->untracked->gitignore_invalidated, dir->untracked->dir_invalidated, dir->untracked->dir_opened); + if (dir->untracked == the_index.untracked && + (dir->untracked->dir_opened || + dir->untracked->gitignore_invalidated || + dir->untracked->dir_invalidated)) + the_index.cache_changed |= UNTRACKED_CHANGED; + if (dir->untracked != the_index.untracked) { + free(dir->untracked); + dir->untracked = NULL; + } } return dir->nr; } diff --git a/read-cache.c b/read-cache.c index 57828bb3f3..705469eb7a 100644 --- a/read-cache.c +++ b/read-cache.c @@ -44,7 +44,7 @@ static struct cache_entry *refresh_cache_entry(struct cache_entry *ce, /* changes that can be kept in $GIT_DIR/index (basically all extensions) */ #define EXTMASK (RESOLVE_UNDO_CHANGED | CACHE_TREE_CHANGED | \ CE_ENTRY_ADDED | CE_ENTRY_REMOVED | CE_ENTRY_CHANGED | \ - SPLIT_INDEX_ORDERED) + SPLIT_INDEX_ORDERED | UNTRACKED_CHANGED) struct index_state the_index; static const char *alternate_index_output; -- cgit v1.3