Compare to v1 (the one on 'pu' as v2 never got to 'pu'): - New cleanup patch 09/23
- New patch 17/23 allows to ignore untracked cache without destroying it (for comparison and verification) - New patches 22/23 and 23/23 add some protection against filesystem or operating system changes - Document UNTR extension - Reorganize UNTR to avoid saving SHA-1/stat data when it's useless - Fix writing UNTR to base index in split index mode - Various review comments since v1 -- 8< -- diff --git a/Documentation/technical/index-format.txt b/Documentation/technical/index-format.txt index fe6f316..5dc2bee 100644 --- a/Documentation/technical/index-format.txt +++ b/Documentation/technical/index-format.txt @@ -233,3 +233,64 @@ Git index format The remaining index entries after replaced ones will be added to the final index. These added entries are also sorted by entry namme then stage. + +== Untracked cache + + Untracked cache saves the untracked file list and necessary data to + verify the cache. The signature for this extension is { 'U', 'N', + 'T', 'R' }. + + The extension starts with + + - A NUL-terminated string describing the environment when the cache + is created. + + - Stat data of $GIT_DIR/info/exclude. See "Index entry" section from + ctime field until "file size". + + - Stat data of core.excludesfile + + - 32-bit dir_flags (see struct dir_struct) + + - 160-bit SHA-1 of $GIT_DIR/info/exclude. Null SHA-1 means the file + does not exist. + + - 160-bit SHA-1 of core.excludesfile. Null SHA-1 means the file does + not exist. + + - NUL-terminated string of per-dir exclude file name. This usually + is ".gitignore". + + - The number of following directory blocks, variable width + encoding. If this number is zero, the extension ends here with a + following NUL. + + - A number of directory blocks in depth-first-search order, each + consists of + + - The number of untracked entries, variable witdh encoding. + + - The number of sub-directory blocks, variable with encoding. + + - The directory name terminated by NUL. + + - A number of untrached file/dir names terminated by NUL. + +The remaining data of each directory block is grouped by type: + + - An ewah bitmap, the n-th bit marks whether the n-th directory has + valid untracked cache entries. + + - An ewah bitmap, the n-th bit records "check-only" bit of + read_directory_recursive() for the n-th directory. + + - An ewah bitmap, the n-th bit indicates whether SHA-1 and stat data + is valid for the n-th directory and exists in the next data. + + - An array of stat data. The n-th data corresponds with the n-th + "one" bit in the previous ewah bitmap. + + - An array of SHA-1. The n-th SHA-1 corresponds with the n-th "one" bit + in the previous ewah bitmap. + + - One NUL. diff --git a/builtin/update-index.c b/builtin/update-index.c index c1c18db..f23ec83 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -115,6 +115,7 @@ static int test_if_untracked_cache_is_supported(void) fd = create_file("dir-mtime-test/newfile"); xstat("dir-mtime-test", &st); if (!match_stat_data(&base, &st)) { + close(fd); fputc('\n', stderr); fprintf_ln(stderr,_("directory stat info does not " "change after adding a new file")); @@ -127,6 +128,7 @@ static int test_if_untracked_cache_is_supported(void) xmkdir("dir-mtime-test/new-dir"); xstat("dir-mtime-test", &st); if (!match_stat_data(&base, &st)) { + close(fd); fputc('\n', stderr); fprintf_ln(stderr, _("directory stat info does not change " "after adding a new directory")); @@ -1094,10 +1096,10 @@ int cmd_update_index(int argc, const char **argv, const char *prefix) /* should be the same flags used by git-status */ uc->dir_flags = DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES; the_index.untracked = uc; - the_index.cache_changed |= SOMETHING_CHANGED; + the_index.cache_changed |= UNTRACKED_CHANGED; } else if (!untracked_cache && the_index.untracked) { the_index.untracked = NULL; - the_index.cache_changed |= SOMETHING_CHANGED; + the_index.cache_changed |= UNTRACKED_CHANGED; } if (active_cache_changed) { diff --git a/cache.h b/cache.h index 201b22e..fca979b 100644 --- a/cache.h +++ b/cache.h @@ -295,7 +295,7 @@ static inline unsigned int canon_mode(unsigned int mode) #define RESOLVE_UNDO_CHANGED (1 << 4) #define CACHE_TREE_CHANGED (1 << 5) #define SPLIT_INDEX_ORDERED (1 << 6) -#define UNTRACKED_CHANGED (1 << 7) +#define UNTRACKED_CHANGED (1 << 7) struct split_index; struct untracked_cache; diff --git a/compat/mingw.c b/compat/mingw.c index c5c37e5..88140e4 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2128,3 +2128,14 @@ void mingw_startup() /* initialize Unicode console */ winansi_init(); } + +int uname(struct utsname *buf) +{ + DWORD v = GetVersion(); + memset(buf, 0, sizeof(*buf)); + strcpy(buf->sysname, "Windows"); + sprintf(buf->release, "%u.%u", v & 0xff, (v >> 8) & 0xff); + /* assuming NT variants only.. */ + sprintf(buf->version, "%u", (v >> 16) & 0x7fff); + return 0; +} diff --git a/compat/mingw.h b/compat/mingw.h index df0e320..d00ba7a 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -77,6 +77,14 @@ struct itimerval { }; #define ITIMER_REAL 0 +struct utsname { + char sysname[16]; + char nodename[1]; + char release[16]; + char version[16]; + char machine[1]; +}; + /* * sanitize preprocessor namespace polluted by Windows headers defining * macros which collide with git local versions @@ -166,6 +174,7 @@ struct passwd *getpwuid(uid_t uid); int setitimer(int type, struct itimerval *in, struct itimerval *out); int sigaction(int sig, struct sigaction *in, struct sigaction *out); int link(const char *oldpath, const char *newpath); +int uname(struct utsname *buf); /* * replacements of existing functions diff --git a/dir.c b/dir.c index 0ae2188..ef58547 100644 --- a/dir.c +++ b/dir.c @@ -13,6 +13,7 @@ #include "wildmatch.h" #include "pathspec.h" #include "varint.h" +#include "ewah/ewok.h" struct path_simplify { int len; @@ -482,7 +483,7 @@ void add_exclude(const char *string, const char *base, } static void *read_skip_worktree_file_from_index(const char *path, size_t *size, - struct sha1_stat *ss) + struct sha1_stat *sha1_stat) { int pos, len; unsigned long sz; @@ -501,9 +502,9 @@ static void *read_skip_worktree_file_from_index(const char *path, size_t *size, return NULL; } *size = xsize_t(sz); - if (ss) { - memset(&ss->stat, 0, sizeof(ss->stat)); - hashcpy(ss->sha1, active_cache[pos]->sha1); + if (sha1_stat) { + memset(&sha1_stat->stat, 0, sizeof(sha1_stat->stat)); + hashcpy(sha1_stat->sha1, active_cache[pos]->sha1); } return data; } @@ -584,9 +585,10 @@ static struct untracked_cache_dir *lookup_untracked(struct untracked_cache *uc, } uc->dir_created++; - d = xmalloc(sizeof(*d) + len); - memset(d, 0, sizeof(*d) + len); + d = xmalloc(sizeof(*d) + len + 1); + memset(d, 0, sizeof(*d)); memcpy(d->name, name, len); + d->name[len] = '\0'; ALLOC_GROW(dir->dirs, dir->dirs_nr + 1, dir->dirs_alloc); memmove(dir->dirs + first + 1, dir->dirs + first, @@ -634,7 +636,7 @@ static void invalidate_directory(struct untracked_cache *uc, */ static int add_excludes(const char *fname, const char *base, int baselen, struct exclude_list *el, int check_index, - struct sha1_stat *ss, int ss_valid) + struct sha1_stat *sha1_stat) { struct stat st; int fd, i, lineno = 1; @@ -648,7 +650,7 @@ static int add_excludes(const char *fname, const char *base, int baselen, if (0 <= fd) close(fd); if (!check_index || - (buf = read_skip_worktree_file_from_index(fname, &size, ss)) == NULL) + (buf = read_skip_worktree_file_from_index(fname, &size, sha1_stat)) == NULL) return -1; if (size == 0) { free(buf); @@ -661,9 +663,10 @@ static int add_excludes(const char *fname, const char *base, int baselen, } else { size = xsize_t(st.st_size); if (size == 0) { - if (ss) { - fill_stat_data(&ss->stat, &st); - hashcpy(ss->sha1, EMPTY_BLOB_SHA1_BIN); + if (sha1_stat) { + fill_stat_data(&sha1_stat->stat, &st); + hashcpy(sha1_stat->sha1, EMPTY_BLOB_SHA1_BIN); + sha1_stat->valid = 1; } close(fd); return 0; @@ -676,19 +679,20 @@ static int add_excludes(const char *fname, const char *base, int baselen, } buf[size++] = '\n'; close(fd); - if (ss) { + if (sha1_stat) { int pos; - if (ss_valid && - !match_stat_data_racy(&the_index, &ss->stat, &st)) + if (sha1_stat->valid && + !match_stat_data_racy(&the_index, &sha1_stat->stat, &st)) ; /* no content change, ss->sha1 still good */ else if (check_index && (pos = cache_name_pos(fname, strlen(fname))) >= 0 && !ce_stage(active_cache[pos]) && ce_uptodate(active_cache[pos])) - hashcpy(ss->sha1, active_cache[pos]->sha1); + hashcpy(sha1_stat->sha1, active_cache[pos]->sha1); else - hash_sha1_file(buf, size, "blob", ss->sha1); - fill_stat_data(&ss->stat, &st); + hash_sha1_file(buf, size, "blob", sha1_stat->sha1); + fill_stat_data(&sha1_stat->stat, &st); + sha1_stat->valid = 1; } } @@ -712,7 +716,7 @@ int add_excludes_from_file_to_list(const char *fname, const char *base, int baselen, struct exclude_list *el, int check_index) { - return add_excludes(fname, base, baselen, el, check_index, NULL, 0); + return add_excludes(fname, base, baselen, el, check_index, NULL); } struct exclude_list *add_exclude_list(struct dir_struct *dir, @@ -733,7 +737,7 @@ struct exclude_list *add_exclude_list(struct dir_struct *dir, * Used to set up core.excludesfile and .git/info/exclude lists. */ static void add_excludes_from_file_1(struct dir_struct *dir, const char *fname, - struct sha1_stat *ss, int ss_valid) + struct sha1_stat *sha1_stat) { struct exclude_list *el; /* @@ -744,14 +748,14 @@ static void add_excludes_from_file_1(struct dir_struct *dir, const char *fname, if (!dir->untracked) dir->unmanaged_exclude_files++; el = add_exclude_list(dir, EXC_FILE, fname); - if (add_excludes(fname, "", 0, el, 0, ss, ss_valid) < 0) + if (add_excludes(fname, "", 0, el, 0, sha1_stat) < 0) die("cannot use %s as an exclude file", fname); } void add_excludes_from_file(struct dir_struct *dir, const char *fname) { dir->unmanaged_exclude_files++; /* see validate_untracked_cache() */ - add_excludes_from_file_1(dir, fname, NULL, 0); + add_excludes_from_file_1(dir, fname, NULL); } int match_basename(const char *basename, int basenamelen, @@ -974,7 +978,7 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen) while (current < baselen) { struct exclude_stack *stk = xcalloc(1, sizeof(*stk)); const char *cp; - struct sha1_stat ss; + struct sha1_stat sha1_stat; if (current < 0) { cp = base; @@ -1015,7 +1019,8 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen) } /* Try to read per-directory file */ - hashclr(ss.sha1); + hashclr(sha1_stat.sha1); + sha1_stat.valid = 0; if (dir->exclude_per_dir && /* * If we know that no files have been added in @@ -1044,7 +1049,7 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen) strbuf_addstr(&sb, dir->exclude_per_dir); el->src = strbuf_detach(&sb, NULL); add_excludes(el->src, el->src, stk->baselen, el, 1, - untracked ? &ss : NULL, 0); + untracked ? &sha1_stat : NULL); } /* * NEEDSWORK: when untracked cache is enabled, prep_exclude() @@ -1060,9 +1065,10 @@ static void prep_exclude(struct dir_struct *dir, const char *base, int baselen) * last_exclude_matching(). Be careful about ignore rule * order, though, if you do that. */ - if (untracked && hashcmp(ss.sha1, untracked->exclude_sha1)) { + if (untracked && + hashcmp(sha1_stat.sha1, untracked->exclude_sha1)) { invalidate_gitignore(dir->untracked, untracked); - hashcpy(untracked->exclude_sha1, ss.sha1); + hashcpy(untracked->exclude_sha1, sha1_stat.sha1); } dir->exclude_stack = stk; current = stk->baselen; @@ -1457,12 +1463,11 @@ static enum path_treatment treat_path_fast(struct dir_struct *dir, int baselen, const struct path_simplify *simplify) { + strbuf_setlen(path, baselen); if (!cdir->ucd) { - strbuf_setlen(path, baselen); strbuf_addstr(path, cdir->file); return path_untracked; } - strbuf_setlen(path, baselen); strbuf_addstr(path, cdir->ucd->name); /* treat_one_path() does this before it calls treat_directory() */ if (path->buf[path->len - 1] != '/') @@ -1474,7 +1479,6 @@ static enum path_treatment treat_path_fast(struct dir_struct *dir, * with check_only set. */ return read_directory_recursive(dir, path->buf, path->len, - cdir->ucd, 1, simplify); /* * We get path_recurse in the first run when @@ -1582,7 +1586,7 @@ static int open_cached_dir(struct cached_dir *cdir, return 0; } -int read_cached_dir(struct cached_dir *cdir) +static int read_cached_dir(struct cached_dir *cdir) { if (cdir->fdir) { cdir->de = readdir(cdir->fdir); @@ -1796,7 +1800,7 @@ static struct untracked_cache_dir *validate_untracked_cache(struct dir_struct *d struct untracked_cache_dir *root; int i; - if (!dir->untracked) + if (!dir->untracked || getenv("GIT_DISABLE_UNTRACKED_CACHE")) return NULL; /* @@ -2101,30 +2105,12 @@ void setup_standard_excludes(struct dir_struct *dir) home_config_paths(NULL, &xdg_path, "ignore"); excludes_file = xdg_path; } - if (!access_or_warn(path, R_OK, 0)) { - struct sha1_stat *ss = NULL; - int ss_valid = 0; - if (dir->untracked) { - ss = &dir->ss_info_exclude; - if (dir->untracked->loaded) { - *ss = dir->untracked->ss_info_exclude; - ss_valid = 1; - } - } - add_excludes_from_file_1(dir, path, ss, ss_valid); - } - if (excludes_file && !access_or_warn(excludes_file, R_OK, 0)) { - struct sha1_stat *ss = NULL; - int ss_valid = 0; - if (dir->untracked) { - ss = &dir->ss_excludes_file; - if (dir->untracked->loaded) { - *ss = dir->untracked->ss_excludes_file; - ss_valid = 1; - } - } - add_excludes_from_file_1(dir, excludes_file, ss, ss_valid); - } + if (!access_or_warn(path, R_OK, 0)) + add_excludes_from_file_1(dir, path, + dir->untracked ? &dir->ss_info_exclude : NULL); + if (excludes_file && !access_or_warn(excludes_file, R_OK, 0)) + add_excludes_from_file_1(dir, excludes_file, + dir->untracked ? &dir->ss_excludes_file : NULL); } int remove_path(const char *name) @@ -2186,6 +2172,16 @@ struct ondisk_untracked_cache { char exclude_per_dir[1]; }; +struct write_data { + int index; /* number of written untracked_cache_dir */ + struct ewah_bitmap *check_only; /* from untracked_cache_dir */ + struct ewah_bitmap *valid; /* from untracked_cache_dir */ + struct ewah_bitmap *sha1_valid; /* set if exclude_sha1 is not null */ + struct strbuf out; + struct strbuf sb_stat; + struct strbuf sb_sha1; +}; + static void stat_data_to_disk(struct stat_data *to, const struct stat_data *from) { to->sd_ctime.sec = htonl(from->sd_ctime.sec); @@ -2199,16 +2195,14 @@ static void stat_data_to_disk(struct stat_data *to, const struct stat_data *from to->sd_size = htonl(from->sd_size); } -static void write_one_dir(struct strbuf *out, struct untracked_cache_dir *untracked) +static void write_one_dir(struct untracked_cache_dir *untracked, + struct write_data *wd) { struct stat_data stat_data; + struct strbuf *out = &wd->out; unsigned char intbuf[16]; unsigned int intlen, value; - int i; - - stat_data_to_disk(&stat_data, &untracked->stat_data); - strbuf_add(out, &stat_data, sizeof(stat_data)); - strbuf_add(out, untracked->exclude_sha1, 20); + int i = wd->index++; /* * untracked_nr should be reset whenever valid is clear, but @@ -2219,10 +2213,19 @@ static void write_one_dir(struct strbuf *out, struct untracked_cache_dir *untrac untracked->check_only = 0; } - value = untracked->valid; - value |= untracked->check_only << 1; - value |= untracked->untracked_nr << 2; - intlen = encode_varint(value, intbuf); + if (untracked->check_only) + ewah_set(wd->check_only, i); + if (untracked->valid) { + ewah_set(wd->valid, i); + stat_data_to_disk(&stat_data, &untracked->stat_data); + strbuf_add(&wd->sb_stat, &stat_data, sizeof(stat_data)); + } + if (!is_null_sha1(untracked->exclude_sha1)) { + ewah_set(wd->sha1_valid, i); + strbuf_add(&wd->sb_sha1, untracked->exclude_sha1, 20); + } + + intlen = encode_varint(untracked->untracked_nr, intbuf); strbuf_add(out, intbuf, intlen); /* skip non-recurse directories */ @@ -2240,13 +2243,26 @@ static void write_one_dir(struct strbuf *out, struct untracked_cache_dir *untrac for (i = 0; i < untracked->dirs_nr; i++) if (untracked->dirs[i]->recurse) - write_one_dir(out, untracked->dirs[i]); + write_one_dir(untracked->dirs[i], wd); +} + +static void get_ident_string(struct strbuf *sb) +{ + struct utsname uts; + + if (uname(&uts)) + die_errno(_("failed to get kernel name and information")); + strbuf_addf(sb, "Location %s, system %s %s %s", get_git_work_tree(), + uts.sysname, uts.release, uts.version); } void write_untracked_extension(struct strbuf *out, struct untracked_cache *untracked) { struct ondisk_untracked_cache *ouc; - int len = 0; + struct write_data wd; + struct strbuf sb = STRBUF_INIT; + unsigned char varbuf[16]; + int len = 0, varint_len; if (untracked->exclude_per_dir) len = strlen(untracked->exclude_per_dir); ouc = xmalloc(sizeof(*ouc) + len); @@ -2256,9 +2272,43 @@ void write_untracked_extension(struct strbuf *out, struct untracked_cache *untra hashcpy(ouc->excludes_file_sha1, untracked->ss_excludes_file.sha1); ouc->dir_flags = htonl(untracked->dir_flags); memcpy(ouc->exclude_per_dir, untracked->exclude_per_dir, len + 1); + + get_ident_string(&sb); + strbuf_add(out, sb.buf, sb.len + 1); + strbuf_release(&sb); + strbuf_add(out, ouc, sizeof(*ouc) + len); - if (untracked->root) - write_one_dir(out, untracked->root); + if (!untracked->root) { + varint_len = encode_varint(0, varbuf); + strbuf_add(out, varbuf, varint_len); + return; + } + + wd.index = 0; + wd.check_only = ewah_new(); + wd.valid = ewah_new(); + wd.sha1_valid = ewah_new(); + strbuf_init(&wd.out, 1024); + strbuf_init(&wd.sb_stat, 1024); + strbuf_init(&wd.sb_sha1, 1024); + write_one_dir(untracked->root, &wd); + + varint_len = encode_varint(wd.index, varbuf); + strbuf_add(out, varbuf, varint_len); + strbuf_addbuf(out, &wd.out); + ewah_serialize_strbuf(wd.valid, out); + ewah_serialize_strbuf(wd.check_only, out); + ewah_serialize_strbuf(wd.sha1_valid, out); + strbuf_addbuf(out, &wd.sb_stat); + strbuf_addbuf(out, &wd.sb_sha1); + strbuf_addch(out, '\0'); /* safe guard for string lists */ + + ewah_free(wd.valid); + ewah_free(wd.check_only); + ewah_free(wd.sha1_valid); + strbuf_release(&wd.out); + strbuf_release(&wd.sb_stat); + strbuf_release(&wd.sb_sha1); } static void free_untracked(struct untracked_cache_dir *ucd) @@ -2282,6 +2332,16 @@ void free_untracked_cache(struct untracked_cache *uc) free(uc); } +struct read_data { + int index; + struct untracked_cache_dir **ucd; + struct ewah_bitmap *check_only; + struct ewah_bitmap *valid; + struct ewah_bitmap *sha1_valid; + const unsigned char *data; + const unsigned char *end; +}; + static void stat_data_from_disk(struct stat_data *to, const struct stat_data *from) { to->sd_ctime.sec = get_be32(&from->sd_ctime.sec); @@ -2296,43 +2356,34 @@ static void stat_data_from_disk(struct stat_data *to, const struct stat_data *fr } static int read_one_dir(struct untracked_cache_dir **untracked_, - const unsigned char *data_, unsigned long sz) + struct read_data *rd) { #define NEXT(x) \ next = data + (x); \ - if (next > data_ + sz) \ + if (next > rd->end) \ return -1; struct untracked_cache_dir ud, *untracked; - const unsigned char *next, *data = data_; + const unsigned char *next, *data = rd->data, *end = rd->end; unsigned int value; int i, len; memset(&ud, 0, sizeof(ud)); - NEXT(sizeof(struct stat_data)); - stat_data_from_disk(&ud.stat_data, (struct stat_data *)data); - data = next; - - NEXT(20); - hashcpy(ud.exclude_sha1, data); - data = next; - next = data; value = decode_varint(&next); - if (next > data_ + sz) + if (next > end) return -1; - ud.recurse = 1; - ud.valid = value & 1; - ud.check_only = (value >> 1) & 1; - ud.untracked_alloc = ud.untracked_nr = value >> 2; + ud.recurse = 1; + ud.untracked_alloc = value; + ud.untracked_nr = value; if (ud.untracked_nr) ud.untracked = xmalloc(sizeof(*ud.untracked) * ud.untracked_nr); data = next; next = data; ud.dirs_alloc = ud.dirs_nr = decode_varint(&next); - if (next > data_ + sz) + if (next > end) return -1; ud.dirs = xmalloc(sizeof(*ud.dirs) * ud.dirs_nr); data = next; @@ -2351,41 +2402,145 @@ static int read_one_dir(struct untracked_cache_dir **untracked_, data = next; } + rd->ucd[rd->index++] = untracked; + rd->data = data; + for (i = 0; i < untracked->dirs_nr; i++) { - len = read_one_dir(untracked->dirs + i, data, sz - (data - data_)); + len = read_one_dir(untracked->dirs + i, rd); if (len < 0) return -1; - data += len; } - return data - data_; + return 0; +} + +static void set_check_only(size_t pos, void *cb) +{ + struct read_data *rd = cb; + struct untracked_cache_dir *ud = rd->ucd[pos]; + ud->check_only = 1; +} + +static void read_stat(size_t pos, void *cb) +{ + struct read_data *rd = cb; + struct untracked_cache_dir *ud = rd->ucd[pos]; + if (rd->data + sizeof(struct stat_data) > rd->end) { + rd->data = rd->end + 1; + return; + } + stat_data_from_disk(&ud->stat_data, (struct stat_data *)rd->data); + rd->data += sizeof(struct stat_data); + ud->valid = 1; +} + +static void read_sha1(size_t pos, void *cb) +{ + struct read_data *rd = cb; + struct untracked_cache_dir *ud = rd->ucd[pos]; + if (rd->data + 20 > rd->end) { + rd->data = rd->end + 1; + return; + } + hashcpy(ud->exclude_sha1, rd->data); + rd->data += 20; +} + +static void load_sha1_stat(struct sha1_stat *sha1_stat, + const struct stat_data *stat, + const unsigned char *sha1) +{ + stat_data_from_disk(&sha1_stat->stat, stat); + hashcpy(sha1_stat->sha1, sha1); + sha1_stat->valid = 1; } struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz) { - const struct ondisk_untracked_cache *ouc = data; + const struct ondisk_untracked_cache *ouc; struct untracked_cache *uc; + struct read_data rd; + const unsigned char *next = data, *end = data + sz; + struct strbuf sb = STRBUF_INIT; int len; - if (sz < sizeof(*ouc)) + if (sz <= 1 || end[-1] != '\0') + return NULL; + end--; + + get_ident_string(&sb); + if (strcmp(sb.buf, (const char *)next)) { + warning(_("system identification does not match, untracked cache disabled.\n" + "Stored: %s\nCurrent: %s\n"), + next, sb.buf); + strbuf_release(&sb); + return NULL; + } + next += sb.len + 1; + strbuf_release(&sb); + + ouc = (const struct ondisk_untracked_cache *)next; + if (next + sizeof(*ouc) > end) return NULL; uc = xcalloc(1, sizeof(*uc)); - stat_data_from_disk(&uc->ss_info_exclude.stat, &ouc->info_exclude_stat); - stat_data_from_disk(&uc->ss_excludes_file.stat, &ouc->excludes_file_stat); - hashcpy(uc->ss_info_exclude.sha1, ouc->info_exclude_sha1); - hashcpy(uc->ss_excludes_file.sha1, ouc->excludes_file_sha1); + load_sha1_stat(&uc->ss_info_exclude, &ouc->info_exclude_stat, + ouc->info_exclude_sha1); + load_sha1_stat(&uc->ss_excludes_file, &ouc->excludes_file_stat, + ouc->excludes_file_sha1); uc->dir_flags = get_be32(&ouc->dir_flags); uc->exclude_per_dir = xstrdup(ouc->exclude_per_dir); - uc->loaded = 1; - len = sizeof(*ouc) + strlen(ouc->exclude_per_dir); - if (sz == len) - return uc; - if (sz > len && - read_one_dir(&uc->root, (const unsigned char *)data + len, - sz - len) == sz - len) - return uc; - free_untracked_cache(uc); - return NULL; + /* NUL after exclude_per_dir is covered by sizeof(*ouc) */ + next += sizeof(*ouc) + strlen(ouc->exclude_per_dir); + if (next >= end) + goto done2; + + len = decode_varint(&next); + if (next > end || len == 0) + goto done2; + + rd.valid = ewah_new(); + rd.check_only = ewah_new(); + rd.sha1_valid = ewah_new(); + rd.data = next; + rd.end = end; + rd.index = 0; + rd.ucd = xmalloc(sizeof(*rd.ucd) * len); + + if (read_one_dir(&uc->root, &rd) || rd.index != len) + goto done; + + next = rd.data; + len = ewah_read_mmap(rd.valid, next, end - next); + if (len < 0) + goto done; + + next += len; + len = ewah_read_mmap(rd.check_only, next, end - next); + if (len < 0) + goto done; + + next += len; + len = ewah_read_mmap(rd.sha1_valid, next, end - next); + if (len < 0) + goto done; + + ewah_each_bit(rd.check_only, set_check_only, &rd); + rd.data = next + len; + ewah_each_bit(rd.valid, read_stat, &rd); + ewah_each_bit(rd.sha1_valid, read_sha1, &rd); + next = rd.data; + +done: + free(rd.ucd); + ewah_free(rd.valid); + ewah_free(rd.check_only); + ewah_free(rd.sha1_valid); +done2: + if (next != end) { + free_untracked_cache(uc); + uc = NULL; + } + return uc; } void untracked_cache_invalidate_path(struct index_state *istate, diff --git a/dir.h b/dir.h index 8c29324..2ce7dd3 100644 --- a/dir.h +++ b/dir.h @@ -77,6 +77,7 @@ struct exclude_list_group { struct sha1_stat { struct stat_data stat; unsigned char sha1[20]; + int valid; }; /* @@ -85,7 +86,7 @@ struct sha1_stat { * The following inputs are sufficient to determine what files in a * directory are excluded: * - * - The list of files and directories of the direction in question + * - The list of files and directories of the directory in question * - The $GIT_DIR/index * - dir_struct flags * - The content of $GIT_DIR/info/exclude @@ -110,16 +111,16 @@ struct sha1_stat { struct untracked_cache_dir { struct untracked_cache_dir **dirs; char **untracked; - /* null SHA-1 means this directory does not have .gitignore */ - unsigned char exclude_sha1[20]; struct stat_data stat_data; - unsigned int recurse : 1; + unsigned int untracked_alloc, dirs_nr, dirs_alloc; + unsigned int untracked_nr; unsigned int check_only : 1; /* all data except 'dirs' in this struct are good */ unsigned int valid : 1; - unsigned int untracked_nr : 29; - unsigned int untracked_alloc, dirs_nr, dirs_alloc; - char name[1]; + unsigned int recurse : 1; + /* null SHA-1 means this directory does not have .gitignore */ + unsigned char exclude_sha1[20]; + char name[FLEX_ARRAY]; }; struct untracked_cache { @@ -137,7 +138,6 @@ struct untracked_cache { int gitignore_invalidated; int dir_invalidated; int dir_opened; - int loaded; }; struct dir_struct { diff --git a/ewah/ewah_io.c b/ewah/ewah_io.c index 1c2d7af..43481b9 100644 --- a/ewah/ewah_io.c +++ b/ewah/ewah_io.c @@ -19,6 +19,7 @@ */ #include "git-compat-util.h" #include "ewok.h" +#include "strbuf.h" int ewah_serialize_native(struct ewah_bitmap *self, int fd) { @@ -110,6 +111,18 @@ int ewah_serialize(struct ewah_bitmap *self, int fd) return ewah_serialize_to(self, write_helper, (void *)(intptr_t)fd); } +static int write_strbuf(void *user_data, const void *data, size_t len) +{ + struct strbuf *sb = user_data; + strbuf_add(sb, data, len); + return len; +} + +int ewah_serialize_strbuf(struct ewah_bitmap *self, struct strbuf *sb) +{ + return ewah_serialize_to(self, write_strbuf, sb); +} + int ewah_read_mmap(struct ewah_bitmap *self, const void *map, size_t len) { const uint8_t *ptr = map; diff --git a/ewah/ewok.h b/ewah/ewok.h index f6ad190..4d7f5e9 100644 --- a/ewah/ewok.h +++ b/ewah/ewok.h @@ -30,6 +30,7 @@ # define ewah_calloc xcalloc #endif +struct strbuf; typedef uint64_t eword_t; #define BITS_IN_WORD (sizeof(eword_t) * 8) @@ -97,6 +98,7 @@ int ewah_serialize_to(struct ewah_bitmap *self, void *out); int ewah_serialize(struct ewah_bitmap *self, int fd); int ewah_serialize_native(struct ewah_bitmap *self, int fd); +int ewah_serialize_strbuf(struct ewah_bitmap *self, struct strbuf *); int ewah_deserialize(struct ewah_bitmap *self, int fd); int ewah_read_mmap(struct ewah_bitmap *self, const void *map, size_t len); diff --git a/git-compat-util.h b/git-compat-util.h index f587749..e9502a1 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -132,6 +132,7 @@ #elif defined(_MSC_VER) #include "compat/msvc.h" #else +#include <sys/utsname.h> #include <sys/wait.h> #include <sys/resource.h> #include <sys/socket.h> diff --git a/read-cache.c b/read-cache.c index 21ae963..71d8e20 100644 --- a/read-cache.c +++ b/read-cache.c @@ -2040,7 +2040,7 @@ static int do_write_index(struct index_state *istate, int newfd, if (err) return -1; } - if (istate->untracked) { + if (!strip_extensions && istate->untracked) { struct strbuf sb = STRBUF_INIT; write_untracked_extension(&sb, istate->untracked); diff --git a/split-index.c b/split-index.c index 21485e2..968b780 100644 --- a/split-index.c +++ b/split-index.c @@ -41,13 +41,6 @@ int read_link_extension(struct index_state *istate, return 0; } -static int write_strbuf(void *user_data, const void *data, size_t len) -{ - struct strbuf *sb = user_data; - strbuf_add(sb, data, len); - return len; -} - int write_link_extension(struct strbuf *sb, struct index_state *istate) { @@ -55,8 +48,8 @@ int write_link_extension(struct strbuf *sb, strbuf_add(sb, si->base_sha1, 20); if (!si->delete_bitmap && !si->replace_bitmap) return 0; - ewah_serialize_to(si->delete_bitmap, write_strbuf, sb); - ewah_serialize_to(si->replace_bitmap, write_strbuf, sb); + ewah_serialize_strbuf(si->delete_bitmap, sb); + ewah_serialize_strbuf(si->replace_bitmap, sb); return 0; } diff --git a/test-dump-untracked-cache.c b/test-dump-untracked-cache.c index 710441e..25d855d 100644 --- a/test-dump-untracked-cache.c +++ b/test-dump-untracked-cache.c @@ -44,6 +44,7 @@ int main(int ac, char **av) { struct untracked_cache *uc; struct strbuf base = STRBUF_INIT; + setup_git_directory(); if (read_cache() < 0) die("unable to read index file"); uc = the_index.untracked; -- 8< -- -- 2.2.0.60.gb7b3c64 -- To unsubscribe from this list: send the line "unsubscribe git" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html