From a70b2807a5108966c8d9aa5b70c271e887139a2a Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 21:03:23 -0700 Subject: [PATCH 001/105] dir: compare all saved metadata for UNTR directories valid_cached_dir() used match_stat_data_racy(), whose treatment of ctime and other fields follows core.trustCtime and core.checkStat. Tracked entries can correct a false stat match with a later content comparison, but cached directories have no equivalent check. Renaming a child and restoring its parent's mtime can therefore hide untracked paths under weak stat settings. Compare every field persisted in directory stat_data regardless of those tracked-file settings, and retain the existing racy-timestamp check. The untracked-cache status test renames a child, restores the directory mtime, and verifies that cached and uncached status agree. Signed-off-by: Taylor Blau --- dir.c | 39 ++++++++++++++++++++++++++++++- t/t7063-status-untracked-cache.sh | 26 +++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/dir.c b/dir.c index 95d8a1cce90f77..f55c9377df6942 100644 --- a/dir.c +++ b/dir.c @@ -71,6 +71,42 @@ struct cached_dir { struct untracked_cache_dir *ucd; }; +/* + * Unlike cache entries, an untracked-cache directory has no later content + * check to correct a false stat match. Compare every field saved in UNTR, + * regardless of core.checkStat or core.trustCtime. + */ +static int match_untracked_dir_stat(const struct stat_data *sd, + const struct stat *st) +{ + return !S_ISDIR(st->st_mode) || + sd->sd_ctime.sec != (unsigned int)st->st_ctime || + sd->sd_ctime.nsec != ST_CTIME_NSEC(*st) || + sd->sd_mtime.sec != (unsigned int)st->st_mtime || + sd->sd_mtime.nsec != ST_MTIME_NSEC(*st) || + sd->sd_dev != (unsigned int)st->st_dev || + sd->sd_ino != (unsigned int)st->st_ino || + sd->sd_uid != (unsigned int)st->st_uid || + sd->sd_gid != (unsigned int)st->st_gid || + sd->sd_size != (unsigned int)st->st_size; +} + +static int match_untracked_dir_stat_racy(const struct cache_time *timestamp, + const struct stat_data *sd, + const struct stat *st) +{ + if (timestamp->sec && +#ifdef USE_NSEC + (timestamp->sec < sd->sd_mtime.sec || + (timestamp->sec == sd->sd_mtime.sec && + timestamp->nsec <= sd->sd_mtime.nsec))) +#else + timestamp->sec <= sd->sd_mtime.sec) +#endif + return MTIME_CHANGED; + return match_untracked_dir_stat(sd, st); +} + static enum path_treatment read_directory_recursive(struct dir_struct *dir, struct index_state *istate, const char *path, int len, struct untracked_cache_dir *untracked, @@ -2541,7 +2577,8 @@ static int valid_cached_dir(struct dir_struct *dir, return 0; } if (!untracked->valid || - match_stat_data_racy(istate, &untracked->stat_data, &st)) { + match_untracked_dir_stat_racy( + &istate->timestamp, &untracked->stat_data, &st)) { fill_stat_data(&untracked->stat_data, &st); return 0; } diff --git a/t/t7063-status-untracked-cache.sh b/t/t7063-status-untracked-cache.sh index 8929ef481f926c..4ab20cc0693dc4 100755 --- a/t/t7063-status-untracked-cache.sh +++ b/t/t7063-status-untracked-cache.sh @@ -991,4 +991,30 @@ test_expect_success 'empty repo (no index) and core.untrackedCache' ' git -C emptyrepo -c core.untrackedCache=true write-tree ' +test_expect_success 'directory snapshots ignore weak file-stat configuration' ' + test_create_repo weak-dir && + ( + cd weak-dir && + mkdir nested && + echo tracked >tracked && + echo one >nested/one && + git add tracked && + git commit -m base && + git config core.untrackedCache true && + git config core.fsmonitor false && + git config core.trustCtime false && + git config core.checkStat minimal && + avoid_racy && + git status --porcelain -uall >/dev/null && + git status --porcelain -uall >/dev/null && + dir_mtime=$(test-tool chmtime --get nested) && + mv nested/one nested/two && + test-tool chmtime =$dir_mtime nested && + GIT_OPTIONAL_LOCKS=0 git -c core.untrackedCache=false \ + status --porcelain -uall >.git/expect && + git status --porcelain -uall >.git/actual && + test_cmp .git/expect .git/actual + ) +' + test_done From 092bd8415233cc662fec75dc1a17872222b8a8de Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 12:57:40 -0500 Subject: [PATCH 002/105] path-namespace: compare complete filesystem object identities Comparing only a pathname, device, and inode does not establish that two observations still describe the same unchanged filesystem object. Users that reopen a path need a reusable comparison covering the represented metadata fields. Represent an object's stat identity as a zero-initialized, fixed-width array containing device, inode, mode, link count, ownership, size, and modification and change timestamps. Include nanoseconds where available and add birth time and generation on Apple platforms. Provide comparison helpers and register both their library source and Clar unit suite in the Makefile and Meson builds. The unit tests check identity equality and reject changes to each represented stat field. No exclude-file validation or production caller is introduced here. Signed-off-by: Taylor Blau --- Makefile | 2 + meson.build | 1 + path-namespace.c | 47 ++++++++++++++++++++++ path-namespace.h | 18 +++++++++ t/meson.build | 1 + t/unit-tests/u-path-namespace.c | 69 +++++++++++++++++++++++++++++++++ 6 files changed, 138 insertions(+) create mode 100644 path-namespace.c create mode 100644 path-namespace.h create mode 100644 t/unit-tests/u-path-namespace.c diff --git a/Makefile b/Makefile index fac3e8879c8377..edac13257671b1 100644 --- a/Makefile +++ b/Makefile @@ -1255,6 +1255,7 @@ LIB_OBJS += parse-options.o LIB_OBJS += patch-delta.o LIB_OBJS += patch-ids.o LIB_OBJS += path.o +LIB_OBJS += path-namespace.o LIB_OBJS += path-walk.o LIB_OBJS += pathspec.o LIB_OBJS += pkt-line.o @@ -1546,6 +1547,7 @@ CLAR_TEST_SUITES += u-odb-inmemory CLAR_TEST_SUITES += u-oid-array CLAR_TEST_SUITES += u-oidmap CLAR_TEST_SUITES += u-oidtree +CLAR_TEST_SUITES += u-path-namespace CLAR_TEST_SUITES += u-prio-queue CLAR_TEST_SUITES += u-reftable-basics CLAR_TEST_SUITES += u-reftable-block diff --git a/meson.build b/meson.build index 7073d5844d2531..a777f4722a2061 100644 --- a/meson.build +++ b/meson.build @@ -460,6 +460,7 @@ libgit_sources = [ 'patch-delta.c', 'patch-ids.c', 'path.c', + 'path-namespace.c', 'path-walk.c', 'pathspec.c', 'pkt-line.c', diff --git a/path-namespace.c b/path-namespace.c new file mode 100644 index 00000000000000..48fc0aae434ef7 --- /dev/null +++ b/path-namespace.c @@ -0,0 +1,47 @@ +#include "git-compat-util.h" +#include "path-namespace.h" + +void path_stat_identity_init(struct path_stat_identity *identity, + const struct stat *st) +{ + memset(identity, 0, sizeof(*identity)); + identity->fields[0] = st->st_dev; + identity->fields[1] = st->st_ino; + identity->fields[2] = st->st_mode; + identity->fields[3] = st->st_nlink; + identity->fields[4] = st->st_uid; + identity->fields[5] = st->st_gid; + identity->fields[6] = st->st_size; + identity->fields[7] = st->st_mtime; +#ifdef __APPLE__ + identity->fields[8] = st->st_mtimespec.tv_nsec; +#else + identity->fields[8] = ST_MTIME_NSEC(*st); +#endif + identity->fields[9] = st->st_ctime; +#ifdef __APPLE__ + identity->fields[10] = st->st_ctimespec.tv_nsec; +#else + identity->fields[10] = ST_CTIME_NSEC(*st); +#endif +#ifdef __APPLE__ + identity->fields[11] = st->st_birthtimespec.tv_sec; + identity->fields[12] = st->st_birthtimespec.tv_nsec; + identity->fields[13] = st->st_gen; +#endif +} + +int path_stat_identity_equal(const struct path_stat_identity *a, + const struct path_stat_identity *b) +{ + return !memcmp(a, b, sizeof(*a)); +} + +int path_namespace_stat_equal(const struct stat *a, const struct stat *b) +{ + struct path_stat_identity first, second; + + path_stat_identity_init(&first, a); + path_stat_identity_init(&second, b); + return path_stat_identity_equal(&first, &second); +} diff --git a/path-namespace.h b/path-namespace.h new file mode 100644 index 00000000000000..0a93683e0b2de1 --- /dev/null +++ b/path-namespace.h @@ -0,0 +1,18 @@ +#ifndef PATH_NAMESPACE_H +#define PATH_NAMESPACE_H + +struct stat; + +#define PATH_STAT_IDENTITY_FIELDS 14 + +struct path_stat_identity { + uint64_t fields[PATH_STAT_IDENTITY_FIELDS]; +}; + +void path_stat_identity_init(struct path_stat_identity *identity, + const struct stat *st); +int path_stat_identity_equal(const struct path_stat_identity *a, + const struct path_stat_identity *b); +int path_namespace_stat_equal(const struct stat *a, const struct stat *b); + +#endif /* PATH_NAMESPACE_H */ diff --git a/t/meson.build b/t/meson.build index a25f37d2f5ae7d..d8734eae5ff77f 100644 --- a/t/meson.build +++ b/t/meson.build @@ -10,6 +10,7 @@ clar_test_suites = [ 'unit-tests/u-oid-array.c', 'unit-tests/u-oidmap.c', 'unit-tests/u-oidtree.c', + 'unit-tests/u-path-namespace.c', 'unit-tests/u-prio-queue.c', 'unit-tests/u-reftable-basics.c', 'unit-tests/u-reftable-block.c', diff --git a/t/unit-tests/u-path-namespace.c b/t/unit-tests/u-path-namespace.c new file mode 100644 index 00000000000000..702104597b1df9 --- /dev/null +++ b/t/unit-tests/u-path-namespace.c @@ -0,0 +1,69 @@ +#include "unit-test.h" + +#include "path-namespace.h" + +#define ASSERT_STAT_FIELD_MATTERS(base, changed, field) do { \ + (changed) = (base); \ + (changed).field = !(base).field; \ + cl_assert(!path_namespace_stat_equal(&(base), &(changed))); \ +} while (0) + +void test_path_namespace__stat_identity(void) +{ + struct stat st = { 0 }; + struct path_stat_identity first, second; + size_t i; + + st.st_dev = 1; + st.st_ino = 2; + st.st_mode = S_IFREG | 0644; + st.st_nlink = 3; + st.st_uid = 4; + st.st_gid = 5; + st.st_size = 6; + st.st_mtime = 7; + st.st_ctime = 8; + + path_stat_identity_init(&first, &st); + path_stat_identity_init(&second, &st); + cl_assert(path_stat_identity_equal(&first, &second)); + cl_assert(path_namespace_stat_equal(&st, &st)); + + for (i = 0; i < PATH_STAT_IDENTITY_FIELDS; i++) { + second = first; + second.fields[i]++; + cl_assert(!path_stat_identity_equal(&first, &second)); + } +} + +void test_path_namespace__stat_fields(void) +{ + struct stat st, changed; + + cl_must_pass(stat(".", &st)); + changed = st; + cl_assert(path_namespace_stat_equal(&st, &changed)); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_dev); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_ino); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_mode); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_nlink); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_uid); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_gid); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_size); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_mtime); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_ctime); +#ifndef NO_NSEC +#ifdef USE_ST_TIMESPEC + ASSERT_STAT_FIELD_MATTERS(st, changed, st_mtimespec.tv_nsec); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_ctimespec.tv_nsec); +#else + ASSERT_STAT_FIELD_MATTERS(st, changed, st_mtim.tv_nsec); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_ctim.tv_nsec); +#endif +#endif +#ifdef __APPLE__ + ASSERT_STAT_FIELD_MATTERS(st, changed, st_birthtimespec.tv_sec); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_birthtimespec.tv_nsec); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_gen); +#endif +} From 405af2f63dc5db13c8e54e68fb1020a0a3c28821 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 16:04:03 -0700 Subject: [PATCH 003/105] dir: consume preloaded UNTR directory stat results valid_cached_dir() performs a synchronous lstat() whenever an untracked-cache directory cannot rely on fsmonitor. A separate validation pass cannot remove that duplicated work unless traversal knows whether a saved result was checked and matched. Add transient stat_checked and stat_matches bits to each cached directory and let traversal consume them only when its caller marks the cache preloaded. Clear that marker after traversal and on the symlink-leading-path exit. Existing callers leave the marker clear, so ordinary lstat() validation and the fsmonitor path remain unchanged. Signed-off-by: Taylor Blau --- dir.c | 28 +++++++++++++++++++--------- dir.h | 4 ++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/dir.c b/dir.c index f55c9377df6942..2bfe4dbd1637e4 100644 --- a/dir.c +++ b/dir.c @@ -2572,15 +2572,23 @@ static int valid_cached_dir(struct dir_struct *dir, */ refresh_fsmonitor(istate); if (!(dir->untracked->use_fsmonitor && untracked->valid)) { - if (lstat(path->len ? path->buf : ".", &st)) { - memset(&untracked->stat_data, 0, sizeof(untracked->stat_data)); - return 0; - } - if (!untracked->valid || - match_untracked_dir_stat_racy( - &istate->timestamp, &untracked->stat_data, &st)) { - fill_stat_data(&untracked->stat_data, &st); - return 0; + if (dir->internal.untracked_cache_preloaded && + untracked->stat_checked) { + if (!untracked->valid || !untracked->stat_matches) + return 0; + } else { + if (lstat(path->len ? path->buf : ".", &st)) { + memset(&untracked->stat_data, 0, + sizeof(untracked->stat_data)); + return 0; + } + if (!untracked->valid || + match_untracked_dir_stat_racy( + &istate->timestamp, + &untracked->stat_data, &st)) { + fill_stat_data(&untracked->stat_data, &st); + return 0; + } } } @@ -3179,6 +3187,7 @@ int read_directory(struct dir_struct *dir, struct index_state *istate, dir->internal.visited_directories = 0; if (has_symlink_leading_path(path, len)) { + dir->internal.untracked_cache_preloaded = 0; trace2_region_leave("dir", "read_directory", istate->repo); return dir->nr; } @@ -3217,6 +3226,7 @@ int read_directory(struct dir_struct *dir, struct index_state *istate, } } + dir->internal.untracked_cache_preloaded = 0; return dir->nr; } diff --git a/dir.h b/dir.h index 83e0f648a81f36..d6fc0c8ef8676a 100644 --- a/dir.h +++ b/dir.h @@ -182,6 +182,9 @@ struct untracked_cache_dir { /* all data except 'dirs' in this struct are good */ unsigned int valid : 1; unsigned int recurse : 1; + /* transient results from directory-stat preloading */ + unsigned int stat_checked : 1; + unsigned int stat_matches : 1; /* null object ID means this directory does not have .gitignore */ struct object_id exclude_oid; char name[FLEX_ARRAY]; @@ -353,6 +356,7 @@ struct dir_struct { /* Stats about the traversal */ unsigned visited_paths; unsigned visited_directories; + unsigned untracked_cache_preloaded : 1; } internal; }; From 63c849b09abb43109757b9d4bcf51063bb5c09f9 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 16:05:22 -0700 Subject: [PATCH 004/105] dir: snapshot UNTR directory validation inputs Tracked-index refresh can invalidate the mutable untracked-cache tree. A concurrent directory-validation worker therefore cannot discover nodes or read their validation inputs directly from that live tree. Capture each node pointer, copied pathname, saved stat data, and prior validity before concurrent work begins. The opaque preload object also retains the cache, root, index timestamp, repository, and directory flags needed to recognize its original context. Expose construction and release as a complete ownership boundary. This preparatory change allocates one snapshot per cached directory but has no production caller and does not start workers or publish results. Signed-off-by: Taylor Blau --- dir.c | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ dir.h | 5 ++++ 2 files changed, 91 insertions(+) diff --git a/dir.c b/dir.c index 2bfe4dbd1637e4..a66050a6f55e9b 100644 --- a/dir.c +++ b/dir.c @@ -71,6 +71,92 @@ struct cached_dir { struct untracked_cache_dir *ucd; }; +struct untracked_cache_preload_task { + struct untracked_cache_dir *ucd; + char *path; + struct stat_data stat_data; + unsigned int was_valid : 1; + unsigned int stat_checked : 1; + unsigned int stat_matches : 1; + unsigned int update_stat_data : 1; +}; + +struct untracked_cache_preload { + struct repository *repo; + struct untracked_cache *uc; + struct untracked_cache_dir *root; + struct untracked_cache_preload_task *tasks; + struct cache_time index_timestamp; + size_t nr; + unsigned int dir_flags; +}; + +static void collect_untracked_cache_preload_tasks( + struct untracked_cache_dir *ucd, + struct strbuf *path, + struct untracked_cache_preload_task **tasks, + size_t *nr, + size_t *alloc) +{ + size_t i; + + ALLOC_GROW(*tasks, *nr + 1, *alloc); + memset(&(*tasks)[*nr], 0, sizeof(**tasks)); + (*tasks)[*nr].ucd = ucd; + (*tasks)[*nr].path = xstrdup(path->len ? path->buf : "."); + (*tasks)[*nr].stat_data = ucd->stat_data; + (*tasks)[*nr].was_valid = ucd->valid; + (*nr)++; + + for (i = 0; i < ucd->dirs_nr; i++) { + struct untracked_cache_dir *child = ucd->dirs[i]; + size_t old_len = path->len; + + if (path->len) + strbuf_addch(path, '/'); + strbuf_addstr(path, child->name); + collect_untracked_cache_preload_tasks(child, path, tasks, nr, + alloc); + strbuf_setlen(path, old_len); + } +} + +struct untracked_cache_preload *untracked_cache_preload_start_ordinary( + struct index_state *istate, unsigned int dir_flags) +{ + struct untracked_cache *uc = istate->untracked; + struct untracked_cache_preload *preload; + struct strbuf path = STRBUF_INIT; + size_t alloc = 0; + + if (!uc || !uc->root || uc->use_fsmonitor || + uc->dir_flags != dir_flags) + return NULL; + + CALLOC_ARRAY(preload, 1); + preload->repo = istate->repo; + preload->uc = uc; + preload->root = uc->root; + preload->index_timestamp = istate->timestamp; + preload->dir_flags = dir_flags; + collect_untracked_cache_preload_tasks( + uc->root, &path, &preload->tasks, &preload->nr, &alloc); + strbuf_release(&path); + return preload; +} + +void untracked_cache_preload_release(struct untracked_cache_preload *preload) +{ + size_t i; + + if (!preload) + return; + for (i = 0; i < preload->nr; i++) + free(preload->tasks[i].path); + free(preload->tasks); + free(preload); +} + /* * Unlike cache entries, an untracked-cache directory has no later content * check to correct a false stat match. Compare every field saved in UNTR, diff --git a/dir.h b/dir.h index d6fc0c8ef8676a..c6273985981a60 100644 --- a/dir.h +++ b/dir.h @@ -611,6 +611,11 @@ void untracked_cache_invalidate_trimmed_path(struct index_state *, void untracked_cache_remove_from_index(struct index_state *, const char *); void untracked_cache_add_to_index(struct index_state *, const char *); +struct untracked_cache_preload; +struct untracked_cache_preload *untracked_cache_preload_start_ordinary( + struct index_state *, unsigned int dir_flags); +void untracked_cache_preload_release(struct untracked_cache_preload *); + void free_untracked_cache(struct untracked_cache *); struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz); void write_untracked_extension(struct strbuf *out, struct untracked_cache *untracked); From 4d84d76a341d2130f4c8b3979424072207c9b38a Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 16:06:00 -0700 Subject: [PATCH 005/105] dir: validate captured UNTR directory stats A captured untracked-cache directory must not be marked reusable from a stale snapshot: tracked-index refresh may invalidate its live node, replace the cache, or change the traversal's directory flags. Run lstat() against each saved pathname and compare its immutable stat snapshot using the strict, racy-aware directory comparison. Publish the checked result only if the current cache, root, and directory flags still match. Preserve any invalidation that happened after capture; failed or changed stats leave ordinary traversal responsible for rescan. Validation remains synchronous, and no status caller invokes the new finish operation at this boundary. Signed-off-by: Taylor Blau --- dir.c | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ dir.h | 2 ++ 2 files changed, 78 insertions(+) diff --git a/dir.c b/dir.c index a66050a6f55e9b..5dbd5659b8a9bb 100644 --- a/dir.c +++ b/dir.c @@ -91,6 +91,10 @@ struct untracked_cache_preload { unsigned int dir_flags; }; +static int match_untracked_dir_stat_racy(const struct cache_time *timestamp, + const struct stat_data *sd, + const struct stat *st); + static void collect_untracked_cache_preload_tasks( struct untracked_cache_dir *ucd, struct strbuf *path, @@ -145,6 +149,78 @@ struct untracked_cache_preload *untracked_cache_preload_start_ordinary( return preload; } +static void validate_untracked_cache_preload( + struct untracked_cache_preload *preload) +{ + size_t i; + + for (i = 0; i < preload->nr; i++) { + struct untracked_cache_preload_task *task = &preload->tasks[i]; + struct stat st; + + if (!task->was_valid) + continue; + task->stat_checked = 1; + if (lstat(task->path, &st)) { + memset(&task->stat_data, 0, sizeof(task->stat_data)); + task->update_stat_data = 1; + continue; + } + if (!match_untracked_dir_stat_racy( + &preload->index_timestamp, &task->stat_data, &st)) { + task->stat_matches = 1; + continue; + } + fill_stat_data(&task->stat_data, &st); + task->update_stat_data = 1; + } +} + +int untracked_cache_preload_finish(struct untracked_cache_preload *preload, + struct index_state *istate, + unsigned int dir_flags) +{ + struct untracked_cache *uc; + size_t i; + int applied = 0; + int valid = 1; + + if (!preload) + return 0; + validate_untracked_cache_preload(preload); + uc = istate->untracked; + if (uc != preload->uc || !uc || uc->root != preload->root || + dir_flags != preload->dir_flags) + goto done; + + for (i = 0; i < preload->nr; i++) { + struct untracked_cache_preload_task *task = &preload->tasks[i]; + struct untracked_cache_dir *ucd = task->ucd; + + ucd->stat_checked = 0; + ucd->stat_matches = 0; + /* Invalidation performed after the snapshot always wins. */ + if (!task->was_valid || !ucd->valid) { + valid = 0; + continue; + } + ucd->stat_checked = task->stat_checked; + ucd->stat_matches = task->stat_matches; + if (!task->stat_checked || !task->stat_matches) + valid = 0; + if (task->update_stat_data) + ucd->stat_data = task->stat_data; + } + trace2_data_intmax("dir", istate->repo, + "preload_untracked_cache/valid", valid); + applied = 1; +done: + trace2_data_intmax("dir", istate->repo, + "preload_untracked_cache/applied", applied); + untracked_cache_preload_release(preload); + return applied; +} + void untracked_cache_preload_release(struct untracked_cache_preload *preload) { size_t i; diff --git a/dir.h b/dir.h index c6273985981a60..631bdad2b2b845 100644 --- a/dir.h +++ b/dir.h @@ -614,6 +614,8 @@ void untracked_cache_add_to_index(struct index_state *, const char *); struct untracked_cache_preload; struct untracked_cache_preload *untracked_cache_preload_start_ordinary( struct index_state *, unsigned int dir_flags); +int untracked_cache_preload_finish(struct untracked_cache_preload *, + struct index_state *, unsigned int dir_flags); void untracked_cache_preload_release(struct untracked_cache_preload *); void free_untracked_cache(struct untracked_cache *); From f09eed871f875d27897d859a4ee7388cefc4223f Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 16:07:06 -0700 Subject: [PATCH 006/105] dir: validate captured UNTR directory stats in parallel Synchronous validation still places every cached-directory lstat() on one execution path. Independent, already-captured directory snapshots can instead be divided among bounded workers without reading the live cache from those workers. Partition the snapshots using approximately 1,000 directories per worker, cap the worker count at six and at three times the available CPU count, and permit a bounded test override. Workers retain results in their own snapshot ranges; the existing finish operation joins them before publishing anything. Record worker count, thread-creation failures, directory count, and elapsed worker time through the threads, thread_failure, dirs, and wall_us Trace2 keys. Publication validity and applied-result counters remain in the earlier finishing boundary. Run the work synchronously for a single worker or without pthreads. If thread creation stops partway through, join started workers and process every unstarted range synchronously. No status caller enables the preload at this boundary. Signed-off-by: Taylor Blau --- dir.c | 123 ++++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 111 insertions(+), 12 deletions(-) diff --git a/dir.c b/dir.c index 5dbd5659b8a9bb..73494b54768602 100644 --- a/dir.c +++ b/dir.c @@ -33,6 +33,8 @@ #include "strbuf.h" #include "submodule-config.h" #include "symlinks.h" +#include "thread-utils.h" +#include "trace.h" #include "trace2.h" #include "tree.h" #include "hex.h" @@ -81,19 +83,35 @@ struct untracked_cache_preload_task { unsigned int update_stat_data : 1; }; +struct untracked_cache_preload; + +struct untracked_cache_preload_data { + pthread_t pthread; + struct untracked_cache_preload *preload; + size_t offset, nr; + int started; +}; + struct untracked_cache_preload { struct repository *repo; struct untracked_cache *uc; struct untracked_cache_dir *root; struct untracked_cache_preload_task *tasks; + struct untracked_cache_preload_data *data; struct cache_time index_timestamp; size_t nr; + int threads; unsigned int dir_flags; + uint64_t started_at; }; +#define UNTRACKED_CACHE_MAX_OVERLAP_THREADS 6 +#define UNTRACKED_CACHE_PRELOAD_COST 1000 + static int match_untracked_dir_stat_racy(const struct cache_time *timestamp, const struct stat_data *sd, const struct stat *st); +static void *preload_untracked_cache_thread(void *data); static void collect_untracked_cache_preload_tasks( struct untracked_cache_dir *ucd, @@ -131,7 +149,9 @@ struct untracked_cache_preload *untracked_cache_preload_start_ordinary( struct untracked_cache *uc = istate->untracked; struct untracked_cache_preload *preload; struct strbuf path = STRBUF_INIT; - size_t alloc = 0; + size_t alloc = 0, offset = 0, work, i; + unsigned long test_threads; + int threads, online, create_threads = 1; if (!uc || !uc->root || uc->use_fsmonitor || uc->dir_flags != dir_flags) @@ -146,15 +166,61 @@ struct untracked_cache_preload *untracked_cache_preload_start_ordinary( collect_untracked_cache_preload_tasks( uc->root, &path, &preload->tasks, &preload->nr, &alloc); strbuf_release(&path); + + threads = HAVE_THREADS ? preload->nr / UNTRACKED_CACHE_PRELOAD_COST : 1; + online = HAVE_THREADS ? online_cpus() : 1; + if (threads > online * 3) + threads = online * 3; + if (threads > UNTRACKED_CACHE_MAX_OVERLAP_THREADS) + threads = UNTRACKED_CACHE_MAX_OVERLAP_THREADS; + test_threads = git_env_ulong("GIT_TEST_UNTRACKED_CACHE_THREADS", 0); + if (test_threads && HAVE_THREADS) + threads = test_threads > UNTRACKED_CACHE_MAX_OVERLAP_THREADS ? + UNTRACKED_CACHE_MAX_OVERLAP_THREADS : test_threads; + if (threads < 1) + threads = 1; + if ((size_t)threads > preload->nr) + threads = preload->nr; + preload->threads = threads; + + preload->started_at = getnanotime(); + trace2_data_intmax("dir", istate->repo, + "preload_untracked_cache/threads", threads); + CALLOC_ARRAY(preload->data, threads); + work = DIV_ROUND_UP(preload->nr, threads); + for (i = 0; i < threads; i++) { + struct untracked_cache_preload_data *data = &preload->data[i]; + int err; + + data->preload = preload; + data->offset = offset; + data->nr = offset < preload->nr ? + (preload->nr - offset < work ? + preload->nr - offset : work) : 0; + offset += data->nr; + if (threads == 1 || !create_threads) + continue; + err = pthread_create(&data->pthread, NULL, + preload_untracked_cache_thread, data); + if (err) { + create_threads = 0; + trace2_data_intmax( + "dir", istate->repo, + "preload_untracked_cache/thread_failure", err); + continue; + } + data->started = 1; + } return preload; } -static void validate_untracked_cache_preload( - struct untracked_cache_preload *preload) +static void *preload_untracked_cache_thread(void *_data) { + struct untracked_cache_preload_data *data = _data; + struct untracked_cache_preload *preload = data->preload; size_t i; - for (i = 0; i < preload->nr; i++) { + for (i = data->offset; i < data->offset + data->nr; i++) { struct untracked_cache_preload_task *task = &preload->tasks[i]; struct stat st; @@ -174,6 +240,43 @@ static void validate_untracked_cache_preload( fill_stat_data(&task->stat_data, &st); task->update_stat_data = 1; } + return NULL; +} + +static void untracked_cache_preload_join( + struct untracked_cache_preload *preload) +{ + int i; + + if (preload->threads == 1) { + preload_untracked_cache_thread(&preload->data[0]); + return; + } + for (i = 0; i < preload->threads; i++) { + if (!preload->data[i].started) { + preload_untracked_cache_thread(&preload->data[i]); + continue; + } + if (pthread_join(preload->data[i].pthread, NULL)) + die(_("unable to join untracked-cache preload thread")); + } +} + +static void untracked_cache_preload_free( + struct untracked_cache_preload *preload) +{ + size_t i; + + trace2_data_intmax("dir", preload->repo, + "preload_untracked_cache/dirs", preload->nr); + trace2_data_intmax("dir", preload->repo, + "preload_untracked_cache/wall_us", + (getnanotime() - preload->started_at) / 1000); + free(preload->data); + for (i = 0; i < preload->nr; i++) + free(preload->tasks[i].path); + free(preload->tasks); + free(preload); } int untracked_cache_preload_finish(struct untracked_cache_preload *preload, @@ -187,7 +290,7 @@ int untracked_cache_preload_finish(struct untracked_cache_preload *preload, if (!preload) return 0; - validate_untracked_cache_preload(preload); + untracked_cache_preload_join(preload); uc = istate->untracked; if (uc != preload->uc || !uc || uc->root != preload->root || dir_flags != preload->dir_flags) @@ -217,20 +320,16 @@ int untracked_cache_preload_finish(struct untracked_cache_preload *preload, done: trace2_data_intmax("dir", istate->repo, "preload_untracked_cache/applied", applied); - untracked_cache_preload_release(preload); + untracked_cache_preload_free(preload); return applied; } void untracked_cache_preload_release(struct untracked_cache_preload *preload) { - size_t i; - if (!preload) return; - for (i = 0; i < preload->nr; i++) - free(preload->tasks[i].path); - free(preload->tasks); - free(preload); + untracked_cache_preload_join(preload); + untracked_cache_preload_free(preload); } /* From 92d76ca52e396ecf725ff322870cfdd4de2757f0 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 12:56:51 -0500 Subject: [PATCH 007/105] status: overlap UNTR validation with tracked-index refresh Directory validation and tracked-index refresh inspect different snapshots, but running them consecutively leaves both operations on the status command's critical path. Start the cached-directory preload after reading the index and before refresh_index(). Join its workers after configuring excludes and before collecting untracked paths, then pass their results into directory traversal. Release any unfinished preload when status buffers are freed. Keep activation behind GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD until production eligibility is defined. The untracked-cache status test checks unchanged and modified directories, preserved output, and the worker count selected by the running build's pthread support. Signed-off-by: Taylor Blau --- builtin/commit.c | 1 + dir.c | 2 ++ t/t7063-status-untracked-cache.sh | 49 +++++++++++++++++++++++++++++++ wt-status.c | 32 ++++++++++++++++++-- wt-status.h | 4 +++ 5 files changed, 86 insertions(+), 2 deletions(-) diff --git a/builtin/commit.c b/builtin/commit.c index 28f61745034506..54e41c8ba578c1 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1627,6 +1627,7 @@ struct repository *repo UNUSED) status_format != STATUS_FORMAT_PORCELAIN_V2) progress_flag = REFRESH_PROGRESS; repo_read_index(the_repository); + wt_status_start_untracked_cache_preload(&s); refresh_index(the_repository->index, REFRESH_QUIET|REFRESH_UNMERGED|progress_flag, &s.pathspec, NULL, NULL); diff --git a/dir.c b/dir.c index 73494b54768602..c97f59cbf3b1de 100644 --- a/dir.c +++ b/dir.c @@ -153,6 +153,8 @@ struct untracked_cache_preload *untracked_cache_preload_start_ordinary( unsigned long test_threads; int threads, online, create_threads = 1; + if (!git_env_bool("GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD", 0)) + return NULL; if (!uc || !uc->root || uc->use_fsmonitor || uc->dir_flags != dir_flags) return NULL; diff --git a/t/t7063-status-untracked-cache.sh b/t/t7063-status-untracked-cache.sh index 4ab20cc0693dc4..8f49cfeebb6896 100755 --- a/t/t7063-status-untracked-cache.sh +++ b/t/t7063-status-untracked-cache.sh @@ -1017,4 +1017,53 @@ test_expect_success 'directory snapshots ignore weak file-stat configuration' ' ) ' +test_expect_success 'status preloads cached-directory validation' ' + test_create_repo auto-preload && + ( + cd auto-preload && + mkdir -p nested/deep && + echo tracked >nested/tracked && + git add nested/tracked && + git commit -m base && + git config core.untrackedCache true && + git config core.fsmonitor false && + echo visible >nested/deep/visible && + git -c core.untrackedCache=false status --porcelain \ + >.git/expect && + avoid_racy && + git status --porcelain >/dev/null && + git status --porcelain >/dev/null && + + GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD=1 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=3 \ + GIT_TRACE2_EVENT="$PWD/.git/normal.trace" \ + git status --porcelain >.git/actual && + test_cmp .git/expect .git/actual && + if test_have_prereq PTHREADS + then + expect_threads=3 + else + expect_threads=1 + fi && + test_grep \ + "preload_untracked_cache/threads.*value.*$expect_threads" \ + .git/normal.trace && + test_grep "preload_untracked_cache/valid.*value.*1" \ + .git/normal.trace && + test_grep "opendir.*value.*0" .git/normal.trace && + echo changed >nested/deep/changed && + GIT_OPTIONAL_LOCKS=0 git -c core.untrackedCache=false \ + status --porcelain >.git/expect-changed && + GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD=1 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=3 \ + GIT_TRACE2_EVENT="$PWD/.git/changed.trace" \ + git status --porcelain >.git/actual-changed && + test_cmp .git/expect-changed .git/actual-changed && + test_grep "preload_untracked_cache/valid.*value.*0" \ + .git/changed.trace && + test_grep "opendir.*value.*[1-9][0-9]*" \ + .git/changed.trace + ) +' + test_done diff --git a/wt-status.c b/wt-status.c index 57772c7501fdba..ffb9ff8f044fd9 100644 --- a/wt-status.c +++ b/wt-status.c @@ -803,6 +803,26 @@ static void wt_status_collect_changes_initial(struct wt_status *s) strbuf_release(&base); } +static unsigned int wt_status_untracked_dir_flags(const struct wt_status *s) +{ + if (s->show_untracked_files == SHOW_ALL_UNTRACKED_FILES) + return 0; + return DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES; +} + +void wt_status_start_untracked_cache_preload(struct wt_status *s) +{ + struct index_state *istate = s->repo->index; + unsigned int dir_flags; + + if (s->untracked_cache_preload) + BUG("untracked-cache preload already started"); + + dir_flags = wt_status_untracked_dir_flags(s); + s->untracked_cache_preload = + untracked_cache_preload_start_ordinary(istate, dir_flags); +} + static void wt_status_collect_untracked(struct wt_status *s) { int i; @@ -814,8 +834,7 @@ static void wt_status_collect_untracked(struct wt_status *s) return; if (s->show_untracked_files != SHOW_ALL_UNTRACKED_FILES) - dir.flags |= - DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES; + dir.flags |= wt_status_untracked_dir_flags(s); if (s->show_ignored_mode) { dir.flags |= DIR_SHOW_IGNORED_TOO; @@ -826,6 +845,13 @@ static void wt_status_collect_untracked(struct wt_status *s) } setup_standard_excludes(&dir); + if (s->untracked_cache_preload) { + s->untracked_cache_preloaded = untracked_cache_preload_finish( + s->untracked_cache_preload, istate, dir.flags); + s->untracked_cache_preload = NULL; + } + dir.internal.untracked_cache_preloaded = + s->untracked_cache_preloaded; fill_directory(&dir, istate, &s->pathspec); @@ -889,6 +915,8 @@ void wt_status_collect(struct wt_status *s) void wt_status_collect_free_buffers(struct wt_status *s) { + untracked_cache_preload_release(s->untracked_cache_preload); + s->untracked_cache_preload = NULL; wt_status_state_free_buffers(&s->state); } diff --git a/wt-status.h b/wt-status.h index e9fe32e98cc18c..e64eda2d9cc666 100644 --- a/wt-status.h +++ b/wt-status.h @@ -8,6 +8,7 @@ struct repository; struct worktree; +struct untracked_cache_preload; enum color_wt_status { WT_STATUS_HEADER = 0, @@ -145,6 +146,8 @@ struct wt_status { struct string_list untracked; struct string_list ignored; uint32_t untracked_in_ms; + struct untracked_cache_preload *untracked_cache_preload; + unsigned untracked_cache_preloaded : 1; }; size_t wt_status_locate_end(const char *s, size_t len); @@ -153,6 +156,7 @@ void wt_status_add_cut_line(struct wt_status *s); void wt_status_prepare(struct repository *r, struct wt_status *s); void wt_status_print(struct wt_status *s); void wt_status_collect(struct wt_status *s); +void wt_status_start_untracked_cache_preload(struct wt_status *s); /* * Collect all changes between the two trees. Changes will be displayed as if From 4b4dc945f3d2830ecfa94c7f5d022e49da995cc7 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 13:08:32 -0500 Subject: [PATCH 008/105] fsmonitor: require content checks for reported paths Clearing CE_FSMONITOR_VALID is not enough to make a provider event authoritative. With core.trustctime disabled, core.checkStat set to minimal, and a restored modification time, stat matching can still accept changed file contents. The same stale match can affect diff, apply, checkout, and unpack-trees. Mark a reported entry with the in-memory CE_CONTENT_CHECK_REQUIRED flag, clear CE_UPTODATE, and discard its cached stat data. Route diff, apply, checkout, and unpack-trees comparisons through ie_match_stat_with_content_check(), which calls ie_modified() only for marked non-gitlinks. Other direct ie_match_stat() callers retain their existing paths. Marking an entry up to date clears the transient flag. Ordinary entries, gitlinks, and unmarked zero-stat entries retain their existing stat behavior. Add hook regressions for restored timestamps, diff and status, indexed apply, checkout, case-insensitive unpacking, unchanged reset, and ordinary zero-stat behavior in t/t7519-status-fsmonitor.sh. Signed-off-by: Taylor Blau --- apply.c | 5 +- diff-lib.c | 5 +- entry.c | 5 +- fsmonitor.c | 17 +++-- fsmonitor.h | 7 ++ read-cache-ll.h | 21 +++++- read-cache.c | 26 +++++++ t/helper/test-read-cache.c | 37 +++++++++ t/t7519-status-fsmonitor.sh | 142 +++++++++++++++++++++++++++++++++++ t/t7527-builtin-fsmonitor.sh | 27 ++++++- unpack-trees.c | 12 ++- 11 files changed, 280 insertions(+), 24 deletions(-) diff --git a/apply.c b/apply.c index f00b7ba4d3a7e6..1f5dda3b6f3fcc 100644 --- a/apply.c +++ b/apply.c @@ -3539,8 +3539,9 @@ static int verify_index_match(struct apply_state *state, return -1; return 0; } - return ie_match_stat(state->repo->index, ce, st, - CE_MATCH_IGNORE_VALID | CE_MATCH_IGNORE_SKIP_WORKTREE); + return ie_match_stat_with_content_check( + state->repo->index, ce, st, + CE_MATCH_IGNORE_VALID | CE_MATCH_IGNORE_SKIP_WORKTREE); } #define SUBMODULE_PATCH_WITHOUT_INDEX 1 diff --git a/diff-lib.c b/diff-lib.c index 46cae637ecda83..748f9ec3f3e7e7 100644 --- a/diff-lib.c +++ b/diff-lib.c @@ -90,7 +90,10 @@ static int match_stat_with_submodule(struct diff_options *diffopt, struct stat *st, unsigned ce_option, unsigned *dirty_submodule) { - int changed = ie_match_stat(diffopt->repo->index, ce, st, ce_option); + int changed; + + changed = ie_match_stat_with_content_check( + diffopt->repo->index, ce, st, ce_option); if (S_ISGITLINK(ce->ce_mode)) { struct diff_flags orig_flags = diffopt->flags; if (!diffopt->flags.override_submodule_config) diff --git a/entry.c b/entry.c index 1c4f0f44070ea3..9284e0d0d49123 100644 --- a/entry.c +++ b/entry.c @@ -512,8 +512,9 @@ int checkout_entry_ca(struct cache_entry *ce, struct conv_attrs *ca, if (!check_path(path.buf, path.len, &st, state->base_dir_len)) { const struct submodule *sub; - unsigned changed = ie_match_stat(state->istate, ce, &st, - CE_MATCH_IGNORE_VALID | CE_MATCH_IGNORE_SKIP_WORKTREE); + unsigned changed = ie_match_stat_with_content_check( + state->istate, ce, &st, + CE_MATCH_IGNORE_VALID | CE_MATCH_IGNORE_SKIP_WORKTREE); /* * Needs to be checked before !changed returns early, * as the possibly empty directory was not changed diff --git a/fsmonitor.c b/fsmonitor.c index 107767527ebec7..175377982d6ec9 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -189,13 +189,14 @@ static int query_fsmonitor_hook(struct repository *r, } /* - * Invalidate the FSM bit on this CE. This is like mark_fsmonitor_invalid() - * but we've already handled the untracked-cache, so let's not repeat that - * work. This also lets us have a different trace message so that we can - * see everything that was done as part of the refresh-callback. + * Strongly invalidate one cache entry without touching attributes or the + * untracked cache. Callers choose those wider invalidation scopes explicitly. */ -static void invalidate_ce_fsm(struct cache_entry *ce) +void fsmonitor_invalidate_cache_entry(struct cache_entry *ce) { + ce->ce_flags &= ~CE_UPTODATE; + memset(&ce->ce_stat_data, 0, sizeof(ce->ce_stat_data)); + ce->ce_flags |= CE_CONTENT_CHECK_REQUIRED; if (ce->ce_flags & CE_FSMONITOR_VALID) { trace_printf_key(&trace_fsmonitor, "fsmonitor_refresh_callback INV: '%s'", @@ -254,7 +255,7 @@ static size_t handle_using_name_hash_icase( */ untracked_cache_invalidate_trimmed_path(istate, ce->name, 0); - invalidate_ce_fsm(ce); + fsmonitor_invalidate_cache_entry(ce); return 1; } @@ -347,7 +348,7 @@ static size_t handle_path_without_trailing_slash( * cache-entry with the same pathname, nor for a cone * at that directory. (That is, assume no D/F conflicts.) */ - invalidate_ce_fsm(istate->cache[pos]); + fsmonitor_invalidate_cache_entry(istate->cache[pos]); return 1; } else { size_t nr_in_cone; @@ -425,7 +426,7 @@ static size_t handle_path_with_trailing_slash( for (i = pos; i < istate->cache_nr; i++) { if (!starts_with(istate->cache[i]->name, name)) break; - invalidate_ce_fsm(istate->cache[i]); + fsmonitor_invalidate_cache_entry(istate->cache[i]); nr_in_cone++; } diff --git a/fsmonitor.h b/fsmonitor.h index 5195a8624db82b..4027ca83f2b8cd 100644 --- a/fsmonitor.h +++ b/fsmonitor.h @@ -8,6 +8,13 @@ #include "read-cache-ll.h" #include "trace.h" +/* + * Force the next stat-aware caller to verify this entry's content. Wider + * invalidation, such as attributes or untracked-cache state, is the caller's + * responsibility. + */ +void fsmonitor_invalidate_cache_entry(struct cache_entry *ce); + /* * Check if refresh_fsmonitor has been called at least once. * refresh_fsmonitor is idempotent. Returns true if fsmonitor is diff --git a/read-cache-ll.h b/read-cache-ll.h index 71b87615ebc6d3..ed7d7aac38257e 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -69,14 +69,18 @@ struct cache_entry { */ #define CE_INTENT_TO_ADD (1 << 29) #define CE_SKIP_WORKTREE (1 << 30) -/* CE_EXTENDED2 is for future extension */ -#define CE_EXTENDED2 (1U << 31) +/* + * In-memory only. The cached stat data cannot be trusted, and callers which + * normally trust stat differences must verify content. This occupies the + * former never-persisted extension slot. + */ +#define CE_CONTENT_CHECK_REQUIRED (1U << 31) #define CE_EXTENDED_FLAGS (CE_INTENT_TO_ADD | CE_SKIP_WORKTREE) /* * Safeguard to avoid saving wrong flags: - * - CE_EXTENDED2 won't get saved until its semantic is known + * - CE_CONTENT_CHECK_REQUIRED is transient and must not be saved * - Bits in 0x0000FFFF have been saved in ce_flags already * - Bits in 0x003F0000 are currently in-memory flags */ @@ -120,7 +124,9 @@ static inline unsigned create_ce_flags(unsigned stage) #define ce_stage(ce) ((CE_STAGEMASK & (ce)->ce_flags) >> CE_STAGESHIFT) #define ce_uptodate(ce) ((ce)->ce_flags & CE_UPTODATE) #define ce_skip_worktree(ce) ((ce)->ce_flags & CE_SKIP_WORKTREE) -#define ce_mark_uptodate(ce) ((ce)->ce_flags |= CE_UPTODATE) +#define ce_mark_uptodate(ce) \ + ((ce)->ce_flags = ((ce)->ce_flags | CE_UPTODATE) & \ + ~CE_CONTENT_CHECK_REQUIRED) #define ce_intent_to_add(ce) ((ce)->ce_flags & CE_INTENT_TO_ADD) #define cache_entry_size(len) (offsetof(struct cache_entry,name) + (len) + 1) @@ -431,6 +437,13 @@ int is_racy_timestamp(const struct index_state *istate, int has_racy_timestamp(struct index_state *istate); int ie_match_stat(struct index_state *, const struct cache_entry *, struct stat *, unsigned int); int ie_modified(struct index_state *, const struct cache_entry *, struct stat *, unsigned int); +/* + * Unlike ie_match_stat(), verify content for marked non-gitlinks. Ordinary + * entries, including unmarked zero-stat entries, retain stat-only matching. + */ +int ie_match_stat_with_content_check(struct index_state *, + const struct cache_entry *, + struct stat *, unsigned int); int match_stat_data_racy(const struct index_state *istate, const struct stat_data *sd, struct stat *st); diff --git a/read-cache.c b/read-cache.c index 6c449f393d8d4d..fe989ce636003b 100644 --- a/read-cache.c +++ b/read-cache.c @@ -492,6 +492,32 @@ int ie_modified(struct index_state *istate, return 0; } +int ie_match_stat_with_content_check(struct index_state *istate, + const struct cache_entry *ce, + struct stat *st, unsigned int options) +{ + struct cache_entry *current; + int changed, pos; + + if (S_ISGITLINK(ce->ce_mode) || + !(ce->ce_flags & CE_CONTENT_CHECK_REQUIRED)) + return ie_match_stat(istate, ce, st, options); + + changed = ie_modified(istate, ce, st, options); + if (changed) + return changed; + + pos = index_name_pos(istate, ce->name, ce_namelen(ce)); + if (pos < 0 || istate->cache[pos] != ce) + return 0; + + current = istate->cache[pos]; + fill_stat_data(¤t->ce_stat_data, st); + current->ce_flags |= CE_UPDATE_IN_BASE; + istate->cache_changed |= CE_ENTRY_CHANGED; + return 0; +} + static int cache_name_stage_compare(const char *name1, int len1, int stage1, const char *name2, int len2, int stage2) { diff --git a/t/helper/test-read-cache.c b/t/helper/test-read-cache.c index 6b08ba8f078d00..f5dae8ecfcc485 100644 --- a/t/helper/test-read-cache.c +++ b/t/helper/test-read-cache.c @@ -3,15 +3,52 @@ #include "test-tool.h" #include "config.h" #include "environment.h" +#include "fsmonitor.h" #include "read-cache-ll.h" #include "repository.h" #include "setup.h" +static int test_fsmonitor_content_recovery(const char *path) +{ + struct index_state *istate; + struct cache_entry *ce; + struct stat_data empty = { 0 }; + struct stat st; + int pos; + + setup_git_directory(the_repository); + repo_config(the_repository, git_default_config, NULL); + if (repo_read_index(the_repository) < 0) + return error("unable to read test index"); + istate = the_repository->index; + pos = index_name_pos(istate, path, strlen(path)); + if (pos < 0) + return error("path is not indexed: %s", path); + ce = istate->cache[pos]; + if (lstat(path, &st)) + return error_errno("unable to stat indexed path"); + + fsmonitor_invalidate_cache_entry(ce); + if (memcmp(&ce->ce_stat_data, &empty, sizeof(empty))) + return error("invalidation did not poison cached stat data"); + if (ie_match_stat_with_content_check(istate, ce, &st, 0)) + return error("clean content did not match"); + if (!memcmp(&ce->ce_stat_data, &empty, sizeof(empty))) + return error("verified clean entry retained poisoned stat data"); + if (!(ce->ce_flags & CE_UPDATE_IN_BASE) || + !(istate->cache_changed & CE_ENTRY_CHANGED)) + return error("verified stat refresh was not marked for persistence"); + return 0; +} + int cmd__read_cache(int argc, const char **argv) { int i, cnt = 1; const char *name = NULL; + if (argc == 3 && + !strcmp(argv[1], "--test-fsmonitor-content-recovery")) + return test_fsmonitor_content_recovery(argv[2]); if (argc > 1 && skip_prefix(argv[1], "--print-and-refresh=", &name)) { argc--; argv++; diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 93973ed25a448b..1160612ea82177 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -477,4 +477,146 @@ test_expect_success 'status succeeds with sparse index' ' ) ' +test_expect_success 'reported events poison weak stat-cache matches' ' + test_create_repo reported-event && + ( + cd reported-event && + printf "aaaa\n" >tracked && + printf "clean\n" >clean && + git add tracked clean && + git commit -m base && + git config core.trustctime false && + git config core.checkStat minimal && + test-tool chmtime =-60 tracked clean && + git update-index --refresh && + mtime=$(test-tool chmtime --get tracked) && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token\0tracked\0clean\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git status --porcelain=v2 >/dev/null && + git status --porcelain=v2 >/dev/null && + printf "bbbb\n" >tracked && + test-tool chmtime =$mtime tracked && + + GIT_OPTIONAL_LOCKS=0 git diff-index --name-status HEAD \ + >.git/diff-index && + test_grep "^M.*tracked$" .git/diff-index && + test_grep ! "clean$" .git/diff-index && + git status --porcelain=v2 >.git/status && + test_grep "^1 \.M .* tracked$" .git/status && + test_grep ! " clean$" .git/status + ) +' + +test_expect_success 'reported path permits apply --index content match' ' + test_create_repo apply-marker && + ( + cd apply-marker && + test_commit base tracked && + test_write_lines next >tracked && + git diff >../apply-marker.patch && + git checkout -- tracked && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token\0tracked\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git apply --index ../apply-marker.patch + ) +' + +test_expect_success 'reported path permits checkout-index content match' ' + test_create_repo checkout-marker && + ( + cd checkout-marker && + test_commit base tracked && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token\0tracked\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git checkout-index tracked + ) +' + +test_expect_success CASE_INSENSITIVE_FS \ + 'reported path permits case-folded unpack match' ' + test_create_repo icase-marker && + ( + cd icase-marker && + test_write_lines same >foo && + git add foo && + git commit -m base && + base=$(git rev-parse HEAD) && + git mv foo intermediate && + git mv intermediate FOO && + git commit -m target && + target=$(git rev-parse HEAD) && + git checkout "$base" && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token\0foo\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git read-tree -m -u "$target" && + echo FOO >expect && + git ls-files >actual && + test_cmp expect actual + ) +' + +test_expect_success 'reported unchanged path avoids reset checkout' ' + test_create_repo reset-marker && + ( + cd reset-marker && + test_commit base tracked && + git config core.trustctime false && + git config core.checkStat minimal && + test-tool chmtime =-60 tracked && + git update-index --refresh && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token\0tracked\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + before=$(test-tool chmtime --get tracked) && + git reset --hard HEAD && + after=$(test-tool chmtime --get tracked) && + test "$before" = "$after" + ) +' + +test_expect_success 'ordinary zero-stat entries retain diff-index behavior' ' + test_create_repo ordinary-zero-stat && + ( + cd ordinary-zero-stat && + echo content >tracked && + git add tracked && + git commit -m base && + oid=$(git rev-parse :tracked) && + git update-index --cacheinfo 100644,$oid,tracked && + git -c core.fsmonitor=false diff-index --name-status HEAD >actual && + test_grep "^M.*tracked$" actual + ) +' + +test_expect_success 'verified reported paths restore poisoned stat data' ' + test_create_repo fsmonitor-stat-recovery && + ( + cd fsmonitor-stat-recovery && + echo content >tracked && + git add tracked && + git commit -m base && + test-tool read-cache \ + --test-fsmonitor-content-recovery tracked + ) +' + test_done diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index 86195770e97779..de7134af8b5c1a 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -1353,12 +1353,31 @@ test_expect_success CASE_INSENSITIVE_FS 'fsmonitor file case wrong on disk' ' test_grep ! -q "fsmonitor_refresh_callback.*FILE-4-A.*pos" "$PWD/file_case_wrong-try2.log" && test_grep ! -q "fsmonitor_refresh_callback.*file-4-a.*pos" "$PWD/file_case_wrong-try2.log" && - # FSM refresh saw nothing, so it will mark all files as valid, - # so they should now have "h" status. + # A late directory event can arrive without repeating the file + # events checked above. Such an event invalidates its entire cone, + # so those entries remain "H" until the next quiet refresh. git -C file_case_wrong ls-files -f >"$PWD/file_case_wrong-lsf2.out" && - test_grep -q "h dir1/dir2/dir3/file-3-a" "$PWD/file_case_wrong-lsf2.out" && - test_grep -q "h dir1/dir2/dir4/FILE-4-A" "$PWD/file_case_wrong-lsf2.out" && + if test_grep -E -q \ + "fsmonitor_refresh_callback .dir1(/dir2(/dir3)?)?/?. .*pos " \ + "$PWD/file_case_wrong-try2.log" >/dev/null 2>&1 4>/dev/null + then + expected_3=H + else + expected_3=h + fi && + test_grep -q "$expected_3 dir1/dir2/dir3/file-3-a" \ + "$PWD/file_case_wrong-lsf2.out" && + if test_grep -E -q \ + "fsmonitor_refresh_callback .dir1(/dir2(/dir4)?)?/?. .*pos " \ + "$PWD/file_case_wrong-try2.log" >/dev/null 2>&1 4>/dev/null + then + expected_4=H + else + expected_4=h + fi && + test_grep -q "$expected_4 dir1/dir2/dir4/FILE-4-A" \ + "$PWD/file_case_wrong-lsf2.out" && # We now have files with clean content, but with case-incorrect diff --git a/unpack-trees.c b/unpack-trees.c index 154d6d40a15934..44d3567c83844b 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -2241,7 +2241,8 @@ static int verify_uptodate_1(const struct cache_entry *ce, if (!lstat(ce->name, &st)) { int flags = CE_MATCH_IGNORE_VALID|CE_MATCH_IGNORE_SKIP_WORKTREE; - unsigned changed = ie_match_stat(o->src_index, ce, &st, flags); + unsigned changed = ie_match_stat_with_content_check( + o->src_index, ce, &st, flags); if (submodule_from_ce(ce)) { int r = check_submodule_move_head(ce, @@ -2407,7 +2408,9 @@ static int icase_exists(struct unpack_trees_options *o, const char *name, int le const struct cache_entry *src; src = index_file_exists(o->src_index, name, len, 1); - return src && !ie_match_stat(o->src_index, src, st, CE_MATCH_IGNORE_VALID|CE_MATCH_IGNORE_SKIP_WORKTREE); + return src && !ie_match_stat_with_content_check( + o->src_index, src, st, + CE_MATCH_IGNORE_VALID | CE_MATCH_IGNORE_SKIP_WORKTREE); } enum absent_checking_type { @@ -3038,7 +3041,10 @@ int oneway_merge(const struct cache_entry * const *src, !(old->ce_flags & CE_FSMONITOR_VALID)) { struct stat st; if (lstat(old->name, &st) || - ie_match_stat(o->src_index, old, &st, CE_MATCH_IGNORE_VALID|CE_MATCH_IGNORE_SKIP_WORKTREE)) + ie_match_stat_with_content_check( + o->src_index, old, &st, + CE_MATCH_IGNORE_VALID | + CE_MATCH_IGNORE_SKIP_WORKTREE)) update |= CE_UPDATE; } if (o->update && S_ISGITLINK(old->ce_mode) && From 8b213971da9454256952d8ab5aea00804e92b950 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 25 Jul 2026 21:48:16 -0500 Subject: [PATCH 009/105] status: gate automatic UNTR validation preloads Starting directory-validation workers for a small cache, restricted pathspec, incompatible traversal, or fsmonitor-managed cache adds work without providing a safe whole-worktree reuse opportunity. Enable automatic preload only when the existing untracked cache and its root are valid, fsmonitor is disabled, traversal flags agree, and a bounded count finds at least 2,000 cached directories. Reject pathspecs, disabled untracked output, ignored-output modes, and incompatible -uall cache settings. Keep the test override for focused small-cache coverage. Status tests exercise both sides of the directory threshold and verify that restricted pathspecs and incompatible -uall requests retain the ordinary traversal path. Signed-off-by: Taylor Blau --- dir.c | 44 ++++++++++++++++++++++++++--- t/t7063-status-untracked-cache.sh | 47 ++++++++++++++++++++++++++++++- wt-status.c | 5 ++++ 3 files changed, 91 insertions(+), 5 deletions(-) diff --git a/dir.c b/dir.c index c97f59cbf3b1de..8ad94248770a39 100644 --- a/dir.c +++ b/dir.c @@ -143,8 +143,33 @@ static void collect_untracked_cache_preload_tasks( } } -struct untracked_cache_preload *untracked_cache_preload_start_ordinary( - struct index_state *istate, unsigned int dir_flags) +static size_t count_untracked_cache_dirs_bounded( + const struct untracked_cache_dir *ucd, + size_t limit) +{ + size_t i, nr = 1; + + for (i = 0; i < ucd->dirs_nr && nr < limit; i++) + nr += count_untracked_cache_dirs_bounded(ucd->dirs[i], limit - nr); + return nr; +} + +#define UNTRACKED_CACHE_AUTO_PRELOAD_MIN_DIRS 2000 + +static int untracked_cache_auto_preload_worthwhile( + const struct untracked_cache *uc) +{ + if (!uc || !uc->root || uc->use_fsmonitor || !uc->root->valid) + return 0; + if (git_env_bool("GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD", 0)) + return 1; + return count_untracked_cache_dirs_bounded( + uc->root, UNTRACKED_CACHE_AUTO_PRELOAD_MIN_DIRS) >= + UNTRACKED_CACHE_AUTO_PRELOAD_MIN_DIRS; +} + +static struct untracked_cache_preload *untracked_cache_preload_start_1( + struct index_state *istate, unsigned int dir_flags, int automatic) { struct untracked_cache *uc = istate->untracked; struct untracked_cache_preload *preload; @@ -153,8 +178,6 @@ struct untracked_cache_preload *untracked_cache_preload_start_ordinary( unsigned long test_threads; int threads, online, create_threads = 1; - if (!git_env_bool("GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD", 0)) - return NULL; if (!uc || !uc->root || uc->use_fsmonitor || uc->dir_flags != dir_flags) return NULL; @@ -188,6 +211,8 @@ struct untracked_cache_preload *untracked_cache_preload_start_ordinary( preload->started_at = getnanotime(); trace2_data_intmax("dir", istate->repo, "preload_untracked_cache/threads", threads); + trace2_data_intmax("dir", istate->repo, + "preload_untracked_cache/automatic", automatic); CALLOC_ARRAY(preload->data, threads); work = DIV_ROUND_UP(preload->nr, threads); for (i = 0; i < threads; i++) { @@ -216,6 +241,17 @@ struct untracked_cache_preload *untracked_cache_preload_start_ordinary( return preload; } +struct untracked_cache_preload *untracked_cache_preload_start_ordinary( + struct index_state *istate, unsigned int dir_flags) +{ + struct untracked_cache *uc = istate->untracked; + + if (!uc || uc->dir_flags != dir_flags || + !untracked_cache_auto_preload_worthwhile(uc)) + return NULL; + return untracked_cache_preload_start_1(istate, dir_flags, 1); +} + static void *preload_untracked_cache_thread(void *_data) { struct untracked_cache_preload_data *data = _data; diff --git a/t/t7063-status-untracked-cache.sh b/t/t7063-status-untracked-cache.sh index 8f49cfeebb6896..9f7c5fee79d639 100755 --- a/t/t7063-status-untracked-cache.sh +++ b/t/t7063-status-untracked-cache.sh @@ -1017,6 +1017,35 @@ test_expect_success 'directory snapshots ignore weak file-stat configuration' ' ) ' +test_expect_success 'automatic preload observes its directory threshold' ' + test_create_repo auto-preload-threshold && + ( + cd auto-preload-threshold && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor false && + sane_unset GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD && + for i in $(test_seq 1 1998) + do + mkdir "d$i" && + >"d$i/file" || return 1 + done && + git status --porcelain >/dev/null && + GIT_TRACE2_EVENT="$PWD/.git/below-threshold.trace" \ + git status --porcelain >/dev/null && + test_grep ! "preload_untracked_cache/automatic" \ + .git/below-threshold.trace && + + mkdir d1999 && + >d1999/file && + git status --porcelain >/dev/null && + GIT_TRACE2_EVENT="$PWD/.git/at-threshold.trace" \ + git status --porcelain >/dev/null && + test_grep \ + "preload_untracked_cache/automatic.*value.*1" \ + .git/at-threshold.trace + ) +' test_expect_success 'status preloads cached-directory validation' ' test_create_repo auto-preload && ( @@ -1062,7 +1091,23 @@ test_expect_success 'status preloads cached-directory validation' ' test_grep "preload_untracked_cache/valid.*value.*0" \ .git/changed.trace && test_grep "opendir.*value.*[1-9][0-9]*" \ - .git/changed.trace + .git/changed.trace && + + GIT_OPTIONAL_LOCKS=0 git -c core.untrackedCache=false \ + status --porcelain -- nested >.git/expect-pathspec && + GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD=1 \ + GIT_TRACE2_EVENT="$PWD/.git/pathspec.trace" \ + git status --porcelain -- nested >.git/actual-pathspec && + test_cmp .git/expect-pathspec .git/actual-pathspec && + test_grep ! 'preload_untracked_cache/threads' \ + .git/pathspec.trace && + GIT_OPTIONAL_LOCKS=0 git -c core.untrackedCache=false \ + status --porcelain -uall >.git/expect-uall && + GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD=1 \ + GIT_TRACE2_EVENT="$PWD/.git/uall.trace" \ + git status --porcelain -uall >.git/actual-uall && + test_cmp .git/expect-uall .git/actual-uall && + test_grep ! 'preload_untracked_cache/threads' .git/uall.trace ) ' diff --git a/wt-status.c b/wt-status.c index ffb9ff8f044fd9..da642642d4a229 100644 --- a/wt-status.c +++ b/wt-status.c @@ -817,6 +817,11 @@ void wt_status_start_untracked_cache_preload(struct wt_status *s) if (s->untracked_cache_preload) BUG("untracked-cache preload already started"); + if (fsm_settings__get_mode(s->repo) > FSMONITOR_MODE_DISABLED || + s->pathspec.nr || + s->show_untracked_files == SHOW_NO_UNTRACKED_FILES || + s->show_ignored_mode) + return; dir_flags = wt_status_untracked_dir_flags(s); s->untracked_cache_preload = From b1fc6305c013e846dfe76a9ab191a7c28dcef8e8 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 11 Jul 2026 11:30:07 -0700 Subject: [PATCH 010/105] fsmonitor: support provider-wide invalidation A filesystem-monitor provider can know that its event history is incomplete without being able to identify every affected path. Treating such a response as an ordinary path leaves tracked entries, cached attributes, and untracked-cache state falsely valid. Reserve // as a provider-only global invalidation record. It cannot collide with a worktree-relative path. When the client receives it, discard cached attribute stacks and untracked-cache state, invalidate every tracked entry, and mark the fsmonitor extension changed. Recognize the existing trivial response only when a complete record consists of a single slash and NUL, newline, or carriage-return terminators. This prevents the new double-slash record from being discarded as a trivial response while preserving existing hook forms. Add a hook regression in t/t7519-status-fsmonitor.sh that changes a tracked file, restores its timestamp, emits the global marker, and requires status to report the change. Global invalidation intentionally scans the tracked index. Signed-off-by: Taylor Blau --- attr.c | 5 +++++ attr.h | 3 +++ dir.c | 9 +++++++++ dir.h | 1 + fsmonitor-ll.h | 3 +++ fsmonitor.c | 39 ++++++++++++++++++++++++++++++++----- t/t7519-status-fsmonitor.sh | 33 +++++++++++++++++++++++++++++++ 7 files changed, 88 insertions(+), 5 deletions(-) diff --git a/attr.c b/attr.c index 0e63f1b6de8f53..87808ba3755d04 100644 --- a/attr.c +++ b/attr.c @@ -536,6 +536,11 @@ static void drop_all_attr_stacks(void) vector_unlock(); } +void git_attr_invalidate_all(void) +{ + drop_all_attr_stacks(); +} + struct attr_check *attr_check_alloc(void) { struct attr_check *c = xcalloc(1, sizeof(struct attr_check)); diff --git a/attr.h b/attr.h index a04a5210921e22..cca94379362f10 100644 --- a/attr.h +++ b/attr.h @@ -227,6 +227,9 @@ enum git_attr_direction { }; void git_attr_set_direction(enum git_attr_direction new_direction); +/* Discard cached attributes after a provider-wide invalidation. */ +void git_attr_invalidate_all(void); + void attr_start(void); /* Return the system gitattributes file. */ diff --git a/dir.c b/dir.c index 95d8a1cce90f77..ad8f43f59536f8 100644 --- a/dir.c +++ b/dir.c @@ -1112,6 +1112,15 @@ static void invalidate_gitignore(struct untracked_cache *uc, do_invalidate_gitignore(dir); } +void untracked_cache_invalidate_all(struct index_state *istate) +{ + if (!istate->untracked || !istate->untracked->root) + return; + invalidate_gitignore(istate->untracked, istate->untracked->root); + istate->untracked->use_fsmonitor = 0; + istate->cache_changed |= UNTRACKED_CHANGED; +} + static void invalidate_directory(struct untracked_cache *uc, struct untracked_cache_dir *dir) { diff --git a/dir.h b/dir.h index 83e0f648a81f36..815225ee147e01 100644 --- a/dir.h +++ b/dir.h @@ -597,6 +597,7 @@ int cmp_dir_entry(const void *p1, const void *p2); int check_dir_entry_contains(const struct dir_entry *out, const struct dir_entry *in); void untracked_cache_invalidate_path(struct index_state *, const char *, int safe_path); +void untracked_cache_invalidate_all(struct index_state *); /* * Invalidate the untracked-cache for this path, but first strip * off a trailing slash, if present. diff --git a/fsmonitor-ll.h b/fsmonitor-ll.h index 0504ca07d62fa1..a409b15e68bc51 100644 --- a/fsmonitor-ll.h +++ b/fsmonitor-ll.h @@ -4,6 +4,9 @@ struct index_state; struct strbuf; +/* A provider-only marker; worktree-relative paths cannot begin with '/'. */ +#define FSMONITOR_PATH_GLOBAL_INVALIDATE "//" + extern struct trace_key trace_fsmonitor; /* diff --git a/fsmonitor.c b/fsmonitor.c index 175377982d6ec9..6c119b17bd391e 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -2,6 +2,7 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "git-compat-util.h" +#include "attr.h" #include "config.h" #include "dir.h" #include "environment.h" @@ -436,12 +437,26 @@ static size_t handle_path_with_trailing_slash( static void fsmonitor_refresh_callback(struct index_state *istate, char *name) { int len = strlen(name); - int pos = index_name_pos(istate, name, len); + int pos; size_t nr_in_cone; trace_printf_key(&trace_fsmonitor, "fsmonitor_refresh_callback '%s' (pos %d)", - name, pos); + name, !strcmp(name, FSMONITOR_PATH_GLOBAL_INVALIDATE) ? + -1 : index_name_pos(istate, name, len)); + if (!strcmp(name, FSMONITOR_PATH_GLOBAL_INVALIDATE)) { + unsigned int i; + + git_attr_invalidate_all(); + untracked_cache_invalidate_all(istate); + for (i = 0; i < istate->cache_nr; i++) + fsmonitor_invalidate_cache_entry(istate->cache[i]); + istate->cache_changed |= FSMONITOR_CHANGED; + trace2_data_intmax("fsmonitor", istate->repo, + "apply/global-invalidation", 1); + return; + } + pos = index_name_pos(istate, name, len); if (name[len - 1] == '/') nr_in_cone = handle_path_with_trailing_slash(istate, name, pos); @@ -504,6 +519,19 @@ static void fsmonitor_refresh_callback(struct index_state *istate, char *name) */ static int fsmonitor_force_update_threshold = 100; +static int is_trivial_response_at(const struct strbuf *result, size_t offset) +{ + size_t i; + + if (offset >= result->len || result->buf[offset] != '/') + return 0; + for (i = offset + 1; i < result->len; i++) + if (result->buf[i] != '\0' && result->buf[i] != '\n' && + result->buf[i] != '\r') + return 0; + return 1; +} + void refresh_fsmonitor(struct index_state *istate) { static int warn_once = 0; @@ -552,7 +580,7 @@ void refresh_fsmonitor(struct index_state *istate) buf = query_result.buf; strbuf_addstr(&last_update_token, buf); bol = last_update_token.len + 1; - is_trivial = query_result.buf[bol] == '/'; + is_trivial = is_trivial_response_at(&query_result, bol); if (is_trivial) trace2_data_intmax("fsm_client", NULL, "query/trivial-response", 1); @@ -613,7 +641,8 @@ void refresh_fsmonitor(struct index_state *istate) query_success = 0; } else { bol = last_update_token.len + 1; - is_trivial = query_result.buf[bol] == '/'; + is_trivial = is_trivial_response_at( + &query_result, bol); } } else if (hook_version < 0) { hook_version = HOOK_INTERFACE_VERSION1; @@ -627,7 +656,7 @@ void refresh_fsmonitor(struct index_state *istate) r, HOOK_INTERFACE_VERSION1, istate->fsmonitor_last_update, &query_result); if (query_success) - is_trivial = query_result.buf[0] == '/'; + is_trivial = is_trivial_response_at(&query_result, 0); } if (is_trivial) diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 1160612ea82177..a0a20aa80e4a8f 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -619,4 +619,37 @@ test_expect_success 'verified reported paths restore poisoned stat data' ' ) ' +test_expect_success 'provider global marker invalidates every tracked entry' ' + test_create_repo global-invalidate && + ( + cd global-invalidate && + printf "aaaa\n" >tracked && + git add tracked && + git commit -m base && + git config core.trustctime false && + git config core.checkStat minimal && + test-tool chmtime =-60 tracked && + git update-index --refresh && + mtime=$(test-tool chmtime --get tracked) && + test_hook --setup fsmonitor-test <<-\EOF && + if test -f .git/global + then + printf "token1\0//\0" + else + printf "token0\0" + fi + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git status --porcelain=v2 >/dev/null && + git status --porcelain=v2 >/dev/null && + printf "bbbb\n" >tracked && + test-tool chmtime =$mtime tracked && + > .git/global && + git status --porcelain=v2 >.git/actual && + test_grep "^1 \.M .* tracked$" .git/actual + ) +' + test_done From 9485e67dd398b09bfa301c7e943b84cbba8490fc Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 01:57:04 -0500 Subject: [PATCH 011/105] fsmonitor: rediscover linked worktrees when starting a daemon An implicitly started fsmonitor daemon inherits its caller's repository environment and current directory. In a linked worktree, inherited Git directory, worktree, common-directory, prefix, and index settings can make the child discover a different repository than the worktree whose status requested the daemon. Resolve the requested worktree to its canonical path, start the child from that directory, and remove repository-addressing variables from its environment. Keep the existing daemon start command and return an error if the worktree cannot be resolved. Add a macOS regression that implicitly starts fsmonitor from a linked worktree and checks the daemon child's working directory in Trace2. Signed-off-by: Taylor Blau --- fsmonitor-ipc.c | 29 ++++++++++++++++++++++++++++- t/t7527-builtin-fsmonitor.sh | 29 +++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/fsmonitor-ipc.c b/fsmonitor-ipc.c index 6112d130644f04..78720fa4aba04c 100644 --- a/fsmonitor-ipc.c +++ b/fsmonitor-ipc.c @@ -1,6 +1,8 @@ #define USE_THE_REPOSITORY_VARIABLE #include "git-compat-util.h" +#include "abspath.h" +#include "environment.h" #include "gettext.h" #include "simple-ipc.h" #include "fsmonitor-ipc.h" @@ -45,6 +47,17 @@ int fsmonitor_ipc__send_command(const char *command UNUSED, #else +static void prepare_spawn_env(struct strvec *env) +{ + /* Let the child rediscover this repository from the worktree. */ + strvec_push(env, GIT_DIR_ENVIRONMENT); + strvec_push(env, GIT_WORK_TREE_ENVIRONMENT); + strvec_push(env, GIT_COMMON_DIR_ENVIRONMENT); + strvec_push(env, GIT_PREFIX_ENVIRONMENT); + strvec_push(env, GIT_IMPLICIT_WORK_TREE_ENVIRONMENT); + strvec_push(env, INDEX_ENVIRONMENT); +} + int fsmonitor_ipc__is_supported(void) { return 1; @@ -58,7 +71,18 @@ enum ipc_active_state fsmonitor_ipc__get_state(void) static int spawn_daemon(void) { struct child_process cmd = CHILD_PROCESS_INIT; + struct strbuf canonical_worktree = STRBUF_INIT; + const char *worktree = repo_get_work_tree(the_repository); + int ret = -1; + if (!worktree || + !strbuf_realpath(&canonical_worktree, worktree, 0)) { + error(_("cannot start fsmonitor daemon without a work tree")); + goto done; + } + + prepare_spawn_env(&cmd.env); + cmd.dir = canonical_worktree.buf; cmd.git_cmd = 1; cmd.no_stdin = 1; cmd.no_stdout = 1; @@ -67,7 +91,10 @@ static int spawn_daemon(void) cmd.trace2_child_class = "fsmonitor"; strvec_pushl(&cmd.args, "fsmonitor--daemon", "start", NULL); - return run_command(&cmd); + ret = run_command(&cmd); +done: + strbuf_release(&canonical_worktree); + return ret; } int fsmonitor_ipc__send_query(const char *since_token, diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index 86195770e97779..9f463a3f7a8dbd 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -1389,4 +1389,33 @@ test_expect_success CASE_INSENSITIVE_FS 'fsmonitor file case wrong on disk' ' test_grep -q " M dir1/dir2/dir4/FILE-4-A" "$PWD/file_case_wrong-try3.out" ' +test_expect_success MACOS 'implicit daemon rediscovers a linked worktree' ' + test_when_finished " + git -C reexec-linked-wt fsmonitor--daemon stop 2>/dev/null || : + git -C reexec-linked-main worktree remove --force \ + ../reexec-linked-wt 2>/dev/null || : + " && + test_create_repo reexec-linked-main && + ( + cd reexec-linked-main && + test_commit base tracked && + git worktree add ../reexec-linked-wt && + git -C ../reexec-linked-wt config core.untrackedCache true && + git -C ../reexec-linked-wt config core.fsmonitor true && + linked_worktree=$(test-tool path-utils real_path \ + ../reexec-linked-wt) && + GIT_TRACE2_EVENT="$PWD/../reexec-linked.trace" \ + git -C ../reexec-linked-wt status --porcelain=v2 \ + >../reexec-linked.actual && + test_must_be_empty ../reexec-linked.actual && + test_subcommand git fsmonitor--daemon start \ + <../reexec-linked.trace && + test_grep \ + "\"child_class\":\"fsmonitor\",\"cd\":\"$linked_worktree\"" \ + ../reexec-linked.trace && + git -C ../reexec-linked-wt fsmonitor--daemon stop && + git worktree remove ../reexec-linked-wt + ) +' + test_done From 270b6f0eabb3d769eba44aa72e1673ae4d397346 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 21:41:23 -0700 Subject: [PATCH 012/105] fsmonitor: keep multiply-linked files invalid A pathname monitor cannot establish that every name for a multiply-linked regular file lies inside its watch cone. Persisting CE_FSMONITOR_VALID after checking the tracked name can therefore hide a later write through an unmonitored hardlink. Use fsmonitor_stat_can_be_valid() to exclude regular files with more than one link from persistent fsmonitor validity when the platform reports real link counts. Apply that decision where index refresh, threaded preload, and diff-files first consume an actual stat. Preserve CE_UPTODATE for the current process and retain existing persistent validity for single-link and nonregular entries. Windows and Cygwin synthesize their link counts, so preserve their existing fsmonitor behavior without claiming the hardlink guarantee there. Add a hardlink regression in t/t7519-status-fsmonitor.sh on platforms with trustworthy stat metadata. It keeps a tracked hardlink outside the fsmonitor-valid bitmap and checks that a write through an alias outside the worktree appears in status. The deliberate cost is another stat in a subsequent process. Signed-off-by: Taylor Blau --- diff-lib.c | 7 ++++++- fsmonitor.h | 13 +++++++++++++ preload-index.c | 3 ++- read-cache.c | 6 ++++-- t/t7519-status-fsmonitor.sh | 31 +++++++++++++++++++++++++++++++ 5 files changed, 56 insertions(+), 4 deletions(-) diff --git a/diff-lib.c b/diff-lib.c index 748f9ec3f3e7e7..33b4233738cac3 100644 --- a/diff-lib.c +++ b/diff-lib.c @@ -130,6 +130,7 @@ void run_diff_files(struct rev_info *revs, unsigned int option) entries = istate->cache_nr; for (i = 0; i < entries; i++) { unsigned int oldmode, newmode; + int fsmonitor_valid = 0; struct cache_entry *ce = istate->cache[i]; int changed; unsigned dirty_submodule = 0; @@ -249,6 +250,8 @@ void run_diff_files(struct rev_info *revs, unsigned int option) if (ce->ce_flags & (CE_VALID | CE_FSMONITOR_VALID)) { changed = 0; newmode = ce->ce_mode; + fsmonitor_valid = + !!(ce->ce_flags & CE_FSMONITOR_VALID); } else { struct stat st; @@ -274,11 +277,13 @@ void run_diff_files(struct rev_info *revs, unsigned int option) changed = match_stat_with_submodule(&revs->diffopt, ce, &st, ce_option, &dirty_submodule); newmode = ce_mode_from_stat(revs->repo, ce, st.st_mode); + fsmonitor_valid = fsmonitor_stat_can_be_valid(&st); } if (!changed && !dirty_submodule) { ce_mark_uptodate(ce); - mark_fsmonitor_valid(istate, ce); + if (fsmonitor_valid) + mark_fsmonitor_valid(istate, ce); if (revs->diffopt.flags.find_copies_harder) diff_same(&revs->diffopt, newmode, &ce->oid, ce->name); diff --git a/fsmonitor.h b/fsmonitor.h index 4027ca83f2b8cd..47ce78de61c508 100644 --- a/fsmonitor.h +++ b/fsmonitor.h @@ -15,6 +15,19 @@ */ void fsmonitor_invalidate_cache_entry(struct cache_entry *ce); +/* + * A pathname monitor cannot prove that every name for a multiply-linked + * inode is inside its watch cone. When the platform reports real link + * counts, keep such regular files out of the persistent valid bitmap so + * that every new process checks their stat data. Platforms that synthesize + * link counts retain their existing fsmonitor behavior. The in-process + * CE_UPTODATE bit is still safe after the caller's lstat(). + */ +static inline int fsmonitor_stat_can_be_valid(const struct stat *st) +{ + return !S_ISREG(st->st_mode) || st->st_nlink <= 1; +} + /* * Check if refresh_fsmonitor has been called at least once. * refresh_fsmonitor is idempotent. Returns true if fsmonitor is diff --git a/preload-index.c b/preload-index.c index b222821b448526..6c675339285257 100644 --- a/preload-index.c +++ b/preload-index.c @@ -90,7 +90,8 @@ static void *preload_thread(void *_data) if (ie_match_stat(index, ce, &st, CE_MATCH_RACY_IS_DIRTY|CE_MATCH_IGNORE_FSMONITOR)) continue; ce_mark_uptodate(ce); - mark_fsmonitor_valid(index, ce); + if (fsmonitor_stat_can_be_valid(&st)) + mark_fsmonitor_valid(index, ce); } while (--nr > 0); if (p->progress) { struct progress_data *pd = p->progress; diff --git a/read-cache.c b/read-cache.c index fe989ce636003b..b853d79c72aa5f 100644 --- a/read-cache.c +++ b/read-cache.c @@ -199,7 +199,8 @@ void fill_stat_cache_info(struct index_state *istate, struct cache_entry *ce, st if (S_ISREG(st->st_mode)) { ce_mark_uptodate(ce); - mark_fsmonitor_valid(istate, ce); + if (fsmonitor_stat_can_be_valid(st)) + mark_fsmonitor_valid(istate, ce); } } @@ -1441,7 +1442,8 @@ static struct cache_entry *refresh_cache_ent(struct index_state *istate, */ if (!S_ISGITLINK(ce->ce_mode)) { ce_mark_uptodate(ce); - mark_fsmonitor_valid(istate, ce); + if (fsmonitor_stat_can_be_valid(&st)) + mark_fsmonitor_valid(istate, ce); } return ce; } diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index a0a20aa80e4a8f..fb2fadc53d5986 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -55,6 +55,11 @@ test_lazy_prereq UNTRACKED_CACHE ' test $ret -ne 1 ' +test_lazy_prereq HARDLINKS ' + : >hardlink-a && + ln hardlink-a hardlink-b +' + # Test that we detect and disallow repos that are incompatible with FSMonitor. test_expect_success 'incompatible bare repo' ' test_when_finished "rm -rf ./bare-clone actual expect" && @@ -652,4 +657,30 @@ test_expect_success 'provider global marker invalidates every tracked entry' ' ) ' +test_expect_success HARDLINKS,!MINGW,!CYGWIN \ + 'multiply-linked files stay fsmonitor-invalid' ' + test_when_finished "rm -f hardlink-alias" && + test_create_repo hardlink-validity && + ( + cd hardlink-validity && + echo content >tracked && + git add tracked && + git commit -m base && + ln tracked ../hardlink-alias && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git status --porcelain=v2 >/dev/null && + git status --porcelain=v2 >/dev/null && + git ls-files -f >.git/flags && + test_grep "^H tracked$" .git/flags && + echo changed >>../hardlink-alias && + git status --porcelain=v2 >.git/actual && + test_grep "^1 \.M .* tracked$" .git/actual + ) +' + test_done From db283634d07f830865acc78b3e8e0e022f2e3343 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 01:57:14 -0500 Subject: [PATCH 013/105] fsmonitor: start implicit daemons with the invoking Git executable Implicit fsmonitor startup resolves a Git command through the execution path and invokes its start subcommand. An overridden execution path can therefore select a different Git than the dispatcher that initiated the query, while adding another launcher between the client and daemon. Retain the absolute executable path during dispatcher initialization and expose it only for a real Git dispatcher. Start that executable directly with fsmonitor--daemon run --detach, then wait until its IPC socket is listening before accepting startup. Respect the configured startup timeout, defaulting to 60 seconds, and retain Git-command lookup when an authoritative dispatcher path is unavailable. The canonical worktree and sanitized environment established by S03/P01 remain in place. Update existing startup Trace2 checks for the direct invocation and add a macOS regression with a fake Git on the execution path to verify that the original executable is used. Signed-off-by: Taylor Blau --- exec-cmd.c | 44 +++++++++++++++++++++++++++++--- exec-cmd.h | 2 ++ fsmonitor-ipc.c | 49 +++++++++++++++++++++++++++++++++--- git.c | 3 +++ t/t7527-builtin-fsmonitor.sh | 37 +++++++++++++++++++++++---- 5 files changed, 123 insertions(+), 12 deletions(-) diff --git a/exec-cmd.c b/exec-cmd.c index 507e67d528b0dd..dc801d1a6d35d4 100644 --- a/exec-cmd.c +++ b/exec-cmd.c @@ -27,6 +27,14 @@ static const char *system_prefix(void); +/* + * Absolute path to the current executable, when it can be determined. Keep + * this separately from executable_dirname because some callers need to + * re-execute this exact Git rather than resolve "git" through PATH. + */ +static const char *executable_path; +static int executable_is_dispatcher; + #ifdef RUNTIME_PREFIX /** @@ -257,7 +265,8 @@ void git_resolve_executable_dir(const char *argv0) return; } - resolved = strbuf_detach(&buf, NULL); + executable_path = strbuf_detach(&buf, NULL); + resolved = xstrdup(executable_path); slash = find_last_dir_sep(resolved); if (slash) resolved[slash - resolved] = '\0'; @@ -278,15 +287,42 @@ static const char *system_prefix(void) } /* - * This is called during initialization, but No work needs to be done here when - * runtime prefix is not being used. + * A non-runtime-prefix build does not need the executable directory for path + * discovery, but an explicit argv[0] is still useful for exact re-execution. */ -void git_resolve_executable_dir(const char *argv0 UNUSED) +void git_resolve_executable_dir(const char *argv0) { + struct strbuf buf = STRBUF_INIT; + + /* A bare argv[0] would require a PATH lookup and is not authoritative. */ + if (!argv0 || !*argv0 || !find_last_dir_sep(argv0)) + return; + strbuf_add_absolute_path(&buf, argv0); + if (strbuf_normalize_path(&buf)) { + trace_printf("trace: could not normalize executable path: %s\n", + buf.buf); + strbuf_release(&buf); + return; + } + executable_path = strbuf_detach(&buf, NULL); + trace2_cmd_path(executable_path); } #endif /* RUNTIME_PREFIX */ +const char *git_executable_path(void) +{ + /* Helpers have an exact path too, but cannot dispatch Git builtins. */ + if (!executable_path || !executable_is_dispatcher) + return NULL; + return executable_path; +} + +void git_mark_executable_as_dispatcher(void) +{ + executable_is_dispatcher = 1; +} + char *system_path(const char *path) { struct strbuf d = STRBUF_INIT; diff --git a/exec-cmd.h b/exec-cmd.h index 330b41d54dec52..0613765ef7e4ee 100644 --- a/exec-cmd.h +++ b/exec-cmd.h @@ -5,6 +5,8 @@ struct strvec; void git_set_exec_path(const char *exec_path); void git_resolve_executable_dir(const char *path); +const char *git_executable_path(void); +void git_mark_executable_as_dispatcher(void); const char *git_exec_path(void); void setup_path(void); const char **prepare_git_cmd(struct strvec *out, const char **argv); diff --git a/fsmonitor-ipc.c b/fsmonitor-ipc.c index 78720fa4aba04c..8957091bfccbb2 100644 --- a/fsmonitor-ipc.c +++ b/fsmonitor-ipc.c @@ -2,8 +2,11 @@ #include "git-compat-util.h" #include "abspath.h" +#include "config.h" #include "environment.h" +#include "exec-cmd.h" #include "gettext.h" +#include "parse.h" #include "simple-ipc.h" #include "fsmonitor-ipc.h" #include "repository.h" @@ -68,10 +71,44 @@ enum ipc_active_state fsmonitor_ipc__get_state(void) return ipc_get_active_state(fsmonitor_ipc__get_path(the_repository)); } +#define FSMONITOR_START_TIMEOUT_KEY "fsmonitor.starttimeout" +#define FSMONITOR_START_TIMEOUT_DEFAULT 60 + +static unsigned int get_start_timeout(void) +{ + const char *value; + int timeout; + + if (!repo_config_get_value(the_repository, + FSMONITOR_START_TIMEOUT_KEY, &value) && + value && git_parse_int(value, &timeout) && timeout >= 0) + return timeout; + return FSMONITOR_START_TIMEOUT_DEFAULT; +} + +static int spawn_wait_cb(const struct child_process *cmd UNUSED, + void *cb_data UNUSED) +{ + switch (fsmonitor_ipc__get_state()) { + case IPC_STATE__LISTENING: + return 0; + case IPC_STATE__NOT_LISTENING: + case IPC_STATE__PATH_NOT_FOUND: + return 1; + default: + case IPC_STATE__INVALID_PATH: + case IPC_STATE__OTHER_ERROR: + return -1; + } +} + static int spawn_daemon(void) { struct child_process cmd = CHILD_PROCESS_INIT; struct strbuf canonical_worktree = STRBUF_INIT; + enum start_bg_result result; + unsigned int timeout = get_start_timeout(); + const char *git = git_executable_path(); const char *worktree = repo_get_work_tree(the_repository); int ret = -1; @@ -83,15 +120,21 @@ static int spawn_daemon(void) prepare_spawn_env(&cmd.env); cmd.dir = canonical_worktree.buf; - cmd.git_cmd = 1; + if (git) + strvec_push(&cmd.args, git); + else + cmd.git_cmd = 1; cmd.no_stdin = 1; cmd.no_stdout = 1; cmd.no_stderr = 1; cmd.close_fd_above_stderr = 1; cmd.trace2_child_class = "fsmonitor"; - strvec_pushl(&cmd.args, "fsmonitor--daemon", "start", NULL); + strvec_pushl(&cmd.args, "fsmonitor--daemon", "run", "--detach", NULL); - ret = run_command(&cmd); + result = start_bg_command(&cmd, spawn_wait_cb, NULL, timeout); + if (result == SBGR_READY || + fsmonitor_ipc__get_state() == IPC_STATE__LISTENING) + ret = 0; done: strbuf_release(&canonical_worktree); return ret; diff --git a/git.c b/git.c index e5f1811b6bb762..956a0c2e4d8f54 100644 --- a/git.c +++ b/git.c @@ -929,6 +929,9 @@ int cmd_main(int argc, const char **argv) if (slash) cmd = slash + 1; } + /* A renamed dispatcher is still safer to re-exec than a Git from PATH. */ + if (!starts_with(cmd, "git-")) + git_mark_executable_as_dispatcher(); trace_command_performance(argv); diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index 9f463a3f7a8dbd..9c96c0e3a6aee8 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -384,7 +384,8 @@ test_expect_success 'update-index implicitly starts daemon' ' # Confirm that the trace2 log contains a record of the # daemon starting. - test_subcommand git fsmonitor--daemon start <.git/trace_implicit_1 + test_grep "\"argv\":.*\"fsmonitor--daemon\",\"run\",\"--detach\"" \ + .git/trace_implicit_1 ' test_expect_success 'status implicitly starts daemon' ' @@ -400,7 +401,8 @@ test_expect_success 'status implicitly starts daemon' ' # Confirm that the trace2 log contains a record of the # daemon starting. - test_subcommand git fsmonitor--daemon start <.git/trace_implicit_2 + test_grep "\"argv\":.*\"fsmonitor--daemon\",\"run\",\"--detach\"" \ + .git/trace_implicit_2 ' edit_files () { @@ -978,7 +980,8 @@ test_expect_success "submodule absorbgitdirs implicitly starts daemon" ' # Confirm that the trace2 log contains a record of the # daemon starting. - test_subcommand git fsmonitor--daemon start "$TRASH_DIRECTORY/fake-git-used" + exit 1 + EOF + ( + cd same-executable-spawn && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_EXEC_PATH="$TRASH_DIRECTORY/fake-exec-path" \ + GIT_TRACE2_EVENT="$PWD/.git/spawn.trace" \ + "$GIT_BUILD_DIR/git" status --porcelain=v2 \ + >.git/actual && + test_must_be_empty .git/actual && + test_path_is_missing "$TRASH_DIRECTORY/fake-git-used" && + test_grep -F "\"argv\":[\"$GIT_BUILD_DIR/git\",\"fsmonitor--daemon\",\"run\",\"--detach\"]" \ + .git/spawn.trace && + git fsmonitor--daemon stop + ) +' + test_expect_success MACOS 'implicit daemon rediscovers a linked worktree' ' test_when_finished " git -C reexec-linked-wt fsmonitor--daemon stop 2>/dev/null || : @@ -1408,8 +1435,8 @@ test_expect_success MACOS 'implicit daemon rediscovers a linked worktree' ' git -C ../reexec-linked-wt status --porcelain=v2 \ >../reexec-linked.actual && test_must_be_empty ../reexec-linked.actual && - test_subcommand git fsmonitor--daemon start \ - <../reexec-linked.trace && + test_grep "\"argv\":.*\"fsmonitor--daemon\",\"run\",\"--detach\"" \ + ../reexec-linked.trace && test_grep \ "\"child_class\":\"fsmonitor\",\"cd\":\"$linked_worktree\"" \ ../reexec-linked.trace && From 3bf0277b494d5cac34143ca1f521b3a0190b8c06 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 12:57:40 -0500 Subject: [PATCH 014/105] dir: verify cached excludes during UNTR preload Matching directory metadata alone cannot prove that its cached ignore rules are unchanged. A rewritten .gitignore with restored timestamps can otherwise leave preload results valid while changing which untracked paths should be visible. Snapshot each cached exclude object ID and validate its per-directory file on the existing preload workers. Open regular files with open_nofollow(), reject files larger than 1 MiB, and compare their raw or trailing-LF blob hash with the cached object ID. Verify the open file's stat identity before and after reading, then reopen its pathname and require the same identity through S01/P08. Publish directory results only when both stat and exclude checks match; otherwise invalidate the cached ignore state and fall back to ordinary traversal. A status test rewrites .gitignore, restores its mtime, and checks the result against uncached status. Signed-off-by: Taylor Blau --- dir.c | 99 +++++++++++++++++++++++++++++-- dir.h | 1 + t/t7063-status-untracked-cache.sh | 31 ++++++++++ 3 files changed, 127 insertions(+), 4 deletions(-) diff --git a/dir.c b/dir.c index 8ad94248770a39..1f923c52396830 100644 --- a/dir.c +++ b/dir.c @@ -19,6 +19,7 @@ #include "name-hash.h" #include "object-file.h" #include "path.h" +#include "path-namespace.h" #include "refs.h" #include "repository.h" #include "wildmatch.h" @@ -77,9 +78,11 @@ struct untracked_cache_preload_task { struct untracked_cache_dir *ucd; char *path; struct stat_data stat_data; + struct object_id exclude_oid; unsigned int was_valid : 1; unsigned int stat_checked : 1; unsigned int stat_matches : 1; + unsigned int exclude_matches : 1; unsigned int update_stat_data : 1; }; @@ -99,6 +102,7 @@ struct untracked_cache_preload { struct untracked_cache_preload_task *tasks; struct untracked_cache_preload_data *data; struct cache_time index_timestamp; + char *exclude_per_dir; size_t nr; int threads; unsigned int dir_flags; @@ -107,6 +111,10 @@ struct untracked_cache_preload { #define UNTRACKED_CACHE_MAX_OVERLAP_THREADS 6 #define UNTRACKED_CACHE_PRELOAD_COST 1000 +#define UNTRACKED_CACHE_MAX_EXCLUDE_SIZE (1024 * 1024) + +static void invalidate_gitignore(struct untracked_cache *uc, + struct untracked_cache_dir *dir); static int match_untracked_dir_stat_racy(const struct cache_time *timestamp, const struct stat_data *sd, @@ -127,6 +135,7 @@ static void collect_untracked_cache_preload_tasks( (*tasks)[*nr].ucd = ucd; (*tasks)[*nr].path = xstrdup(path->len ? path->buf : "."); (*tasks)[*nr].stat_data = ucd->stat_data; + oidcpy(&(*tasks)[*nr].exclude_oid, &ucd->exclude_oid); (*tasks)[*nr].was_valid = ucd->valid; (*nr)++; @@ -168,6 +177,63 @@ static int untracked_cache_auto_preload_worthwhile( UNTRACKED_CACHE_AUTO_PRELOAD_MIN_DIRS; } +static int exclude_path_matches_fd(const char *path, + const struct stat *expected) +{ + struct stat st; + int fd = open_nofollow(path, O_RDONLY); + int ret = fd >= 0 && !fstat(fd, &st) && S_ISREG(st.st_mode) && + path_namespace_stat_equal(expected, &st); + + if (fd >= 0) + close(fd); + return ret; +} + +static int cached_exclude_file_matches( + const struct git_hash_algo *algo, + const char *path, const struct object_id *cached_oid) +{ + struct object_id raw_oid, normalized_oid; + struct stat st, st_after; + char *buf; + size_t size; + int fd, ret = 0; + + fd = open_nofollow(path, O_RDONLY); + if (fd < 0) + return 0; + if (fstat(fd, &st) < 0 || !S_ISREG(st.st_mode) || st.st_size < 0 || + st.st_size > UNTRACKED_CACHE_MAX_EXCLUDE_SIZE) + goto out_close; + + size = xsize_t(st.st_size); + buf = xmallocz(size + 1); + if (read_in_full(fd, buf, size) != size) + goto out; + /* Prove both the opened file and its pathname stayed unchanged. */ + if (fstat(fd, &st_after) || + !path_namespace_stat_equal(&st, &st_after) || + !exclude_path_matches_fd(path, &st_after)) + goto out; + + /* add_patterns() may record either the blob or its LF-normalized form. */ + hash_object_file(algo, buf, size, OBJ_BLOB, &raw_oid); + if (oideq(&raw_oid, cached_oid)) { + ret = 1; + goto out; + } + buf[size] = '\n'; + hash_object_file(algo, buf, size + 1, OBJ_BLOB, + &normalized_oid); + ret = oideq(&normalized_oid, cached_oid); +out: + free(buf); +out_close: + close(fd); + return ret; +} + static struct untracked_cache_preload *untracked_cache_preload_start_1( struct index_state *istate, unsigned int dir_flags, int automatic) { @@ -187,6 +253,7 @@ static struct untracked_cache_preload *untracked_cache_preload_start_1( preload->uc = uc; preload->root = uc->root; preload->index_timestamp = istate->timestamp; + preload->exclude_per_dir = xstrdup_or_null(uc->exclude_per_dir); preload->dir_flags = dir_flags; collect_untracked_cache_preload_tasks( uc->root, &path, &preload->tasks, &preload->nr, &alloc); @@ -260,6 +327,7 @@ static void *preload_untracked_cache_thread(void *_data) for (i = data->offset; i < data->offset + data->nr; i++) { struct untracked_cache_preload_task *task = &preload->tasks[i]; + struct strbuf exclude_path = STRBUF_INIT; struct stat st; if (!task->was_valid) @@ -273,10 +341,26 @@ static void *preload_untracked_cache_thread(void *_data) if (!match_untracked_dir_stat_racy( &preload->index_timestamp, &task->stat_data, &st)) { task->stat_matches = 1; + } else { + fill_stat_data(&task->stat_data, &st); + task->update_stat_data = 1; continue; } - fill_stat_data(&task->stat_data, &st); - task->update_stat_data = 1; + if (is_null_oid(&task->exclude_oid)) { + task->exclude_matches = 1; + continue; + } + if (!preload->exclude_per_dir) + continue; + if (strcmp(task->path, ".")) + strbuf_addstr(&exclude_path, task->path); + if (exclude_path.len) + strbuf_addch(&exclude_path, '/'); + strbuf_addstr(&exclude_path, preload->exclude_per_dir); + task->exclude_matches = cached_exclude_file_matches( + preload->repo->hash_algo, exclude_path.buf, + &task->exclude_oid); + strbuf_release(&exclude_path); } return NULL; } @@ -314,6 +398,7 @@ static void untracked_cache_preload_free( for (i = 0; i < preload->nr; i++) free(preload->tasks[i].path); free(preload->tasks); + free(preload->exclude_per_dir); free(preload); } @@ -340,6 +425,7 @@ int untracked_cache_preload_finish(struct untracked_cache_preload *preload, ucd->stat_checked = 0; ucd->stat_matches = 0; + ucd->exclude_matches = 0; /* Invalidation performed after the snapshot always wins. */ if (!task->was_valid || !ucd->valid) { valid = 0; @@ -347,10 +433,14 @@ int untracked_cache_preload_finish(struct untracked_cache_preload *preload, } ucd->stat_checked = task->stat_checked; ucd->stat_matches = task->stat_matches; - if (!task->stat_checked || !task->stat_matches) + ucd->exclude_matches = task->exclude_matches; + if (!task->stat_checked || !task->stat_matches || + !task->exclude_matches) valid = 0; if (task->update_stat_data) ucd->stat_data = task->stat_data; + if (task->stat_matches && !task->exclude_matches) + invalidate_gitignore(uc, ucd); } trace2_data_intmax("dir", istate->repo, "preload_untracked_cache/valid", valid); @@ -2873,7 +2963,8 @@ static int valid_cached_dir(struct dir_struct *dir, if (!(dir->untracked->use_fsmonitor && untracked->valid)) { if (dir->internal.untracked_cache_preloaded && untracked->stat_checked) { - if (!untracked->valid || !untracked->stat_matches) + if (!untracked->valid || !untracked->stat_matches || + !untracked->exclude_matches) return 0; } else { if (lstat(path->len ? path->buf : ".", &st)) { diff --git a/dir.h b/dir.h index 631bdad2b2b845..60b63bbf304782 100644 --- a/dir.h +++ b/dir.h @@ -185,6 +185,7 @@ struct untracked_cache_dir { /* transient results from directory-stat preloading */ unsigned int stat_checked : 1; unsigned int stat_matches : 1; + unsigned int exclude_matches : 1; /* null object ID means this directory does not have .gitignore */ struct object_id exclude_oid; char name[FLEX_ARRAY]; diff --git a/t/t7063-status-untracked-cache.sh b/t/t7063-status-untracked-cache.sh index 9f7c5fee79d639..70cd5b7dfdd2d2 100755 --- a/t/t7063-status-untracked-cache.sh +++ b/t/t7063-status-untracked-cache.sh @@ -1046,6 +1046,37 @@ test_expect_success 'automatic preload observes its directory threshold' ' .git/at-threshold.trace ) ' + +test_expect_success 'preload verifies cached per-directory excludes' ' + test_create_repo auto-exclude && + ( + cd auto-exclude && + test_write_lines hide-a >.gitignore && + test_write_lines tracked >tracked && + git add .gitignore tracked && + git commit -m base && + git config core.untrackedCache true && + git config core.fsmonitor false && + test_write_lines a >hide-a && + test_write_lines b >hide-b && + git status --porcelain >/dev/null && + git status --porcelain >/dev/null && + avoid_racy && + mtime=$(test-tool chmtime --get .gitignore) && + test_write_lines hide-b >.gitignore && + test-tool chmtime =$mtime .gitignore && + GIT_OPTIONAL_LOCKS=0 git -c core.untrackedCache=false \ + status --porcelain >.git/expect && + GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD=1 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=4 \ + GIT_TRACE2_EVENT="$PWD/.git/actual.trace" \ + git status --porcelain >.git/actual && + test_cmp .git/expect .git/actual && + test_grep "preload_untracked_cache/valid.*value.*0" \ + .git/actual.trace + ) +' + test_expect_success 'status preloads cached-directory validation' ' test_create_repo auto-preload && ( From 60e6747af82e65deb60725d22f84546cc13c1b45 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 21:42:25 -0700 Subject: [PATCH 015/105] fsmonitor--daemon: invalidate globally for worktree hardlink events Darwin FSEvents identifies the pathname associated with a hardlink event, not every name referring to the same inode. Invalidating only that pathname can leave another tracked hardlink trusted after its contents change. Classify the event's absolute path before handling its hardlink flags. For worktree events, enqueue the provider-wide marker introduced by S04/P02 so clients content-check the tracked set. Leave gitdir events in the existing cookie and gitdir handling; otherwise reads of hardlinked object files could repeatedly trigger global invalidation. Add a MACOS,HARDLINKS daemon regression that rejects a marker for a gitdir hardlink, then verifies the marker and correct status for a changed worktree hardlink with its timestamp restored. Signed-off-by: Taylor Blau --- compat/fsmonitor/fsm-listen-darwin.c | 25 ++++++++++++++++- t/t7527-builtin-fsmonitor.sh | 40 ++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c index 43c3a915a0edfc..ffd8392262261b 100644 --- a/compat/fsmonitor/fsm-listen-darwin.c +++ b/compat/fsmonitor/fsm-listen-darwin.c @@ -138,6 +138,12 @@ static int ef_is_dropped(const FSEventStreamEventFlags ef) ef & kFSEventStreamEventFlagUserDropped); } +static int ef_is_hardlink(const FSEventStreamEventFlags ef) +{ + return ef & (kFSEventStreamEventFlagItemIsHardlink | + kFSEventStreamEventFlagItemIsLastHardlink); +} + /* * If an `xattr` change is the only reason we received this event, * then silently ignore it. Git doesn't care about xattr's. We @@ -208,6 +214,7 @@ static void fsevent_callback(ConstFSEventStreamRef streamRef UNUSED, const char *slash; char *resolved = NULL; struct strbuf tmp = STRBUF_INIT; + enum fsmonitor_path_type path_type; /* * Build a list of all filesystem changes into a private/local @@ -290,7 +297,23 @@ static void fsevent_callback(ConstFSEventStreamRef streamRef UNUSED, continue; } - switch (fsmonitor_classify_path_absolute(state, path_k)) { + path_type = fsmonitor_classify_path_absolute(state, path_k); + if (ef_is_hardlink(event_flags[k]) && + path_type == IS_WORKDIR_PATH) { + /* + * An event for one name does not prove that all names of the + * inode are in this watch cone. Make the client content-check + * the entire tracked set rather than trusting path-local stats. + */ + if (trace_pass_fl(&trace_fsmonitor)) + log_flags_set(path_k, event_flags[k]); + if (!batch) + batch = fsmonitor_batch__new(); + my_add_path(batch, FSMONITOR_PATH_GLOBAL_INVALIDATE); + continue; + } + + switch (path_type) { case IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX: case IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX: diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index de7134af8b5c1a..9b3d9305310848 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -53,6 +53,11 @@ test_lazy_prereq FSMONITOR_WORKS ' return $ret ' +test_lazy_prereq HARDLINKS ' + : >hardlink-a && + ln hardlink-a hardlink-b +' + if ! test_have_prereq FSMONITOR_WORKS then skip_all="filesystem does not deliver fsmonitor events (container/overlayfs?)" @@ -1408,4 +1413,39 @@ test_expect_success CASE_INSENSITIVE_FS 'fsmonitor file case wrong on disk' ' test_grep -q " M dir1/dir2/dir4/FILE-4-A" "$PWD/file_case_wrong-try3.out" ' +test_expect_success MACOS,HARDLINKS 'hardlink events invalidate all tracked paths' ' + test_when_finished "git -C hardlink-event fsmonitor--daemon stop 2>/dev/null || :" && + test_create_repo hardlink-event && + ( + cd hardlink-event && + printf "AAAA\\n" >tracked && + git add tracked && + git commit -m base && + git config core.fsmonitor true && + git config core.untrackedCache true && + git config core.trustctime false && + git config core.checkStat minimal && + cp -p tracked .git/mtime-reference && + start_daemon --tf "$PWD/../hardlink-event.trace" && + git update-index --fsmonitor && + git status --porcelain=v2 >/dev/null && + git status --porcelain=v2 >/dev/null && + printf "ignore\n" >.git/hardlink-source && + ln .git/hardlink-source .git/hardlink-alias && + printf "still-ignore\n" >.git/hardlink-alias && + test-tool fsmonitor-client query >.git/gitdir-query && + test_grep ! "^event: //$" ../hardlink-event.trace && + ln tracked alias && + printf "BBBB\\n" >alias && + GIT_OPTIONAL_LOCKS=0 git -c core.fsmonitor=false \ + -c core.untrackedCache=false status --porcelain=v2 \ + >.git/expect && + touch -r .git/mtime-reference alias && + GIT_OPTIONAL_LOCKS=0 git status --porcelain=v2 >.git/actual && + test_cmp .git/expect .git/actual && + test_grep "^event: //$" ../hardlink-event.trace && + git fsmonitor--daemon stop + ) +' + test_done From d0493c0833734d3ac98612152ed9448f6f1b2f09 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Thu, 23 Jul 2026 23:41:39 -0500 Subject: [PATCH 016/105] fsmonitor: ignore startup timeout configuration in daemon run The fsmonitor.startTimeout setting controls how long a client waits for daemon startup; the daemon's run subcommand does not consume it. Nevertheless, daemon configuration parsing validates that setting for every subcommand. A malformed value can consequently kill an implicitly started daemon before it opens its IPC socket. Pass a run-specific configuration flag into the callback and skip startup-timeout parsing only for run. Continue parsing other daemon settings normally, and preserve strict timeout validation for the explicit start subcommand. Add a macOS regression that verifies implicit status still starts the daemon with a malformed timeout while explicit daemon start rejects the same configuration. Signed-off-by: Taylor Blau --- builtin/fsmonitor--daemon.c | 20 +++++++++++++++++--- t/t7527-builtin-fsmonitor.sh | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 4161dd82825b4c..659ce0b2621010 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -42,9 +42,15 @@ static int fsmonitor__start_timeout_sec = 60; #define FSMONITOR__ANNOUNCE_STARTUP "fsmonitor.announcestartup" static int fsmonitor__announce_startup = 0; +struct fsmonitor_config_data { + unsigned int ignore_start_timeout : 1; +}; + static int fsmonitor_config(const char *var, const char *value, const struct config_context *ctx, void *cb) { + struct fsmonitor_config_data *data = cb; + if (!strcmp(var, FSMONITOR__IPC_THREADS)) { int i = git_config_int(var, value, ctx->kvi); if (i < 1) @@ -55,7 +61,12 @@ static int fsmonitor_config(const char *var, const char *value, } if (!strcmp(var, FSMONITOR__START_TIMEOUT)) { - int i = git_config_int(var, value, ctx->kvi); + int i; + + /* The run process does not consume this client-only setting. */ + if (data && data->ignore_start_timeout) + return 0; + i = git_config_int(var, value, ctx->kvi); if (i < 0) return error(_("value of '%s' out of range: %d"), FSMONITOR__START_TIMEOUT, i); @@ -73,7 +84,7 @@ static int fsmonitor_config(const char *var, const char *value, return 0; } - return git_default_config(var, value, ctx, cb); + return git_default_config(var, value, ctx, NULL); } /* @@ -1570,6 +1581,9 @@ int cmd_fsmonitor__daemon(int argc, const char *prefix, struct repository *repo UNUSED) { + struct fsmonitor_config_data config_data = { + .ignore_start_timeout = argc > 1 && !strcmp(argv[1], "run"), + }; const char *subcmd; enum fsmonitor_reason reason; int detach_console = 0; @@ -1586,7 +1600,7 @@ int cmd_fsmonitor__daemon(int argc, OPT_END() }; - repo_config(the_repository, fsmonitor_config, NULL); + repo_config(the_repository, fsmonitor_config, &config_data); argc = parse_options(argc, argv, prefix, options, builtin_fsmonitor__daemon_usage, 0); diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index 9c96c0e3a6aee8..b02df8b5ce5cd3 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -1445,4 +1445,22 @@ test_expect_success MACOS 'implicit daemon rediscovers a linked worktree' ' ) ' +test_expect_success MACOS 'implicit startup treats a bad timeout as best effort' ' + test_create_repo reexec-timeout && + ( + cd reexec-timeout && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + git config fsmonitor.starttimeout nonsense && + git status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual && + git config --unset fsmonitor.starttimeout && + git fsmonitor--daemon stop && + git config fsmonitor.starttimeout nonsense && + test_must_fail git fsmonitor--daemon start 2>.git/err && + test_grep "bad numeric config value" .git/err + ) +' + test_done From efb09a1f0f285dc018b45cdbaaebd00bef4b3d21 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Thu, 23 Jul 2026 23:35:52 -0500 Subject: [PATCH 017/105] dir: rescan invalid collapsed UNTR witnesses In collapsed-directory mode, an untracked-cache parent may represent an entire directory by one descendant witness. If that witness becomes invalid or disappears, removing it without inspecting the directory can also hide another unvisited child that remains untracked. Compute cached validity from descendants upward after preload and invalidate collapsed ancestors when a required child proof fails. Before removing a stale collapsed witness, rescan its directory and retain the parent as untracked whenever another child survives. A focused untracked-cache test removes the cached witness while leaving a sibling present and verifies that status still reports the collapsed directory. Signed-off-by: Taylor Blau --- dir.c | 103 +++++++++++++++++++++++++++--- t/t7063-status-untracked-cache.sh | 31 +++++++++ 2 files changed, 124 insertions(+), 10 deletions(-) diff --git a/dir.c b/dir.c index 1f923c52396830..cda3dc208cdfd2 100644 --- a/dir.c +++ b/dir.c @@ -115,6 +115,8 @@ struct untracked_cache_preload { static void invalidate_gitignore(struct untracked_cache *uc, struct untracked_cache_dir *dir); +static void invalidate_directory(struct untracked_cache *uc, + struct untracked_cache_dir *dir); static int match_untracked_dir_stat_racy(const struct cache_time *timestamp, const struct stat_data *sd, @@ -365,6 +367,53 @@ static void *preload_untracked_cache_thread(void *_data) return NULL; } +static int untracked_cache_has_collapsed_child( + const struct untracked_cache_dir *parent, + const struct untracked_cache_dir *child) +{ + size_t i, len = strlen(child->name); + + for (i = 0; i < parent->untracked_nr; i++) { + const char *name = parent->untracked[i]; + + if (strlen(name) == len + 1 && name[len] == '/' && + !strncmp(name, child->name, len)) + return 1; + } + return 0; +} + +static int compute_untracked_cache_valid_recursive( + struct untracked_cache *uc, + struct untracked_cache_dir *ucd, + int invalidate_ancestors) +{ + size_t i; + int local_valid = ucd->valid && ucd->stat_checked && + ucd->stat_matches && ucd->exclude_matches; + int valid = local_valid; + int invalidate_self = !local_valid; + + for (i = 0; i < ucd->dirs_nr; i++) { + int child_valid = compute_untracked_cache_valid_recursive( + uc, ucd->dirs[i], invalidate_ancestors); + + if (!child_valid) { + valid = 0; + if (untracked_cache_has_collapsed_child(ucd, ucd->dirs[i])) + invalidate_self = 1; + } + } + if (invalidate_self && invalidate_ancestors) { + if (!local_valid && ucd->valid && ucd->stat_checked && + ucd->stat_matches && !ucd->exclude_matches) + invalidate_gitignore(uc, ucd); + else + invalidate_directory(uc, ucd); + } + return valid; +} + static void untracked_cache_preload_join( struct untracked_cache_preload *preload) { @@ -409,7 +458,6 @@ int untracked_cache_preload_finish(struct untracked_cache_preload *preload, struct untracked_cache *uc; size_t i; int applied = 0; - int valid = 1; if (!preload) return 0; @@ -427,23 +475,19 @@ int untracked_cache_preload_finish(struct untracked_cache_preload *preload, ucd->stat_matches = 0; ucd->exclude_matches = 0; /* Invalidation performed after the snapshot always wins. */ - if (!task->was_valid || !ucd->valid) { - valid = 0; + if (!task->was_valid || !ucd->valid) continue; - } ucd->stat_checked = task->stat_checked; ucd->stat_matches = task->stat_matches; ucd->exclude_matches = task->exclude_matches; - if (!task->stat_checked || !task->stat_matches || - !task->exclude_matches) - valid = 0; if (task->update_stat_data) ucd->stat_data = task->stat_data; - if (task->stat_matches && !task->exclude_matches) - invalidate_gitignore(uc, ucd); } trace2_data_intmax("dir", istate->repo, - "preload_untracked_cache/valid", valid); + "preload_untracked_cache/valid", + compute_untracked_cache_valid_recursive( + uc, preload->root, + dir_flags & DIR_SHOW_OTHER_DIRECTORIES)); applied = 1; done: trace2_data_intmax("dir", istate->repo, @@ -3063,6 +3107,28 @@ static int read_cached_dir(struct cached_dir *cdir) return -1; } +static void remove_collapsed_untracked_child( + struct untracked_cache *uc, + struct untracked_cache_dir *parent, + const struct untracked_cache_dir *child) +{ + size_t i, len = strlen(child->name); + + for (i = 0; i < parent->untracked_nr; i++) { + char *name = parent->untracked[i]; + + if (strlen(name) != len + 1 || name[len] != '/' || + strncmp(name, child->name, len)) + continue; + free(name); + MOVE_ARRAY(parent->untracked + i, parent->untracked + i + 1, + parent->untracked_nr - i - 1); + parent->untracked_nr--; + uc->dir_invalidated++; + return; + } +} + static void close_cached_dir(struct cached_dir *cdir) { if (cdir->fdir) @@ -3157,6 +3223,23 @@ static enum path_treatment read_directory_recursive(struct dir_struct *dir, /* check how the file or directory should be treated */ state = treat_path(dir, untracked, &cdir, istate, &path, baselen, pathspec); + if (!cdir.d_name && cdir.ucd && cdir.ucd->check_only && + state < path_untracked && untracked && + untracked_cache_has_collapsed_child(untracked, cdir.ucd) && + (dir->flags & DIR_SHOW_OTHER_DIRECTORIES)) { + /* + * A collapsed directory retains one descendant as its + * untracked witness. Rescan before dropping a stale witness; + * an unvisited sibling may still make the directory untracked. + */ + invalidate_directory(dir->untracked, cdir.ucd); + state = read_directory_recursive( + dir, istate, path.buf, path.len, cdir.ucd, + 1, 0, pathspec); + if (state < path_untracked) + remove_collapsed_untracked_child( + dir->untracked, untracked, cdir.ucd); + } dir->internal.visited_paths++; if (state > dir_state) diff --git a/t/t7063-status-untracked-cache.sh b/t/t7063-status-untracked-cache.sh index 70cd5b7dfdd2d2..5948a579b5783b 100755 --- a/t/t7063-status-untracked-cache.sh +++ b/t/t7063-status-untracked-cache.sh @@ -1077,6 +1077,37 @@ test_expect_success 'preload verifies cached per-directory excludes' ' ) ' +test_expect_success 'recursive preload rescans a vanished collapsed witness' ' + test_create_repo collapsed-witness && + ( + cd collapsed-witness && + test_write_lines "*.ignored" >.gitignore && + git add .gitignore && + git commit -m base && + git config core.untrackedCache true && + git config core.fsmonitor false && + for i in 00 01 + do + mkdir -p "scratch/d$i" && + test_write_lines "$i" >"scratch/d$i/file" || return 1 + done && + echo "?? scratch/" >.git/expect && + git status --porcelain >/dev/null && + git status --porcelain >/dev/null && + test-tool dump-untracked-cache >.git/cache && + witness=$(sed -n \ + "s#^/scratch/\\(d[0-9][0-9]*\\)/ .*#\\1#p" \ + .git/cache | sed -n 1p) && + test -n "$witness" && + avoid_racy && + rm "scratch/$witness/file" && + GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD=1 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=4 \ + git status --porcelain >.git/actual && + test_cmp .git/expect .git/actual + ) +' + test_expect_success 'status preloads cached-directory validation' ' test_create_repo auto-preload && ( From 359df782c35143939cce69cb3ecda746fe16c59a Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 02:00:28 -0500 Subject: [PATCH 018/105] fsmonitor: invalidate conversion state for attribute-file events Changing a .gitattributes file can change how tracked content is converted without changing the tracked file's stat data. Invalidating the attribute-file path alone therefore leaves cached conversion state and affected fsmonitor-valid tracked entries falsely reusable. Recognize an exact .gitattributes basename in the refresh callback. Discard cached attribute stacks globally and strongly invalidate only tracked entries beneath that file's parent directory. A root attribute file invalidates all tracked entries; tracked entries in sibling directories remain valid after a nested attribute-file event. Mark the fsmonitor extension changed only when an entry is invalidated. Add Clar unit coverage for unrelated paths, nested-directory scope, root-directory scope, cleared validity, zeroed stat data, and the content-check marker. Register u-fsmonitor-attributes in both Makefile and t/meson.build so the suite is included in both build systems. Signed-off-by: Taylor Blau --- Makefile | 1 + fsmonitor-ll.h | 2 + fsmonitor.c | 34 +++++++++++++ t/meson.build | 1 + t/unit-tests/u-fsmonitor-attributes.c | 72 +++++++++++++++++++++++++++ 5 files changed, 110 insertions(+) create mode 100644 t/unit-tests/u-fsmonitor-attributes.c diff --git a/Makefile b/Makefile index fac3e8879c8377..91090f730b14ae 100644 --- a/Makefile +++ b/Makefile @@ -1538,6 +1538,7 @@ THIRD_PARTY_SOURCES += $(UNIT_TEST_DIR)/clar/clar/% CLAR_TEST_SUITES += u-ctype CLAR_TEST_SUITES += u-dir CLAR_TEST_SUITES += u-example-decorate +CLAR_TEST_SUITES += u-fsmonitor-attributes CLAR_TEST_SUITES += u-hash CLAR_TEST_SUITES += u-hashmap CLAR_TEST_SUITES += u-list-objects-filter-options diff --git a/fsmonitor-ll.h b/fsmonitor-ll.h index a409b15e68bc51..7f78ad21c8d0b0 100644 --- a/fsmonitor-ll.h +++ b/fsmonitor-ll.h @@ -47,6 +47,8 @@ void tweak_fsmonitor(struct index_state *istate); */ void refresh_fsmonitor(struct index_state *istate); +int fsmonitor_invalidate_attributes_path(struct index_state *istate, + const char *name); /* * Does the received result contain the "trivial" response? */ diff --git a/fsmonitor.c b/fsmonitor.c index 6c119b17bd391e..2fd070b1d5b22a 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -209,6 +209,39 @@ void fsmonitor_invalidate_cache_entry(struct cache_entry *ce) static size_t handle_path_with_trailing_slash( struct index_state *istate, const char *name, int pos); +int fsmonitor_invalidate_attributes_path(struct index_state *istate, + const char *name) +{ + size_t len = strlen(name), base, attr_len = strlen(GITATTRIBUTES_FILE); + size_t invalidated = 0; + unsigned int i; + + while (len && is_dir_sep(name[len - 1])) + len--; + base = len; + while (base && !is_dir_sep(name[base - 1])) + base--; + if (len - base != attr_len || + fspathncmp(name + base, GITATTRIBUTES_FILE, attr_len)) + return 0; + + git_attr_invalidate_all(); + for (i = 0; i < istate->cache_nr; i++) { + struct cache_entry *ce = istate->cache[i]; + + if (base && (ce->ce_namelen < base || + fspathncmp(ce->name, name, base))) + continue; + fsmonitor_invalidate_cache_entry(ce); + invalidated++; + } + if (invalidated) + istate->cache_changed |= FSMONITOR_CHANGED; + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/attributes-scope", base); + return invalidated > 0; +} + /* * Use the name-hash to do a case-insensitive cache-entry lookup with * the pathname and invalidate the cache-entry. @@ -457,6 +490,7 @@ static void fsmonitor_refresh_callback(struct index_state *istate, char *name) return; } pos = index_name_pos(istate, name, len); + fsmonitor_invalidate_attributes_path(istate, name); if (name[len - 1] == '/') nr_in_cone = handle_path_with_trailing_slash(istate, name, pos); diff --git a/t/meson.build b/t/meson.build index a25f37d2f5ae7d..c9b35c051949ff 100644 --- a/t/meson.build +++ b/t/meson.build @@ -2,6 +2,7 @@ clar_test_suites = [ 'unit-tests/u-ctype.c', 'unit-tests/u-dir.c', 'unit-tests/u-example-decorate.c', + 'unit-tests/u-fsmonitor-attributes.c', 'unit-tests/u-hash.c', 'unit-tests/u-hashmap.c', 'unit-tests/u-list-objects-filter-options.c', diff --git a/t/unit-tests/u-fsmonitor-attributes.c b/t/unit-tests/u-fsmonitor-attributes.c new file mode 100644 index 00000000000000..5a2b7a25f137b3 --- /dev/null +++ b/t/unit-tests/u-fsmonitor-attributes.c @@ -0,0 +1,72 @@ +#include "unit-test.h" +#include "fsmonitor-ll.h" +#include "read-cache-ll.h" +#include "repository.h" + +static void add_entry(struct index_state *istate, size_t pos, + const char *path) +{ + size_t len = strlen(path); + struct cache_entry *ce = make_empty_cache_entry(istate, len); + + ce->ce_mode = S_IFREG | 0644; + ce->ce_namelen = len; + ce->ce_flags = CE_FSMONITOR_VALID | CE_UPTODATE; + memset(&ce->ce_stat_data, 1, sizeof(ce->ce_stat_data)); + memcpy(ce->name, path, len + 1); + istate->cache[pos] = ce; +} + +static int stat_data_is_zero(const struct cache_entry *ce) +{ + struct stat_data zero = { 0 }; + + return !memcmp(&ce->ce_stat_data, &zero, sizeof(zero)); +} + +void test_fsmonitor_attributes__invalidates_only_the_affected_scope(void) +{ + struct repository repo = { .hash_algo = &hash_algos[GIT_HASH_SHA1] }; + struct index_state istate = INDEX_STATE_INIT(&repo); + + CALLOC_ARRAY(istate.cache, 3); + istate.cache_alloc = istate.cache_nr = 3; + add_entry(&istate, 0, "a/file"); + add_entry(&istate, 1, "a/sub/file"); + add_entry(&istate, 2, "b/file"); + + cl_assert(!fsmonitor_invalidate_attributes_path( + &istate, "a/not-attributes")); + cl_assert(istate.cache[0]->ce_flags & CE_FSMONITOR_VALID); + cl_assert(fsmonitor_invalidate_attributes_path( + &istate, "a/.gitattributes")); + for (size_t i = 0; i < 2; i++) { + cl_assert(!(istate.cache[i]->ce_flags & CE_FSMONITOR_VALID)); + cl_assert(!(istate.cache[i]->ce_flags & CE_UPTODATE)); + cl_assert(istate.cache[i]->ce_flags & CE_CONTENT_CHECK_REQUIRED); + cl_assert(stat_data_is_zero(istate.cache[i])); + } + cl_assert(istate.cache[2]->ce_flags & CE_FSMONITOR_VALID); + cl_assert(istate.cache[2]->ce_flags & CE_UPTODATE); + cl_assert(!stat_data_is_zero(istate.cache[2])); + cl_assert(istate.cache_changed & FSMONITOR_CHANGED); + release_index(&istate); +} + +void test_fsmonitor_attributes__root_source_invalidates_every_entry(void) +{ + struct repository repo = { .hash_algo = &hash_algos[GIT_HASH_SHA1] }; + struct index_state istate = INDEX_STATE_INIT(&repo); + + CALLOC_ARRAY(istate.cache, 2); + istate.cache_alloc = istate.cache_nr = 2; + add_entry(&istate, 0, "a/file"); + add_entry(&istate, 1, "b/file"); + cl_assert(fsmonitor_invalidate_attributes_path( + &istate, ".gitattributes")); + for (size_t i = 0; i < istate.cache_nr; i++) { + cl_assert(!(istate.cache[i]->ce_flags & CE_FSMONITOR_VALID)); + cl_assert(istate.cache[i]->ce_flags & CE_CONTENT_CHECK_REQUIRED); + } + release_index(&istate); +} From e89815a0df5d4f502fa4b69cf872adcc01c00f3a Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 21:19:27 -0700 Subject: [PATCH 019/105] fsmonitor: bind daemon queries to the canonical worktree root An fsmonitor socket is selected through the Git directory, so separate worktree paths can reach the same daemon when they share that directory. A client in the second worktree can then consume change history from a daemon that watches the first, incorrectly treating changed files in its own worktree as clean. Hash the canonical worktree path together with its device and inode, plus birth time and generation on Apple platforms. Cache the resulting 64-character SHA-256 identity in the daemon and attach it to every client query. Check the identity before interpreting the requested token; reject missing or mismatched bindings with a cookie-synchronized trivial response that forces the ordinary refresh path. The protocol change must also tolerate a daemon left running by an older Git. Such a daemon treats a bound query as an opaque token and can return a plausible trivial response. After that exact response, query an unbound capability command. If the daemon does not advertise query-v1, serialize replacement through a per-socket restart lock, stop it, and start the invoking Git executable before retrying the bound query. Keep quit, flush, and capability control commands unbound. Bound daemon lifecycle retries, and fail the query instead of trusting history when the root cannot be identified or an incompatible daemon cannot be replaced. Regression tests cover shared-gitdir worktree aliases, replacement of a legacy daemon, and acceptance of a daemon that advertises a capability superset. The replacement test also verifies that the next status neither refreshes tracked entries nor starts another daemon. Signed-off-by: Taylor Blau --- builtin/fsmonitor--daemon.c | 46 ++++++- fsmonitor--daemon.h | 1 + fsmonitor-ipc.c | 251 ++++++++++++++++++++++++++++++++--- fsmonitor-ipc.h | 9 ++ t/helper/test-simple-ipc.c | 54 ++++++++ t/t7527-builtin-fsmonitor.sh | 100 ++++++++++++++ 6 files changed, 437 insertions(+), 24 deletions(-) diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 659ce0b2621010..953f68b4fc1185 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -705,18 +705,48 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, int do_trivial = 0; int do_flush = 0; int do_cookie = 0; + int invalid_binding = 0; enum fsmonitor_cookie_item_result cookie_result; + if (strcmp(command, "quit") && + strcmp(command, "flush") && + strcmp(command, FSMONITOR_IPC_CAPABILITY_COMMAND)) { + const char *identity; + const char *query; + + if (!skip_prefix(command, FSMONITOR_IPC_QUERY_PREFIX, + &identity) || + !(query = strchr(identity, '\n')) || + query - identity != FSMONITOR_IPC_WORKTREE_ID_HEX || + state->worktree_identity.len != FSMONITOR_IPC_WORKTREE_ID_HEX || + memcmp(identity, state->worktree_identity.buf, + FSMONITOR_IPC_WORKTREE_ID_HEX)) { + invalid_binding = 1; + trace2_data_intmax("fsmonitor", the_repository, + "query/worktree-mismatch", 1); + } else { + command = query + 1; + } + } + /* * We expect `command` to be of the form: * - * := quit NUL + * := get-capabilities NUL + * | quit NUL * | flush NUL * | NUL * | NUL */ - if (!strcmp(command, "quit")) { + if (!strcmp(command, FSMONITOR_IPC_CAPABILITY_COMMAND)) { + static const char capabilities[] = + FSMONITOR_IPC_QUERY_VERSION "\n"; + + return reply(reply_data, capabilities, + sizeof(capabilities) - 1); + + } else if (!strcmp(command, "quit")) { /* * A client has requested over the socket/pipe that the * daemon shutdown. @@ -739,6 +769,11 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, do_flush = 1; do_trivial = 1; + } else if (invalid_binding) { + /* Never trust a token from an unbound or different worktree. */ + do_trivial = 1; + do_cookie = 1; + } else if (!skip_prefix(command, "builtin:", &p)) { /* assume V1 timestamp or garbage */ @@ -1322,6 +1357,12 @@ static int fsmonitor_run_daemon(void) strbuf_init(&state.path_worktree_watch, 0); strbuf_addstr(&state.path_worktree_watch, absolute_path(repo_get_work_tree(the_repository))); + strbuf_init(&state.worktree_identity, 0); + if (fsmonitor_ipc__get_worktree_identity(the_repository, + &state.worktree_identity)) { + err = error(_("could not identify worktree root")); + goto done; + } state.nr_paths_watching = 1; strbuf_init(&state.alias.alias, 0); @@ -1448,6 +1489,7 @@ static int fsmonitor_run_daemon(void) ipc_server_free(state.ipc_server_data); strbuf_release(&state.path_worktree_watch); + strbuf_release(&state.worktree_identity); strbuf_release(&state.path_gitdir_watch); strbuf_release(&state.path_cookie_prefix); strbuf_release(&state.path_ipc); diff --git a/fsmonitor--daemon.h b/fsmonitor--daemon.h index 5cbbec8d940ba7..850188f872b783 100644 --- a/fsmonitor--daemon.h +++ b/fsmonitor--daemon.h @@ -40,6 +40,7 @@ struct fsmonitor_daemon_state { pthread_mutex_t main_lock; struct strbuf path_worktree_watch; + struct strbuf worktree_identity; struct strbuf path_gitdir_watch; struct alias_info alias; int nr_paths_watching; diff --git a/fsmonitor-ipc.c b/fsmonitor-ipc.c index 8957091bfccbb2..f6eb03cfd9442f 100644 --- a/fsmonitor-ipc.c +++ b/fsmonitor-ipc.c @@ -6,6 +6,8 @@ #include "environment.h" #include "exec-cmd.h" #include "gettext.h" +#include "hash.h" +#include "lockfile.h" #include "parse.h" #include "simple-ipc.h" #include "fsmonitor-ipc.h" @@ -14,6 +16,46 @@ #include "strbuf.h" #include "trace2.h" +int fsmonitor_ipc__get_worktree_identity(struct repository *r, + struct strbuf *identity) +{ + static const char hex[] = "0123456789abcdef"; + struct strbuf canonical = STRBUF_INIT; + struct strbuf stable = STRBUF_INIT; + git_SHA256_CTX ctx; + unsigned char hash[GIT_SHA256_RAWSZ]; + struct stat st; + const char *worktree = repo_get_work_tree(r); + int ret = -1; + + if (!worktree || + !strbuf_realpath(&canonical, worktree, 0) || + stat(canonical.buf, &st)) + goto done; + strbuf_addf(&stable, "v1\n%"PRIuMAX":", (uintmax_t)canonical.len); + strbuf_addbuf(&stable, &canonical); + strbuf_addf(&stable, "\n%"PRIuMAX"\n%"PRIuMAX, + (uintmax_t)st.st_dev, (uintmax_t)st.st_ino); +#ifdef __APPLE__ + strbuf_addf(&stable, "\n%"PRIdMAX"\n%ld\n%"PRIu32, + (intmax_t)st.st_birthtimespec.tv_sec, + st.st_birthtimespec.tv_nsec, st.st_gen); +#endif + git_SHA256_Init(&ctx); + git_SHA256_Update(&ctx, stable.buf, stable.len); + git_SHA256_Final(hash, &ctx); + strbuf_reset(identity); + for (size_t i = 0; i < ARRAY_SIZE(hash); i++) { + strbuf_addch(identity, hex[hash[i] >> 4]); + strbuf_addch(identity, hex[hash[i] & 0xf]); + } + ret = 0; +done: + strbuf_release(&stable); + strbuf_release(&canonical); + return ret; +} + #ifndef HAVE_FSMONITOR_DAEMON_BACKEND /* @@ -73,6 +115,7 @@ enum ipc_active_state fsmonitor_ipc__get_state(void) #define FSMONITOR_START_TIMEOUT_KEY "fsmonitor.starttimeout" #define FSMONITOR_START_TIMEOUT_DEFAULT 60 +#define FSMONITOR_RESTART_ATTEMPTS 3 static unsigned int get_start_timeout(void) { @@ -140,44 +183,221 @@ static int spawn_daemon(void) return ret; } +static int try_send_command(const char *command, struct strbuf *answer, + enum ipc_active_state *state_out) +{ + struct ipc_client_connection *connection = NULL; + struct ipc_client_connect_options options + = IPC_CLIENT_CONNECT_OPTIONS_INIT; + enum ipc_active_state state; + int ret = -1; + + strbuf_reset(answer); + options.wait_if_busy = 1; + options.wait_if_not_found = 0; + + state = ipc_client_try_connect(fsmonitor_ipc__get_path(the_repository), + &options, &connection); + if (state == IPC_STATE__LISTENING) { + ret = ipc_client_send_command_to_connection( + connection, command, strlen(command), answer); + ipc_client_close_connection(connection); + } + + if (state_out) + *state_out = state; + return ret; +} + +static int is_trivial_response(const struct strbuf *answer) +{ + const char *nul = memchr(answer->buf, '\0', answer->len); + + return nul && nul != answer->buf && + answer->len == (size_t)(nul - answer->buf) + 3 && + nul[1] == '/' && nul[2] == '\0'; +} + +static int has_capability(const struct strbuf *answer, + const char *capability) +{ + const char *p = answer->buf; + const char *end = answer->buf + answer->len; + size_t capability_len = strlen(capability); + + while (p < end) { + const char *eol = memchr(p, '\n', end - p); + const char *line_end = eol ? eol : end; + + if ((size_t)(line_end - p) == capability_len && + !memcmp(p, capability, capability_len)) + return 1; + if (!eol) + break; + p = eol + 1; + } + return 0; +} + +static int server_supports_bound_queries(void) +{ + struct strbuf answer = STRBUF_INIT; + int ret; + + ret = !try_send_command(FSMONITOR_IPC_CAPABILITY_COMMAND, + &answer, NULL) && + has_capability(&answer, FSMONITOR_IPC_QUERY_VERSION); + strbuf_release(&answer); + return ret; +} + +static int wait_for_daemon_exit(void) +{ + uintmax_t elapsed_ms = 0; + uintmax_t timeout_ms = (uintmax_t)get_start_timeout() * 1000; + + while (fsmonitor_ipc__get_state() == IPC_STATE__LISTENING) { + if (elapsed_ms >= timeout_ms) + return -1; + sleep_millisec(50); + elapsed_ms += 50; + } + return 0; +} + +static int restart_incompatible_daemon(void) +{ + struct strbuf answer = STRBUF_INIT; + struct strbuf lock_path = STRBUF_INIT; + struct lock_file restart_lock = LOCK_INIT; + uintmax_t timeout_ms = (uintmax_t)get_start_timeout() * 1000; + long lock_timeout_ms = timeout_ms > LONG_MAX ? + LONG_MAX : (long)timeout_ms; + int have_lock = 0; + int ret = -1; + + /* + * Serialize the re-probe, quit, wait, and spawn sequence. This uses a + * different lock from the one used briefly while binding the socket. + */ + strbuf_addf(&lock_path, "%s.restart", + fsmonitor_ipc__get_path(the_repository)); + if (hold_lock_file_for_update_timeout(&restart_lock, lock_path.buf, + LOCK_NO_DEREF, + lock_timeout_ms) < 0) { + if (server_supports_bound_queries()) + ret = 0; + goto done; + } + have_lock = 1; + + /* Another client may have replaced the daemon while we waited. */ + if (server_supports_bound_queries()) + goto success; + + trace2_data_intmax("fsm_client", NULL, + "query/incompatible-daemon", 1); + if (try_send_command("quit", &answer, NULL)) { + /* + * The connection state describes the failed attempt, not + * necessarily the state after the failure. Re-read it before + * deciding whether there is still a daemon to replace. + */ + if (fsmonitor_ipc__get_state() == IPC_STATE__LISTENING) { + if (server_supports_bound_queries()) + ret = 0; + goto done; + } + } + + if (wait_for_daemon_exit()) + goto done; + + /* + * A concurrent client may already have started a replacement. + * The retried bound query will verify its capability if needed. + */ + if (fsmonitor_ipc__get_state() != IPC_STATE__LISTENING && + spawn_daemon()) + goto done; + +success: + ret = 0; + +done: + if (have_lock) + rollback_lock_file(&restart_lock); + strbuf_release(&lock_path); + strbuf_release(&answer); + return ret; +} + int fsmonitor_ipc__send_query(const char *since_token, struct strbuf *answer) { + struct strbuf command = STRBUF_INIT; + struct strbuf identity = STRBUF_INIT; int ret = -1; - int tried_to_spawn = 0; + int lifecycle_attempts = 0; enum ipc_active_state state = IPC_STATE__OTHER_ERROR; struct ipc_client_connection *connection = NULL; struct ipc_client_connect_options options = IPC_CLIENT_CONNECT_OPTIONS_INIT; const char *tok = since_token ? since_token : ""; - size_t tok_len = since_token ? strlen(since_token) : 0; + + trace2_region_enter("fsm_client", "query", NULL); + if (fsmonitor_ipc__get_worktree_identity(the_repository, &identity)) { + trace2_data_intmax("fsm_client", NULL, + "query/worktree-identity-error", 1); + goto done; + } + strbuf_addstr(&command, FSMONITOR_IPC_QUERY_PREFIX); + strbuf_addbuf(&command, &identity); + strbuf_addch(&command, '\n'); + strbuf_addstr(&command, tok); options.wait_if_busy = 1; options.wait_if_not_found = 0; - trace2_region_enter("fsm_client", "query", NULL); trace2_data_string("fsm_client", NULL, "query/command", tok); try_again: + strbuf_reset(answer); state = ipc_client_try_connect(fsmonitor_ipc__get_path(the_repository), &options, &connection); switch (state) { case IPC_STATE__LISTENING: ret = ipc_client_send_command_to_connection( - connection, tok, tok_len, answer); + connection, command.buf, command.len, answer); ipc_client_close_connection(connection); + connection = NULL; trace2_data_intmax("fsm_client", NULL, "query/response-length", answer->len); + if (!ret && is_trivial_response(answer) && + !server_supports_bound_queries()) { + /* + * A daemon predating bound queries treats query-v1 as + * garbage and returns a valid trivial response. Never + * accept that unbound result. Replace the daemon with + * the invoking Git executable and retry instead. + */ + strbuf_reset(answer); + ret = -1; + if (lifecycle_attempts++ >= FSMONITOR_RESTART_ATTEMPTS || + restart_incompatible_daemon()) + goto done; + options.wait_if_not_found = 1; + goto try_again; + } goto done; case IPC_STATE__NOT_LISTENING: case IPC_STATE__PATH_NOT_FOUND: - if (tried_to_spawn) + if (lifecycle_attempts++ >= FSMONITOR_RESTART_ATTEMPTS) goto done; - tried_to_spawn++; if (spawn_daemon()) goto done; @@ -207,6 +427,8 @@ int fsmonitor_ipc__send_query(const char *since_token, done: trace2_region_leave("fsm_client", "query", NULL); + strbuf_release(&identity); + strbuf_release(&command); return ret; } @@ -214,30 +436,15 @@ int fsmonitor_ipc__send_query(const char *since_token, int fsmonitor_ipc__send_command(const char *command, struct strbuf *answer) { - struct ipc_client_connection *connection = NULL; - struct ipc_client_connect_options options - = IPC_CLIENT_CONNECT_OPTIONS_INIT; - int ret; enum ipc_active_state state; const char *c = command ? command : ""; - size_t c_len = command ? strlen(command) : 0; + int ret = try_send_command(c, answer, &state); - strbuf_reset(answer); - - options.wait_if_busy = 1; - options.wait_if_not_found = 0; - - state = ipc_client_try_connect(fsmonitor_ipc__get_path(the_repository), - &options, &connection); if (state != IPC_STATE__LISTENING) { die(_("fsmonitor--daemon is not running")); return -1; } - ret = ipc_client_send_command_to_connection(connection, c, c_len, - answer); - ipc_client_close_connection(connection); - if (ret == -1) { die(_("could not send '%s' command to fsmonitor--daemon"), c); return -1; diff --git a/fsmonitor-ipc.h b/fsmonitor-ipc.h index 8b489da762b047..006ee0750cf134 100644 --- a/fsmonitor-ipc.h +++ b/fsmonitor-ipc.h @@ -5,6 +5,15 @@ struct repository; +#define FSMONITOR_IPC_QUERY_VERSION "query-v1" +#define FSMONITOR_IPC_QUERY_PREFIX FSMONITOR_IPC_QUERY_VERSION " " +#define FSMONITOR_IPC_CAPABILITY_COMMAND "get-capabilities" +#define FSMONITOR_IPC_WORKTREE_ID_HEX 64 + +/* Hash the canonical worktree root and its stable filesystem identity. */ +int fsmonitor_ipc__get_worktree_identity(struct repository *r, + struct strbuf *identity); + /* * Returns true if built-in file system monitor daemon is defined * for this platform. diff --git a/t/helper/test-simple-ipc.c b/t/helper/test-simple-ipc.c index 442ad6b16f18d8..3be92e4fbd01ca 100644 --- a/t/helper/test-simple-ipc.c +++ b/t/helper/test-simple-ipc.c @@ -159,9 +159,40 @@ static int app__sendbytes_command(const char *received, size_t received_len, * data is handled properly. */ static int my_app_data = 42; +static int fsmonitor_legacy; +static int fsmonitor_capability_superset; static ipc_server_application_cb test_app_cb; +static int app__fsmonitor_capability_superset( + const char *command, size_t command_len, + ipc_server_reply_cb *reply_cb, + struct ipc_server_reply_data *reply_data) +{ + static const char capability_command[] = "get-capabilities"; + static const char capabilities[] = "query-v1\nquery-v2\n"; + static const char query_prefix[] = "query-v1 "; + static const char token[] = "builtin:test-capable:0"; + const char *query; + size_t query_len; + int ret; + + if (command_len == sizeof(capability_command) - 1 && + !memcmp(command, capability_command, command_len)) + return reply_cb(reply_data, capabilities, + sizeof(capabilities) - 1); + + query = memchr(command, '\n', command_len); + query_len = query ? command_len - (query + 1 - command) : 0; + ret = reply_cb(reply_data, token, sizeof(token)); + if (!ret && + (!starts_with(command, query_prefix) || + query_len != sizeof(token) - 1 || + memcmp(query + 1, token, query_len))) + ret = reply_cb(reply_data, "/", 2); + return ret; +} + /* * This is the "application callback" that sits on top of the * "ipc-server". It completely defines the set of commands supported @@ -201,6 +232,20 @@ static int test_app_cb(void *application_data, return SIMPLE_IPC_QUIT; } + if (fsmonitor_capability_superset) + return app__fsmonitor_capability_superset( + command, command_len, reply_cb, reply_data); + + if (fsmonitor_legacy) { + static const char token[] = "builtin:test-legacy:0"; + int ret; + + ret = reply_cb(reply_data, token, sizeof(token)); + if (!ret && !starts_with(command, "builtin:")) + ret = reply_cb(reply_data, "/", 2); + return ret; + } + if (command_len == 4 && !strncmp(command, "ping", 4)) { const char *answer = "pong"; return reply_cb(reply_data, answer, strlen(answer)); @@ -310,6 +355,10 @@ static int daemon__start_server(void) strvec_push(&cp.args, "run-daemon"); strvec_pushf(&cp.args, "--name=%s", cl_args.path); strvec_pushf(&cp.args, "--threads=%d", cl_args.nr_threads); + if (fsmonitor_legacy) + strvec_push(&cp.args, "--fsmonitor-legacy"); + if (fsmonitor_capability_superset) + strvec_push(&cp.args, "--fsmonitor-capability-superset"); cp.no_stdin = 1; cp.no_stdout = 1; @@ -602,6 +651,11 @@ int cmd__simple_ipc(int argc, const char **argv) OPT_INTEGER(0, "bytecount", &cl_args.bytecount, N_("number of bytes")), OPT_INTEGER(0, "batchsize", &cl_args.batchsize, N_("number of requests per thread")), + OPT_BOOL(0, "fsmonitor-legacy", &fsmonitor_legacy, + N_("emulate the legacy fsmonitor query protocol")), + OPT_BOOL(0, "fsmonitor-capability-superset", + &fsmonitor_capability_superset, + N_("advertise multiple fsmonitor query versions")), /* * The "byte" string here is not marked for translation and diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index b02df8b5ce5cd3..198aca8b1f5e14 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -1463,4 +1463,104 @@ test_expect_success MACOS 'implicit startup treats a bad timeout as best effort' ) ' +test_expect_success 'bound query replaces a legacy daemon' ' + test_when_finished \ + "stop_daemon_delete_repo legacy-daemon-upgrade" && + test_create_repo legacy-daemon-upgrade && + ( + cd legacy-daemon-upgrade && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.preloadIndex false && + git config core.untrackedCache true && + git status --porcelain=v2 >/dev/null && + git config core.fsmonitor true && + ipc_path=$(git rev-parse --path-format=absolute \ + --git-path fsmonitor--daemon.ipc) && + test-tool simple-ipc start-daemon \ + --name="$ipc_path" --threads=1 --fsmonitor-legacy && + + GIT_TRACE2_EVENT="$PWD/.git/upgrade.trace" \ + git status >.git/upgrade.out && + test_trace2_data fsm_client query/incompatible-daemon 1 \ + <.git/upgrade.trace && + test-tool dump-fsmonitor >.git/fsmonitor && + test_grep ! "builtin:test-legacy:0" .git/fsmonitor && + test_grep \ + "\"argv\":.*\"fsmonitor--daemon\",\"run\",\"--detach\"" \ + .git/upgrade.trace && + + GIT_TRACE2_EVENT="$PWD/.git/warm.trace" \ + git status >.git/warm.out && + ! test_trace2_data index refresh/sum_lstat \ + "[1-9][0-9]*" <.git/warm.trace && + ! test_trace2_data fsm_client query/trivial-response 1 \ + <.git/warm.trace && + test_grep ! \ + "\"argv\":.*\"fsmonitor--daemon\",\"run\",\"--detach\"" \ + .git/warm.trace + ) +' + +test_expect_success 'bound query accepts a capability superset' ' + test_when_finished \ + "stop_daemon_delete_repo capability-superset" && + test_create_repo capability-superset && + ( + cd capability-superset && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.preloadIndex false && + git config core.untrackedCache true && + git status --porcelain=v2 >/dev/null && + git config core.fsmonitor true && + ipc_path=$(git rev-parse --path-format=absolute \ + --git-path fsmonitor--daemon.ipc) && + test-tool simple-ipc start-daemon \ + --name="$ipc_path" --threads=1 \ + --fsmonitor-capability-superset && + + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status >.git/status.out && + test-tool dump-fsmonitor >.git/fsmonitor && + test_grep \ + "^fsmonitor last update builtin:test-capable:0" \ + .git/fsmonitor && + test_grep ! \ + "\"key\":\"query/incompatible-daemon\"" \ + .git/status.trace && + test_grep ! \ + "\"argv\":.*\"fsmonitor--daemon\",\"run\",\"--detach\"" \ + .git/status.trace + ) +' + +test_expect_success MACOS 'worktree binding rejects same-gitdir aliases' ' + test_when_finished "git -C binding-a fsmonitor--daemon stop 2>/dev/null || :" && + git init --separate-git-dir="$PWD/binding-gitdir" binding-a && + mkdir binding-b && + cp binding-a/.git binding-b/.git && + ( + cd binding-a && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TRACE2_EVENT="$PWD/../binding-daemon.trace" \ + git status --porcelain=v2 >/dev/null && + git status --porcelain=v2 >/dev/null + ) && + cp binding-a/tracked binding-b/tracked && + echo changed >>binding-b/tracked && + GIT_OPTIONAL_LOCKS=0 git -c core.fsmonitor=false \ + -c core.untrackedCache=false -C binding-b \ + status --porcelain=v2 >binding.expect && + GIT_OPTIONAL_LOCKS=0 git -C binding-b \ + status --porcelain=v2 >binding.actual && + test_cmp binding.expect binding.actual && + test_grep "^1 \.M .* tracked$" binding.actual && + test_grep "\"key\":\"query/worktree-mismatch\",\"value\":\"1\"" \ + binding-daemon.trace && + git -C binding-a fsmonitor--daemon stop +' + test_done From 3a2262680123609ea614ae6b3343bf3ce5fc5661 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Thu, 23 Jul 2026 23:37:23 -0500 Subject: [PATCH 020/105] dir: skip recursively valid empty UNTR subtrees Even after every directory and ignore input has been validated, collapsed-directory traversal still reopens cached subtrees that are known to contain no untracked paths. That walk repeats work the successful preload has already established. Record recursive validation and whether each cached subtree contains untracked output. In collapsed-directory mode, skip reopening a subtree only when its directory, descendants, check-only mode, and ignore inputs remain valid and no cached untracked entry exists. Clear the recursive proof when directory or ignore state is invalidated. The untracked-cache status test verifies that an unchanged empty subtree visits no directories and that a changed descendant still falls back to traversal and reports the new untracked path. Signed-off-by: Taylor Blau --- dir.c | 27 ++++++++++++++++++++++++ dir.h | 3 +++ t/t7063-status-untracked-cache.sh | 34 +++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+) diff --git a/dir.c b/dir.c index cda3dc208cdfd2..6670b5f4ff869b 100644 --- a/dir.c +++ b/dir.c @@ -411,6 +411,7 @@ static int compute_untracked_cache_valid_recursive( else invalidate_directory(uc, ucd); } + ucd->valid_recursive = valid; return valid; } @@ -474,6 +475,7 @@ int untracked_cache_preload_finish(struct untracked_cache_preload *preload, ucd->stat_checked = 0; ucd->stat_matches = 0; ucd->exclude_matches = 0; + ucd->valid_recursive = 0; /* Invalidation performed after the snapshot always wins. */ if (!task->was_valid || !ucd->valid) continue; @@ -1567,6 +1569,8 @@ static void do_invalidate_gitignore(struct untracked_cache_dir *dir) { int i; dir->valid = 0; + dir->valid_recursive = 0; + dir->has_untracked = 0; for (size_t i = 0; i < dir->untracked_nr; i++) free(dir->untracked[i]); dir->untracked_nr = 0; @@ -1596,6 +1600,7 @@ static void invalidate_directory(struct untracked_cache *uc, uc->dir_invalidated++; dir->valid = 0; + dir->valid_recursive = 0; for (size_t i = 0; i < dir->untracked_nr; i++) free(dir->untracked[i]); dir->untracked_nr = 0; @@ -2987,6 +2992,7 @@ static void add_untracked(struct untracked_cache_dir *dir, const char *name) ALLOC_GROW(dir->untracked, dir->untracked_nr + 1, dir->untracked_alloc); dir->untracked[dir->untracked_nr++] = xstrdup(name); + dir->has_untracked = 1; } static int valid_cached_dir(struct dir_struct *dir, @@ -3131,6 +3137,8 @@ static void remove_collapsed_untracked_child( static void close_cached_dir(struct cached_dir *cdir) { + int i; + if (cdir->fdir) closedir(cdir->fdir); /* @@ -3140,6 +3148,12 @@ static void close_cached_dir(struct cached_dir *cdir) if (cdir->untracked) { cdir->untracked->valid = 1; cdir->untracked->recurse = 1; + cdir->untracked->has_untracked = !!cdir->untracked->untracked_nr; + for (i = 0; !cdir->untracked->has_untracked && + i < cdir->untracked->dirs_nr; i++) + cdir->untracked->has_untracked = + cdir->untracked->dirs[i]->recurse && + cdir->untracked->dirs[i]->has_untracked; } } @@ -3211,6 +3225,14 @@ static enum path_treatment read_directory_recursive(struct dir_struct *dir, struct strbuf path = STRBUF_INIT; strbuf_add(&path, base, baselen); + if (untracked && dir->internal.untracked_cache_preloaded && + untracked->valid && untracked->valid_recursive && + untracked->check_only == !!check_only && + !untracked->has_untracked && + (dir->flags & DIR_SHOW_OTHER_DIRECTORIES)) { + untracked->recurse = 1; + goto out; + } if (open_cached_dir(&cdir, dir, untracked, istate, &path, check_only)) goto out; @@ -4344,7 +4366,11 @@ static int read_one_dir(struct untracked_cache_dir **untracked_, for (i = 0; i < untracked->dirs_nr; i++) { if (read_one_dir(untracked->dirs + i, rd) < 0) return -1; + if (untracked->dirs[i]->has_untracked) + untracked->has_untracked = 1; } + if (untracked->untracked_nr) + untracked->has_untracked = 1; return 0; } @@ -4486,6 +4512,7 @@ static void invalidate_one_directory(struct untracked_cache *uc, { uc->dir_invalidated++; ucd->valid = 0; + ucd->valid_recursive = 0; for (size_t i = 0; i < ucd->untracked_nr; i++) free(ucd->untracked[i]); ucd->untracked_nr = 0; diff --git a/dir.h b/dir.h index 60b63bbf304782..717e96386ee06c 100644 --- a/dir.h +++ b/dir.h @@ -182,10 +182,13 @@ struct untracked_cache_dir { /* all data except 'dirs' in this struct are good */ unsigned int valid : 1; unsigned int recurse : 1; + /* this subtree contains at least one cached untracked entry */ + unsigned int has_untracked : 1; /* transient results from directory-stat preloading */ unsigned int stat_checked : 1; unsigned int stat_matches : 1; unsigned int exclude_matches : 1; + unsigned int valid_recursive : 1; /* null object ID means this directory does not have .gitignore */ struct object_id exclude_oid; char name[FLEX_ARRAY]; diff --git a/t/t7063-status-untracked-cache.sh b/t/t7063-status-untracked-cache.sh index 5948a579b5783b..1cf25b5088284e 100755 --- a/t/t7063-status-untracked-cache.sh +++ b/t/t7063-status-untracked-cache.sh @@ -1077,6 +1077,40 @@ test_expect_success 'preload verifies cached per-directory excludes' ' ) ' +test_expect_success 'recursive preload checks descendant directory mtimes' ' + test_create_repo recursive-preload && + ( + cd recursive-preload && + mkdir -p a/b && + echo tracked >a/b/tracked && + git add a/b/tracked && + git commit -m base && + git config core.untrackedCache true && + git config core.fsmonitor false && + git status --porcelain >/dev/null && + git status --porcelain >/dev/null && + avoid_racy && + git status --porcelain >/dev/null && + git status --porcelain >/dev/null && + GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD=1 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=4 \ + GIT_TRACE2_EVENT="$PWD/.git/pruned.trace" \ + git status --porcelain >.git/pruned && + test_must_be_empty .git/pruned && + test_grep \ + "directories-visited.*value.*0" \ + .git/pruned.trace && + avoid_racy && + echo untracked >a/b/new-untracked && + GIT_OPTIONAL_LOCKS=0 git -c core.untrackedCache=false \ + status --porcelain >.git/expect && + GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD=1 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=4 \ + git status --porcelain >.git/actual && + test_cmp .git/expect .git/actual + ) +' + test_expect_success 'recursive preload rescans a vanished collapsed witness' ' test_create_repo collapsed-witness && ( From eb042ff3d65452219b62ea73e8607db0213aea65 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 02:00:58 -0500 Subject: [PATCH 021/105] fsmonitor: invalidate attributes for matched directory summaries A provider can report a directory move or modification without naming a changed .gitattributes file beneath it. Existing directory handling invalidates tracked entries in the reported cone but can leave cached attribute stacks describing the old conversion rules. Discard cached attribute stacks only after directory handling matches at least one tracked index entry. Record semantic/attributes-cone with the number of matched entries. An unmatched directory keeps its existing case-correction and untracked-path fallback without speculatively flushing attribute state. Extend t/helper/test-read-cache.c to cache an old attribute, process a directory event, and require the new attribute value. Add hook regressions in t/t7519-status-fsmonitor.sh for both an indexed cone and an unmatched directory, including their distinct Trace2 behavior. Signed-off-by: Taylor Blau --- fsmonitor.c | 10 ++++++ t/helper/test-read-cache.c | 45 ++++++++++++++++++++++++ t/t7519-status-fsmonitor.sh | 69 +++++++++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+) diff --git a/fsmonitor.c b/fsmonitor.c index 2fd070b1d5b22a..df716a26b85499 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -464,6 +464,16 @@ static size_t handle_path_with_trailing_slash( nr_in_cone++; } + if (nr_in_cone) { + /* + * A matched directory event may stand in for a nested + * attribute-file change. + */ + git_attr_invalidate_all(); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/attributes-cone", nr_in_cone); + } + return nr_in_cone; } diff --git a/t/helper/test-read-cache.c b/t/helper/test-read-cache.c index f5dae8ecfcc485..c7631a204c8b2a 100644 --- a/t/helper/test-read-cache.c +++ b/t/helper/test-read-cache.c @@ -1,9 +1,11 @@ #define USE_THE_REPOSITORY_VARIABLE #include "test-tool.h" +#include "attr.h" #include "config.h" #include "environment.h" #include "fsmonitor.h" +#include "fsmonitor-ll.h" #include "read-cache-ll.h" #include "repository.h" #include "setup.h" @@ -41,6 +43,45 @@ static int test_fsmonitor_content_recovery(const char *path) return 0; } +static int test_fsmonitor_directory_attributes(void) +{ + struct attr_check *check; + int ret = 1; + + setup_git_directory(the_repository); + repo_config(the_repository, git_default_config, NULL); + if (repo_read_index(the_repository) < 0) + return error("unable to read test index"); + + check = attr_check_initl("marker", NULL); + git_check_attr(the_repository->index, "tracked-dir/tracked", check); + if (!check->items[0].value || + strcmp(check->items[0].value, "old")) { + error("initial attribute value was not cached"); + goto done; + } + + write_file("tracked-dir/.gitattributes", "tracked marker=new\n"); + /* + * repo_read_index() consumed the normal refresh. Re-arm it after + * caching the pre-event attribute value. + */ + the_repository->index->fsmonitor_has_run_once = 0; + refresh_fsmonitor(the_repository->index); + git_check_attr(the_repository->index, "tracked-dir/tracked", check); + if (!check->items[0].value || + strcmp(check->items[0].value, "new")) { + error("directory event did not invalidate cached attributes"); + goto done; + } + ret = 0; + +done: + attr_check_free(check); + discard_index(the_repository->index); + return ret; +} + int cmd__read_cache(int argc, const char **argv) { int i, cnt = 1; @@ -49,6 +90,10 @@ int cmd__read_cache(int argc, const char **argv) if (argc == 3 && !strcmp(argv[1], "--test-fsmonitor-content-recovery")) return test_fsmonitor_content_recovery(argv[2]); + if (argc == 2 && + !strcmp(argv[1], "--test-fsmonitor-directory-attributes")) + return test_fsmonitor_directory_attributes(); + if (argc > 1 && skip_prefix(argv[1], "--print-and-refresh=", &name)) { argc--; argv++; diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index fb2fadc53d5986..691148ae677113 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -657,6 +657,75 @@ test_expect_success 'provider global marker invalidates every tracked entry' ' ) ' +test_expect_success \ + 'directory attribute invalidation requires an indexed cone' ' + test_create_repo directory-attributes && + ( + cd directory-attributes && + mkdir tracked-dir && + test_commit base tracked-dir/tracked && + test_hook --setup fsmonitor-test <<-\EOF && + if test -f .git/report-cone + then + printf "cone-token\0tracked-dir/\0" + elif test -f .git/report-unmatched + then + printf "unmatched-token\0untracked/\0" + else + printf "base-token\0" + fi + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git status --porcelain=v2 >/dev/null && + git status --porcelain=v2 >/dev/null && + + > .git/report-cone && + GIT_TRACE2_EVENT="$PWD/.git/cone.trace" \ + GIT_TRACE_FSMONITOR="$PWD/.git/cone.fsm" \ + git status --porcelain=v2 >.git/cone.actual && + test_must_be_empty .git/cone.actual && + test_grep "fsmonitor_refresh_callback.*tracked-dir/" \ + .git/cone.fsm && + test_trace2_data fsmonitor semantic/attributes-cone 1 \ + <.git/cone.trace && + + rm .git/report-cone && + > .git/report-unmatched && + GIT_TRACE2_EVENT="$PWD/.git/unmatched.trace" \ + GIT_TRACE_FSMONITOR="$PWD/.git/unmatched.fsm" \ + git status --porcelain=v2 >.git/unmatched.actual && + test_must_be_empty .git/unmatched.actual && + test_grep "fsmonitor_refresh_callback.*untracked/" \ + .git/unmatched.fsm && + test_grep ! \ + "\"category\":\"fsmonitor\",\"key\":\"semantic/attributes-cone\"" \ + .git/unmatched.trace + ) +' + +test_expect_success 'directory events invalidate cached attributes' ' + test_create_repo directory-attribute-cache && + ( + cd directory-attribute-cache && + mkdir tracked-dir && + test_write_lines "tracked marker=old" \ + >tracked-dir/.gitattributes && + test_write_lines tracked >tracked-dir/tracked && + git add tracked-dir && + git commit -m base && + test_hook --setup fsmonitor-test <<-\EOF && + printf "new-token\0tracked-dir/\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + test-tool read-cache \ + --test-fsmonitor-directory-attributes + ) +' + test_expect_success HARDLINKS,!MINGW,!CYGWIN \ 'multiply-linked files stay fsmonitor-invalid' ' test_when_finished "rm -f hardlink-alias" && From 37d0ae4167dff07ed63239ce389cc936b9d3c75f Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Thu, 23 Jul 2026 23:47:44 -0500 Subject: [PATCH 022/105] preload-index: queue bounded bulk directory scans Ordinary index preload assigns existing paths directly to workers. A physical directory walk instead discovers new tasks while it runs, so an unbounded queue can exhaust descriptors or strand tasks when worker creation fails. Add a directory-task queue that retains parent and child identities, budgets descriptors against RLIMIT_NOFILE, and tracks queued as well as in-flight work. Reserve at most 128 task descriptors, leave up to 16 for the rest of the process, and run a worker synchronously when extra threads cannot start. Register the common queue for Darwin in Make, CMake, and Meson. No bulk scan is invoked from preload_index(), so existing behavior is unchanged. Signed-off-by: Taylor Blau --- Makefile | 8 + config.mak.uname | 1 + contrib/buildsystems/CMakeLists.txt | 6 + meson.build | 3 + preload-index-bulk-thread.c | 232 ++++++++++++++++++++++++++++ preload-index-bulk.h | 76 +++++++++ 6 files changed, 326 insertions(+) create mode 100644 preload-index-bulk-thread.c create mode 100644 preload-index-bulk.h diff --git a/Makefile b/Makefile index edac13257671b1..f9bd42aba71222 100644 --- a/Makefile +++ b/Makefile @@ -413,6 +413,9 @@ include shared.mak # `compat/fsmonitor/fsm-health-.c` files # that implement the `fsm_listen__*()` and `fsm_health__*()` routines. # +# If a platform supports bulk worktree scans during index preload, set +# PRELOAD_INDEX_BULK_BACKEND to the name of its backend. +# # If your platform has OS-specific ways to tell if a repo is incompatible with # fsmonitor (whether the hook or IPC daemon version), set FSMONITOR_OS_SETTINGS # to the "" of the corresponding `compat/fsmonitor/fsm-settings-.c` @@ -1378,6 +1381,11 @@ LIB_OBJS += worktree.o LIB_OBJS += wrapper.o LIB_OBJS += write-or-die.o LIB_OBJS += ws.o +ifdef PRELOAD_INDEX_BULK_BACKEND +PRELOAD_INDEX_BULK_OBJS += preload-index-bulk-thread.o +PRELOAD_INDEX_BULK_OBJS += $(PRELOAD_INDEX_BULK_PLATFORM_OBJS) +endif +LIB_OBJS += $(PRELOAD_INDEX_BULK_OBJS) LIB_OBJS += wt-status.o LIB_OBJS += xdiff-interface.o LIB_OBJS += xdiff/xdiffi.o diff --git a/config.mak.uname b/config.mak.uname index 9ebd240378ca59..af3f927d8b8a50 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -162,6 +162,7 @@ ifeq ($(uname_S),Darwin) USE_ENHANCED_BASIC_REGULAR_EXPRESSIONS = YesPlease HAVE_PLATFORM_PROCINFO = YesPlease COMPAT_OBJS += compat/darwin/procinfo.o + PRELOAD_INDEX_BULK_BACKEND = darwin ifeq ($(uname_M),arm64) HOMEBREW_PREFIX = /opt/homebrew diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index a57c4b464fa456..1eb0a9fc0ef646 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -103,6 +103,7 @@ macro(parse_makefile_for_sources list_var makefile regex) file(STRINGS ${makefile} ${list_var} REGEX "^${regex} \\+=(.*)") string(REPLACE "${regex} +=" "" ${list_var} ${${list_var}}) string(REPLACE "$(COMPAT_OBJS)" "" ${list_var} ${${list_var}}) #remove "$(COMPAT_OBJS)" This is only for libgit. + string(REPLACE "$(PRELOAD_INDEX_BULK_OBJS)" "" ${list_var} ${${list_var}}) string(STRIP ${${list_var}} ${list_var}) #remove trailing/leading whitespaces string(REPLACE ".o" ".c;" ${list_var} ${${list_var}}) #change .o to .c, ; is for converting the string into a list list(TRANSFORM ${list_var} STRIP) #remove trailing/leading whitespaces for each element in list @@ -664,6 +665,11 @@ include_directories(${CMAKE_BINARY_DIR}) #libgit parse_makefile_for_sources(libgit_SOURCES ${CMAKE_SOURCE_DIR}/Makefile "LIB_OBJS") +if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + list(APPEND libgit_SOURCES + preload-index-bulk-thread.c) +endif() + list(TRANSFORM libgit_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") list(TRANSFORM compat_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") diff --git a/meson.build b/meson.build index a777f4722a2061..68cd5819811f3e 100644 --- a/meson.build +++ b/meson.build @@ -1345,6 +1345,9 @@ elif host_machine.system() == 'windows' compat_sources += 'compat/win32/trace2_win32_process_info.c' elif host_machine.system() == 'darwin' compat_sources += 'compat/darwin/procinfo.c' + libgit_sources += [ + 'preload-index-bulk-thread.c', + ] else compat_sources += 'compat/stub/procinfo.c' endif diff --git a/preload-index-bulk-thread.c b/preload-index-bulk-thread.c new file mode 100644 index 00000000000000..0a73d5a1fdeafd --- /dev/null +++ b/preload-index-bulk-thread.c @@ -0,0 +1,232 @@ +#include "git-compat-util.h" + +#include + +#include "preload-index-bulk.h" + +#define PRELOAD_INDEX_BULK_OPEN_FD_CAP 128 +#define PRELOAD_INDEX_BULK_OPEN_FD_RESERVE 16 + +static void queue_set_failed(struct preload_bulk_queue *queue) +{ + pthread_mutex_lock(&queue->mutex); + queue->failed = 1; + pthread_mutex_unlock(&queue->mutex); +} + +static void enqueue_task(struct preload_bulk_scan *scan, + struct preload_bulk_task *task) +{ + struct preload_bulk_queue *queue = &scan->queue; + + pthread_mutex_lock(&queue->mutex); + task->next = queue->head; + queue->head = task; + queue->pending++; + pthread_cond_signal(&queue->cond); + pthread_mutex_unlock(&queue->mutex); +} + +static int reserve_open_fd(struct preload_bulk_queue *queue) +{ + int reserved = 0; + + pthread_mutex_lock(&queue->mutex); + if (queue->open_fds < queue->open_fd_limit) { + queue->open_fds++; + reserved = 1; + } + pthread_mutex_unlock(&queue->mutex); + return reserved; +} + +static void release_open_fd(struct preload_bulk_queue *queue) +{ + pthread_mutex_lock(&queue->mutex); + if (!queue->open_fds) + BUG("bulk preload open-fd count underflow"); + queue->open_fds--; + pthread_mutex_unlock(&queue->mutex); +} + +void preload_bulk_schedule_directory( + struct preload_bulk_worker *worker, int parent_fd, + const struct preload_bulk_dir_identity *parent_identity, + const struct preload_bulk_dir_identity *child_identity, + const char *name, const char *path, size_t path_len) +{ + struct preload_bulk_scan *scan = worker->scan; + struct preload_bulk_task *task; + + FLEX_ALLOC_MEM(task, path, path, path_len); + if (parent_identity) { + task->parent_identity = *parent_identity; + task->has_parent_identity = 1; + } + if (child_identity) { + task->child_identity = *child_identity; + task->has_child_identity = 1; + } + task->fd = -1; + if (reserve_open_fd(&scan->queue)) { + task->reserved_fd = 1; + task->fd = scan->backend->open_dir_at(worker, parent_fd, name); + if (task->fd < 0) { + int saved_errno = errno; + + task->reserved_fd = 0; + release_open_fd(&scan->queue); + if (saved_errno == EXDEV) { + free(task); + return; + } + if (saved_errno != EMFILE && saved_errno != ENFILE) { + free(task); + queue_set_failed(&scan->queue); + return; + } + } + } + enqueue_task(scan, task); +} + +static size_t preload_bulk_open_fd_limit(void) +{ + struct rlimit limit; + rlim_t value; + + if (getrlimit(RLIMIT_NOFILE, &limit)) + return 1; + if (limit.rlim_cur == RLIM_INFINITY) + return PRELOAD_INDEX_BULK_OPEN_FD_CAP; + if (limit.rlim_cur <= PRELOAD_INDEX_BULK_OPEN_FD_RESERVE) + return 1; + value = limit.rlim_cur - PRELOAD_INDEX_BULK_OPEN_FD_RESERVE; + if (value > PRELOAD_INDEX_BULK_OPEN_FD_CAP) + value = PRELOAD_INDEX_BULK_OPEN_FD_CAP; + return value; +} + +static int queue_init(struct preload_bulk_queue *queue) +{ + memset(queue, 0, sizeof(*queue)); +#if HAVE_THREADS + if (pthread_mutex_init(&queue->mutex, NULL)) + return -1; + if (pthread_cond_init(&queue->cond, NULL)) { + pthread_mutex_destroy(&queue->mutex); + return -1; + } +#endif + queue->open_fd_limit = preload_bulk_open_fd_limit(); + return 0; +} + +static void queue_release(struct preload_bulk_queue *queue) +{ + if (queue->head || queue->pending || queue->open_fds) + BUG("releasing non-empty bulk preload queue"); +#if HAVE_THREADS + pthread_cond_destroy(&queue->cond); + pthread_mutex_destroy(&queue->mutex); +#endif + memset(queue, 0, sizeof(*queue)); +} + +static void *preload_bulk_worker_main(void *data) +{ + struct preload_bulk_worker *worker = data; + struct preload_bulk_queue *queue = &worker->scan->queue; + + for (;;) { + struct preload_bulk_task *task; + int failed, reserved_fd; + + pthread_mutex_lock(&queue->mutex); + while (!queue->head && queue->pending) + pthread_cond_wait(&queue->cond, &queue->mutex); + if (!queue->pending) { + pthread_mutex_unlock(&queue->mutex); + break; + } + task = queue->head; + queue->head = task->next; + pthread_mutex_unlock(&queue->mutex); + + failed = + worker->scan->backend->scan_directory(worker, task); + reserved_fd = task->reserved_fd; + free(task); + + pthread_mutex_lock(&queue->mutex); + if (failed) + queue->failed = 1; + if (reserved_fd) { + if (!queue->open_fds) + BUG("bulk preload open-fd count underflow"); + queue->open_fds--; + } + if (!queue->pending) + BUG("bulk preload task count underflow"); + queue->pending--; + if (!queue->pending) + pthread_cond_broadcast(&queue->cond); + pthread_mutex_unlock(&queue->mutex); + } + return NULL; +} + +int preload_bulk_run_scan(struct preload_bulk_scan *scan, + struct preload_bulk_run_result *result) +{ + struct preload_bulk_task *root_task; + int failed, started_threads = 1; + + if (scan->threads < 1) + BUG("bulk preload scan requires at least one worker"); + memset(result, 0, sizeof(*result)); + if (queue_init(&scan->queue)) + return -1; + CALLOC_ARRAY(scan->workers, scan->threads); + for (int i = 0; i < scan->threads; i++) + scan->workers[i].scan = scan; + + FLEX_ALLOC_STR(root_task, path, "."); + if (!reserve_open_fd(&scan->queue)) + BUG("bulk preload queue cannot reserve its root descriptor"); + root_task->reserved_fd = 1; + root_task->fd = fcntl(scan->root_fd, F_DUPFD_CLOEXEC, 0); + if (root_task->fd < 0) { + release_open_fd(&scan->queue); + free(root_task); + free(scan->workers); + scan->workers = NULL; + queue_release(&scan->queue); + return -1; + } + enqueue_task(scan, root_task); + + for (int i = 1; i < scan->threads; i++) { + int err = pthread_create(&scan->workers[i].thread, NULL, + preload_bulk_worker_main, + &scan->workers[i]); + + if (err) + break; + scan->workers[i].started = 1; + started_threads++; + } + preload_bulk_worker_main(&scan->workers[0]); + for (int i = 1; i < scan->threads; i++) + if (scan->workers[i].started && + pthread_join(scan->workers[i].thread, NULL)) + BUG("unable to join bulk preload worker"); + + result->threads = started_threads; + failed = scan->queue.failed; + + free(scan->workers); + scan->workers = NULL; + queue_release(&scan->queue); + return failed ? -1 : 0; +} diff --git a/preload-index-bulk.h b/preload-index-bulk.h new file mode 100644 index 00000000000000..64cb000474413a --- /dev/null +++ b/preload-index-bulk.h @@ -0,0 +1,76 @@ +#ifndef PRELOAD_INDEX_BULK_H +#define PRELOAD_INDEX_BULK_H + +#include "git-compat-util.h" +#include "thread-utils.h" + +struct preload_bulk_dir_identity { + struct stat stat; + unsigned complete : 1; +}; + +struct preload_bulk_task { + struct preload_bulk_task *next; + struct preload_bulk_dir_identity parent_identity; + struct preload_bulk_dir_identity child_identity; + int fd; + unsigned reserved_fd : 1; + unsigned has_parent_identity : 1; + unsigned has_child_identity : 1; + char path[FLEX_ARRAY]; +}; + +struct preload_bulk_queue { + pthread_mutex_t mutex; + pthread_cond_t cond; + struct preload_bulk_task *head; + /* + * pending includes queued and in-flight tasks. open_fds counts only + * descriptor reservations held by tasks. + */ + size_t pending; + size_t open_fds; + size_t open_fd_limit; + int failed; +}; + +struct preload_bulk_scan; + +struct preload_bulk_worker { + struct preload_bulk_scan *scan; + pthread_t thread; + unsigned started : 1; +}; + +struct preload_bulk_backend { + int (*open_dir_at)(struct preload_bulk_worker *worker, int parent_fd, + const char *name); + /* + * Consume task->fd when it is non-negative, and close it before + * returning. + */ + int (*scan_directory)(struct preload_bulk_worker *worker, + struct preload_bulk_task *task); +}; + +struct preload_bulk_scan { + const struct preload_bulk_backend *backend; + struct preload_bulk_queue queue; + struct preload_bulk_worker *workers; + int root_fd; + int threads; +}; + +struct preload_bulk_run_result { + int threads; +}; + +void preload_bulk_schedule_directory( + struct preload_bulk_worker *worker, int parent_fd, + const struct preload_bulk_dir_identity *parent_identity, + const struct preload_bulk_dir_identity *child_identity, + const char *name, const char *path, size_t path_len); +int preload_bulk_run_scan(struct preload_bulk_scan *scan, + struct preload_bulk_run_result *result); + +#endif /* PRELOAD_INDEX_BULK_H */ From ab71232b3fecc452255081db1807c0c7c4c71c71 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Thu, 23 Jul 2026 23:48:20 -0500 Subject: [PATCH 023/105] preload-index: classify sparse-aware bulk stat observations A bulk directory worker must locate each observed tracked path and decide whether a directory has tracked descendants. Plain pathname ordering cannot answer either question correctly for sparse indexes. Add sparse-aware entry and descendant lookups with unseen, clean, content-check, and fallback states. Compare observed metadata with ie_match_stat(), and make duplicate observations fall back through an atomic compare-and-exchange or the existing queue mutex. Skip staged, intent-to-add, skip-worktree, removed, and otherwise ineligible entries. Register the index classifier in Make, CMake, and Meson without introducing deletion outcomes or content proofs. Signed-off-by: Taylor Blau --- Makefile | 1 + contrib/buildsystems/CMakeLists.txt | 1 + meson.build | 1 + preload-index-bulk-index.c | 101 ++++++++++++++++++++++++++++ preload-index-bulk.h | 10 +++ preload-index.h | 7 ++ 6 files changed, 121 insertions(+) create mode 100644 preload-index-bulk-index.c diff --git a/Makefile b/Makefile index f9bd42aba71222..51059dadddf171 100644 --- a/Makefile +++ b/Makefile @@ -1382,6 +1382,7 @@ LIB_OBJS += wrapper.o LIB_OBJS += write-or-die.o LIB_OBJS += ws.o ifdef PRELOAD_INDEX_BULK_BACKEND +PRELOAD_INDEX_BULK_OBJS += preload-index-bulk-index.o PRELOAD_INDEX_BULK_OBJS += preload-index-bulk-thread.o PRELOAD_INDEX_BULK_OBJS += $(PRELOAD_INDEX_BULK_PLATFORM_OBJS) endif diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 1eb0a9fc0ef646..a9e4e6aaa9d10c 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -667,6 +667,7 @@ parse_makefile_for_sources(libgit_SOURCES ${CMAKE_SOURCE_DIR}/Makefile "LIB_OBJS if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") list(APPEND libgit_SOURCES + preload-index-bulk-index.c preload-index-bulk-thread.c) endif() diff --git a/meson.build b/meson.build index 68cd5819811f3e..1f8667e551a9c9 100644 --- a/meson.build +++ b/meson.build @@ -1346,6 +1346,7 @@ elif host_machine.system() == 'windows' elif host_machine.system() == 'darwin' compat_sources += 'compat/darwin/procinfo.c' libgit_sources += [ + 'preload-index-bulk-index.c', 'preload-index-bulk-thread.c', ] else diff --git a/preload-index-bulk-index.c b/preload-index-bulk-index.c new file mode 100644 index 00000000000000..3c8bfad7c2a631 --- /dev/null +++ b/preload-index-bulk-index.c @@ -0,0 +1,101 @@ +#include "git-compat-util.h" +#include "preload-index-bulk.h" +#include "read-cache-ll.h" + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +int preload_bulk_index_position(struct preload_bulk_scan *scan, + const char *path, size_t path_len) +{ + if (path_len > INT_MAX) + return -1; + return index_name_pos_sparse(scan->istate, path, path_len); +} + +int preload_bulk_index_pos_has_tracked_descendants( + struct preload_bulk_scan *scan, const char *path, size_t path_len, + int pos) +{ + struct index_state *istate = scan->istate; + const struct cache_entry *ce; + + if (pos >= 0) + return 0; + pos = -pos - 1; + while ((unsigned int)pos < istate->cache_nr) { + ce = istate->cache[pos]; + if (ce_namelen(ce) < path_len || + memcmp(ce->name, path, path_len)) + return 0; + if (ce_namelen(ce) == path_len) { + pos++; + continue; + } + if (ce->name[path_len] == '/') + return 1; + if ((unsigned char)ce->name[path_len] > '/') + return 0; + pos++; + } + return 0; +} + +static int record_tracked_state(struct preload_bulk_worker *worker, int pos, + unsigned char state) +{ + struct preload_bulk_scan *scan = worker->scan; + int recorded = 1; + +#if GIT_GNUC_PREREQ(4, 7) || \ + (__has_builtin(__atomic_compare_exchange_n) && \ + __has_builtin(__atomic_store_n)) + unsigned char expected = PRELOAD_BULK_TRACKED_UNSEEN; + + if (!__atomic_compare_exchange_n(&scan->tracked_state[pos], &expected, + state, 0, __ATOMIC_RELAXED, + __ATOMIC_RELAXED)) { + __atomic_store_n(&scan->tracked_state[pos], + PRELOAD_BULK_TRACKED_FALLBACK, + __ATOMIC_RELAXED); + recorded = 0; + } +#else + pthread_mutex_lock(&scan->queue.mutex); + if (scan->tracked_state[pos] != PRELOAD_BULK_TRACKED_UNSEEN) { + state = PRELOAD_BULK_TRACKED_FALLBACK; + recorded = 0; + } + scan->tracked_state[pos] = state; + pthread_mutex_unlock(&scan->queue.mutex); +#endif + return recorded; +} + +static int tracked_entry_is_eligible(const struct cache_entry *ce) +{ + return !ce_stage(ce) && + !ce_intent_to_add(ce) && + !ce_skip_worktree(ce) && + !(ce->ce_flags & (CE_VALID | CE_REMOVE)) && + (S_ISREG(ce->ce_mode) || S_ISLNK(ce->ce_mode)); +} + +void preload_bulk_record_tracked( + struct preload_bulk_worker *worker, int pos, const struct stat *st) +{ + struct preload_bulk_scan *scan = worker->scan; + struct cache_entry *ce = scan->istate->cache[pos]; + unsigned int changed; + unsigned char state; + + if (!tracked_entry_is_eligible(ce)) + return; + changed = ie_match_stat( + scan->istate, ce, (struct stat *)st, + CE_MATCH_RACY_IS_DIRTY | CE_MATCH_IGNORE_FSMONITOR); + state = changed ? PRELOAD_BULK_TRACKED_CONTENT_CHECK : + PRELOAD_BULK_TRACKED_CLEAN; + record_tracked_state(worker, pos, state); +} diff --git a/preload-index-bulk.h b/preload-index-bulk.h index 64cb000474413a..64f5e9cc9a167d 100644 --- a/preload-index-bulk.h +++ b/preload-index-bulk.h @@ -2,6 +2,7 @@ #define PRELOAD_INDEX_BULK_H #include "git-compat-util.h" +#include "preload-index.h" #include "thread-utils.h" struct preload_bulk_dir_identity { @@ -54,9 +55,11 @@ struct preload_bulk_backend { }; struct preload_bulk_scan { + struct index_state *istate; const struct preload_bulk_backend *backend; struct preload_bulk_queue queue; struct preload_bulk_worker *workers; + unsigned char *tracked_state; int root_fd; int threads; }; @@ -70,6 +73,13 @@ void preload_bulk_schedule_directory( const struct preload_bulk_dir_identity *parent_identity, const struct preload_bulk_dir_identity *child_identity, const char *name, const char *path, size_t path_len); +int preload_bulk_index_position(struct preload_bulk_scan *scan, + const char *path, size_t path_len); +int preload_bulk_index_pos_has_tracked_descendants( + struct preload_bulk_scan *scan, const char *path, size_t path_len, + int pos); +void preload_bulk_record_tracked( + struct preload_bulk_worker *worker, int pos, const struct stat *st); int preload_bulk_run_scan(struct preload_bulk_scan *scan, struct preload_bulk_run_result *result); diff --git a/preload-index.h b/preload-index.h index 251b1ed88e9820..4b21e22b6afb19 100644 --- a/preload-index.h +++ b/preload-index.h @@ -5,6 +5,13 @@ struct index_state; struct pathspec; struct repository; +enum preload_bulk_tracked_state { + PRELOAD_BULK_TRACKED_UNSEEN = 0, + PRELOAD_BULK_TRACKED_CLEAN, + PRELOAD_BULK_TRACKED_CONTENT_CHECK, + PRELOAD_BULK_TRACKED_FALLBACK, +}; + void preload_index(struct index_state *index, const struct pathspec *pathspec, unsigned int refresh_flags); From aa37846a33bab0c7372189c2d87605e855456b1c Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 07:26:38 -0500 Subject: [PATCH 024/105] preload-index: bind APFS scans to a stable worktree root A pathname-based directory walk can cross into a replacement worktree or another mount after the scan begins. Metadata from that namespace cannot safely certify entries from the original worktree. Open the worktree directory with O_NOFOLLOW, accept only a local APFS root, and capture its device, filesystem identity, and stat data. Reopen the configured worktree at completion and compare its identity using the namespace helper supplied by S01/P08. Register the Darwin root helpers in Make, CMake, and Meson. Their presence does not yet activate bulk preload. Signed-off-by: Taylor Blau --- compat/preload-index/bulk-darwin-root.c | 100 ++++++++++++++++++++++++ compat/preload-index/bulk-darwin.h | 24 ++++++ config.mak.uname | 1 + contrib/buildsystems/CMakeLists.txt | 5 +- meson.build | 5 +- preload-index-bulk.h | 2 + 6 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 compat/preload-index/bulk-darwin-root.c create mode 100644 compat/preload-index/bulk-darwin.h diff --git a/compat/preload-index/bulk-darwin-root.c b/compat/preload-index/bulk-darwin-root.c new file mode 100644 index 00000000000000..f12f730761c99c --- /dev/null +++ b/compat/preload-index/bulk-darwin-root.c @@ -0,0 +1,100 @@ +#include "git-compat-util.h" + +#include + +#include "compat/preload-index/bulk-darwin.h" +#include "path-namespace.h" +#include "repository.h" +#include "preload-index-bulk.h" + +static int same_fsid(const fsid_t *a, const fsid_t *b) +{ + return !memcmp(a, b, sizeof(*a)); +} + +static int stat_local_apfs(int fd, struct stat *st, struct statfs *fs) +{ + if (fstat(fd, st) || fstatfs(fd, fs)) + return -1; + if (!S_ISDIR(st->st_mode) || !(fs->f_flags & MNT_LOCAL) || + strcmp(fs->f_fstypename, "apfs")) { + errno = EXDEV; + return -1; + } + return 0; +} + +int preload_bulk_darwin_fd_on_root_mount(struct preload_bulk_scan *scan, + int fd, struct stat *st_out) +{ + struct preload_bulk_darwin_data *data = scan->platform_data; + struct statfs fs; + struct stat st; + + if (stat_local_apfs(fd, &st, &fs)) + return -1; + if (st.st_dev != data->root_stat.st_dev || + !same_fsid(&fs.f_fsid, &data->root_fsid)) { + errno = EXDEV; + return -1; + } + if (st_out) + *st_out = st; + return 0; +} + +const char *preload_bulk_darwin_open_root(struct preload_bulk_scan *scan) +{ + struct preload_bulk_darwin_data *data; + struct statfs fs; + struct stat st; + + CALLOC_ARRAY(data, 1); + scan->platform_data = data; + scan->root_fd = open(repo_get_work_tree(scan->repo), + O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (scan->root_fd < 0 || + stat_local_apfs(scan->root_fd, &st, &fs)) + return "unsupported-filesystem"; + return NULL; +} + +const char *preload_bulk_darwin_snapshot_root(struct preload_bulk_scan *scan) +{ + struct preload_bulk_darwin_data *data = scan->platform_data; + struct statfs fs; + struct stat st; + + if (stat_local_apfs(scan->root_fd, &st, &fs)) + return "unsupported-filesystem"; + data->root_stat = st; + data->root_fsid = fs.f_fsid; + return NULL; +} + +const char *preload_bulk_darwin_validate_root(struct preload_bulk_scan *scan) +{ + struct preload_bulk_darwin_data *data = scan->platform_data; + struct stat root_after; + int fd = open(repo_get_work_tree(scan->repo), + O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + + if (fd < 0 || + preload_bulk_darwin_fd_on_root_mount(scan, fd, &root_after) || + !path_namespace_stat_equal(&data->root_stat, &root_after)) { + if (fd >= 0) + close(fd); + return "namespace-race"; + } + close(fd); + return NULL; +} + +void preload_bulk_darwin_release(struct preload_bulk_scan *scan) +{ + if (scan->root_fd >= 0) { + close(scan->root_fd); + scan->root_fd = -1; + } + FREE_AND_NULL(scan->platform_data); +} diff --git a/compat/preload-index/bulk-darwin.h b/compat/preload-index/bulk-darwin.h new file mode 100644 index 00000000000000..d96ed99240c3d7 --- /dev/null +++ b/compat/preload-index/bulk-darwin.h @@ -0,0 +1,24 @@ +#ifndef PRELOAD_INDEX_BULK_DARWIN_H +#define PRELOAD_INDEX_BULK_DARWIN_H + +#ifdef __APPLE__ + +#include + +struct preload_bulk_scan; + +struct preload_bulk_darwin_data { + struct stat root_stat; + fsid_t root_fsid; +}; + +int preload_bulk_darwin_fd_on_root_mount(struct preload_bulk_scan *scan, + int fd, struct stat *st_out); +const char *preload_bulk_darwin_open_root(struct preload_bulk_scan *scan); +const char *preload_bulk_darwin_snapshot_root(struct preload_bulk_scan *scan); +const char *preload_bulk_darwin_validate_root(struct preload_bulk_scan *scan); +void preload_bulk_darwin_release(struct preload_bulk_scan *scan); + +#endif /* __APPLE__ */ + +#endif /* PRELOAD_INDEX_BULK_DARWIN_H */ diff --git a/config.mak.uname b/config.mak.uname index af3f927d8b8a50..31f4524d1158ab 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -163,6 +163,7 @@ ifeq ($(uname_S),Darwin) HAVE_PLATFORM_PROCINFO = YesPlease COMPAT_OBJS += compat/darwin/procinfo.o PRELOAD_INDEX_BULK_BACKEND = darwin + PRELOAD_INDEX_BULK_PLATFORM_OBJS += compat/preload-index/bulk-darwin-root.o ifeq ($(uname_M),arm64) HOMEBREW_PREFIX = /opt/homebrew diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index a9e4e6aaa9d10c..67fb3f2d9ba7c8 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -275,7 +275,10 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_compile_definitions(PROCFS_EXECUTABLE_PATH="/proc/self/exe" HAVE_DEV_TTY ) list(APPEND compat_SOURCES unix-socket.c unix-stream-server.c compat/linux/procinfo.c) elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") - list(APPEND compat_SOURCES compat/darwin/procinfo.c) + add_compile_definitions(USE_ST_TIMESPEC) + list(APPEND compat_SOURCES + compat/darwin/procinfo.c + compat/preload-index/bulk-darwin-root.c) endif() if(CMAKE_SYSTEM_NAME STREQUAL "Windows") diff --git a/meson.build b/meson.build index 1f8667e551a9c9..9134428deb5d31 100644 --- a/meson.build +++ b/meson.build @@ -1344,7 +1344,10 @@ if host_machine.system() == 'linux' elif host_machine.system() == 'windows' compat_sources += 'compat/win32/trace2_win32_process_info.c' elif host_machine.system() == 'darwin' - compat_sources += 'compat/darwin/procinfo.c' + compat_sources += [ + 'compat/darwin/procinfo.c', + 'compat/preload-index/bulk-darwin-root.c', + ] libgit_sources += [ 'preload-index-bulk-index.c', 'preload-index-bulk-thread.c', diff --git a/preload-index-bulk.h b/preload-index-bulk.h index 64f5e9cc9a167d..9e8b085160a873 100644 --- a/preload-index-bulk.h +++ b/preload-index-bulk.h @@ -55,8 +55,10 @@ struct preload_bulk_backend { }; struct preload_bulk_scan { + struct repository *repo; struct index_state *istate; const struct preload_bulk_backend *backend; + void *platform_data; struct preload_bulk_queue queue; struct preload_bulk_worker *workers; unsigned char *tracked_state; From 08641fe8ecc75fd1a56f8154e57529e4836a56bc Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 08:30:18 -0500 Subject: [PATCH 025/105] precompose: prepare Unicode configuration before parallel conversion precompose_string_if_needed() lazily reads core.precomposeUnicode on its first non-ASCII input. Concurrent directory workers must not race while initializing repository configuration. Add repo_precompose_utf8_prepare() to resolve that policy before workers start, and add repo_precompose_string_if_needed() for conversion against an explicit repository. Preserve precompose_string_if_needed() as the existing one-argument wrapper. Existing callers retain their behavior; only a caller that opts into explicit preparation separates configuration from parallel conversion. Signed-off-by: Taylor Blau --- compat/precompose_utf8.c | 24 +++++++++++++++++++++--- compat/precompose_utf8.h | 5 +++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/compat/precompose_utf8.c b/compat/precompose_utf8.c index 8077f6235b0cae..2be9f7577f9519 100644 --- a/compat/precompose_utf8.c +++ b/compat/precompose_utf8.c @@ -72,19 +72,32 @@ void probe_utf8_pathname_composition(void) strbuf_release(&path); } -const char *precompose_string_if_needed(const char *in) +void repo_precompose_utf8_prepare(struct repository *repo) +{ + struct repo_config_values *cfg = repo_config_values(repo); + + if (cfg->precomposed_unicode < 0 && + repo_config_get_bool(repo, "core.precomposeunicode", + &cfg->precomposed_unicode)) + cfg->precomposed_unicode = 0; +} + +const char *repo_precompose_string_if_needed(struct repository *repo, + const char *in) { size_t inlen; size_t outlen; - struct repo_config_values *cfg = repo_config_values(the_repository); + struct repo_config_values *cfg = repo_config_values(repo); if (!in) return NULL; if (has_non_ascii(in, (size_t)-1, &inlen)) { iconv_t ic_prec; char *out; + if (cfg->precomposed_unicode < 0) - repo_config_get_bool(the_repository, "core.precomposeunicode", &cfg->precomposed_unicode); + repo_config_get_bool(repo, "core.precomposeunicode", + &cfg->precomposed_unicode); if (cfg->precomposed_unicode != 1) return in; ic_prec = iconv_open(repo_encoding, path_encoding); @@ -104,6 +117,11 @@ const char *precompose_string_if_needed(const char *in) return in; } +const char *precompose_string_if_needed(const char *in) +{ + return repo_precompose_string_if_needed(the_repository, in); +} + const char *precompose_argv_prefix(int argc, const char **argv, const char *prefix) { int i = 0; diff --git a/compat/precompose_utf8.h b/compat/precompose_utf8.h index c7c3cc211e5031..6ec1fa973b7db9 100644 --- a/compat/precompose_utf8.h +++ b/compat/precompose_utf8.h @@ -29,8 +29,13 @@ typedef struct { struct dirent_prec_psx *dirent_nfc; } PREC_DIR; +struct repository; + const char *precompose_argv_prefix(int argc, const char **argv, const char *prefix); const char *precompose_string_if_needed(const char *in); +const char *repo_precompose_string_if_needed(struct repository *repo, + const char *in); +void repo_precompose_utf8_prepare(struct repository *repo); void probe_utf8_pathname_composition(void); PREC_DIR *precompose_utf8_opendir(const char *dirname); From caa0279fe70dd82937b1525016bb0db0d29af0e5 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 21:44:02 -0700 Subject: [PATCH 026/105] fsmonitor: validate FSMN before publishing it The index reader consumed an optional FSMN token and EWAH bitmap before checking their complete framing. A truncated or duplicate record could publish partial monitor state; an impossible bitmap length could allocate out of bounds or cover nonexistent index entries. Validate both FSMN versions against the extension bounds, cap version-2 tokens at 4 KiB, and check EWAH word counts, run lengths, padding, and the final running-length word. Reject a bitmap wider than a non-split index. Publish the token and bitmap only after every check succeeds, and clear all existing FSMN state on failure. Extend the read-cache helper to exercise valid records, duplicates, truncation, invalid literal and set-bit runs, nonzero padding, and an invalid final running-length-word pointer. Register the helper regression in t/t7519-status-fsmonitor.sh. Malformed optional state falls back without making the worktree appear clean. Signed-off-by: Taylor Blau --- fsmonitor.c | 113 ++++++++++++++++++++++++++++--- read-cache-ll.h | 3 +- t/helper/test-read-cache.c | 130 ++++++++++++++++++++++++++++++++++++ t/t7519-status-fsmonitor.sh | 4 ++ 4 files changed, 238 insertions(+), 12 deletions(-) diff --git a/fsmonitor.c b/fsmonitor.c index df716a26b85499..ebec5620dbb630 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -7,6 +7,7 @@ #include "dir.h" #include "environment.h" #include "ewah/ewok.h" +#include "ewah/ewok_rlw.h" #include "fsmonitor.h" #include "fsmonitor-ipc.h" #include "name-hash.h" @@ -17,6 +18,7 @@ #define INDEX_EXTENSION_VERSION1 (1) #define INDEX_EXTENSION_VERSION2 (2) +#define FSMONITOR_TOKEN_MAX (4096) #define HOOK_INTERFACE_VERSION1 (1) #define HOOK_INTERFACE_VERSION2 (2) @@ -40,6 +42,46 @@ static void fsmonitor_ewah_callback(size_t pos, void *is) ce->ce_flags &= ~CE_FSMONITOR_VALID; } +static int fsmonitor_ewah_is_valid(struct ewah_bitmap *bitmap) +{ + size_t pointer = 0, expanded_words = 0; + size_t logical_words = bitmap->bit_size / BITS_IN_EWORD + + !!(bitmap->bit_size % BITS_IN_EWORD); + size_t padding = bitmap->bit_size % BITS_IN_EWORD; + eword_t *last_rlw = NULL; + + while (pointer < bitmap->buffer_size) { + eword_t *rlw = &bitmap->buffer[pointer]; + size_t running_words = rlw_get_running_len(rlw); + size_t literal_words = rlw_get_literal_words(rlw); + size_t i; + + last_rlw = rlw; + if (literal_words > bitmap->buffer_size - pointer - 1) + return 0; + if (running_words > logical_words - expanded_words) + return 0; + expanded_words += running_words; + if (rlw_get_run_bit(rlw) && running_words && padding && + expanded_words == logical_words) + return 0; + if (literal_words > logical_words - expanded_words) + return 0; + for (i = 0; i < literal_words; i++) { + eword_t literal = bitmap->buffer[pointer + 1 + i]; + + if (padding && + expanded_words + i + 1 == logical_words && + literal >> padding) + return 0; + } + expanded_words += literal_words; + pointer += 1 + literal_words; + } + + return expanded_words == logical_words && bitmap->rlw == last_rlw; +} + static int fsmonitor_hook_version(void) { int hook_version; @@ -60,44 +102,81 @@ int read_fsmonitor_extension(struct index_state *istate, const void *data, unsigned long sz) { const char *index = data; + const char *end = index + sz; + const char *nul; uint32_t hdr_version; uint32_t ewah_size; + uint32_t ewah_words; + uint32_t ewah_rlw; struct ewah_bitmap *fsmonitor_dirty; int ret; uint64_t timestamp; struct strbuf last_update = STRBUF_INIT; - if (sz < sizeof(uint32_t) + 1 + sizeof(uint32_t)) - return error("corrupt fsmonitor extension (too short)"); + if (istate->fsmonitor_extension_seen) + goto invalid; + istate->fsmonitor_extension_seen = 1; + if (end - index < sizeof(uint32_t)) + goto invalid; hdr_version = get_be32(index); index += sizeof(uint32_t); if (hdr_version == INDEX_EXTENSION_VERSION1) { + if (end - index < sizeof(uint64_t)) + goto invalid; timestamp = get_be64(index); strbuf_addf(&last_update, "%"PRIu64"", timestamp); index += sizeof(uint64_t); } else if (hdr_version == INDEX_EXTENSION_VERSION2) { - strbuf_addstr(&last_update, index); - index += last_update.len + 1; + nul = memchr(index, '\0', end - index); + if (!nul || nul == index || nul - index > FSMONITOR_TOKEN_MAX) + goto invalid; + strbuf_add(&last_update, index, nul - index); + index = nul + 1; } else { - return error("bad fsmonitor version %d", hdr_version); + goto invalid; } - istate->fsmonitor_last_update = strbuf_detach(&last_update, NULL); - + if (end - index < sizeof(uint32_t)) + goto invalid; ewah_size = get_be32(index); index += sizeof(uint32_t); + if (ewah_size != end - index || ewah_size < 3 * sizeof(uint32_t)) + goto invalid; + + /* Reject impossible EWAH lengths before its parser allocates memory. */ + ewah_words = get_be32(index + sizeof(uint32_t)); + if (ewah_words > (ewah_size - 3 * sizeof(uint32_t)) / + sizeof(eword_t) || + 3 * sizeof(uint32_t) + (size_t)ewah_words * sizeof(eword_t) != + ewah_size) + goto invalid; + ewah_rlw = get_be32(index + ewah_size - sizeof(uint32_t)); + if (ewah_rlw >= ewah_words) + goto invalid; fsmonitor_dirty = ewah_new(); ret = ewah_read_mmap(fsmonitor_dirty, index, ewah_size); if (ret != ewah_size) { ewah_free(fsmonitor_dirty); - return error("failed to parse ewah bitmap reading fsmonitor index extension"); + goto invalid; + } + if (!fsmonitor_ewah_is_valid(fsmonitor_dirty)) { + ewah_free(fsmonitor_dirty); + goto invalid; + } + if (!istate->split_index && + fsmonitor_dirty->bit_size > istate->cache_nr) { + ewah_free(fsmonitor_dirty); + goto invalid; } - istate->fsmonitor_dirty = fsmonitor_dirty; - if (!istate->split_index) - assert_index_minimum(istate, istate->fsmonitor_dirty->bit_size); + /* Publish only after the complete optional extension is validated. */ + FREE_AND_NULL(istate->fsmonitor_last_update); + if (istate->fsmonitor_dirty) + ewah_free(istate->fsmonitor_dirty); + istate->fsmonitor_last_update = strbuf_detach(&last_update, NULL); + istate->fsmonitor_dirty = fsmonitor_dirty; trace2_data_string("index", NULL, "extension/fsmn/read/token", istate->fsmonitor_last_update); @@ -105,6 +184,18 @@ int read_fsmonitor_extension(struct index_state *istate, const void *data, "read fsmonitor extension successful '%s'", istate->fsmonitor_last_update); return 0; + +invalid: + istate->fsmonitor_extension_seen = 1; + FREE_AND_NULL(istate->fsmonitor_last_update); + if (istate->fsmonitor_dirty) { + ewah_free(istate->fsmonitor_dirty); + istate->fsmonitor_dirty = NULL; + } + strbuf_release(&last_update); + trace2_data_intmax("fsmonitor", istate->repo, + "extension/invalid", 1); + return 0; } void fill_fsmonitor_bitmap(struct index_state *istate) diff --git a/read-cache-ll.h b/read-cache-ll.h index ed7d7aac38257e..d728e050e5fb23 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -182,7 +182,8 @@ struct index_state { drop_cache_tree : 1, updated_workdir : 1, updated_skipworktree : 1, - fsmonitor_has_run_once : 1; + fsmonitor_has_run_once : 1, + fsmonitor_extension_seen : 1; enum sparse_index_mode sparse_index; struct hashmap name_hash; struct hashmap dir_hash; diff --git a/t/helper/test-read-cache.c b/t/helper/test-read-cache.c index c7631a204c8b2a..7034e30c80d2ac 100644 --- a/t/helper/test-read-cache.c +++ b/t/helper/test-read-cache.c @@ -4,11 +4,62 @@ #include "attr.h" #include "config.h" #include "environment.h" +#include "ewah/ewok.h" +#include "ewah/ewok_rlw.h" #include "fsmonitor.h" #include "fsmonitor-ll.h" #include "read-cache-ll.h" #include "repository.h" #include "setup.h" +#include "strbuf.h" + +static void wrap_fsmn_ewah(struct strbuf *out, const struct strbuf *ewah) +{ + uint32_t value; + + put_be32(&value, 2); + strbuf_add(out, &value, sizeof(value)); + strbuf_addstr(out, "token"); + strbuf_addch(out, '\0'); + put_be32(&value, ewah->len); + strbuf_add(out, &value, sizeof(value)); + strbuf_addbuf(out, ewah); +} + +static void make_valid_fsmn(struct strbuf *out) +{ + struct ewah_bitmap *dirty = ewah_new(); + struct strbuf ewah = STRBUF_INIT; + + ewah_set(dirty, 0); + ewah_serialize_strbuf(dirty, &ewah); + wrap_fsmn_ewah(out, &ewah); + ewah_free(dirty); + strbuf_release(&ewah); +} + +static void make_raw_fsmn(struct strbuf *out, uint32_t bit_size, + const eword_t *words, uint32_t word_count, + uint32_t rlw) +{ + struct strbuf ewah = STRBUF_INIT; + uint32_t value; + uint32_t i; + + put_be32(&value, bit_size); + strbuf_add(&ewah, &value, sizeof(value)); + put_be32(&value, word_count); + strbuf_add(&ewah, &value, sizeof(value)); + for (i = 0; i < word_count; i++) { + eword_t word = htonll(words[i]); + + strbuf_add(&ewah, &word, sizeof(word)); + } + put_be32(&value, rlw); + strbuf_add(&ewah, &value, sizeof(value)); + wrap_fsmn_ewah(out, &ewah); + strbuf_release(&ewah); +} static int test_fsmonitor_content_recovery(const char *path) { @@ -43,6 +94,83 @@ static int test_fsmonitor_content_recovery(const char *path) return 0; } +static int fsmn_failed_closed(const struct index_state *istate) +{ + return istate->fsmonitor_extension_seen && + !istate->fsmonitor_last_update && !istate->fsmonitor_dirty; +} + +static int check_invalid_fsmn(const struct strbuf *encoded, + const char *description) +{ + struct index_state invalid = INDEX_STATE_INIT(the_repository); + + invalid.cache_nr = 1; + invalid.fsmonitor_last_update = xstrdup("old"); + invalid.fsmonitor_dirty = ewah_new(); + read_fsmonitor_extension(&invalid, encoded->buf, encoded->len); + if (!fsmn_failed_closed(&invalid)) + return error("%s FSMN was published", description); + return 0; +} + +static int test_fsmn_parser(void) +{ + struct index_state duplicate = INDEX_STATE_INIT(the_repository); + struct index_state truncated = INDEX_STATE_INIT(the_repository); + struct strbuf encoded = STRBUF_INIT; + struct strbuf malformed = STRBUF_INIT; + eword_t words[2] = { 0 }; + + duplicate.cache_nr = truncated.cache_nr = 1; + make_valid_fsmn(&encoded); + read_fsmonitor_extension(&duplicate, encoded.buf, encoded.len); + if (!duplicate.fsmonitor_last_update || + strcmp(duplicate.fsmonitor_last_update, "token") || + !duplicate.fsmonitor_dirty) + return error("valid FSMN was not published"); + read_fsmonitor_extension(&duplicate, encoded.buf, encoded.len); + if (!fsmn_failed_closed(&duplicate)) + return error("duplicate FSMN did not fail closed"); + + truncated.fsmonitor_last_update = xstrdup("old"); + truncated.fsmonitor_dirty = ewah_new(); + read_fsmonitor_extension(&truncated, encoded.buf, encoded.len - 1); + if (!fsmn_failed_closed(&truncated)) + return error("truncated FSMN was partially published"); + + rlw_set_literal_words(&words[0], 1); + make_raw_fsmn(&malformed, 1, words, 1, 0); + if (check_invalid_fsmn(&malformed, "out-of-bounds literal")) + return 1; + strbuf_reset(&malformed); + + words[0] = 0; + rlw_set_run_bit(&words[0], 1); + rlw_set_running_len(&words[0], 1); + make_raw_fsmn(&malformed, 1, words, 1, 0); + if (check_invalid_fsmn(&malformed, "oversized set-bit run")) + return 1; + strbuf_reset(&malformed); + + words[0] = words[1] = 0; + rlw_set_literal_words(&words[0], 1); + words[1] = 2; + make_raw_fsmn(&malformed, 1, words, 2, 0); + if (check_invalid_fsmn(&malformed, "set padding bit")) + return 1; + strbuf_reset(&malformed); + + words[1] = 1; + make_raw_fsmn(&malformed, 1, words, 2, 1); + if (check_invalid_fsmn(&malformed, "non-final RLW")) + return 1; + + strbuf_release(&malformed); + strbuf_release(&encoded); + return 0; +} + static int test_fsmonitor_directory_attributes(void) { struct attr_check *check; @@ -90,6 +218,8 @@ int cmd__read_cache(int argc, const char **argv) if (argc == 3 && !strcmp(argv[1], "--test-fsmonitor-content-recovery")) return test_fsmonitor_content_recovery(argv[2]); + if (argc == 2 && !strcmp(argv[1], "--test-fsmn-parser")) + return test_fsmn_parser(); if (argc == 2 && !strcmp(argv[1], "--test-fsmonitor-directory-attributes")) return test_fsmonitor_directory_attributes(); diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 691148ae677113..f257a05f92930e 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -60,6 +60,10 @@ test_lazy_prereq HARDLINKS ' ln hardlink-a hardlink-b ' +test_expect_success 'FSMN parser fails closed' ' + test-tool read-cache --test-fsmn-parser +' + # Test that we detect and disallow repos that are incompatible with FSMonitor. test_expect_success 'incompatible bare repo' ' test_when_finished "rm -rf ./bare-clone actual expect" && From cf4c103c838915ee1396fccd4a0362ee7e96b462 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Thu, 23 Jul 2026 23:49:15 -0500 Subject: [PATCH 027/105] preload-index: validate packed APFS directory records getattrlistbulk() returns variable-length records with attribute sets and name offsets supplied by the filesystem. A truncated record, entry error, unexpected attributes, or invalid name reference cannot safely describe an index entry. Add a bounded decoder that checks record alignment and length, required attribute sets, entry errors, record-local name offsets, valid path components, and file metadata before accepting a record. Add six Darwin unit checks for valid file and directory records, entry errors, short records, unexpected attributes, and invalid names. Register the Darwin decoder with Make, CMake, and Meson. Register its six unit checks with Make and Meson. Existing preload behavior remains unchanged. Signed-off-by: Taylor Blau --- Makefile | 2 + compat/preload-index/bulk-darwin.c | 134 ++++++++++++++ compat/preload-index/bulk-darwin.h | 4 + contrib/buildsystems/CMakeLists.txt | 1 + meson.build | 1 + t/meson.build | 1 + t/unit-tests/u-preload-index-bulk-darwin.c | 193 +++++++++++++++++++++ 7 files changed, 336 insertions(+) create mode 100644 compat/preload-index/bulk-darwin.c create mode 100644 t/unit-tests/u-preload-index-bulk-darwin.c diff --git a/Makefile b/Makefile index 51059dadddf171..a5a0dabc80e45f 100644 --- a/Makefile +++ b/Makefile @@ -1384,6 +1384,7 @@ LIB_OBJS += ws.o ifdef PRELOAD_INDEX_BULK_BACKEND PRELOAD_INDEX_BULK_OBJS += preload-index-bulk-index.o PRELOAD_INDEX_BULK_OBJS += preload-index-bulk-thread.o +PRELOAD_INDEX_BULK_OBJS += compat/preload-index/bulk-$(PRELOAD_INDEX_BULK_BACKEND).o PRELOAD_INDEX_BULK_OBJS += $(PRELOAD_INDEX_BULK_PLATFORM_OBJS) endif LIB_OBJS += $(PRELOAD_INDEX_BULK_OBJS) @@ -1558,6 +1559,7 @@ CLAR_TEST_SUITES += u-oidmap CLAR_TEST_SUITES += u-oidtree CLAR_TEST_SUITES += u-path-namespace CLAR_TEST_SUITES += u-prio-queue +CLAR_TEST_SUITES += u-preload-index-bulk-darwin CLAR_TEST_SUITES += u-reftable-basics CLAR_TEST_SUITES += u-reftable-block CLAR_TEST_SUITES += u-reftable-merged diff --git a/compat/preload-index/bulk-darwin.c b/compat/preload-index/bulk-darwin.c new file mode 100644 index 00000000000000..d766d0600151b2 --- /dev/null +++ b/compat/preload-index/bulk-darwin.c @@ -0,0 +1,134 @@ +#include "git-compat-util.h" + +#include +#include + +#include "compat/preload-index/bulk-darwin.h" + +static const attrgroup_t required_common = + ATTR_CMN_RETURNED_ATTRS | ATTR_CMN_ERROR | ATTR_CMN_NAME | + ATTR_CMN_DEVID | ATTR_CMN_OBJTYPE | + ATTR_CMN_CRTIME | ATTR_CMN_MODTIME | ATTR_CMN_CHGTIME | + ATTR_CMN_OWNERID | ATTR_CMN_GRPID | ATTR_CMN_ACCESSMASK | + ATTR_CMN_FLAGS | ATTR_CMN_FILEID; +static const attrgroup_t required_dir = ATTR_DIR_MOUNTSTATUS; +static const attrgroup_t required_file = + ATTR_FILE_LINKCOUNT | ATTR_FILE_DATALENGTH; + +static int valid_component(const char *component, size_t len) +{ + return len && + !(len == 1 && component[0] == '.') && + !(len == 2 && component[0] == '.' && component[1] == '.'); +} + +struct preload_bulk_darwin_entry { + const char *name; + uint32_t record_len; + dev_t dev; + fsobj_type_t type; + struct timespec birthtime; + struct timespec mtime; + struct timespec ctime; + uid_t uid; + gid_t gid; + uint32_t access; + uint32_t flags; + uint32_t linkcount; + uint32_t mountstatus; + uint64_t fileid; + off_t size; +}; + +static int decode_entry(const char *record, size_t remaining, + struct preload_bulk_darwin_entry *entry) +{ + uint32_t entry_error = 0; + attribute_set_t returned; + attrreference_t name_ref; + const char *p, *end, *name_ref_at; + size_t name_ref_offset, name_offset, name_remaining; + + if (remaining < sizeof(entry->record_len) + sizeof(returned)) + return -1; + memcpy(&entry->record_len, record, sizeof(entry->record_len)); + if ((entry->record_len % sizeof(uint64_t)) || + entry->record_len < sizeof(entry->record_len) + sizeof(returned) || + entry->record_len > remaining) + return -1; + + p = record + sizeof(entry->record_len); + end = record + entry->record_len; + memcpy(&returned, p, sizeof(returned)); + p += sizeof(returned); + if (returned.commonattr != required_common || + returned.volattr || returned.forkattr) + return -1; + +#define TAKE_ATTR(value) do { \ + if ((size_t)(end - p) < sizeof(value)) \ + return -1; \ + memcpy(&(value), p, sizeof(value)); \ + p += sizeof(value); \ +} while (0) + TAKE_ATTR(entry_error); + if (entry_error) + return -1; + name_ref_at = p; + TAKE_ATTR(name_ref); + TAKE_ATTR(entry->dev); + TAKE_ATTR(entry->type); + TAKE_ATTR(entry->birthtime); + TAKE_ATTR(entry->mtime); + TAKE_ATTR(entry->ctime); + TAKE_ATTR(entry->uid); + TAKE_ATTR(entry->gid); + TAKE_ATTR(entry->access); + TAKE_ATTR(entry->flags); + TAKE_ATTR(entry->fileid); + + if (entry->type == VDIR) { + if (returned.dirattr != required_dir || + returned.fileattr) + return -1; + TAKE_ATTR(entry->mountstatus); + } else { + if (returned.dirattr || + (returned.fileattr & ~required_file)) + return -1; + TAKE_ATTR(entry->linkcount); + TAKE_ATTR(entry->size); + if ((entry->type == VREG || entry->type == VLNK) && + returned.fileattr != required_file) + return -1; + } +#undef TAKE_ATTR + + if (name_ref.attr_dataoffset < 0 || + (name_ref.attr_dataoffset % (int32_t)sizeof(uint32_t))) + return -1; + name_ref_offset = name_ref_at - record; + if ((uint32_t)name_ref.attr_dataoffset > + entry->record_len - name_ref_offset) + return -1; + name_offset = name_ref_offset + name_ref.attr_dataoffset; + name_remaining = entry->record_len - name_offset; + entry->name = record + name_offset; + if (!name_ref.attr_length || + name_ref.attr_length > name_remaining || + entry->name < p) + return -1; + if (entry->name[name_ref.attr_length - 1] || + memchr(entry->name, '\0', name_ref.attr_length - 1) || + !valid_component(entry->name, name_ref.attr_length - 1) || + memchr(entry->name, '/', name_ref.attr_length - 1)) + return -1; + return 0; +} + +int preload_bulk_darwin_decode_record(const char *record, size_t len) +{ + struct preload_bulk_darwin_entry entry; + + return decode_entry(record, len, &entry); +} diff --git a/compat/preload-index/bulk-darwin.h b/compat/preload-index/bulk-darwin.h index d96ed99240c3d7..c69886ac972268 100644 --- a/compat/preload-index/bulk-darwin.h +++ b/compat/preload-index/bulk-darwin.h @@ -12,6 +12,10 @@ struct preload_bulk_darwin_data { fsid_t root_fsid; }; +/* + * Exposed so that tests can validate kernel-supplied records directly. + */ +int preload_bulk_darwin_decode_record(const char *record, size_t len); int preload_bulk_darwin_fd_on_root_mount(struct preload_bulk_scan *scan, int fd, struct stat *st_out); const char *preload_bulk_darwin_open_root(struct preload_bulk_scan *scan); diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 67fb3f2d9ba7c8..e53ff08cd07b23 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -278,6 +278,7 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") add_compile_definitions(USE_ST_TIMESPEC) list(APPEND compat_SOURCES compat/darwin/procinfo.c + compat/preload-index/bulk-darwin.c compat/preload-index/bulk-darwin-root.c) endif() diff --git a/meson.build b/meson.build index 9134428deb5d31..2bdff5bb374325 100644 --- a/meson.build +++ b/meson.build @@ -1346,6 +1346,7 @@ elif host_machine.system() == 'windows' elif host_machine.system() == 'darwin' compat_sources += [ 'compat/darwin/procinfo.c', + 'compat/preload-index/bulk-darwin.c', 'compat/preload-index/bulk-darwin-root.c', ] libgit_sources += [ diff --git a/t/meson.build b/t/meson.build index d8734eae5ff77f..d22603637bc5a6 100644 --- a/t/meson.build +++ b/t/meson.build @@ -11,6 +11,7 @@ clar_test_suites = [ 'unit-tests/u-oidmap.c', 'unit-tests/u-oidtree.c', 'unit-tests/u-path-namespace.c', + 'unit-tests/u-preload-index-bulk-darwin.c', 'unit-tests/u-prio-queue.c', 'unit-tests/u-reftable-basics.c', 'unit-tests/u-reftable-block.c', diff --git a/t/unit-tests/u-preload-index-bulk-darwin.c b/t/unit-tests/u-preload-index-bulk-darwin.c new file mode 100644 index 00000000000000..f20a7bbd704ac5 --- /dev/null +++ b/t/unit-tests/u-preload-index-bulk-darwin.c @@ -0,0 +1,193 @@ +#include "unit-test.h" + +#ifdef __APPLE__ + +#include +#include + +#include "compat/preload-index/bulk-darwin.h" + +#define REQUIRED_COMMON \ + (ATTR_CMN_RETURNED_ATTRS | ATTR_CMN_ERROR | ATTR_CMN_NAME | \ + ATTR_CMN_DEVID | ATTR_CMN_OBJTYPE | \ + ATTR_CMN_CRTIME | ATTR_CMN_MODTIME | ATTR_CMN_CHGTIME | \ + ATTR_CMN_OWNERID | ATTR_CMN_GRPID | ATTR_CMN_ACCESSMASK | \ + ATTR_CMN_FLAGS | ATTR_CMN_FILEID) +#define REQUIRED_FILE (ATTR_FILE_LINKCOUNT | ATTR_FILE_DATALENGTH) + +struct test_record { + uint32_t record_len; + attribute_set_t returned; + uint32_t error; + attrreference_t name_ref; + dev_t dev; + fsobj_type_t type; + struct timespec birthtime; + struct timespec mtime; + struct timespec ctime; + uid_t uid; + gid_t gid; + uint32_t access; + uint32_t flags; + uint64_t fileid; + uint32_t linkcount; + off_t size; + char name[8]; +} __attribute__((packed)); + +static struct test_record make_record(void) +{ + struct test_record record = { + .record_len = sizeof(record), + .returned = { + .commonattr = REQUIRED_COMMON, + .fileattr = REQUIRED_FILE, + }, + .name_ref = { + .attr_dataoffset = offsetof(struct test_record, name) - + offsetof(struct test_record, name_ref), + .attr_length = 5, + }, + .dev = 1, + .type = VREG, + .uid = 1, + .gid = 1, + .access = 0644, + .fileid = 1, + .linkcount = 1, + .size = 1, + .name = "file", + }; + + return record; +} + +static void check_malformed(struct test_record *record, size_t len) +{ + cl_assert(preload_bulk_darwin_decode_record((char *)record, len) < 0); +} + +#endif /* __APPLE__ */ + +void test_preload_index_bulk_darwin__accepts_valid_record(void) +{ +#ifndef __APPLE__ + cl_skip(); +#else + struct test_record record = make_record(); + + cl_assert_equal_i(0, preload_bulk_darwin_decode_record((char *)&record, + sizeof(record))); +#endif +} + +void test_preload_index_bulk_darwin__accepts_valid_directory_record(void) +{ +#ifndef __APPLE__ + cl_skip(); +#else + struct test_record record = make_record(); + + record.returned.fileattr = 0; + record.returned.dirattr = ATTR_DIR_MOUNTSTATUS; + record.type = VDIR; + cl_assert_equal_i(0, preload_bulk_darwin_decode_record((char *)&record, + sizeof(record))); +#endif +} + +void test_preload_index_bulk_darwin__rejects_entry_error(void) +{ +#ifndef __APPLE__ + cl_skip(); +#else + struct test_record record = make_record(); + + record.error = EIO; + check_malformed(&record, sizeof(record)); +#endif +} + +void test_preload_index_bulk_darwin__rejects_short_record(void) +{ +#ifndef __APPLE__ + cl_skip(); +#else + struct test_record record = make_record(); + + check_malformed(&record, + sizeof(uint32_t) + sizeof(attribute_set_t) - 1); + + record.record_len = sizeof(record) + sizeof(uint64_t); + check_malformed(&record, sizeof(record)); + + record.record_len = sizeof(uint64_t); + check_malformed(&record, sizeof(record)); + + record.record_len = sizeof(record) - 1; + check_malformed(&record, sizeof(record)); + + record.record_len = (offsetof(struct test_record, type) + + sizeof(uint64_t) - 1) & + ~(sizeof(uint64_t) - 1); + check_malformed(&record, sizeof(record)); +#endif +} + +void test_preload_index_bulk_darwin__rejects_wrong_returned_attributes(void) +{ +#ifndef __APPLE__ + cl_skip(); +#else + struct test_record record; + + record = make_record(); + record.returned.commonattr &= ~ATTR_CMN_NAME; + check_malformed(&record, sizeof(record)); + + record = make_record(); + record.returned.volattr = 1; + check_malformed(&record, sizeof(record)); + + record = make_record(); + record.returned.fileattr &= ~ATTR_FILE_DATALENGTH; + check_malformed(&record, sizeof(record)); +#endif +} + +void test_preload_index_bulk_darwin__rejects_invalid_name_reference(void) +{ +#ifndef __APPLE__ + cl_skip(); +#else + struct test_record record; + + record = make_record(); + record.name_ref.attr_dataoffset = -4; + check_malformed(&record, sizeof(record)); + + record = make_record(); + record.name_ref.attr_dataoffset = 2; + check_malformed(&record, sizeof(record)); + + record = make_record(); + record.name_ref.attr_dataoffset = INT32_MAX; + check_malformed(&record, sizeof(record)); + + record = make_record(); + record.name_ref.attr_length = UINT32_MAX; + check_malformed(&record, sizeof(record)); + + record = make_record(); + record.name_ref.attr_dataoffset = 0; + check_malformed(&record, sizeof(record)); + + record = make_record(); + record.name[1] = '\0'; + check_malformed(&record, sizeof(record)); + + record = make_record(); + record.name[1] = '/'; + check_malformed(&record, sizeof(record)); +#endif +} From 1a074bcbba2b78a20fe81df8319bac68ab1d3e2b Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 22:11:47 -0700 Subject: [PATCH 028/105] status: pin semantic verification to the worktree root A worktree pathname can be replaced while a content verifier is opening files. Resolving later paths against that name can therefore hash a different tree from the one the index was meant to describe. Retain a no-follow descriptor and the complete stat identity of the repository worktree. On Linux, probe openat2() and require beneath-root resolution without symlink, magic-link, or mount crossings. Resolve SYS_openat2 through __NR_openat2 or the x86 syscall number 437 when older headers omit it; still require the complete runtime probe. On macOS, provide no-follow descriptor-relative opens. If the required platform support or the stable root is unavailable, return an error instead of attempting an unanchored proof. Register the root implementation with both Make and Meson. This patch introduces a compiled, isolated root primitive; it does not yet publish a proof, add a status caller, or check final root stability. Signed-off-by: Taylor Blau --- Makefile | 1 + meson.build | 1 + semantic-verify-internal.h | 43 ++++++++++ semantic-verify-root.c | 155 +++++++++++++++++++++++++++++++++++++ 4 files changed, 200 insertions(+) create mode 100644 semantic-verify-internal.h create mode 100644 semantic-verify-root.c diff --git a/Makefile b/Makefile index 97a4936b146df3..d31f118893a214 100644 --- a/Makefile +++ b/Makefile @@ -1317,6 +1317,7 @@ LIB_OBJS += resolve-undo.o LIB_OBJS += revision.o LIB_OBJS += run-command.o LIB_OBJS += send-pack.o +LIB_OBJS += semantic-verify-root.o LIB_OBJS += sequencer.o LIB_OBJS += serve.o LIB_OBJS += server-info.o diff --git a/meson.build b/meson.build index a777f4722a2061..408b0c27ca288d 100644 --- a/meson.build +++ b/meson.build @@ -523,6 +523,7 @@ libgit_sources = [ 'run-command.c', 'send-pack.c', 'sequencer.c', + 'semantic-verify-root.c', 'serve.c', 'server-info.c', 'setup.c', diff --git a/semantic-verify-internal.h b/semantic-verify-internal.h new file mode 100644 index 00000000000000..fd7dd9897f8e45 --- /dev/null +++ b/semantic-verify-internal.h @@ -0,0 +1,43 @@ +#ifndef SEMANTIC_VERIFY_INTERNAL_H +#define SEMANTIC_VERIFY_INTERNAL_H + +#include "statinfo.h" + +#ifdef __linux__ +#include +#if !defined(SYS_openat2) && defined(__NR_openat2) +#define SYS_openat2 __NR_openat2 +#elif !defined(SYS_openat2) && \ + (defined(__x86_64__) || defined(__i386__)) +#define SYS_openat2 437 +#endif +#endif + +#if defined(__APPLE__) && defined(O_NONBLOCK) && \ + defined(O_NOFOLLOW) && defined(O_DIRECTORY) && \ + defined(AT_SYMLINK_NOFOLLOW) +#define SEMANTIC_VERIFY_HAS_ANCHORED_OPEN 1 +#elif defined(__linux__) && defined(SYS_openat2) && \ + defined(O_CLOEXEC) && defined(O_NONBLOCK) && \ + defined(O_NOFOLLOW) && defined(O_DIRECTORY) && \ + defined(AT_SYMLINK_NOFOLLOW) +#define SEMANTIC_VERIFY_HAS_ANCHORED_OPEN 1 +#else +#define SEMANTIC_VERIFY_HAS_ANCHORED_OPEN 0 +#endif + +struct repository; + +struct semantic_verify_root { + int fd; + char *path; + struct stat stat; +}; + +int semantic_verify_root_init(struct repository *repo, + struct semantic_verify_root **root_out); +void semantic_verify_root_clear(struct semantic_verify_root *root); + +int semantic_verify_openat(int dirfd, const char *path, int flags); + +#endif /* SEMANTIC_VERIFY_INTERNAL_H */ diff --git a/semantic-verify-root.c b/semantic-verify-root.c new file mode 100644 index 00000000000000..e896cbe51f7c94 --- /dev/null +++ b/semantic-verify-root.c @@ -0,0 +1,155 @@ +#include "git-compat-util.h" +#include "path-namespace.h" +#include "repository.h" +#include "semantic-verify-internal.h" +#include "wrapper.h" + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN && defined(__linux__) +struct semantic_open_how { + uint64_t flags; + uint64_t mode; + uint64_t resolve; +}; + +#define SEMANTIC_RESOLVE_NO_XDEV 0x01 +#define SEMANTIC_RESOLVE_NO_MAGICLINKS 0x02 +#define SEMANTIC_RESOLVE_NO_SYMLINKS 0x04 +#define SEMANTIC_RESOLVE_BENEATH 0x08 +#endif + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN && !defined(__linux__) +static int set_fd_cloexec(int fd) +{ +#if defined(F_GETFD) && defined(F_SETFD) && defined(FD_CLOEXEC) + int flags = fcntl(fd, F_GETFD); + + if (flags < 0 || fcntl(fd, F_SETFD, flags | FD_CLOEXEC) < 0) + return -1; +#endif + return 0; +} +#endif + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +int semantic_verify_openat(int dirfd, const char *path, int flags) +{ +#ifdef __linux__ + struct semantic_open_how how = { + .flags = flags | O_CLOEXEC, + .resolve = SEMANTIC_RESOLVE_BENEATH | + SEMANTIC_RESOLVE_NO_SYMLINKS | + SEMANTIC_RESOLVE_NO_MAGICLINKS | + SEMANTIC_RESOLVE_NO_XDEV, + }; + + return syscall(SYS_openat2, dirfd, path, &how, sizeof(how)); +#else + int fd; + int saved_errno; + +#ifdef O_CLOEXEC + fd = openat(dirfd, path, flags | O_CLOEXEC); + if (fd >= 0) + return fd; + if (errno != EINVAL) + return -1; +#endif + fd = openat(dirfd, path, flags); + if (fd < 0) + return -1; + if (!set_fd_cloexec(fd)) + return fd; + saved_errno = errno; + close(fd); + errno = saved_errno; + return -1; +#endif +} +#else +int semantic_verify_openat(int dirfd UNUSED, const char *path UNUSED, + int flags UNUSED) +{ + errno = ENOSYS; + return -1; +} +#endif + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +int semantic_verify_root_init(struct repository *repo, + struct semantic_verify_root **root_out) +{ + struct semantic_verify_root *root; + const char *path = repo_get_work_tree(repo); + + if (!path) { + errno = ENOENT; + return -1; + } + CALLOC_ARRAY(root, 1); + root->fd = -1; + root->path = xstrdup(path); + root->fd = git_open_cloexec(root->path, + O_RDONLY | O_DIRECTORY | O_NOFOLLOW); + if (root->fd < 0 || fstat(root->fd, &root->stat) || + !S_ISDIR(root->stat.st_mode)) { + int saved_errno = errno ? errno : ENOTDIR; + + semantic_verify_root_clear(root); + errno = saved_errno; + return -1; + } +#ifdef __linux__ + { + struct stat probe_stat; + int probe_fd = semantic_verify_openat( + root->fd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW); + int saved_errno; + + if (probe_fd < 0) { + saved_errno = errno; + semantic_verify_root_clear(root); + errno = saved_errno; + return -1; + } + if (fstat(probe_fd, &probe_stat)) { + saved_errno = errno; + close(probe_fd); + semantic_verify_root_clear(root); + errno = saved_errno; + return -1; + } + if (!path_namespace_stat_equal(&root->stat, &probe_stat)) { + close(probe_fd); + semantic_verify_root_clear(root); + errno = EAGAIN; + return -1; + } + if (close(probe_fd)) { + saved_errno = errno; + semantic_verify_root_clear(root); + errno = saved_errno; + return -1; + } + } +#endif + *root_out = root; + return 0; +} +#else +int semantic_verify_root_init(struct repository *repo UNUSED, + struct semantic_verify_root **root_out UNUSED) +{ + errno = ENOSYS; + return -1; +} +#endif + +void semantic_verify_root_clear(struct semantic_verify_root *root) +{ + if (!root) + return; + if (root->fd >= 0) + close(root->fd); + free(root->path); + free(root); +} From 77db739276032480156e2335e52cbfbf41808ad1 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 21:46:00 -0700 Subject: [PATCH 029/105] fsmonitor: add an untracked-cache token extension FSMN identifies the token associated with tracked fsmonitor state, but the independently serialized UNTR extension cannot identify the provider boundary associated with its directory snapshot. The mere presence of both extensions cannot prove that their states agree. Define and document FSUC as a versioned optional index extension containing one NUL-terminated provider token. Register its reader with index-extension dispatch; reject empty tokens, tokens longer than 4 KiB, duplicate records, unsupported versions, truncation, and trailing data before publishing state. Provide the matching serializer and release the retained token with the index. Add a read-cache helper regression for a valid record, serializer round trip, duplicate, and truncated record. Register that helper in t/t7519-status-fsmonitor.sh. The format is independently testable; deciding when its token authenticates UNTR is a separate change. Signed-off-by: Taylor Blau --- Documentation/gitformat-index.adoc | 13 ++++ fsmonitor-ll.h | 5 ++ fsmonitor.c | 48 ++++++++++++ read-cache-ll.h | 5 +- read-cache.c | 5 ++ t/helper/test-read-cache.c | 116 +++++++++++++++++++++-------- t/t7519-status-fsmonitor.sh | 4 + 7 files changed, 162 insertions(+), 34 deletions(-) diff --git a/Documentation/gitformat-index.adoc b/Documentation/gitformat-index.adoc index f6a427cb495990..aaa9c29b4653b8 100644 --- a/Documentation/gitformat-index.adoc +++ b/Documentation/gitformat-index.adoc @@ -366,6 +366,19 @@ The remaining data of each directory block is grouped by type: - An ewah bitmap, the n-th bit indicates whether the n-th index entry is not CE_FSMONITOR_VALID. +== File System Monitor untracked-cache token + + The file system monitor untracked-cache token records the provider + token associated with an untracked-cache snapshot. The signature for + this extension is { 'F', 'S', 'U', 'C' }. + + The extension consists of: + + - 32-bit version number: the current version is 1. + + - A NUL-terminated string containing the opaque file system monitor + token associated with the untracked-cache data. + == End of Index Entry The End of Index Entry (EOIE) is used to locate the end of the variable diff --git a/fsmonitor-ll.h b/fsmonitor-ll.h index 7f78ad21c8d0b0..1028e630e9a912 100644 --- a/fsmonitor-ll.h +++ b/fsmonitor-ll.h @@ -15,6 +15,11 @@ extern struct trace_key trace_fsmonitor; */ int read_fsmonitor_extension(struct index_state *istate, const void *data, unsigned long sz); +int read_fsmonitor_untracked_extension(struct index_state *istate, + const void *data, unsigned long sz); +void write_fsmonitor_untracked_extension(struct strbuf *sb, + struct index_state *istate); + /* * Fill the fsmonitor_dirty ewah bits with their state from the index, * before it is split during writing. diff --git a/fsmonitor.c b/fsmonitor.c index ebec5620dbb630..26d00b5d912dfb 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -198,6 +198,54 @@ int read_fsmonitor_extension(struct index_state *istate, const void *data, return 0; } +#define FSMONITOR_UNTRACKED_EXTENSION_VERSION 1 + +int read_fsmonitor_untracked_extension(struct index_state *istate, + const void *data, unsigned long sz) +{ + const char *p = data; + const char *nul; + uint32_t version; + + if (istate->fsmonitor_untracked_extension_seen) + goto invalid; + istate->fsmonitor_untracked_extension_seen = 1; + if (sz < sizeof(version) + 2) + goto invalid; + version = get_be32(p); + p += sizeof(version); + sz -= sizeof(version); + if (version != FSMONITOR_UNTRACKED_EXTENSION_VERSION) + goto invalid; + nul = memchr(p, '\0', sz); + if (!nul || nul == p || (size_t)(nul - p + 1) != sz || + nul - p > FSMONITOR_TOKEN_MAX) + goto invalid; + + FREE_AND_NULL(istate->fsmonitor_untracked_token); + istate->fsmonitor_untracked_token = xstrdup(p); + return 0; + +invalid: + istate->fsmonitor_untracked_extension_seen = 1; + istate->fsmonitor_untracked_extension_invalid = 1; + FREE_AND_NULL(istate->fsmonitor_untracked_token); + trace2_data_intmax("fsmonitor", istate->repo, + "untracked/invalid-extension", 1); + return 0; +} + +void write_fsmonitor_untracked_extension(struct strbuf *sb, + struct index_state *istate) +{ + uint32_t version; + + put_be32(&version, FSMONITOR_UNTRACKED_EXTENSION_VERSION); + strbuf_add(sb, &version, sizeof(version)); + strbuf_addstr(sb, istate->fsmonitor_last_update); + strbuf_addch(sb, '\0'); +} + void fill_fsmonitor_bitmap(struct index_state *istate) { unsigned int i, skipped = 0; diff --git a/read-cache-ll.h b/read-cache-ll.h index d728e050e5fb23..f71889101e2775 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -183,13 +183,16 @@ struct index_state { updated_workdir : 1, updated_skipworktree : 1, fsmonitor_has_run_once : 1, - fsmonitor_extension_seen : 1; + fsmonitor_extension_seen : 1, + fsmonitor_untracked_extension_seen : 1, + fsmonitor_untracked_extension_invalid : 1; enum sparse_index_mode sparse_index; struct hashmap name_hash; struct hashmap dir_hash; struct object_id oid; struct untracked_cache *untracked; char *fsmonitor_last_update; + char *fsmonitor_untracked_token; struct ewah_bitmap *fsmonitor_dirty; struct mem_pool *ce_mem_pool; struct progress *progress; diff --git a/read-cache.c b/read-cache.c index b853d79c72aa5f..43d4b15d54ad7e 100644 --- a/read-cache.c +++ b/read-cache.c @@ -71,6 +71,7 @@ #define CACHE_EXT_LINK 0x6c696e6b /* "link" */ #define CACHE_EXT_UNTRACKED 0x554E5452 /* "UNTR" */ #define CACHE_EXT_FSMONITOR 0x46534D4E /* "FSMN" */ +#define CACHE_EXT_FSMONITOR_UNTRACKED 0x46535543 /* "FSUC" */ #define CACHE_EXT_ENDOFINDEXENTRIES 0x454F4945 /* "EOIE" */ #define CACHE_EXT_INDEXENTRYOFFSETTABLE 0x49454F54 /* "IEOT" */ #define CACHE_EXT_SPARSE_DIRECTORIES 0x73646972 /* "sdir" */ @@ -1777,6 +1778,9 @@ static int read_index_extension(struct index_state *istate, case CACHE_EXT_FSMONITOR: read_fsmonitor_extension(istate, data, sz); break; + case CACHE_EXT_FSMONITOR_UNTRACKED: + read_fsmonitor_untracked_extension(istate, data, sz); + break; case CACHE_EXT_ENDOFINDEXENTRIES: case CACHE_EXT_INDEXENTRYOFFSETTABLE: /* already handled in do_read_index() */ @@ -2466,6 +2470,7 @@ void release_index(struct index_state *istate) free_name_hash(istate); cache_tree_free(&(istate->cache_tree)); free(istate->fsmonitor_last_update); + free(istate->fsmonitor_untracked_token); free(istate->cache); discard_split_index(istate); free_untracked_cache(istate->untracked); diff --git a/t/helper/test-read-cache.c b/t/helper/test-read-cache.c index 7034e30c80d2ac..4698265f5c090f 100644 --- a/t/helper/test-read-cache.c +++ b/t/helper/test-read-cache.c @@ -13,6 +13,87 @@ #include "setup.h" #include "strbuf.h" +static int test_fsmonitor_content_recovery(const char *path) +{ + struct index_state *istate; + struct cache_entry *ce; + struct stat_data empty = { 0 }; + struct stat st; + int pos; + + setup_git_directory(the_repository); + repo_config(the_repository, git_default_config, NULL); + if (repo_read_index(the_repository) < 0) + return error("unable to read test index"); + istate = the_repository->index; + pos = index_name_pos(istate, path, strlen(path)); + if (pos < 0) + return error("path is not indexed: %s", path); + ce = istate->cache[pos]; + if (lstat(path, &st)) + return error_errno("unable to stat indexed path"); + + fsmonitor_invalidate_cache_entry(ce); + if (memcmp(&ce->ce_stat_data, &empty, sizeof(empty))) + return error("invalidation did not poison cached stat data"); + if (ie_match_stat_with_content_check(istate, ce, &st, 0)) + return error("clean content did not match"); + if (!memcmp(&ce->ce_stat_data, &empty, sizeof(empty))) + return error("verified clean entry retained poisoned stat data"); + if (!(ce->ce_flags & CE_UPDATE_IN_BASE) || + !(istate->cache_changed & CE_ENTRY_CHANGED)) + return error("verified stat refresh was not marked for persistence"); + return 0; +} + +static int fsuc_failed_closed(const struct index_state *istate) +{ + return istate->fsmonitor_untracked_extension_seen && + istate->fsmonitor_untracked_extension_invalid && + !istate->fsmonitor_untracked_token; +} + +static int test_fsuc_parser(void) +{ + struct index_state duplicate = INDEX_STATE_INIT(the_repository); + struct index_state truncated = INDEX_STATE_INIT(the_repository); + struct strbuf encoded = STRBUF_INIT; + struct strbuf written = STRBUF_INIT; + uint32_t version; + + put_be32(&version, 1); + strbuf_add(&encoded, &version, sizeof(version)); + strbuf_addstr(&encoded, "token"); + strbuf_addch(&encoded, '\0'); + read_fsmonitor_untracked_extension( + &duplicate, encoded.buf, encoded.len); + if (duplicate.fsmonitor_untracked_extension_invalid || + !duplicate.fsmonitor_untracked_token || + strcmp(duplicate.fsmonitor_untracked_token, "token")) + return error("valid FSUC was not published"); + + duplicate.fsmonitor_last_update = xstrdup("token"); + write_fsmonitor_untracked_extension(&written, &duplicate); + if (written.len != encoded.len || + memcmp(written.buf, encoded.buf, encoded.len)) + return error("FSUC did not round-trip"); + read_fsmonitor_untracked_extension( + &duplicate, encoded.buf, encoded.len); + if (!fsuc_failed_closed(&duplicate)) + return error("duplicate FSUC did not fail closed"); + + truncated.fsmonitor_untracked_token = xstrdup("old"); + read_fsmonitor_untracked_extension( + &truncated, encoded.buf, sizeof(version)); + if (!fsuc_failed_closed(&truncated)) + return error("truncated FSUC was partially published"); + + free(duplicate.fsmonitor_last_update); + strbuf_release(&written); + strbuf_release(&encoded); + return 0; +} + static void wrap_fsmn_ewah(struct strbuf *out, const struct strbuf *ewah) { uint32_t value; @@ -61,39 +142,6 @@ static void make_raw_fsmn(struct strbuf *out, uint32_t bit_size, strbuf_release(&ewah); } -static int test_fsmonitor_content_recovery(const char *path) -{ - struct index_state *istate; - struct cache_entry *ce; - struct stat_data empty = { 0 }; - struct stat st; - int pos; - - setup_git_directory(the_repository); - repo_config(the_repository, git_default_config, NULL); - if (repo_read_index(the_repository) < 0) - return error("unable to read test index"); - istate = the_repository->index; - pos = index_name_pos(istate, path, strlen(path)); - if (pos < 0) - return error("path is not indexed: %s", path); - ce = istate->cache[pos]; - if (lstat(path, &st)) - return error_errno("unable to stat indexed path"); - - fsmonitor_invalidate_cache_entry(ce); - if (memcmp(&ce->ce_stat_data, &empty, sizeof(empty))) - return error("invalidation did not poison cached stat data"); - if (ie_match_stat_with_content_check(istate, ce, &st, 0)) - return error("clean content did not match"); - if (!memcmp(&ce->ce_stat_data, &empty, sizeof(empty))) - return error("verified clean entry retained poisoned stat data"); - if (!(ce->ce_flags & CE_UPDATE_IN_BASE) || - !(istate->cache_changed & CE_ENTRY_CHANGED)) - return error("verified stat refresh was not marked for persistence"); - return 0; -} - static int fsmn_failed_closed(const struct index_state *istate) { return istate->fsmonitor_extension_seen && @@ -218,6 +266,8 @@ int cmd__read_cache(int argc, const char **argv) if (argc == 3 && !strcmp(argv[1], "--test-fsmonitor-content-recovery")) return test_fsmonitor_content_recovery(argv[2]); + if (argc == 2 && !strcmp(argv[1], "--test-fsuc-parser")) + return test_fsuc_parser(); if (argc == 2 && !strcmp(argv[1], "--test-fsmn-parser")) return test_fsmn_parser(); if (argc == 2 && diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index f257a05f92930e..f29bea912efd18 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -64,6 +64,10 @@ test_expect_success 'FSMN parser fails closed' ' test-tool read-cache --test-fsmn-parser ' +test_expect_success 'FSUC parser fails closed' ' + test-tool read-cache --test-fsuc-parser +' + # Test that we detect and disallow repos that are incompatible with FSMonitor. test_expect_success 'incompatible bare repo' ' test_when_finished "rm -rf ./bare-clone actual expect" && From f1f20ba304835d2d15a7f0171ac90b2d78428d96 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Thu, 23 Jul 2026 23:49:16 -0500 Subject: [PATCH 030/105] preload-index: walk APFS directories beneath a held root Descriptor limits can force a queued directory to be reopened after its parent was scanned. Reopening through an ordinary worktree pathname could follow a replacement directory or symlink into another namespace. Open immediate children with O_NOFOLLOW and reopen relative paths below the held root with O_NOFOLLOW_ANY. Visit only directories with tracked descendants on the original APFS mount, and compare parent and child identities before and after enumeration. Discard the complete scan after malformed records or changed directory identities. Leave multiply-linked tracked files to ordinary lstat. Allocate one 1 MiB record buffer per active worker; the backend is still not invoked from preload_index(), so existing behavior is unchanged. Signed-off-by: Taylor Blau --- compat/preload-index/bulk-darwin.c | 326 ++++++++++++++++++++++++++++ contrib/buildsystems/CMakeLists.txt | 3 +- preload-index-bulk-index.c | 13 ++ preload-index-bulk-thread.c | 31 ++- preload-index-bulk.h | 17 ++ 5 files changed, 383 insertions(+), 7 deletions(-) diff --git a/compat/preload-index/bulk-darwin.c b/compat/preload-index/bulk-darwin.c index d766d0600151b2..22a0e16def307f 100644 --- a/compat/preload-index/bulk-darwin.c +++ b/compat/preload-index/bulk-darwin.c @@ -3,7 +3,16 @@ #include #include +#include "compat/precompose_utf8.h" #include "compat/preload-index/bulk-darwin.h" +#include "path-namespace.h" +#include "preload-index-bulk.h" + +#ifndef SF_FIRMLINK +#define SF_FIRMLINK 0x00800000 +#endif + +#define PRELOAD_INDEX_BULK_BUFFER_SIZE (1024 * 1024) static const attrgroup_t required_common = ATTR_CMN_RETURNED_ATTRS | ATTR_CMN_ERROR | ATTR_CMN_NAME | @@ -22,6 +31,92 @@ static int valid_component(const char *component, size_t len) !(len == 2 && component[0] == '.' && component[1] == '.'); } +static int valid_relative_path(const char *path) +{ + const char *component = path; + + if (!strcmp(path, ".")) + return 1; + if (!*path || *path == '/') + return 0; + for (;;) { + const char *slash = strchr(component, '/'); + size_t len = slash ? (size_t)(slash - component) : + strlen(component); + + if (!valid_component(component, len)) + return 0; + if (!slash) + return 1; + component = slash + 1; + } +} + +static int preload_bulk_darwin_open_dir_at( + struct preload_bulk_worker *worker UNUSED, + int parent_fd, const char *name) +{ + if (!valid_component(name, strlen(name)) || strchr(name, '/')) { + errno = EINVAL; + return -1; + } + return openat(parent_fd, name, + O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); +} + +static int preload_bulk_darwin_open_relative(struct preload_bulk_scan *scan, + const char *path) +{ + if (!valid_relative_path(path)) { + errno = EINVAL; + return -1; + } + +#ifdef O_NOFOLLOW_ANY + return openat(scan->root_fd, path, + O_RDONLY | O_DIRECTORY | O_NOFOLLOW_ANY | O_CLOEXEC); +#else + errno = ENOTSUP; + return -1; +#endif +} + +static mode_t vnode_mode(fsobj_type_t type) +{ + switch (type) { + case VREG: + return S_IFREG; + case VLNK: + return S_IFLNK; + default: + return 0; + } +} + +static int fill_file_stat(struct stat *st, dev_t dev, uint64_t fileid, + fsobj_type_t type, struct timespec mtime, + struct timespec ctime, uid_t uid, gid_t gid, + uint32_t access, uint32_t linkcount, off_t size) +{ + mode_t mode = vnode_mode(type); + + if (!mode || size < 0 || + ((access & S_IFMT) && (access & S_IFMT) != mode) || + (access & ~(S_IFMT | 07777))) + return -1; + memset(st, 0, sizeof(*st)); + st->st_dev = dev; + st->st_ino = fileid; + st->st_mode = mode | (access & 07777); + st->st_uid = uid; + st->st_gid = gid; + st->st_nlink = linkcount; + st->st_size = size; + st->st_mtimespec = mtime; + st->st_ctimespec = ctime; + return 0; +} + struct preload_bulk_darwin_entry { const char *name; uint32_t record_len; @@ -132,3 +227,234 @@ int preload_bulk_darwin_decode_record(const char *record, size_t len) return decode_entry(record, len, &entry); } + +static struct preload_bulk_dir_identity directory_identity( + const struct stat *st) +{ + struct preload_bulk_dir_identity result = { + .stat = *st, + .complete = 1, + }; + + return result; +} + +static int directory_identity_matches( + const struct preload_bulk_dir_identity *before, + const struct stat *after) +{ + if (before->complete) + return path_namespace_stat_equal(&before->stat, after); + return S_ISDIR(after->st_mode) && + before->stat.st_dev == after->st_dev && + before->stat.st_ino == after->st_ino && + before->stat.st_birthtimespec.tv_sec == + after->st_birthtimespec.tv_sec && + before->stat.st_birthtimespec.tv_nsec == + after->st_birthtimespec.tv_nsec && + before->stat.st_mtimespec.tv_sec == after->st_mtimespec.tv_sec && + before->stat.st_mtimespec.tv_nsec == + after->st_mtimespec.tv_nsec && + before->stat.st_ctimespec.tv_sec == after->st_ctimespec.tv_sec && + before->stat.st_ctimespec.tv_nsec == + after->st_ctimespec.tv_nsec; +} + +static int enumerate_directory(struct preload_bulk_worker *worker, + const struct preload_bulk_task *task, int fd, + const struct preload_bulk_dir_identity *parent_identity) +{ + struct preload_bulk_scan *scan = worker->scan; + struct preload_bulk_darwin_data *data = scan->platform_data; + struct attrlist attrs = { 0 }; + char *buf = worker->buffer; + size_t path_prefix_len; + + if (!buf) { + buf = xmalloc(PRELOAD_INDEX_BULK_BUFFER_SIZE); + worker->buffer = buf; + } + + attrs.bitmapcount = ATTR_BIT_MAP_COUNT; + attrs.commonattr = required_common; + attrs.dirattr = required_dir; + attrs.fileattr = required_file; + worker->dirs++; + strbuf_reset(&worker->path); + if (strcmp(task->path, ".")) { + strbuf_addstr(&worker->path, task->path); + strbuf_addch(&worker->path, '/'); + } + path_prefix_len = worker->path.len; + + for (;;) { + int nr = getattrlistbulk(fd, &attrs, buf, + PRELOAD_INDEX_BULK_BUFFER_SIZE, + FSOPT_NOFOLLOW | + FSOPT_PACK_INVAL_ATTRS); + char *record = buf; + + worker->bulk_calls++; + if (nr < 0) + return -1; + if (!nr) + return 0; + + for (int i = 0; i < nr; i++) { + struct preload_bulk_darwin_entry entry; + struct stat st; + const char *path_name; + size_t remaining; + int pos; + + remaining = buf + PRELOAD_INDEX_BULK_BUFFER_SIZE - record; + if (decode_entry(record, remaining, &entry)) + goto malformed; + worker->entries++; + + /* + * The caller prepares the repository's Unicode policy + * before starting workers, so this is read-only here. + */ + path_name = repo_precompose_string_if_needed(scan->repo, + entry.name); + strbuf_setlen(&worker->path, path_prefix_len); + strbuf_addstr(&worker->path, path_name); + if (path_name != entry.name) + free((char *)path_name); + + if (entry.type == VDIR) { + struct preload_bulk_dir_identity child_identity = { + .stat = { + .st_dev = entry.dev, + .st_ino = entry.fileid, + .st_birthtimespec = + entry.birthtime, + .st_mtimespec = entry.mtime, + .st_ctimespec = entry.ctime, + }, + }; + + if (!preload_bulk_index_has_tracked_descendants( + scan, worker->path.buf, + worker->path.len)) + goto next_record; + if (((entry.access & S_IFMT) && + (entry.access & S_IFMT) != S_IFDIR) || + (entry.access & ~(S_IFMT | 07777))) + goto malformed_record; + if (entry.dev != data->root_stat.st_dev || + entry.mountstatus || + (entry.flags & SF_FIRMLINK)) { + goto next_record; + } + preload_bulk_schedule_directory( + worker, fd, parent_identity, + &child_identity, entry.name, + worker->path.buf, + worker->path.len); + goto next_record; + } + + pos = preload_bulk_index_position(scan, worker->path.buf, + worker->path.len); + if (pos < 0) + goto next_record; + if (entry.dev != data->root_stat.st_dev) { + goto next_record; + } + if (entry.type != VREG && entry.type != VLNK) + goto next_record; + if (entry.linkcount != 1) + goto next_record; + if (fill_file_stat(&st, entry.dev, entry.fileid, + entry.type, entry.mtime, entry.ctime, + entry.uid, entry.gid, entry.access, + entry.linkcount, entry.size)) + goto malformed_record; + preload_bulk_record_tracked(worker, pos, &st); + +next_record: + record += entry.record_len; + continue; + +malformed_record: + worker->malformed++; + goto next_record; + } + } + +malformed: + worker->malformed++; + return -1; +} + +static int scan_directory(struct preload_bulk_worker *worker, + struct preload_bulk_task *task) +{ + struct preload_bulk_scan *scan = worker->scan; + struct preload_bulk_dir_identity before_identity; + struct stat before, after; + int fd = task->fd; + int ret = -1; + + if (fd < 0) + fd = preload_bulk_darwin_open_relative(scan, task->path); + if (fd < 0) + goto out; + if (preload_bulk_darwin_fd_on_root_mount(scan, fd, &before)) { + if (errno != EXDEV) + goto out; + ret = 0; + goto out; + } + /* + * A child may have been replaced after its parent returned the bulk + * record, or while this task waited in the queue. + */ + if (task->has_child_identity && + !directory_identity_matches(&task->child_identity, &before)) { + worker->changed_dirs++; + ret = 0; + goto out; + } + before_identity = directory_identity(&before); + if (enumerate_directory(worker, task, fd, &before_identity)) + goto out; + if (fstat(fd, &after)) + goto out; + if (!directory_identity_matches(&before_identity, &after)) + worker->changed_dirs++; + ret = 0; + +out: + if (task->has_parent_identity) { + struct stat parent_after; + int parent_changed = fd < 0; + + /* + * Resolve ".." through the child descriptor, not the worktree + * path, so a rename cannot redirect this parent check. + */ + if (!parent_changed) + parent_changed = fstatat(fd, "..", &parent_after, + AT_SYMLINK_NOFOLLOW); + if (parent_changed || + !directory_identity_matches(&task->parent_identity, + &parent_after)) + worker->changed_dirs++; + } + if (fd >= 0) + close(fd); + return ret; +} + +static const struct preload_bulk_backend darwin_backend = { + .open_dir_at = preload_bulk_darwin_open_dir_at, + .scan_directory = scan_directory, +}; + +const struct preload_bulk_backend *preload_bulk_platform_backend(void) +{ + return &darwin_backend; +} diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index e53ff08cd07b23..e24ec739720dc5 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -275,9 +275,10 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_compile_definitions(PROCFS_EXECUTABLE_PATH="/proc/self/exe" HAVE_DEV_TTY ) list(APPEND compat_SOURCES unix-socket.c unix-stream-server.c compat/linux/procinfo.c) elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") - add_compile_definitions(USE_ST_TIMESPEC) + add_compile_definitions(PRECOMPOSE_UNICODE USE_ST_TIMESPEC) list(APPEND compat_SOURCES compat/darwin/procinfo.c + compat/precompose_utf8.c compat/preload-index/bulk-darwin.c compat/preload-index/bulk-darwin-root.c) endif() diff --git a/preload-index-bulk-index.c b/preload-index-bulk-index.c index 3c8bfad7c2a631..623164822b5fdb 100644 --- a/preload-index-bulk-index.c +++ b/preload-index-bulk-index.c @@ -14,6 +14,19 @@ int preload_bulk_index_position(struct preload_bulk_scan *scan, return index_name_pos_sparse(scan->istate, path, path_len); } +int preload_bulk_index_has_tracked_descendants(struct preload_bulk_scan *scan, + const char *path, + size_t path_len) +{ + int pos; + + if (path_len > INT_MAX) + return 0; + pos = index_name_pos_sparse(scan->istate, path, path_len); + return preload_bulk_index_pos_has_tracked_descendants( + scan, path, path_len, pos); +} + int preload_bulk_index_pos_has_tracked_descendants( struct preload_bulk_scan *scan, const char *path, size_t path_len, int pos) diff --git a/preload-index-bulk-thread.c b/preload-index-bulk-thread.c index 0a73d5a1fdeafd..8f57f143d4761b 100644 --- a/preload-index-bulk-thread.c +++ b/preload-index-bulk-thread.c @@ -176,6 +176,15 @@ static void *preload_bulk_worker_main(void *data) return NULL; } +static void release_workers(struct preload_bulk_scan *scan) +{ + for (int i = 0; i < scan->threads; i++) { + free(scan->workers[i].buffer); + strbuf_release(&scan->workers[i].path); + } + FREE_AND_NULL(scan->workers); +} + int preload_bulk_run_scan(struct preload_bulk_scan *scan, struct preload_bulk_run_result *result) { @@ -188,8 +197,10 @@ int preload_bulk_run_scan(struct preload_bulk_scan *scan, if (queue_init(&scan->queue)) return -1; CALLOC_ARRAY(scan->workers, scan->threads); - for (int i = 0; i < scan->threads; i++) + for (int i = 0; i < scan->threads; i++) { scan->workers[i].scan = scan; + strbuf_init(&scan->workers[i].path, 0); + } FLEX_ALLOC_STR(root_task, path, "."); if (!reserve_open_fd(&scan->queue)) @@ -199,8 +210,7 @@ int preload_bulk_run_scan(struct preload_bulk_scan *scan, if (root_task->fd < 0) { release_open_fd(&scan->queue); free(root_task); - free(scan->workers); - scan->workers = NULL; + release_workers(scan); queue_release(&scan->queue); return -1; } @@ -222,11 +232,20 @@ int preload_bulk_run_scan(struct preload_bulk_scan *scan, pthread_join(scan->workers[i].thread, NULL)) BUG("unable to join bulk preload worker"); + for (int i = 0; i < scan->threads; i++) { + struct preload_bulk_worker *worker = &scan->workers[i]; + + result->dirs += worker->dirs; + result->entries += worker->entries; + result->bulk_calls += worker->bulk_calls; + result->changed_dirs += worker->changed_dirs; + result->malformed += worker->malformed; + } result->threads = started_threads; - failed = scan->queue.failed; + failed = scan->queue.failed || result->malformed || + result->changed_dirs; - free(scan->workers); - scan->workers = NULL; + release_workers(scan); queue_release(&scan->queue); return failed ? -1 : 0; } diff --git a/preload-index-bulk.h b/preload-index-bulk.h index 9e8b085160a873..3272ce41ee81ec 100644 --- a/preload-index-bulk.h +++ b/preload-index-bulk.h @@ -3,6 +3,7 @@ #include "git-compat-util.h" #include "preload-index.h" +#include "strbuf.h" #include "thread-utils.h" struct preload_bulk_dir_identity { @@ -40,6 +41,13 @@ struct preload_bulk_scan; struct preload_bulk_worker { struct preload_bulk_scan *scan; pthread_t thread; + void *buffer; + struct strbuf path; + uint64_t dirs; + uint64_t entries; + uint64_t bulk_calls; + uint64_t changed_dirs; + uint64_t malformed; unsigned started : 1; }; @@ -67,6 +75,11 @@ struct preload_bulk_scan { }; struct preload_bulk_run_result { + uint64_t dirs; + uint64_t entries; + uint64_t bulk_calls; + uint64_t changed_dirs; + uint64_t malformed; int threads; }; @@ -77,6 +90,9 @@ void preload_bulk_schedule_directory( const char *name, const char *path, size_t path_len); int preload_bulk_index_position(struct preload_bulk_scan *scan, const char *path, size_t path_len); +int preload_bulk_index_has_tracked_descendants(struct preload_bulk_scan *scan, + const char *path, + size_t path_len); int preload_bulk_index_pos_has_tracked_descendants( struct preload_bulk_scan *scan, const char *path, size_t path_len, int pos); @@ -84,5 +100,6 @@ void preload_bulk_record_tracked( struct preload_bulk_worker *worker, int pos, const struct stat *st); int preload_bulk_run_scan(struct preload_bulk_scan *scan, struct preload_bulk_run_result *result); +const struct preload_bulk_backend *preload_bulk_platform_backend(void); #endif /* PRELOAD_INDEX_BULK_H */ From fc93346a5ac29a8e0272998f63df4373dbca386b Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 22:12:41 -0700 Subject: [PATCH 031/105] status: resolve semantic proof paths through pinned parents Holding the worktree root does not keep a nested directory from being renamed or replaced while indexed paths are visited. Reopening a whole pathname can silently switch verification into the replacement tree. Resolve each indexed path one component at a time beneath the retained root. Keep matching ancestor descriptors while walking sorted index names, reject empty and dot components and device crossings, and reopen each outgoing directory through its still-pinned parent. If the reopened identity changes, record the earliest affected index position so a caller cannot retain a clean result from that namespace. Register the resolver in both Make and Meson. Unsupported platforms return ENOSYS rather than falling back to ordinary pathname resolution. The resolver is an internal, independently buildable primitive; this patch does not yet run a status scan or construct a complete proof. Signed-off-by: Taylor Blau --- Makefile | 1 + meson.build | 1 + semantic-verify-internal.h | 10 ++ semantic-verify-path.c | 186 +++++++++++++++++++++++++++++++++++++ 4 files changed, 198 insertions(+) create mode 100644 semantic-verify-path.c diff --git a/Makefile b/Makefile index d31f118893a214..afb57d02604aec 100644 --- a/Makefile +++ b/Makefile @@ -1317,6 +1317,7 @@ LIB_OBJS += resolve-undo.o LIB_OBJS += revision.o LIB_OBJS += run-command.o LIB_OBJS += send-pack.o +LIB_OBJS += semantic-verify-path.o LIB_OBJS += semantic-verify-root.o LIB_OBJS += sequencer.o LIB_OBJS += serve.o diff --git a/meson.build b/meson.build index 408b0c27ca288d..35c693e1f1503e 100644 --- a/meson.build +++ b/meson.build @@ -523,6 +523,7 @@ libgit_sources = [ 'run-command.c', 'send-pack.c', 'sequencer.c', + 'semantic-verify-path.c', 'semantic-verify-root.c', 'serve.c', 'server-info.c', diff --git a/semantic-verify-internal.h b/semantic-verify-internal.h index fd7dd9897f8e45..1f881af44fba6f 100644 --- a/semantic-verify-internal.h +++ b/semantic-verify-internal.h @@ -27,6 +27,7 @@ #endif struct repository; +struct semantic_verify_path; struct semantic_verify_root { int fd; @@ -40,4 +41,13 @@ void semantic_verify_root_clear(struct semantic_verify_root *root); int semantic_verify_openat(int dirfd, const char *path, int flags); +struct semantic_verify_path *semantic_verify_path_new( + struct semantic_verify_root *root); +int semantic_verify_resolve_parent(struct semantic_verify_path *path, + const char *name, size_t cache_pos, + int *parent_fd, const char **basename); +void semantic_verify_path_free(struct semantic_verify_path *path, + unsigned int *namespace_unstable, + size_t *namespace_unstable_from); + #endif /* SEMANTIC_VERIFY_INTERNAL_H */ diff --git a/semantic-verify-path.c b/semantic-verify-path.c new file mode 100644 index 00000000000000..db7fd874084c42 --- /dev/null +++ b/semantic-verify-path.c @@ -0,0 +1,186 @@ +#include "git-compat-util.h" +#include "path-namespace.h" +#include "semantic-verify-internal.h" +#include "strbuf.h" + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +struct anchored_dir { + char *component; + int fd; + struct stat stat; + size_t first_cache_pos; +}; + +struct semantic_verify_path { + struct semantic_verify_root *root; + struct anchored_dir *dirs; + size_t dirs_nr; + size_t dirs_alloc; + size_t namespace_unstable_from; + unsigned int namespace_unstable; + struct strbuf component; +}; + +static void note_namespace_unstable(struct semantic_verify_path *path, + size_t from) +{ + path->namespace_unstable = 1; + if (from < path->namespace_unstable_from) + path->namespace_unstable_from = from; +} + +static void pop_anchored_dir(struct semantic_verify_path *path) +{ + struct anchored_dir *dir = &path->dirs[path->dirs_nr - 1]; + int parent_fd = path->dirs_nr == 1 ? path->root->fd : + path->dirs[path->dirs_nr - 2].fd; + int named_fd; + struct stat named_stat; + + named_fd = semantic_verify_openat(parent_fd, dir->component, + O_RDONLY | O_DIRECTORY | O_NOFOLLOW); + if (named_fd < 0 || fstat(named_fd, &named_stat) || + !path_namespace_stat_equal(&dir->stat, &named_stat)) + note_namespace_unstable(path, dir->first_cache_pos); + if (named_fd >= 0) + close(named_fd); + close(dir->fd); + free(dir->component); + path->dirs_nr--; +} + +struct semantic_verify_path *semantic_verify_path_new( + struct semantic_verify_root *root) +{ + struct semantic_verify_path *path; + + CALLOC_ARRAY(path, 1); + path->root = root; + path->namespace_unstable_from = SIZE_MAX; + path->component = (struct strbuf)STRBUF_INIT; + return path; +} + +int semantic_verify_resolve_parent(struct semantic_verify_path *path, + const char *name, size_t cache_pos, + int *parent_fd, const char **basename) +{ + const char *slash = strrchr(name, '/'); + size_t parent_len = slash ? (size_t)(slash - name) : 0; + size_t begin = 0, depth = 0; + + *basename = slash ? slash + 1 : name; + if (!**basename) { + errno = EINVAL; + return -1; + } + + /* Find the component-aligned prefix already pinned by this worker. */ + while (begin < parent_len && depth < path->dirs_nr) { + size_t end = begin; + struct anchored_dir *dir = &path->dirs[depth]; + + while (end < parent_len && name[end] != '/') + end++; + if (strlen(dir->component) != end - begin || + memcmp(dir->component, name + begin, end - begin)) + break; + depth++; + begin = end + 1; + } + while (path->dirs_nr > depth) + pop_anchored_dir(path); + + while (begin < parent_len) { + size_t end = begin; + struct anchored_dir *dir; + int dirfd, fd; + struct stat st; + + while (end < parent_len && name[end] != '/') + end++; + if (end == begin || + (end - begin == 1 && name[begin] == '.') || + (end - begin == 2 && name[begin] == '.' && + name[begin + 1] == '.')) { + errno = EINVAL; + return -1; + } + strbuf_reset(&path->component); + strbuf_add(&path->component, name + begin, end - begin); + dirfd = path->dirs_nr ? path->dirs[path->dirs_nr - 1].fd : + path->root->fd; + fd = semantic_verify_openat(dirfd, path->component.buf, + O_RDONLY | O_DIRECTORY | O_NOFOLLOW); + if (fd < 0) + return -1; + if (fstat(fd, &st)) { + int saved_errno = errno; + + close(fd); + errno = saved_errno; + return -1; + } + if (!S_ISDIR(st.st_mode) || st.st_dev != path->root->stat.st_dev) { + close(fd); + errno = EXDEV; + return -1; + } + ALLOC_GROW(path->dirs, path->dirs_nr + 1, path->dirs_alloc); + dir = &path->dirs[path->dirs_nr++]; + dir->component = xstrdup(path->component.buf); + dir->fd = fd; + memcpy(&dir->stat, &st, sizeof(st)); + dir->first_cache_pos = cache_pos; + begin = end + 1; + } + + *parent_fd = path->dirs_nr ? path->dirs[path->dirs_nr - 1].fd : + path->root->fd; + return 0; +} + +void semantic_verify_path_free(struct semantic_verify_path *path, + unsigned int *namespace_unstable, + size_t *namespace_unstable_from) +{ + if (!path) + return; + while (path->dirs_nr) + pop_anchored_dir(path); + if (namespace_unstable) + *namespace_unstable = path->namespace_unstable; + if (namespace_unstable_from) + *namespace_unstable_from = path->namespace_unstable_from; + free(path->dirs); + strbuf_release(&path->component); + free(path); +} +#else +struct semantic_verify_path *semantic_verify_path_new( + struct semantic_verify_root *root UNUSED) +{ + errno = ENOSYS; + return NULL; +} + +int semantic_verify_resolve_parent( + struct semantic_verify_path *path UNUSED, + const char *name UNUSED, size_t cache_pos UNUSED, + int *parent_fd UNUSED, const char **basename UNUSED) +{ + errno = ENOSYS; + return -1; +} + +void semantic_verify_path_free( + struct semantic_verify_path *path UNUSED, + unsigned int *namespace_unstable, + size_t *namespace_unstable_from) +{ + if (namespace_unstable) + *namespace_unstable = 0; + if (namespace_unstable_from) + *namespace_unstable_from = SIZE_MAX; +} +#endif From 5952e41c34b3ccd5ded9a07f4d336c0d80586892 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 21:47:29 -0700 Subject: [PATCH 032/105] fsmonitor: bind untracked-cache state to its token A well-formed FSMN bitmap and a well-formed FSUC record still do not prove that a populated untracked-cache root and tracked entries were observed at the same provider boundary. Trusting mismatched tokens can suppress the directory validation needed to detect a change. After all index extensions have been read, trust a populated untracked-cache root only when a valid on-disk FSMN token matches its FSUC token. An absent cache or root needs no token pairing. Clear the untracked proof when either extension is invalid, and write FSUC beside FSMN only when an untracked cache, a current FSMN token, and valid untracked state are present. Extend the existing read-cache parser regression to check matching and mismatched tokens and to verify that a rejected FSMN clears tracked-token validity. An invalid pair continues through ordinary untracked-cache validation. Signed-off-by: Taylor Blau --- fsmonitor-ll.h | 1 + fsmonitor.c | 16 ++++++++++++++++ read-cache-ll.h | 2 ++ read-cache.c | 17 +++++++++++++++++ t/helper/test-read-cache.c | 22 ++++++++++++++++++++-- 5 files changed, 56 insertions(+), 2 deletions(-) diff --git a/fsmonitor-ll.h b/fsmonitor-ll.h index 1028e630e9a912..8591a166665bd5 100644 --- a/fsmonitor-ll.h +++ b/fsmonitor-ll.h @@ -19,6 +19,7 @@ int read_fsmonitor_untracked_extension(struct index_state *istate, const void *data, unsigned long sz); void write_fsmonitor_untracked_extension(struct strbuf *sb, struct index_state *istate); +void prepare_fsmonitor_untracked(struct index_state *istate); /* * Fill the fsmonitor_dirty ewah bits with their state from the index, diff --git a/fsmonitor.c b/fsmonitor.c index 26d00b5d912dfb..7b90f80405909c 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -177,6 +177,7 @@ int read_fsmonitor_extension(struct index_state *istate, const void *data, ewah_free(istate->fsmonitor_dirty); istate->fsmonitor_last_update = strbuf_detach(&last_update, NULL); istate->fsmonitor_dirty = fsmonitor_dirty; + istate->fsmonitor_token_valid = 1; trace2_data_string("index", NULL, "extension/fsmn/read/token", istate->fsmonitor_last_update); @@ -187,6 +188,8 @@ int read_fsmonitor_extension(struct index_state *istate, const void *data, invalid: istate->fsmonitor_extension_seen = 1; + istate->fsmonitor_token_valid = 0; + istate->fsmonitor_untracked_valid = 0; FREE_AND_NULL(istate->fsmonitor_last_update); if (istate->fsmonitor_dirty) { ewah_free(istate->fsmonitor_dirty); @@ -229,6 +232,7 @@ int read_fsmonitor_untracked_extension(struct index_state *istate, invalid: istate->fsmonitor_untracked_extension_seen = 1; istate->fsmonitor_untracked_extension_invalid = 1; + istate->fsmonitor_untracked_valid = 0; FREE_AND_NULL(istate->fsmonitor_untracked_token); trace2_data_intmax("fsmonitor", istate->repo, "untracked/invalid-extension", 1); @@ -246,6 +250,18 @@ void write_fsmonitor_untracked_extension(struct strbuf *sb, strbuf_addch(sb, '\0'); } +void prepare_fsmonitor_untracked(struct index_state *istate) +{ + istate->fsmonitor_untracked_valid = + !istate->fsmonitor_untracked_extension_invalid && + (!istate->untracked || !istate->untracked->root || + (istate->fsmonitor_token_valid && + istate->fsmonitor_last_update && + istate->fsmonitor_untracked_token && + !strcmp(istate->fsmonitor_last_update, + istate->fsmonitor_untracked_token))); +} + void fill_fsmonitor_bitmap(struct index_state *istate) { unsigned int i, skipped = 0; diff --git a/read-cache-ll.h b/read-cache-ll.h index f71889101e2775..db5b13d06b946c 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -183,7 +183,9 @@ struct index_state { updated_workdir : 1, updated_skipworktree : 1, fsmonitor_has_run_once : 1, + fsmonitor_token_valid : 1, fsmonitor_extension_seen : 1, + fsmonitor_untracked_valid : 1, fsmonitor_untracked_extension_seen : 1, fsmonitor_untracked_extension_invalid : 1; enum sparse_index_mode sparse_index; diff --git a/read-cache.c b/read-cache.c index 43d4b15d54ad7e..57cc73dec90a0e 100644 --- a/read-cache.c +++ b/read-cache.c @@ -1982,6 +1982,7 @@ static void post_read_index_from(struct index_state *istate) check_ce_order(istate); tweak_untracked_cache(istate); tweak_split_index(istate); + prepare_fsmonitor_untracked(istate); tweak_fsmonitor(istate); } @@ -3077,6 +3078,22 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, goto out; } } + if (write_extensions & WRITE_FSMONITOR_EXTENSION && + istate->untracked && + istate->fsmonitor_last_update && + istate->fsmonitor_untracked_valid) { + strbuf_reset(&sb); + + write_fsmonitor_untracked_extension(&sb, istate); + err = write_index_ext_header(f, eoie_c, + CACHE_EXT_FSMONITOR_UNTRACKED, + sb.len) < 0; + hashwrite(f, sb.buf, sb.len); + if (err) { + ret = -1; + goto out; + } + } if (istate->sparse_index) { if (write_index_ext_header(f, eoie_c, CACHE_EXT_SPARSE_DIRECTORIES, 0) < 0) { ret = -1; diff --git a/t/helper/test-read-cache.c b/t/helper/test-read-cache.c index 4698265f5c090f..372b55b419d6b4 100644 --- a/t/helper/test-read-cache.c +++ b/t/helper/test-read-cache.c @@ -3,6 +3,7 @@ #include "test-tool.h" #include "attr.h" #include "config.h" +#include "dir.h" #include "environment.h" #include "ewah/ewok.h" #include "ewah/ewok_rlw.h" @@ -57,6 +58,8 @@ static int test_fsuc_parser(void) { struct index_state duplicate = INDEX_STATE_INIT(the_repository); struct index_state truncated = INDEX_STATE_INIT(the_repository); + struct untracked_cache untracked = { 0 }; + struct untracked_cache_dir root = { 0 }; struct strbuf encoded = STRBUF_INIT; struct strbuf written = STRBUF_INIT; uint32_t version; @@ -77,6 +80,17 @@ static int test_fsuc_parser(void) if (written.len != encoded.len || memcmp(written.buf, encoded.buf, encoded.len)) return error("FSUC did not round-trip"); + duplicate.fsmonitor_token_valid = 1; + duplicate.untracked = &untracked; + untracked.root = &root; + prepare_fsmonitor_untracked(&duplicate); + if (!duplicate.fsmonitor_untracked_valid) + return error("matching FSMN and FSUC tokens were not paired"); + free(duplicate.fsmonitor_last_update); + duplicate.fsmonitor_last_update = xstrdup("other"); + prepare_fsmonitor_untracked(&duplicate); + if (duplicate.fsmonitor_untracked_valid) + return error("mismatched FSMN and FSUC tokens were paired"); read_fsmonitor_untracked_extension( &duplicate, encoded.buf, encoded.len); if (!fsuc_failed_closed(&duplicate)) @@ -145,7 +159,8 @@ static void make_raw_fsmn(struct strbuf *out, uint32_t bit_size, static int fsmn_failed_closed(const struct index_state *istate) { return istate->fsmonitor_extension_seen && - !istate->fsmonitor_last_update && !istate->fsmonitor_dirty; + !istate->fsmonitor_last_update && !istate->fsmonitor_dirty && + !istate->fsmonitor_token_valid; } static int check_invalid_fsmn(const struct strbuf *encoded, @@ -156,6 +171,7 @@ static int check_invalid_fsmn(const struct strbuf *encoded, invalid.cache_nr = 1; invalid.fsmonitor_last_update = xstrdup("old"); invalid.fsmonitor_dirty = ewah_new(); + invalid.fsmonitor_token_valid = 1; read_fsmonitor_extension(&invalid, encoded->buf, encoded->len); if (!fsmn_failed_closed(&invalid)) return error("%s FSMN was published", description); @@ -173,7 +189,8 @@ static int test_fsmn_parser(void) duplicate.cache_nr = truncated.cache_nr = 1; make_valid_fsmn(&encoded); read_fsmonitor_extension(&duplicate, encoded.buf, encoded.len); - if (!duplicate.fsmonitor_last_update || + if (!duplicate.fsmonitor_token_valid || + !duplicate.fsmonitor_last_update || strcmp(duplicate.fsmonitor_last_update, "token") || !duplicate.fsmonitor_dirty) return error("valid FSMN was not published"); @@ -183,6 +200,7 @@ static int test_fsmn_parser(void) truncated.fsmonitor_last_update = xstrdup("old"); truncated.fsmonitor_dirty = ewah_new(); + truncated.fsmonitor_token_valid = 1; read_fsmonitor_extension(&truncated, encoded.buf, encoded.len - 1); if (!fsmn_failed_closed(&truncated)) return error("truncated FSMN was partially published"); From 5e416905bb82c564d448ae6e2f68d70249299c3a Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 12:06:20 -0500 Subject: [PATCH 033/105] preload-index: factor the ordinary preload eligibility predicate preload_thread() spells out which index entries require a filesystem lookup. A bulk preloader must begin with those same exclusions before applying its stricter publication rules. Extract the existing checks into preload_entry_needs_stat(), covering staged entries, gitlinks, up-to-date entries, skip-worktree entries, and fsmonitor-valid entries. Keep the ordinary preload loop and its ordering unchanged. Signed-off-by: Taylor Blau --- preload-index.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/preload-index.c b/preload-index.c index b222821b448526..10bd66affe169c 100644 --- a/preload-index.c +++ b/preload-index.c @@ -44,6 +44,15 @@ struct thread_data { int t2_nr_lstat; }; +static int preload_entry_needs_stat(const struct cache_entry *ce) +{ + return !ce_stage(ce) && + !S_ISGITLINK(ce->ce_mode) && + !ce_uptodate(ce) && + !ce_skip_worktree(ce) && + !(ce->ce_flags & CE_FSMONITOR_VALID); +} + static void *preload_thread(void *_data) { int nr, last_nr; @@ -61,15 +70,7 @@ static void *preload_thread(void *_data) struct cache_entry *ce = *cep++; struct stat st; - if (ce_stage(ce)) - continue; - if (S_ISGITLINK(ce->ce_mode)) - continue; - if (ce_uptodate(ce)) - continue; - if (ce_skip_worktree(ce)) - continue; - if (ce->ce_flags & CE_FSMONITOR_VALID) + if (!preload_entry_needs_stat(ce)) continue; if (p->progress && !(nr & 31)) { struct progress_data *pd = p->progress; From 0ef5138fc47efa1b0e39d75e3a3c5c0a623b6184 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 00:45:06 -0500 Subject: [PATCH 034/105] path-namespace: check reopened component identity A successful descriptor-relative reopen proves only that a component currently exists. If its parent entry has been replaced, the reopened descriptor can refer to a different object from the file that was originally observed. Add path_namespace_reopen_component() to reopen exactly one component with a caller-supplied anchored-open function and compare the complete stat identity with the expected object. Reject empty components, dot components, and embedded separators with EINVAL; report a replacement as EAGAIN and close the reopened descriptor on every outcome. When file identity is unreliable, return EAGAIN before opening the component. Extend the already registered path-namespace unit suite with matching and replaced temporary objects and a parent-traversal attempt. The primitive does not infer that equal content or a successful open is sufficient to establish namespace identity. Signed-off-by: Taylor Blau --- path-namespace.c | 42 +++++++++++++++++++++++++++++ path-namespace.h | 5 ++++ t/unit-tests/u-path-namespace.c | 48 +++++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+) diff --git a/path-namespace.c b/path-namespace.c index 48fc0aae434ef7..151634b886b663 100644 --- a/path-namespace.c +++ b/path-namespace.c @@ -45,3 +45,45 @@ int path_namespace_stat_equal(const struct stat *a, const struct stat *b) path_stat_identity_init(&second, b); return path_stat_identity_equal(&first, &second); } + +int path_namespace_reopen_component( + int parent_fd, const char *component, int flags, + path_namespace_open_fn open_fn, const struct stat *expected) +{ + struct stat reopened; + int fd, saved_errno; + + if (!open_fn || !component || !*component || + !strcmp(component, ".") || !strcmp(component, "..")) { + errno = EINVAL; + return -1; + } + for (const char *p = component; *p; p++) { + if (is_dir_sep(*p)) { + errno = EINVAL; + return -1; + } + } + if (!fstat_is_reliable()) { + errno = EAGAIN; + return -1; + } + + fd = open_fn(parent_fd, component, flags); + if (fd < 0) + return -1; + if (fstat(fd, &reopened)) { + saved_errno = errno; + goto error; + } + if (!path_namespace_stat_equal(expected, &reopened)) { + saved_errno = EAGAIN; + goto error; + } + return close(fd); + +error: + close(fd); + errno = saved_errno; + return -1; +} diff --git a/path-namespace.h b/path-namespace.h index 0a93683e0b2de1..c26f4f12aebd48 100644 --- a/path-namespace.h +++ b/path-namespace.h @@ -3,6 +3,8 @@ struct stat; +typedef int (*path_namespace_open_fn)(int dirfd, const char *path, int flags); + #define PATH_STAT_IDENTITY_FIELDS 14 struct path_stat_identity { @@ -14,5 +16,8 @@ void path_stat_identity_init(struct path_stat_identity *identity, int path_stat_identity_equal(const struct path_stat_identity *a, const struct path_stat_identity *b); int path_namespace_stat_equal(const struct stat *a, const struct stat *b); +int path_namespace_reopen_component( + int parent_fd, const char *component, int flags, + path_namespace_open_fn open_fn, const struct stat *expected); #endif /* PATH_NAMESPACE_H */ diff --git a/t/unit-tests/u-path-namespace.c b/t/unit-tests/u-path-namespace.c index 702104597b1df9..4e0d9dfab24d5d 100644 --- a/t/unit-tests/u-path-namespace.c +++ b/t/unit-tests/u-path-namespace.c @@ -1,6 +1,7 @@ #include "unit-test.h" #include "path-namespace.h" +#include "tempfile.h" #define ASSERT_STAT_FIELD_MATTERS(base, changed, field) do { \ (changed) = (base); \ @@ -67,3 +68,50 @@ void test_path_namespace__stat_fields(void) ASSERT_STAT_FIELD_MATTERS(st, changed, st_gen); #endif } + +static int source_fd = -1; + +static int reopen_source(int dirfd UNUSED, const char *path, int flags UNUSED) +{ + if (strcmp(path, "source")) { + errno = ENOENT; + return -1; + } + return dup(source_fd); +} + +void test_path_namespace__reopen_component(void) +{ + struct tempfile *first = mks_tempfile_t("path-namespace-one-XXXXXX"); + struct tempfile *second = mks_tempfile_t("path-namespace-two-XXXXXX"); + struct stat expected; + + cl_assert(first != NULL); + cl_assert(second != NULL); + cl_must_pass(fstat(get_tempfile_fd(first), &expected)); + + source_fd = get_tempfile_fd(first); + if (!fstat_is_reliable()) { + cl_assert(path_namespace_reopen_component( + -1, "source", O_RDONLY, + reopen_source, &expected) < 0); + cl_assert_equal_i(errno, EAGAIN); + goto invalid_component; + } + cl_must_pass(path_namespace_reopen_component( + -1, "source", O_RDONLY, reopen_source, &expected)); + + source_fd = get_tempfile_fd(second); + cl_assert(path_namespace_reopen_component( + -1, "source", O_RDONLY, reopen_source, &expected) < 0); + cl_assert_equal_i(errno, EAGAIN); + +invalid_component: + cl_assert(path_namespace_reopen_component( + -1, "../source", O_RDONLY, reopen_source, &expected) < 0); + cl_assert_equal_i(errno, EINVAL); + + source_fd = -1; + cl_must_pass(delete_tempfile(&first)); + cl_must_pass(delete_tempfile(&second)); +} From 6b66f3adabd44aa6c891133bcc332b5b74be8953 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 12:08:32 -0700 Subject: [PATCH 035/105] fsmonitor: validate builtin daemon responses before applying them The builtin fsmonitor client interpreted an IPC reply as unbounded C strings. A truncated token or pathname could read past the reply; an empty pathname could enter invalidation code expecting at least one byte; and a slash response could be confused with a real path. Parse the complete reply into an explicit error, delta, or trivial outcome before exposing a builtin token or path. Require a bounded builtin-prefixed token and fully terminated, nonempty, worktree- relative path records. Reserve an exact single slash for a trivial reply and retain the separate double-slash global invalidation marker. Route malformed replies through the existing scan fallback. Add unit coverage for valid paths, trivial and global responses, missing delimiters, oversized tokens, empty records, absolute paths, parent traversal, and malformed separators. Register the new unit suite in both the Makefile and t/meson.build. Hook parsing and token adoption remain unchanged. Signed-off-by: Taylor Blau --- Makefile | 1 + fsmonitor.c | 112 ++++++++++++++++++++++++---- fsmonitor.h | 23 ++++++ t/meson.build | 1 + t/unit-tests/u-fsmonitor-response.c | 86 +++++++++++++++++++++ 5 files changed, 207 insertions(+), 16 deletions(-) create mode 100644 t/unit-tests/u-fsmonitor-response.c diff --git a/Makefile b/Makefile index 97a4936b146df3..2d4754b4643417 100644 --- a/Makefile +++ b/Makefile @@ -1540,6 +1540,7 @@ CLAR_TEST_SUITES += u-ctype CLAR_TEST_SUITES += u-dir CLAR_TEST_SUITES += u-example-decorate CLAR_TEST_SUITES += u-fsmonitor-attributes +CLAR_TEST_SUITES += u-fsmonitor-response CLAR_TEST_SUITES += u-hash CLAR_TEST_SUITES += u-hashmap CLAR_TEST_SUITES += u-list-objects-filter-options diff --git a/fsmonitor.c b/fsmonitor.c index 7b90f80405909c..b88a5c377894af 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -731,6 +731,90 @@ static int is_trivial_response_at(const struct strbuf *result, size_t offset) return 1; } +void fsmonitor_query_result_release(struct fsmonitor_query_result *result) +{ + strbuf_release(&result->token); + strbuf_release(&result->paths); +} + +static int fsmonitor_valid_worktree_path(const char *path, size_t len) +{ + struct strbuf copy = STRBUF_INIT; + int valid = 0; + + if (!len || is_dir_sep(path[0]) || has_dos_drive_prefix(path)) + return 0; + strbuf_add(©, path, len); + if (is_dir_sep(copy.buf[copy.len - 1])) + strbuf_setlen(©, copy.len - 1); + if (!copy.len || is_dir_sep(copy.buf[copy.len - 1])) + goto done; + valid = verify_path(copy.buf, 0); + +done: + strbuf_release(©); + return valid; +} + +enum fsmonitor_query_outcome fsmonitor_parse_builtin_response( + const struct strbuf *raw, struct fsmonitor_query_result *result) +{ + const char *nul, *p, *end; + + if (!raw->len) + goto malformed; + nul = memchr(raw->buf, '\0', raw->len); + if (!nul || nul == raw->buf || nul - raw->buf > FSMONITOR_TOKEN_MAX) + goto malformed; + strbuf_add(&result->token, raw->buf, nul - raw->buf); + if (!starts_with(result->token.buf, "builtin:")) + goto malformed; + + p = nul + 1; + end = raw->buf + raw->len; + if (p == end) { + result->outcome = FSMONITOR_QUERY_DELTA; + return result->outcome; + } + if (end[-1] != '\0') + goto malformed; + if (end - p == 2 && p[0] == '/' && p[1] == '\0') { + result->outcome = FSMONITOR_QUERY_TRIVIAL; + return result->outcome; + } + + while (p < end) { + nul = memchr(p, '\0', end - p); + if (!nul || nul == p) + goto malformed; + if (strcmp(p, FSMONITOR_PATH_GLOBAL_INVALIDATE) && + !fsmonitor_valid_worktree_path(p, nul - p)) + goto malformed; + p = nul + 1; + } + strbuf_add(&result->paths, raw->buf + result->token.len + 1, + end - (raw->buf + result->token.len + 1)); + result->outcome = FSMONITOR_QUERY_DELTA; + return result->outcome; + +malformed: + strbuf_reset(&result->token); + strbuf_reset(&result->paths); + trace2_data_intmax("fsm_client", NULL, "query/invalid-response", 1); + return FSMONITOR_QUERY_ERROR; +} + +static enum fsmonitor_query_outcome query_builtin_fsmonitor( + const char *since_token, struct fsmonitor_query_result *result) +{ + struct strbuf raw = STRBUF_INIT; + + if (!fsmonitor_ipc__send_query(since_token, &raw)) + fsmonitor_parse_builtin_response(&raw, result); + strbuf_release(&raw); + return result->outcome; +} + void refresh_fsmonitor(struct index_state *istate) { static int warn_once = 0; @@ -762,24 +846,19 @@ void refresh_fsmonitor(struct index_state *istate) trace_printf_key(&trace_fsmonitor, "refresh fsmonitor"); if (fsm_mode == FSMONITOR_MODE_IPC) { - query_success = !fsmonitor_ipc__send_query( + struct fsmonitor_query_result result = + FSMONITOR_QUERY_RESULT_INIT; + + query_builtin_fsmonitor( istate->fsmonitor_last_update ? istate->fsmonitor_last_update : "builtin:fake", - &query_result); - if (query_success) { - /* - * The response contains a series of nul terminated - * strings. The first is the new token. - * - * Use `char *buf` as an interlude to trick the CI - * static analysis to let us use `strbuf_addstr()` - * here (and only copy the token) rather than - * `strbuf_addbuf()`. - */ - buf = query_result.buf; - strbuf_addstr(&last_update_token, buf); - bol = last_update_token.len + 1; - is_trivial = is_trivial_response_at(&query_result, bol); + &result); + if (result.outcome != FSMONITOR_QUERY_ERROR) { + query_success = 1; + strbuf_addbuf(&last_update_token, &result.token); + is_trivial = result.outcome == FSMONITOR_QUERY_TRIVIAL; + if (!is_trivial) + strbuf_addbuf(&query_result, &result.paths); if (is_trivial) trace2_data_intmax("fsm_client", NULL, "query/trivial-response", 1); @@ -795,6 +874,7 @@ void refresh_fsmonitor(struct index_state *istate) */ strbuf_addstr(&last_update_token, "builtin:fake"); } + fsmonitor_query_result_release(&result); goto apply_results; } diff --git a/fsmonitor.h b/fsmonitor.h index 47ce78de61c508..e20d280e06a220 100644 --- a/fsmonitor.h +++ b/fsmonitor.h @@ -6,6 +6,7 @@ #include "fsmonitor-settings.h" #include "object.h" #include "read-cache-ll.h" +#include "strbuf.h" #include "trace.h" /* @@ -15,6 +16,28 @@ */ void fsmonitor_invalidate_cache_entry(struct cache_entry *ce); +enum fsmonitor_query_outcome { + FSMONITOR_QUERY_ERROR = 0, + FSMONITOR_QUERY_DELTA, + FSMONITOR_QUERY_TRIVIAL, +}; + +struct fsmonitor_query_result { + enum fsmonitor_query_outcome outcome; + struct strbuf token; + struct strbuf paths; +}; + +#define FSMONITOR_QUERY_RESULT_INIT { \ + .outcome = FSMONITOR_QUERY_ERROR, \ + .token = STRBUF_INIT, \ + .paths = STRBUF_INIT, \ +} + +void fsmonitor_query_result_release(struct fsmonitor_query_result *result); +enum fsmonitor_query_outcome fsmonitor_parse_builtin_response( + const struct strbuf *raw, struct fsmonitor_query_result *result); + /* * A pathname monitor cannot prove that every name for a multiply-linked * inode is inside its watch cone. When the platform reports real link diff --git a/t/meson.build b/t/meson.build index 6d4c7c4bed8fdf..d7dacb4063aff1 100644 --- a/t/meson.build +++ b/t/meson.build @@ -3,6 +3,7 @@ clar_test_suites = [ 'unit-tests/u-dir.c', 'unit-tests/u-example-decorate.c', 'unit-tests/u-fsmonitor-attributes.c', + 'unit-tests/u-fsmonitor-response.c', 'unit-tests/u-hash.c', 'unit-tests/u-hashmap.c', 'unit-tests/u-list-objects-filter-options.c', diff --git a/t/unit-tests/u-fsmonitor-response.c b/t/unit-tests/u-fsmonitor-response.c new file mode 100644 index 00000000000000..dda747aa764097 --- /dev/null +++ b/t/unit-tests/u-fsmonitor-response.c @@ -0,0 +1,86 @@ +#include "unit-test.h" + +#include "fsmonitor.h" + +static void check_response(const void *data, size_t len, + enum fsmonitor_query_outcome expected, + const char *token, const void *paths, + size_t paths_len) +{ + struct fsmonitor_query_result result = FSMONITOR_QUERY_RESULT_INIT; + struct strbuf raw = STRBUF_INIT; + + strbuf_add(&raw, data, len); + cl_assert_equal_i(fsmonitor_parse_builtin_response(&raw, &result), + expected); + cl_assert_equal_i(result.outcome, expected); + cl_assert_equal_s(result.token.buf, token); + cl_assert_equal_i(result.paths.len, paths_len); + cl_assert(!paths_len || !memcmp(result.paths.buf, paths, paths_len)); + + fsmonitor_query_result_release(&result); + strbuf_release(&raw); +} + +static void check_malformed(const void *data, size_t len) +{ + check_response(data, len, FSMONITOR_QUERY_ERROR, "", NULL, 0); +} + +void test_fsmonitor_response__rejects_malformed_framing(void) +{ + static const char missing_nul[] = "builtin:1"; + static const char empty_token[] = "\0"; + static const char non_builtin[] = "other:1\0"; + static const char unterminated_path[] = "builtin:1\0path"; + static const char empty_path[] = "builtin:1\0\0"; + static const char absolute_path[] = "builtin:1\0/absolute\0"; + static const char parent_path[] = "builtin:1\0../outside\0"; + static const char embedded_parent[] = + "builtin:1\0dir/../tracked\0"; + static const char dot_path[] = "builtin:1\0./tracked\0"; + static const char repeated_separator[] = + "builtin:1\0dir//tracked\0"; + static const char drive_path[] = "builtin:1\0C:/absolute\0"; + static const char backslash_path[] = "builtin:1\0\\absolute\0"; + struct strbuf overlong = STRBUF_INIT; + + check_malformed("", 0); + check_malformed(missing_nul, sizeof(missing_nul) - 1); + check_malformed(empty_token, sizeof(empty_token) - 1); + check_malformed(non_builtin, sizeof(non_builtin) - 1); + check_malformed(unterminated_path, sizeof(unterminated_path) - 1); + check_malformed(empty_path, sizeof(empty_path) - 1); + check_malformed(absolute_path, sizeof(absolute_path) - 1); + check_malformed(parent_path, sizeof(parent_path) - 1); + check_malformed(embedded_parent, sizeof(embedded_parent) - 1); + check_malformed(dot_path, sizeof(dot_path) - 1); + check_malformed(repeated_separator, + sizeof(repeated_separator) - 1); + if (has_dos_drive_prefix(drive_path + sizeof("builtin:1"))) + check_malformed(drive_path, sizeof(drive_path) - 1); + if (is_dir_sep('\\')) + check_malformed(backslash_path, sizeof(backslash_path) - 1); + + strbuf_addstr(&overlong, "builtin:"); + strbuf_addchars(&overlong, 'x', 4096); + strbuf_addch(&overlong, '\0'); + check_malformed(overlong.buf, overlong.len); + strbuf_release(&overlong); +} + +void test_fsmonitor_response__accepts_valid_builtin_responses(void) +{ + static const char delta[] = "builtin:2\0a\0dir/file\0dir/\0"; + static const char global[] = "builtin:3\0//\0"; + static const char trivial[] = "builtin:4\0/\0"; + + check_response(delta, sizeof(delta) - 1, FSMONITOR_QUERY_DELTA, + "builtin:2", delta + sizeof("builtin:2"), + sizeof(delta) - 1 - sizeof("builtin:2")); + check_response(global, sizeof(global) - 1, FSMONITOR_QUERY_DELTA, + "builtin:3", global + sizeof("builtin:3"), + sizeof(global) - 1 - sizeof("builtin:3")); + check_response(trivial, sizeof(trivial) - 1, FSMONITOR_QUERY_TRIVIAL, + "builtin:4", NULL, 0); +} From 22911e24fe9c34dd7c1f48331bcbc45ce97a01dd Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 28 Jul 2026 10:30:31 -0500 Subject: [PATCH 036/105] preload-index: enable opt-in APFS bulk preloading The APFS backend can enumerate tracked paths, but preload_index() never invokes it. Publishing incomplete or inconclusive directory-scan results would bypass the existing per-entry correctness checks. Run the bulk collector before ordinary threaded preload only for full-index requests with core.preloadIndexBulk and core.preloadIndex enabled and no active fsmonitor provider. Require local APFS, Darwin 20 or newer, and O_NOFOLLOW_ANY support; leave the option off by default. Retain an O_NOFOLLOW worktree-root descriptor, reopen relative paths with O_NOFOLLOW_ANY, and validate the root namespace again when the scan finishes. Publish only clean entries from a completed scan, and leave missing, changed, or multiply-linked entries to ordinary lstat. Register the backend in Make, CMake, and Meson. Add APFS integration coverage for configuration and test overrides, provider exclusion, descriptor pressure, prefix siblings, index states, sparse entries, Unicode names, and hardlink fallback. Signed-off-by: Taylor Blau --- Documentation/config/core.adoc | 10 ++ Makefile | 2 + compat/preload-index/bulk-darwin.c | 47 ++++- compat/preload-index/bulk-darwin.h | 1 + contrib/buildsystems/CMakeLists.txt | 6 +- meson.build | 2 + preload-index-bulk.c | 85 +++++++++ preload-index-bulk.h | 15 ++ preload-index.c | 152 +++++++++++++++- t/README | 3 + t/meson.build | 1 + t/t7529-preload-index-apfs.sh | 262 ++++++++++++++++++++++++++++ 12 files changed, 582 insertions(+), 4 deletions(-) create mode 100644 preload-index-bulk.c create mode 100755 t/t7529-preload-index-apfs.sh diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index 340329edc38143..5f01b603e5761a 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -729,6 +729,16 @@ relatively high IO latencies. When enabled, Git will do the index comparison to the filesystem data in parallel, allowing overlapping IO's. Defaults to true. +core.preloadIndexBulk:: + On supported filesystems, scan working tree directories in bulk before + the parallel index preload. ++ +This replaces per-entry filesystem lookups with a physical directory scan, +but may cost more than normal preload depending on filesystem and cache +state. Inconclusive scans are discarded before continuing with the normal +preload. Currently this is supported on APFS and only has an effect when +`core.preloadIndex` is enabled. Defaults to false. + core.unsetenvvars:: Windows-only: comma-separated list of environment variables' names that need to be unset before spawning any other process. diff --git a/Makefile b/Makefile index a5a0dabc80e45f..94ad0f342c92b0 100644 --- a/Makefile +++ b/Makefile @@ -1382,8 +1382,10 @@ LIB_OBJS += wrapper.o LIB_OBJS += write-or-die.o LIB_OBJS += ws.o ifdef PRELOAD_INDEX_BULK_BACKEND +BASIC_CFLAGS += -DHAVE_PRELOAD_INDEX_BULK PRELOAD_INDEX_BULK_OBJS += preload-index-bulk-index.o PRELOAD_INDEX_BULK_OBJS += preload-index-bulk-thread.o +PRELOAD_INDEX_BULK_OBJS += preload-index-bulk.o PRELOAD_INDEX_BULK_OBJS += compat/preload-index/bulk-$(PRELOAD_INDEX_BULK_BACKEND).o PRELOAD_INDEX_BULK_OBJS += $(PRELOAD_INDEX_BULK_PLATFORM_OBJS) endif diff --git a/compat/preload-index/bulk-darwin.c b/compat/preload-index/bulk-darwin.c index 22a0e16def307f..9b9adba79b2ccd 100644 --- a/compat/preload-index/bulk-darwin.c +++ b/compat/preload-index/bulk-darwin.c @@ -1,6 +1,7 @@ #include "git-compat-util.h" #include +#include #include #include "compat/precompose_utf8.h" @@ -64,6 +65,27 @@ static int preload_bulk_darwin_open_dir_at( O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); } +int preload_bulk_darwin_supports_nofollow_any(void) +{ +#ifdef O_NOFOLLOW_ANY + struct utsname uts; + char *end; + unsigned long major; + + /* + * O_NOFOLLOW_ANY arrived in Darwin 20. Older kernels accept the + * same bit as O_ALERT without enforcing no-follow semantics. + */ + if (uname(&uts) || !isdigit((unsigned char)uts.release[0])) + return 0; + errno = 0; + major = strtoul(uts.release, &end, 10); + return !errno && end != uts.release && *end == '.' && major >= 20; +#else + return 0; +#endif +} + static int preload_bulk_darwin_open_relative(struct preload_bulk_scan *scan, const char *path) { @@ -449,12 +471,35 @@ static int scan_directory(struct preload_bulk_worker *worker, return ret; } +static const char *start_scan(struct preload_bulk_scan *scan) +{ + const char *error; + + repo_precompose_utf8_prepare(scan->repo); + error = preload_bulk_darwin_open_root(scan); + if (error) + return error; + return preload_bulk_darwin_snapshot_root(scan); +} + +static const char *finish_scan(struct preload_bulk_scan *scan) +{ + return preload_bulk_darwin_validate_root(scan); +} + static const struct preload_bulk_backend darwin_backend = { + .start = start_scan, + .finish = finish_scan, + .release = preload_bulk_darwin_release, .open_dir_at = preload_bulk_darwin_open_dir_at, .scan_directory = scan_directory, }; const struct preload_bulk_backend *preload_bulk_platform_backend(void) { - return &darwin_backend; +#ifdef O_NOFOLLOW_ANY + if (preload_bulk_darwin_supports_nofollow_any()) + return &darwin_backend; +#endif + return NULL; } diff --git a/compat/preload-index/bulk-darwin.h b/compat/preload-index/bulk-darwin.h index c69886ac972268..7c5c45ee947651 100644 --- a/compat/preload-index/bulk-darwin.h +++ b/compat/preload-index/bulk-darwin.h @@ -12,6 +12,7 @@ struct preload_bulk_darwin_data { fsid_t root_fsid; }; +int preload_bulk_darwin_supports_nofollow_any(void); /* * Exposed so that tests can validate kernel-supplied records directly. */ diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index e24ec739720dc5..87152a27fe7cb1 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -275,7 +275,8 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_compile_definitions(PROCFS_EXECUTABLE_PATH="/proc/self/exe" HAVE_DEV_TTY ) list(APPEND compat_SOURCES unix-socket.c unix-stream-server.c compat/linux/procinfo.c) elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") - add_compile_definitions(PRECOMPOSE_UNICODE USE_ST_TIMESPEC) + add_compile_definitions(HAVE_PRELOAD_INDEX_BULK PRECOMPOSE_UNICODE + USE_ST_TIMESPEC) list(APPEND compat_SOURCES compat/darwin/procinfo.c compat/precompose_utf8.c @@ -673,7 +674,8 @@ parse_makefile_for_sources(libgit_SOURCES ${CMAKE_SOURCE_DIR}/Makefile "LIB_OBJS if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") list(APPEND libgit_SOURCES preload-index-bulk-index.c - preload-index-bulk-thread.c) + preload-index-bulk-thread.c + preload-index-bulk.c) endif() list(TRANSFORM libgit_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") diff --git a/meson.build b/meson.build index 2bdff5bb374325..74a1548af19a6f 100644 --- a/meson.build +++ b/meson.build @@ -1297,6 +1297,7 @@ endif if host_machine.system() == 'darwin' compat_sources += 'compat/precompose_utf8.c' + libgit_c_args += '-DHAVE_PRELOAD_INDEX_BULK' libgit_c_args += '-DPRECOMPOSE_UNICODE' libgit_c_args += '-DPROTECT_HFS_DEFAULT' endif @@ -1352,6 +1353,7 @@ elif host_machine.system() == 'darwin' libgit_sources += [ 'preload-index-bulk-index.c', 'preload-index-bulk-thread.c', + 'preload-index-bulk.c', ] else compat_sources += 'compat/stub/procinfo.c' diff --git a/preload-index-bulk.c b/preload-index-bulk.c new file mode 100644 index 00000000000000..31c370e2813e58 --- /dev/null +++ b/preload-index-bulk.c @@ -0,0 +1,85 @@ +#include "git-compat-util.h" +#include "preload-index-bulk.h" +#include "read-cache-ll.h" + +static int backend_available(const struct preload_bulk_backend *backend) +{ + return backend && backend->start && backend->finish && + backend->release && backend->open_dir_at && + backend->scan_directory; +} + +int preload_bulk_available(void) +{ + return backend_available(preload_bulk_platform_backend()); +} + +int preload_bulk_collect(struct index_state *istate, int threads, + struct preload_bulk_result *result) +{ + const struct preload_bulk_backend *backend = + preload_bulk_platform_backend(); + struct preload_bulk_scan scan = { + .repo = istate->repo, + .istate = istate, + .backend = backend, + .root_fd = -1, + .threads = threads, + }; + struct preload_bulk_run_result run_result = { 0 }; + const char *start_error, *finish_error = NULL; + int scan_error = -1; + int clean; + + memset(result, 0, sizeof(*result)); + result->outcome = "start-fallback"; + result->reason = "backend-unavailable"; + if (!backend_available(backend)) + return -1; + + CALLOC_ARRAY(scan.tracked_state, istate->cache_nr); + start_error = backend->start(&scan); + if (!start_error) { + scan_error = preload_bulk_run_scan(&scan, &run_result); + finish_error = backend->finish(&scan); + } + + clean = !start_error && !scan_error && !finish_error && + !run_result.changed_dirs && + !run_result.malformed; + result->run = run_result; + if (start_error) { + result->outcome = "start-fallback"; + result->reason = start_error; + } else if (run_result.changed_dirs) { + result->outcome = "scan-fallback"; + result->reason = "filesystem-race"; + } else if (run_result.malformed) { + result->outcome = "scan-fallback"; + result->reason = "malformed-record"; + } else if (scan_error) { + result->outcome = "scan-fallback"; + result->reason = "scan-error"; + } else if (finish_error) { + result->outcome = "finish-fallback"; + result->reason = finish_error; + } else { + result->outcome = "complete"; + result->reason = NULL; + } + if (clean) { + result->tracked_state = scan.tracked_state; + result->nr = istate->cache_nr; + scan.tracked_state = NULL; + } + + backend->release(&scan); + free(scan.tracked_state); + return clean ? 0 : -1; +} + +void preload_bulk_result_release(struct preload_bulk_result *result) +{ + FREE_AND_NULL(result->tracked_state); + memset(result, 0, sizeof(*result)); +} diff --git a/preload-index-bulk.h b/preload-index-bulk.h index 3272ce41ee81ec..45899e56e58a46 100644 --- a/preload-index-bulk.h +++ b/preload-index-bulk.h @@ -52,6 +52,9 @@ struct preload_bulk_worker { }; struct preload_bulk_backend { + const char *(*start)(struct preload_bulk_scan *scan); + const char *(*finish)(struct preload_bulk_scan *scan); + void (*release)(struct preload_bulk_scan *scan); int (*open_dir_at)(struct preload_bulk_worker *worker, int parent_fd, const char *name); /* @@ -83,6 +86,14 @@ struct preload_bulk_run_result { int threads; }; +struct preload_bulk_result { + unsigned char *tracked_state; + size_t nr; + const char *outcome; + const char *reason; + struct preload_bulk_run_result run; +}; + void preload_bulk_schedule_directory( struct preload_bulk_worker *worker, int parent_fd, const struct preload_bulk_dir_identity *parent_identity, @@ -101,5 +112,9 @@ void preload_bulk_record_tracked( int preload_bulk_run_scan(struct preload_bulk_scan *scan, struct preload_bulk_run_result *result); const struct preload_bulk_backend *preload_bulk_platform_backend(void); +int preload_bulk_collect(struct index_state *istate, int threads, + struct preload_bulk_result *result); +int preload_bulk_available(void); +void preload_bulk_result_release(struct preload_bulk_result *result); #endif /* PRELOAD_INDEX_BULK_H */ diff --git a/preload-index.c b/preload-index.c index 10bd66affe169c..f77760f05def9b 100644 --- a/preload-index.c +++ b/preload-index.c @@ -12,6 +12,9 @@ #include "gettext.h" #include "parse.h" #include "preload-index.h" +#ifdef HAVE_PRELOAD_INDEX_BULK +#include "preload-index-bulk.h" +#endif #include "progress.h" #include "read-cache.h" #include "thread-utils.h" @@ -28,6 +31,8 @@ */ #define MAX_PARALLEL (20) #define THREAD_COST (500) +#define BULK_MAX_PARALLEL (32) +#define BULK_ENTRIES_PER_THREAD (5000) struct progress_data { unsigned long n; @@ -104,6 +109,144 @@ static void *preload_thread(void *_data) return NULL; } +#ifdef HAVE_PRELOAD_INDEX_BULK +static int stat_data_is_zero(const struct stat_data *sd) +{ + return !sd->sd_ctime.sec && + !sd->sd_ctime.nsec && + !sd->sd_mtime.sec && + !sd->sd_mtime.nsec && + !sd->sd_dev && + !sd->sd_ino && + !sd->sd_uid && + !sd->sd_gid && + !sd->sd_size; +} + +static int preload_bulk_entry_is_useful(const struct cache_entry *ce) +{ + return preload_entry_needs_stat(ce) && + !ce_intent_to_add(ce) && + !(ce->ce_flags & (CE_VALID | CE_REMOVE)) && + (S_ISREG(ce->ce_mode) || S_ISLNK(ce->ce_mode)) && + !stat_data_is_zero(&ce->ce_stat_data); +} + +static size_t preload_bulk_useful_candidates(struct index_state *index) +{ + size_t useful = 0; + + for (size_t i = 0; i < index->cache_nr; i++) + if (preload_bulk_entry_is_useful(index->cache[i])) + useful++; + return useful; +} + +static size_t preload_bulk_publish_clean( + struct index_state *index, + const struct preload_bulk_result *result) +{ + size_t applied = 0; + + if (result->nr != index->cache_nr) + BUG("bulk preload result does not match the index"); + + for (size_t i = 0; i < result->nr; i++) { + struct cache_entry *ce; + unsigned char state = result->tracked_state[i]; + + if (state != PRELOAD_BULK_TRACKED_CLEAN) + continue; + ce = index->cache[i]; + if (!preload_bulk_entry_is_useful(ce)) + continue; + ce_mark_uptodate(ce); + mark_fsmonitor_valid(index, ce); + applied++; + } + return applied; +} + +static int preload_bulk_threads(size_t useful) +{ + int cpus = online_cpus(); + int threads = DIV_ROUND_UP(useful, BULK_ENTRIES_PER_THREAD); + + if (threads < 1) + threads = 1; + if (cpus > 0) { + int cpu_limit = cpus > BULK_MAX_PARALLEL / 2 ? + BULK_MAX_PARALLEL : cpus * 2; + + if (threads > cpu_limit) + threads = cpu_limit; + } + if (threads > BULK_MAX_PARALLEL) + threads = BULK_MAX_PARALLEL; + return threads; +} + +static void preload_bulk_trace_result( + struct index_state *index, + const struct preload_bulk_result *result, + size_t applied) +{ + trace2_data_string("index", index->repo, "preload/bulk_result", + result->outcome); + if (result->reason) + trace2_data_string("index", index->repo, + "preload/bulk_reason", result->reason); + trace2_data_intmax("index", index->repo, "preload/bulk_applied", + applied); + trace2_data_intmax("index", index->repo, "preload/bulk_dirs", + result->run.dirs); + trace2_data_intmax("index", index->repo, "preload/bulk_entries", + result->run.entries); + trace2_data_intmax("index", index->repo, "preload/bulk_calls", + result->run.bulk_calls); + trace2_data_intmax("index", index->repo, "preload/bulk_workers", + result->run.threads); +} + +static void preload_bulk_try(struct index_state *index) +{ + struct preload_bulk_result result = { 0 }; + size_t useful; + size_t applied = 0; + int enabled = 0; + int control, threads; + + /* + * Let the test variable override configuration without bypassing + * any of the proof checks. + */ + control = git_env_bool("GIT_TEST_PRELOAD_INDEX_BULK", -1); + if (control < 0) + repo_config_get_bool(index->repo, "core.preloadindexbulk", + &enabled); + else + enabled = control; + if (!enabled || + fsm_settings__get_mode(index->repo) != FSMONITOR_MODE_DISABLED || + !preload_bulk_available()) + return; + useful = preload_bulk_useful_candidates(index); + trace2_data_intmax("index", index->repo, "preload/bulk_useful", + useful); + trace2_data_intmax("index", index->repo, "preload/bulk_cache_nr", + index->cache_nr); + if (!useful) + return; + threads = preload_bulk_threads(useful); + trace2_region_enter("index", "preload/bulk", index->repo); + if (!preload_bulk_collect(index, threads, &result)) + applied = preload_bulk_publish_clean(index, &result); + preload_bulk_trace_result(index, &result, applied); + trace2_region_leave("index", "preload/bulk", index->repo); + preload_bulk_result_release(&result); +} +#endif + void preload_index(struct index_state *index, const struct pathspec *pathspec, unsigned int refresh_flags) @@ -116,7 +259,14 @@ void preload_index(struct index_state *index, repo_config_get_bool(index->repo, "core.preloadindex", &core_preload_index); - if (!HAVE_THREADS || !core_preload_index) + if (!core_preload_index) + return; + +#ifdef HAVE_PRELOAD_INDEX_BULK + if (!pathspec || !pathspec->nr) + preload_bulk_try(index); +#endif + if (!HAVE_THREADS) return; threads = index->cache_nr / THREAD_COST; diff --git a/t/README b/t/README index 4252774f86c8ce..6ba21aead64858 100644 --- a/t/README +++ b/t/README @@ -422,6 +422,9 @@ overridden by the --no-path-walk command-line argument. GIT_TEST_PRELOAD_INDEX= exercises the preload-index code path by overriding the minimum number of cache entries required per thread. +GIT_TEST_PRELOAD_INDEX_BULK= overrides the +`core.preloadIndexBulk` setting. + GIT_TEST_INDEX_THREADS= enables exercising the multi-threaded loading of the index for the whole test suite by bypassing the default number of cache entries and thread minimums. Setting this to 1 will make the diff --git a/t/meson.build b/t/meson.build index d22603637bc5a6..2507f29aeec5ff 100644 --- a/t/meson.build +++ b/t/meson.build @@ -945,6 +945,7 @@ integration_tests = [ 't7526-commit-pathspec-file.sh', 't7527-builtin-fsmonitor.sh', 't7528-signed-commit-ssh.sh', + 't7529-preload-index-apfs.sh', 't7600-merge.sh', 't7601-merge-pull-config.sh', 't7602-merge-octopus-many.sh', diff --git a/t/t7529-preload-index-apfs.sh b/t/t7529-preload-index-apfs.sh new file mode 100755 index 00000000000000..87b49869515151 --- /dev/null +++ b/t/t7529-preload-index-apfs.sh @@ -0,0 +1,262 @@ +#!/bin/sh + +test_description='APFS bulk index preload' + +. ./test-lib.sh + +test_lazy_prereq APFS_BULK_PRELOAD ' + test_have_prereq MACOS && + darwin_major=$(uname -r) && + darwin_major=${darwin_major%%.*} && + test "$darwin_major" -ge 20 && + /bin/df -t apfs "$TRASH_DIRECTORY" >/dev/null +' + +if ! test_have_prereq APFS_BULK_PRELOAD +then + skip_all='bulk index preload requires macOS on APFS' + test_done +fi + +setup_repo () { + repo=$1 && + git init "$repo" && + mkdir -p "$repo/nested/deep" "$repo/other" && + test_write_lines root >"$repo/root" && + test_write_lines root-peer >"$repo/root-peer" && + test_write_lines nested >"$repo/nested/tracked" && + test_write_lines nested-peer >"$repo/nested/peer" && + test_write_lines deep >"$repo/nested/deep/tracked" && + test_write_lines deep-peer >"$repo/nested/deep/peer" && + test_write_lines other >"$repo/other/tracked" && + test_write_lines other-peer >"$repo/other/peer" && + git -C "$repo" add . && + git -C "$repo" commit -m base && + git -C "$repo" config core.fsmonitor false && + test-tool chmtime -120 \ + "$repo/root" "$repo/root-peer" \ + "$repo/nested/tracked" "$repo/nested/peer" \ + "$repo/nested/deep/tracked" "$repo/nested/deep/peer" \ + "$repo/other/tracked" "$repo/other/peer" && + git -C "$repo" update-index --refresh +} + +ordinary_status () { + GIT_OPTIONAL_LOCKS=0 \ + git -C "$1" -c core.preloadIndex=false \ + status --porcelain=v2 >"$2" +} + +bulk_status () { + repo=$1 && + output=$2 && + bulk_trace=$TRASH_DIRECTORY/$3 && + rm -f "$bulk_trace" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TRACE2_EVENT="$bulk_trace" \ + git -C "$repo" status --porcelain=v2 >"$output" +} + +check_data () { + test_trace2_data index "$2" "$3" <"$TRASH_DIRECTORY/$1" +} + +check_lstat_data () { + test_have_prereq !PTHREADS || + check_data "$1" preload/sum_lstat "$2" +} + +compare_status () { + ordinary_status "$1" expect && + bulk_status "$1" actual "$2" && + test_cmp expect actual +} + +configured_bulk_status () { + repo=$1 && + output=$2 && + bulk_trace=$TRASH_DIRECTORY/$3 && + bulk=${4-true} && + preload=${5-true} && + rm -f "$bulk_trace" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TRACE2_EVENT="$bulk_trace" \ + git -C "$repo" \ + -c core.preloadIndex="$preload" \ + -c core.preloadIndexBulk="$bulk" \ + status --porcelain=v2 >"$output" +} + +test_expect_success 'bulk preload follows its configuration' ' + setup_repo opt-in && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/default.trace" \ + git -C opt-in status --porcelain=v2 >actual && + test_must_be_empty actual && + test_grep ! "\"key\":\"preload/bulk_result\"" default.trace && + configured_bulk_status opt-in actual enabled.trace && + test_must_be_empty actual && + test_grep "\"key\":\"preload/bulk_result\"" enabled.trace && + configured_bulk_status opt-in actual preload-disabled.trace true false && + test_must_be_empty actual && + test_grep ! "\"key\":\"preload/bulk_result\"" preload-disabled.trace +' + +test_expect_success 'test variable overrides bulk preload configuration' ' + test_env GIT_TEST_PRELOAD_INDEX_BULK=0 \ + configured_bulk_status opt-in actual disabled.trace && + test_must_be_empty actual && + test_grep ! "\"key\":\"preload/bulk_result\"" disabled.trace && + test_env GIT_TEST_PRELOAD_INDEX_BULK=1 \ + configured_bulk_status opt-in actual forced.trace false && + test_must_be_empty actual && + check_data forced.trace preload/bulk_applied 8 +' + +test_expect_success 'bulk preload waits for fsmonitor provider closure' ' + write_script opt-in/.git/hooks/fsmonitor-test <<-\EOF && + printf "token\\0" + EOF + git -C opt-in config core.fsmonitor .git/hooks/fsmonitor-test && + configured_bulk_status opt-in actual fsmonitor.trace && + test_must_be_empty actual && + test_grep ! "\"key\":\"preload/bulk_result\"" fsmonitor.trace +' + +test_expect_success 'clean entries are published without lstat' ' + setup_repo clean && + bulk_status clean actual clean.trace && + test_must_be_empty actual && + check_data clean.trace preload/bulk_applied 8 && + check_lstat_data clean.trace 0 +' + +test_expect_success ULIMIT_FILE_DESCRIPTORS \ + 'bulk preload reopens directories under a low descriptor limit' ' + git init low-fd && + for i in $(test_seq 1 64) + do + mkdir "low-fd/$i" && + test_write_lines "$i" >"low-fd/$i/tracked" || + return 1 + done && + git -C low-fd add . && + git -C low-fd commit -m base && + test-tool chmtime -120 low-fd/*/tracked && + git -C low-fd update-index --refresh && + run_with_limited_open_files \ + bulk_status low-fd actual low-fd.trace && + test_must_be_empty actual && + check_data low-fd.trace preload/bulk_applied 64 +' + +test_expect_success 'prefix siblings do not hide tracked descendants' ' + git init prefix-order && + mkdir prefix-order/feather prefix-order/feather-db && + test_write_lines tracked >prefix-order/feather/tracked && + test_write_lines sibling >prefix-order/feather-db/tracked && + git -C prefix-order add . && + git -C prefix-order commit -m base && + test-tool chmtime -120 prefix-order/feather/tracked \ + prefix-order/feather-db/tracked && + git -C prefix-order update-index --refresh && + bulk_status prefix-order actual prefix-order.trace && + test_must_be_empty actual && + check_data prefix-order.trace preload/bulk_applied 2 && + check_lstat_data prefix-order.trace 0 +' + +test_expect_success SYMLINKS \ + 'modified, deleted, typechanged, and symlink entries agree' ' + setup_repo worktree-states && + ln -s root worktree-states/link && + git -C worktree-states add link && + git -C worktree-states commit -m symlink && + mtime=$(test-tool chmtime --get worktree-states/root) && + sleep 1 && + test_write_lines moot >worktree-states/root && + test-tool chmtime "=$mtime" worktree-states/root && + rm worktree-states/nested/tracked && + rm worktree-states/nested/deep/tracked && + mkdir worktree-states/nested/deep/tracked && + rm worktree-states/other/tracked worktree-states/other/peer && + rmdir worktree-states/other && + test_write_lines other >worktree-states/other && + rm worktree-states/link && + ln -s nested/deep/peer worktree-states/link && + compare_status worktree-states worktree-states.trace && + test_file_not_empty actual +' + +test_expect_success 'staged and unmerged entries agree' ' + setup_repo index-states && + test_write_lines staged >index-states/root && + test_write_lines added >index-states/added && + git -C index-states add root added && + git -C index-states rm nested/tracked && + base=$(git -C index-states rev-parse HEAD:nested/peer) && + ours=$(printf "ours\n" | + git -C index-states hash-object -w --stdin) && + theirs=$(printf "theirs\n" | + git -C index-states hash-object -w --stdin) && + { + printf "0 %s\tnested/peer\n" "$(test_oid zero)" && + printf "100644 %s 1\tnested/peer\n" "$base" && + printf "100644 %s 2\tnested/peer\n" "$ours" && + printf "100644 %s 3\tnested/peer\n" "$theirs" + } | git -C index-states update-index --index-info && + compare_status index-states index-states.trace && + test_file_not_empty actual +' + +test_expect_success 'sparse-index entries are left unexpanded' ' + setup_repo sparse && + git -C sparse sparse-checkout init --cone --sparse-index && + git -C sparse sparse-checkout set nested && + git -C sparse ls-files --sparse >before && + test_grep "^other/$" before && + test_write_lines changed >sparse/nested/tracked && + compare_status sparse sparse.trace && + git -C sparse ls-files --sparse >after && + test_cmp before after +' + +test_expect_success UTF8_NFD_TO_NFC \ + 'decomposed Unicode names agree' ' + setup_repo unicode && + nfc=$(printf "\303\244") && + nfd=$(printf "\141\314\210") && + git -C unicode config core.precomposeunicode true && + test_write_lines unicode >"unicode/$nfd" && + git -C unicode add "$nfc" && + git -C unicode commit -m unicode && + test-tool chmtime -120 "unicode/$nfd" && + git -C unicode update-index --refresh && + compare_status unicode unicode.trace && + test_must_be_empty actual && + check_data unicode.trace preload/bulk_applied 9 && + check_lstat_data unicode.trace 0 +' + +test_expect_success 'multiply-linked entries are left to lstat' ' + setup_repo hardlink && + # Mutate through a name outside the watched worktree, then restore + # mtime. The bulk scan must reject the multiply-linked entry. + ln hardlink/root hardlink-alias && + test-tool chmtime -120 hardlink/root && + git -C hardlink update-index --refresh && + mtime=$(test-tool chmtime --get hardlink/root) && + sleep 1 && + test_write_lines moot >hardlink-alias && + test-tool chmtime "=$mtime" hardlink/root && + compare_status hardlink hardlink.trace && + test_file_not_empty actual && + check_data hardlink.trace preload/bulk_applied 7 && + check_lstat_data hardlink.trace 1 +' + +test_done From 58de3de12e4e483f421e75795bc0ee02ad87a430 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 22:13:45 -0700 Subject: [PATCH 037/105] status: hash tracked files through anchored descriptors Size and modification time cannot establish that worktree content matches an indexed blob. A replacement can preserve those values, and reopening a pathname after hashing can reach a different object. Open a regular file relative to its pinned parent and compare the pathname observation with the held descriptor before hashing. Stream the Git blob header and exact observed bytes using the repository hash algorithm, reject short reads and concurrent appends, then repeat the descriptor and pathname identity checks and reopen the final component. Record a matching object as raw-clean only after every check succeeds. A multiply-linked clean file is not persistable, because an unobserved alias can later change its contents. Structural errors, replacements, unsupported anchored opens, and hash mismatches remain explicit results. Register the file verifier with both Make and Meson. Its 256 KiB hash buffer is supplied by its caller; this patch does not yet classify conversion attributes, run a worker, or apply an index update. Signed-off-by: Taylor Blau --- Makefile | 1 + meson.build | 1 + semantic-verify-file.c | 210 +++++++++++++++++++++++++++++++++++++ semantic-verify-internal.h | 24 +++++ semantic-verify.h | 15 +++ 5 files changed, 251 insertions(+) create mode 100644 semantic-verify-file.c create mode 100644 semantic-verify.h diff --git a/Makefile b/Makefile index afb57d02604aec..b7ee8bcb8e9379 100644 --- a/Makefile +++ b/Makefile @@ -1317,6 +1317,7 @@ LIB_OBJS += resolve-undo.o LIB_OBJS += revision.o LIB_OBJS += run-command.o LIB_OBJS += send-pack.o +LIB_OBJS += semantic-verify-file.o LIB_OBJS += semantic-verify-path.o LIB_OBJS += semantic-verify-root.o LIB_OBJS += sequencer.o diff --git a/meson.build b/meson.build index 35c693e1f1503e..194f2f11456e93 100644 --- a/meson.build +++ b/meson.build @@ -523,6 +523,7 @@ libgit_sources = [ 'run-command.c', 'send-pack.c', 'sequencer.c', + 'semantic-verify-file.c', 'semantic-verify-path.c', 'semantic-verify-root.c', 'serve.c', diff --git a/semantic-verify-file.c b/semantic-verify-file.c new file mode 100644 index 00000000000000..5a06809060e58d --- /dev/null +++ b/semantic-verify-file.c @@ -0,0 +1,210 @@ +#include "git-compat-util.h" +#include "environment.h" +#include "object-file.h" +#include "path-namespace.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "semantic-verify.h" +#include "semantic-verify-internal.h" + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +static int mode_matches_ce(struct repository *repo, + const struct cache_entry *ce, + const struct stat *st) +{ + if (!S_ISREG(st->st_mode)) + return 0; + if (repo_trust_executable_bit(repo) && + ((ce->ce_mode ^ st->st_mode) & 0100)) + return 0; + return 1; +} + +static int hash_raw_blob(int fd, size_t size, + const struct git_hash_algo *algo, + struct object_id *oid, void *buffer, + size_t *bytes_hashed) +{ + struct git_hash_ctx ctx; + char header[MAX_HEADER_LEN]; + int header_len; + size_t remaining = size; + + header_len = format_object_header(header, sizeof(header), OBJ_BLOB, size); + git_hash_init(&ctx, algo); + git_hash_update(&ctx, header, header_len); + + while (remaining) { + size_t want = remaining < SEMANTIC_VERIFY_HASH_BUFFER_SIZE ? + remaining : SEMANTIC_VERIFY_HASH_BUFFER_SIZE; + ssize_t nr = xread(fd, buffer, want); + + if (nr < 0) + return -1; + if (!nr) { + errno = EIO; + return -1; + } + git_hash_update(&ctx, buffer, nr); + remaining -= nr; + *bytes_hashed += nr; + } + + /* Do not silently omit an append which raced with the declared size. */ + { + char extra; + ssize_t nr = xread(fd, &extra, 1); + + if (nr < 0) + return -1; + if (nr) { + errno = EAGAIN; + return -1; + } + } + + git_hash_final_oid(oid, &ctx); + return 0; +} + +static unsigned int classify_resolve_error(int error) +{ + if (error == ENOENT) + return SEMANTIC_VERIFY_RAW_MODIFIED; + if (error == ELOOP || error == ENOTDIR || error == EXDEV || + error == EINVAL) + return SEMANTIC_VERIFY_STRUCTURAL; + return SEMANTIC_VERIFY_ERROR; +} +#endif + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +void semantic_verify_file_at(int parent_fd, const char *basename, + const struct stat *observed, + dev_t root_dev, + const struct cache_entry *ce, + struct repository *repo, void *buffer, + struct semantic_verify_file_result *result) +{ + struct stat path_before = *observed, fd_before, fd_after, path_after; + struct object_id oid; + int fd = -1; + int saved_errno; + + memset(result, 0, sizeof(*result)); + if (!mode_matches_ce(repo, ce, &path_before)) { + result->kind = SEMANTIC_VERIFY_RAW_MODIFIED; + return; + } + if (path_before.st_size < 0 || + (uintmax_t)path_before.st_size > (uintmax_t)SIZE_MAX) { + result->kind = SEMANTIC_VERIFY_SENSITIVE; + return; + } + + fd = semantic_verify_openat(parent_fd, basename, + O_RDONLY | O_NONBLOCK | O_NOFOLLOW); + if (fd < 0) { + result->error = errno; + result->kind = errno == ENOENT || errno == ENOTDIR || + errno == ELOOP ? + SEMANTIC_VERIFY_UNSTABLE : SEMANTIC_VERIFY_ERROR; + return; + } + if (fstat(fd, &fd_before)) + goto unstable; + if (fd_before.st_dev != root_dev || + !path_namespace_stat_equal(&path_before, &fd_before)) { + errno = EAGAIN; + goto unstable; + } + if (hash_raw_blob(fd, (size_t)fd_before.st_size, repo->hash_algo, &oid, + buffer, &result->bytes_hashed)) + goto unstable; + if (fstat(fd, &fd_after)) + goto unstable; + if (fstatat(parent_fd, basename, &path_after, AT_SYMLINK_NOFOLLOW)) + goto unstable; + if (!path_namespace_stat_equal(&fd_before, &fd_after) || + !path_namespace_stat_equal(&fd_after, &path_after)) { + errno = EAGAIN; + goto unstable; + } + if (path_namespace_reopen_component( + parent_fd, basename, O_RDONLY | O_NONBLOCK | O_NOFOLLOW, + semantic_verify_openat, &fd_after)) + goto unstable; + close(fd); + + if (!oideq(&oid, &ce->oid)) { + result->kind = SEMANTIC_VERIFY_RAW_MODIFIED; + return; + } + result->kind = SEMANTIC_VERIFY_RAW_CLEAN; + result->persistable = fd_after.st_nlink == 1; + fill_stat_data(&result->stat_data, &fd_after); + return; + +unstable: + saved_errno = errno; + close(fd); + result->kind = SEMANTIC_VERIFY_UNSTABLE; + result->error = saved_errno; +} + +void semantic_verify_file(struct semantic_verify_root *root, + struct semantic_verify_path *path, + const struct cache_entry *ce, size_t cache_pos, + struct repository *repo, void *buffer, + struct semantic_verify_file_result *result) +{ + struct stat path_before; + const char *basename; + int parent_fd; + + memset(result, 0, sizeof(*result)); + if (semantic_verify_resolve_parent(path, ce->name, cache_pos, + &parent_fd, &basename)) { + result->error = errno; + result->kind = classify_resolve_error(errno); + return; + } + if (fstatat(parent_fd, basename, &path_before, AT_SYMLINK_NOFOLLOW)) { + result->error = errno; + result->kind = errno == ENOENT || errno == ENOTDIR ? + SEMANTIC_VERIFY_RAW_MODIFIED : SEMANTIC_VERIFY_ERROR; + return; + } + semantic_verify_file_at(parent_fd, basename, &path_before, + root->stat.st_dev, ce, repo, buffer, result); +} +#else +static void semantic_verify_file_unavailable( + struct semantic_verify_file_result *result) +{ + memset(result, 0, sizeof(*result)); + result->kind = SEMANTIC_VERIFY_ERROR; + result->error = ENOSYS; +} + +void semantic_verify_file_at( + int parent_fd UNUSED, const char *basename UNUSED, + const struct stat *observed UNUSED, + dev_t root_dev UNUSED, + const struct cache_entry *ce UNUSED, + struct repository *repo UNUSED, void *buffer UNUSED, + struct semantic_verify_file_result *result) +{ + semantic_verify_file_unavailable(result); +} + +void semantic_verify_file( + struct semantic_verify_root *root UNUSED, + struct semantic_verify_path *path UNUSED, + const struct cache_entry *ce UNUSED, size_t cache_pos UNUSED, + struct repository *repo UNUSED, void *buffer UNUSED, + struct semantic_verify_file_result *result) +{ + semantic_verify_file_unavailable(result); +} +#endif diff --git a/semantic-verify-internal.h b/semantic-verify-internal.h index 1f881af44fba6f..a1451feb0e0e30 100644 --- a/semantic-verify-internal.h +++ b/semantic-verify-internal.h @@ -27,8 +27,12 @@ #endif struct repository; +struct cache_entry; +struct git_hash_algo; struct semantic_verify_path; +#define SEMANTIC_VERIFY_HASH_BUFFER_SIZE (256 * 1024) + struct semantic_verify_root { int fd; char *path; @@ -50,4 +54,24 @@ void semantic_verify_path_free(struct semantic_verify_path *path, unsigned int *namespace_unstable, size_t *namespace_unstable_from); +struct semantic_verify_file_result { + struct stat_data stat_data; + size_t bytes_hashed; + int error; + unsigned int kind; + unsigned int persistable; +}; + +void semantic_verify_file(struct semantic_verify_root *root, + struct semantic_verify_path *path, + const struct cache_entry *ce, size_t cache_pos, + struct repository *repo, void *buffer, + struct semantic_verify_file_result *result); +void semantic_verify_file_at(int parent_fd, const char *basename, + const struct stat *observed, + dev_t root_dev, + const struct cache_entry *ce, + struct repository *repo, void *buffer, + struct semantic_verify_file_result *result); + #endif /* SEMANTIC_VERIFY_INTERNAL_H */ diff --git a/semantic-verify.h b/semantic-verify.h new file mode 100644 index 00000000000000..17b13de3765c53 --- /dev/null +++ b/semantic-verify.h @@ -0,0 +1,15 @@ +#ifndef SEMANTIC_VERIFY_H +#define SEMANTIC_VERIFY_H + +enum semantic_verify_kind { + SEMANTIC_VERIFY_UNCHECKED = 0, + SEMANTIC_VERIFY_SKIPPED, + SEMANTIC_VERIFY_RAW_CLEAN, + SEMANTIC_VERIFY_RAW_MODIFIED, + SEMANTIC_VERIFY_SENSITIVE, + SEMANTIC_VERIFY_STRUCTURAL, + SEMANTIC_VERIFY_UNSTABLE, + SEMANTIC_VERIFY_ERROR, +}; + +#endif /* SEMANTIC_VERIFY_H */ From 533f8fc820a144443b748c89889d44e80ba00878 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 00:44:49 -0500 Subject: [PATCH 038/105] fsmonitor: apply validated builtin path records directly A validated builtin daemon response already contains complete, nonempty, NUL-terminated path records. Sending those records through the hook-oriented byte-by-byte offset scanner repeats framing work and obscures the distinction between builtin and hook protocols. Introduce apply_fsmonitor_paths() and call it immediately from the builtin branch of refresh_fsmonitor(). Walk the already validated path buffer, invalidate each reported path exactly once, and retain the resulting path count. Preserve hook token offsets, malformed-response handling, and global invalidation. The parser and unit tests from S05/P04 provide the bounded input; this refactor adds no separate benchmark or test execution claim. Signed-off-by: Taylor Blau --- fsmonitor.c | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/fsmonitor.c b/fsmonitor.c index b88a5c377894af..d2d369e6a1d2e5 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -815,6 +815,23 @@ static enum fsmonitor_query_outcome query_builtin_fsmonitor( return result->outcome; } +static int apply_fsmonitor_paths(struct index_state *istate, + const struct strbuf *paths) +{ + const char *p = paths->buf; + const char *end = paths->buf + paths->len; + int count = 0; + + while (p < end) { + size_t len = strlen(p); + + fsmonitor_refresh_callback(istate, (char *)p); + count++; + p += len + 1; + } + return count; +} + void refresh_fsmonitor(struct index_state *istate) { static int warn_once = 0; @@ -974,17 +991,21 @@ void refresh_fsmonitor(struct index_state *istate) */ int count = 0; - buf = query_result.buf; - for (i = bol; i < query_result.len; i++) { - if (buf[i] != '\0') - continue; - fsmonitor_refresh_callback(istate, buf + bol); - bol = i + 1; - count++; - } - if (bol < query_result.len) { - fsmonitor_refresh_callback(istate, buf + bol); - count++; + if (fsm_mode == FSMONITOR_MODE_IPC) { + count = apply_fsmonitor_paths(istate, &query_result); + } else { + buf = query_result.buf; + for (i = bol; i < query_result.len; i++) { + if (buf[i] != '\0') + continue; + fsmonitor_refresh_callback(istate, buf + bol); + bol = i + 1; + count++; + } + if (bol < query_result.len) { + fsmonitor_refresh_callback(istate, buf + bol); + count++; + } } /* Now mark the untracked cache for fsmonitor usage */ From 9bb03806ce75a5220d677b92a0f740d207831e1b Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 28 Jul 2026 10:31:40 -0500 Subject: [PATCH 039/105] t7529: cover APFS preload directory and root replacement A queued directory can be replaced after its parent is enumerated, and the configured worktree root can change after the bulk walk completes. Neither race may leave previously collected observations published. Add a test-only barrier after opening a selected directory or after the complete walk. Arm it only when the existing bulk-preload test override is enabled, and use it to replace a queued child or the worktree root while status is paused. Require both integration cases to discard all bulk observations and produce the same output as ordinary status. These tests check namespace identity; they do not assert a scan-wide snapshot of individual files. Signed-off-by: Taylor Blau --- compat/preload-index/bulk-darwin.c | 2 + preload-index-bulk.c | 31 ++++++++++++ preload-index-bulk.h | 5 ++ t/README | 9 ++++ t/t7529-preload-index-apfs.sh | 79 ++++++++++++++++++++++++++++++ 5 files changed, 126 insertions(+) diff --git a/compat/preload-index/bulk-darwin.c b/compat/preload-index/bulk-darwin.c index 9b9adba79b2ccd..f5fb62af26f4d3 100644 --- a/compat/preload-index/bulk-darwin.c +++ b/compat/preload-index/bulk-darwin.c @@ -424,6 +424,8 @@ static int scan_directory(struct preload_bulk_worker *worker, fd = preload_bulk_darwin_open_relative(scan, task->path); if (fd < 0) goto out; + if (preload_bulk_test_barrier(scan, task->path)) + goto out; if (preload_bulk_darwin_fd_on_root_mount(scan, fd, &before)) { if (errno != EXDEV) goto out; diff --git a/preload-index-bulk.c b/preload-index-bulk.c index 31c370e2813e58..97fbd4b6ef22ff 100644 --- a/preload-index-bulk.c +++ b/preload-index-bulk.c @@ -1,4 +1,5 @@ #include "git-compat-util.h" +#include "parse.h" #include "preload-index-bulk.h" #include "read-cache-ll.h" @@ -14,6 +15,25 @@ int preload_bulk_available(void) return backend_available(preload_bulk_platform_backend()); } +int preload_bulk_test_barrier(struct preload_bulk_scan *scan, + const char *path) +{ + struct strbuf buf = STRBUF_INIT; + int result; + + if (!scan->test_barrier_path || + strcmp(scan->test_barrier_path, path)) + return 0; + if (!scan->test_barrier_ready || !scan->test_barrier_resume) + return -1; + + write_file(scan->test_barrier_ready, "ready"); + result = strbuf_read_file(&buf, scan->test_barrier_resume, 1) > 0 ? + 0 : -1; + strbuf_release(&buf); + return result; +} + int preload_bulk_collect(struct index_state *istate, int threads, struct preload_bulk_result *result) { @@ -37,10 +57,21 @@ int preload_bulk_collect(struct index_state *istate, int threads, if (!backend_available(backend)) return -1; + if (git_env_bool("GIT_TEST_PRELOAD_INDEX_BULK", 0)) { + scan.test_barrier_path = getenv( + "GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_PATH"); + scan.test_barrier_ready = getenv( + "GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_READY"); + scan.test_barrier_resume = getenv( + "GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_RESUME"); + } + CALLOC_ARRAY(scan.tracked_state, istate->cache_nr); start_error = backend->start(&scan); if (!start_error) { scan_error = preload_bulk_run_scan(&scan, &run_result); + if (!scan_error) + scan_error = preload_bulk_test_barrier(&scan, ""); finish_error = backend->finish(&scan); } diff --git a/preload-index-bulk.h b/preload-index-bulk.h index 45899e56e58a46..61bce71a454268 100644 --- a/preload-index-bulk.h +++ b/preload-index-bulk.h @@ -70,6 +70,9 @@ struct preload_bulk_scan { struct index_state *istate; const struct preload_bulk_backend *backend; void *platform_data; + const char *test_barrier_path; + const char *test_barrier_ready; + const char *test_barrier_resume; struct preload_bulk_queue queue; struct preload_bulk_worker *workers; unsigned char *tracked_state; @@ -115,6 +118,8 @@ const struct preload_bulk_backend *preload_bulk_platform_backend(void); int preload_bulk_collect(struct index_state *istate, int threads, struct preload_bulk_result *result); int preload_bulk_available(void); +int preload_bulk_test_barrier(struct preload_bulk_scan *scan, + const char *path); void preload_bulk_result_release(struct preload_bulk_result *result); #endif /* PRELOAD_INDEX_BULK_H */ diff --git a/t/README b/t/README index 6ba21aead64858..e063266240fda9 100644 --- a/t/README +++ b/t/README @@ -425,6 +425,15 @@ by overriding the minimum number of cache entries required per thread. GIT_TEST_PRELOAD_INDEX_BULK= overrides the `core.preloadIndexBulk` setting. +GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_PATH=, +GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_READY=, and +GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_RESUME=, when +GIT_TEST_PRELOAD_INDEX_BULK is enabled, pause a bulk preload before +scanning the named directory. An empty directory path pauses after the +complete walk. Git writes `ready` to the ready path, then waits until it +can read from the resume path. Tests which set one barrier variable must +set all three. + GIT_TEST_INDEX_THREADS= enables exercising the multi-threaded loading of the index for the whole test suite by bypassing the default number of cache entries and thread minimums. Setting this to 1 will make the diff --git a/t/t7529-preload-index-apfs.sh b/t/t7529-preload-index-apfs.sh index 87b49869515151..a54b049e75c5b2 100755 --- a/t/t7529-preload-index-apfs.sh +++ b/t/t7529-preload-index-apfs.sh @@ -127,6 +127,61 @@ test_expect_success 'bulk preload waits for fsmonitor provider closure' ' test_grep ! "\"key\":\"preload/bulk_result\"" fsmonitor.trace ' +cleanup_race () { + exec 9>&- + if test -n "$status_pid" + then + kill "$status_pid" 2>/dev/null || : + wait "$status_pid" 2>/dev/null || : + fi + status_pid= && + rm -f "$ready" "$resume" +} + +wait_for_ready () { + for i in $(test_seq 1 1000) + do + test "$(cat "$ready" 2>/dev/null)" = ready && return 0 + kill -0 "$status_pid" 2>/dev/null || return 1 + sleep 0.01 + done + return 1 +} + +start_raced_status () { + repo=$1 && + barrier=$2 && + ready=$TRASH_DIRECTORY/$repo.ready && + resume=$TRASH_DIRECTORY/$repo.resume && + race_trace=$TRASH_DIRECTORY/$repo.trace && + status_pid= && + rm -f "$ready" "$resume" "$race_trace" && + mkfifo "$resume" && + exec 9<>"$resume" && + { + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_PATH="$barrier" \ + GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_READY="$ready" \ + GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_RESUME="$resume" \ + GIT_TRACE2_EVENT="$race_trace" \ + git -C "$repo" status --porcelain=v2 >actual 9>&- & + status_pid=$! + } && + wait_for_ready +} + +finish_raced_status () { + printf "resume\n" >&9 && + exec 9>&- && + wait "$status_pid" && + status_pid= && + ordinary_status "$1" expect && + test_cmp expect actual && + test_trace2_data index preload/bulk_applied 0 <"$race_trace" +} + test_expect_success 'clean entries are published without lstat' ' setup_repo clean && bulk_status clean actual clean.trace && @@ -259,4 +314,28 @@ test_expect_success 'multiply-linked entries are left to lstat' ' check_lstat_data hardlink.trace 1 ' +test_expect_success PIPE 'queued child replacement discards observations' ' + setup_repo child-race && + test_when_finished cleanup_race && + start_raced_status child-race nested/deep && + mv child-race/nested/deep child-race/nested/deep-away && + mkdir child-race/nested/deep && + test_write_lines dirty >child-race/nested/deep/tracked && + test_write_lines deep-peer >child-race/nested/deep/peer && + finish_raced_status child-race && + test_file_not_empty actual +' + +test_expect_success PIPE,SYMLINKS \ + 'worktree root replacement discards observations' ' + setup_repo root-race && + test_when_finished cleanup_race && + start_raced_status root-race "" && + mv root-race root-race-away && + ln -s root-race-away root-race && + test_write_lines dirty >root-race-away/root && + finish_raced_status root-race && + test_file_not_empty actual +' + test_done From a1c1756073378169f261a96a8cadb6fc775c56c8 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 12:45:25 -0700 Subject: [PATCH 040/105] convert: support independent conversion attribute checks convert_attrs() evaluates conversion attributes through a single process-global attr_check. Sharing that mutable check between workers would let concurrent path evaluations overwrite each other's results. Separate singleton initialization from attribute evaluation. Provide an allocator for the same six conversion attributes and an evaluator that accepts a caller-owned check. Keep convert_attrs() on its existing initialized singleton, so ordinary filters, encoding, ident expansion, and line-ending decisions retain their previous behavior. Each concurrent caller must provide a distinct six-attribute check while global conversion and attribute state remains unchanged. This patch does not introduce the raw-safe predicate, prepare conversion state for workers, or claim that existing conversion tests were run at this intermediate commit. Signed-off-by: Taylor Blau --- convert.c | 33 +++++++++++++++++++++++++++------ convert.h | 13 +++++++++++++ 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/convert.c b/convert.c index 036506842c3d41..0b3100aeb6b8ee 100644 --- a/convert.c +++ b/convert.c @@ -1318,11 +1318,8 @@ static int git_path_check_ident(struct attr_check_item *check) static struct attr_check *check; -void convert_attrs(struct index_state *istate, - struct conv_attrs *ca, const char *path) +static void convert_attrs_init(void) { - struct attr_check_item *ccheck = NULL; - if (!check) { check = attr_check_initl("crlf", "ident", "filter", "eol", "text", "working-tree-encoding", @@ -1330,9 +1327,26 @@ void convert_attrs(struct index_state *istate, user_convert_tail = &user_convert; repo_config(the_repository, read_convert_config, NULL); } +} - git_check_attr(istate, path, check); - ccheck = check->items; +struct attr_check *convert_attrs_check_alloc(void) +{ + return attr_check_initl("crlf", "ident", "filter", + "eol", "text", "working-tree-encoding", + NULL); +} + +void convert_attrs_with_check(struct index_state *istate, + struct conv_attrs *ca, const char *path, + struct attr_check *attr_check) +{ + struct attr_check_item *ccheck; + + if (!attr_check || attr_check->nr != 6) + BUG("invalid per-thread conversion attribute check"); + + git_check_attr(istate, path, attr_check); + ccheck = attr_check->items; ca->crlf_action = git_path_check_crlf(ccheck + 4); if (ca->crlf_action == CRLF_UNDEFINED) ca->crlf_action = git_path_check_crlf(ccheck + 0); @@ -1363,6 +1377,13 @@ void convert_attrs(struct index_state *istate, ca->crlf_action = CRLF_AUTO_INPUT; } +void convert_attrs(struct index_state *istate, + struct conv_attrs *ca, const char *path) +{ + convert_attrs_init(); + convert_attrs_with_check(istate, ca, path, check); +} + void reset_parsed_attributes(void) { struct convert_driver *drv, *next; diff --git a/convert.h b/convert.h index 0a6e4086b8f932..58baf24853a8a3 100644 --- a/convert.h +++ b/convert.h @@ -8,6 +8,7 @@ #include "string-list.h" struct index_state; +struct attr_check; struct strbuf; #define CONV_EOL_RNDTRP_DIE (1<<0) /* Die if CRLF to LF to CRLF is different */ @@ -91,6 +92,18 @@ struct conv_attrs { void convert_attrs(struct index_state *istate, struct conv_attrs *ca, const char *path); +/* Allocate the exact six-attribute check used by convert_attrs(). */ +struct attr_check *convert_attrs_check_alloc(void); + +/* + * Thread-friendly variant. Each concurrent caller must supply a distinct + * check allocated by convert_attrs_check_alloc(), and conversion/attribute + * global state must remain immutable until all callers have finished. + */ +void convert_attrs_with_check(struct index_state *istate, + struct conv_attrs *ca, const char *path, + struct attr_check *check); + extern enum eol core_eol; extern char *check_roundtrip_encoding; const char *get_cached_convert_stats_ascii(struct index_state *istate, From 61540f1c662c8ad64a351cbc5d75d35d8cdaca05 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 00:44:57 -0500 Subject: [PATCH 041/105] fsmonitor: ignore empty hook path records A version-2 fsmonitor hook can return consecutive NUL delimiters after its token. The hook parser passed the resulting empty record to pathname invalidation, whose callback inspects the last byte of a nonempty path. Skip zero-length hook records and count only pathnames that actually reach fsmonitor_refresh_callback(). Preserve valid reported paths, the existing treatment of a final unterminated hook record, and the separately validated builtin response path. Add a t/t7519-status-fsmonitor.sh regression whose hook emits an empty record before a modified tracked path. Require status to report the real modification without processing the empty pathname. Signed-off-by: Taylor Blau --- fsmonitor.c | 6 ++++-- t/t7519-status-fsmonitor.sh | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/fsmonitor.c b/fsmonitor.c index d2d369e6a1d2e5..4d4979770a27c6 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -998,9 +998,11 @@ void refresh_fsmonitor(struct index_state *istate) for (i = bol; i < query_result.len; i++) { if (buf[i] != '\0') continue; - fsmonitor_refresh_callback(istate, buf + bol); + if (i > bol) { + fsmonitor_refresh_callback(istate, buf + bol); + count++; + } bol = i + 1; - count++; } if (bol < query_result.len) { fsmonitor_refresh_callback(istate, buf + bol); diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index f29bea912efd18..e8cc70c428b181 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -68,6 +68,26 @@ test_expect_success 'FSUC parser fails closed' ' test-tool read-cache --test-fsuc-parser ' +test_expect_success 'hook parser ignores empty path records' ' + test_when_finished "rm -rf empty-hook-record" && + test_create_repo empty-hook-record && + ( + cd empty-hook-record && + test_commit base tracked && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token\0" + printf "\0" + printf "tracked\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + echo changed >tracked && + git status --porcelain --untracked-files=no >actual && + echo " M tracked" >expect && + test_cmp expect actual + ) +' + # Test that we detect and disallow repos that are incompatible with FSMonitor. test_expect_success 'incompatible bare repo' ' test_when_finished "rm -rf ./bare-clone actual expect" && From dda64373743915cae0cd790b6d9e2ec78c1fa7f8 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 22:15:13 -0700 Subject: [PATCH 042/105] status: classify candidates for semantic verification Comparing raw worktree bytes with an indexed object is valid only when the indexed entry is an ordinary file and conversion cannot alter its canonical content. Index promises, sparse entries, staged conflicts, intent-to-add entries, and active conversion cannot share that proof. Introduce convert_attrs_is_raw_safe() and require a binary conversion action with no filter, working-tree encoding, or ident expansion. Classify skip-worktree and assumed-valid entries as skipped; classify conflicts, intent-to-add entries, and sparse directories as structural; leave converted and nonregular files to the ordinary refresh path. Run eligible entries over one contiguous index range using an independent attribute check, a private hash buffer, and a pinned parent resolver. Collect results and replacement stat data without modifying the index. If a cached ancestor changes, downgrade affected clean results to unstable. Register the worker in both Make and Meson. This patch introduces the raw-safe predicate and internal range worker; it does not yet expose a complete proof, add the test helper, or start verifier threads. Signed-off-by: Taylor Blau --- Makefile | 1 + convert.c | 10 ++++ convert.h | 4 ++ meson.build | 1 + semantic-verify-file.c | 24 +++++++++ semantic-verify-internal.h | 35 ++++++++++++ semantic-verify-worker.c | 108 +++++++++++++++++++++++++++++++++++++ semantic-verify.h | 5 ++ 8 files changed, 188 insertions(+) create mode 100644 semantic-verify-worker.c diff --git a/Makefile b/Makefile index b7ee8bcb8e9379..9e159a28fb6959 100644 --- a/Makefile +++ b/Makefile @@ -1320,6 +1320,7 @@ LIB_OBJS += send-pack.o LIB_OBJS += semantic-verify-file.o LIB_OBJS += semantic-verify-path.o LIB_OBJS += semantic-verify-root.o +LIB_OBJS += semantic-verify-worker.o LIB_OBJS += sequencer.o LIB_OBJS += serve.o LIB_OBJS += server-info.o diff --git a/convert.c b/convert.c index 0b3100aeb6b8ee..04cb3251bedfc1 100644 --- a/convert.c +++ b/convert.c @@ -1384,6 +1384,16 @@ void convert_attrs(struct index_state *istate, convert_attrs_with_check(istate, ca, path, check); } +int convert_attrs_is_raw_safe(struct index_state *istate, const char *path, + struct attr_check *attr_check) +{ + struct conv_attrs ca; + + convert_attrs_with_check(istate, &ca, path, attr_check); + return !ca.drv && !ca.working_tree_encoding && !ca.ident && + ca.crlf_action == CRLF_BINARY; +} + void reset_parsed_attributes(void) { struct convert_driver *drv, *next; diff --git a/convert.h b/convert.h index 58baf24853a8a3..72686753600c05 100644 --- a/convert.h +++ b/convert.h @@ -104,6 +104,10 @@ void convert_attrs_with_check(struct index_state *istate, struct conv_attrs *ca, const char *path, struct attr_check *check); +/* True only when hashing the worktree bytes verbatim is exact. */ +int convert_attrs_is_raw_safe(struct index_state *istate, const char *path, + struct attr_check *check); + extern enum eol core_eol; extern char *check_roundtrip_encoding; const char *get_cached_convert_stats_ascii(struct index_state *istate, diff --git a/meson.build b/meson.build index 194f2f11456e93..91fde4044ad768 100644 --- a/meson.build +++ b/meson.build @@ -526,6 +526,7 @@ libgit_sources = [ 'semantic-verify-file.c', 'semantic-verify-path.c', 'semantic-verify-root.c', + 'semantic-verify-worker.c', 'serve.c', 'server-info.c', 'setup.c', diff --git a/semantic-verify-file.c b/semantic-verify-file.c index 5a06809060e58d..72389ac10c065c 100644 --- a/semantic-verify-file.c +++ b/semantic-verify-file.c @@ -1,4 +1,5 @@ #include "git-compat-util.h" +#include "convert.h" #include "environment.h" #include "object-file.h" #include "path-namespace.h" @@ -78,6 +79,29 @@ static unsigned int classify_resolve_error(int error) } #endif +int semantic_verify_classify_entry(struct index_state *istate, + const struct cache_entry *ce, + struct attr_check *check, + struct semantic_verify_file_result *result) +{ + memset(result, 0, sizeof(*result)); + if (ce_skip_worktree(ce) || (ce->ce_flags & CE_VALID)) { + result->kind = SEMANTIC_VERIFY_SKIPPED; + return 0; + } + if (ce_stage(ce) || ce_intent_to_add(ce) || + S_ISSPARSEDIR(ce->ce_mode)) { + result->kind = SEMANTIC_VERIFY_STRUCTURAL; + return 0; + } + if (!S_ISREG(ce->ce_mode) || + !convert_attrs_is_raw_safe(istate, ce->name, check)) { + result->kind = SEMANTIC_VERIFY_SENSITIVE; + return 0; + } + return 1; +} + #if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN void semantic_verify_file_at(int parent_fd, const char *basename, const struct stat *observed, diff --git a/semantic-verify-internal.h b/semantic-verify-internal.h index a1451feb0e0e30..e8d9f01b3907f8 100644 --- a/semantic-verify-internal.h +++ b/semantic-verify-internal.h @@ -26,9 +26,12 @@ #define SEMANTIC_VERIFY_HAS_ANCHORED_OPEN 0 #endif +struct attr_check; struct repository; struct cache_entry; struct git_hash_algo; +struct index_state; +struct semantic_verify_result; struct semantic_verify_path; #define SEMANTIC_VERIFY_HASH_BUFFER_SIZE (256 * 1024) @@ -62,6 +65,10 @@ struct semantic_verify_file_result { unsigned int persistable; }; +int semantic_verify_classify_entry(struct index_state *istate, + const struct cache_entry *ce, + struct attr_check *check, + struct semantic_verify_file_result *result); void semantic_verify_file(struct semantic_verify_root *root, struct semantic_verify_path *path, const struct cache_entry *ce, size_t cache_pos, @@ -74,4 +81,32 @@ void semantic_verify_file_at(int parent_fd, const char *basename, struct repository *repo, void *buffer, struct semantic_verify_file_result *result); +struct semantic_verify_stat_update { + uint32_t cache_pos; + struct stat_data stat_data; +}; + +struct semantic_verify_worker { + struct index_state *istate; + struct semantic_verify_root *root; + struct semantic_verify_result *results; + size_t start; + size_t end; + struct semantic_verify_stat_update *updates; + size_t updates_nr; + size_t updates_alloc; + size_t bytes_hashed; + size_t raw_clean; + size_t raw_modified; + size_t sensitive; + size_t structural; + size_t skipped; + size_t unstable; + size_t errors; + size_t hardlinks; + unsigned int namespace_unstable; +}; + +void semantic_verify_worker_run(struct semantic_verify_worker *worker); + #endif /* SEMANTIC_VERIFY_INTERNAL_H */ diff --git a/semantic-verify-worker.c b/semantic-verify-worker.c new file mode 100644 index 00000000000000..f4102926ae9ccb --- /dev/null +++ b/semantic-verify-worker.c @@ -0,0 +1,108 @@ +#define USE_THE_REPOSITORY_VARIABLE + +#include "git-compat-util.h" +#include "attr.h" +#include "convert.h" +#include "object.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "semantic-verify.h" +#include "semantic-verify-internal.h" + +static void record_stat_update(struct semantic_verify_worker *worker, + uint32_t cache_pos, + const struct stat_data *stat_data) +{ + struct semantic_verify_stat_update *update; + + ALLOC_GROW(worker->updates, worker->updates_nr + 1, + worker->updates_alloc); + update = &worker->updates[worker->updates_nr++]; + update->cache_pos = cache_pos; + memcpy(&update->stat_data, stat_data, sizeof(*stat_data)); +} + +static void count_result(struct semantic_verify_worker *worker, + enum semantic_verify_kind kind) +{ + switch (kind) { + case SEMANTIC_VERIFY_SKIPPED: + worker->skipped++; + break; + case SEMANTIC_VERIFY_RAW_CLEAN: + worker->raw_clean++; + break; + case SEMANTIC_VERIFY_RAW_MODIFIED: + worker->raw_modified++; + break; + case SEMANTIC_VERIFY_SENSITIVE: + worker->sensitive++; + break; + case SEMANTIC_VERIFY_STRUCTURAL: + worker->structural++; + break; + case SEMANTIC_VERIFY_UNSTABLE: + worker->unstable++; + break; + case SEMANTIC_VERIFY_ERROR: + worker->errors++; + break; + case SEMANTIC_VERIFY_UNCHECKED: + BUG("cannot count an unchecked semantic result"); + } +} + +void semantic_verify_worker_run(struct semantic_verify_worker *worker) +{ + struct semantic_verify_path *path = + semantic_verify_path_new(worker->root); + struct attr_check *check = convert_attrs_check_alloc(); + void *buffer = xmalloc(SEMANTIC_VERIFY_HASH_BUFFER_SIZE); + size_t unstable_from = SIZE_MAX; + + for (size_t i = worker->start; i < worker->end; i++) { + struct cache_entry *ce = worker->istate->cache[i]; + struct semantic_verify_result *result = &worker->results[i]; + struct semantic_verify_file_result file; + + if (!semantic_verify_classify_entry(worker->istate, ce, check, + &file)) { + result->kind = file.kind; + count_result(worker, result->kind); + continue; + } + + semantic_verify_file(worker->root, path, ce, i, + worker->istate->repo, + buffer, &file); + result->kind = file.kind; + result->error = file.error > UINT16_MAX ? EIO : file.error; + worker->bytes_hashed += file.bytes_hashed; + if (result->kind == SEMANTIC_VERIFY_RAW_CLEAN) { + if (!file.persistable) + worker->hardlinks++; + if (memcmp(&file.stat_data, &ce->ce_stat_data, + sizeof(file.stat_data))) + record_stat_update(worker, i, &file.stat_data); + } + count_result(worker, result->kind); + } + + semantic_verify_path_free(path, &worker->namespace_unstable, + &unstable_from); + if (worker->namespace_unstable) { + for (size_t i = unstable_from; i < worker->end; i++) { + struct semantic_verify_result *result = &worker->results[i]; + + if (result->kind != SEMANTIC_VERIFY_RAW_CLEAN) + continue; + result->kind = SEMANTIC_VERIFY_UNSTABLE; + result->error = EAGAIN; + worker->raw_clean--; + worker->unstable++; + } + } + + free(buffer); + attr_check_free(check); +} diff --git a/semantic-verify.h b/semantic-verify.h index 17b13de3765c53..f6b63a2c2b220b 100644 --- a/semantic-verify.h +++ b/semantic-verify.h @@ -12,4 +12,9 @@ enum semantic_verify_kind { SEMANTIC_VERIFY_ERROR, }; +struct semantic_verify_result { + uint16_t error; + uint8_t kind; +}; + #endif /* SEMANTIC_VERIFY_H */ From 71685550b5ca0cb8a54ddf267a1d1669c533dd5c Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 21:48:46 -0700 Subject: [PATCH 043/105] fsmonitor: centralize complete fsmonitor invalidation The fsmonitor failure path cleared tracked validity bits and disabled untracked-cache monitoring, but left the separate fsmonitor_untracked_valid proof intact. An untrusted cache token could therefore outlive the tracked state it was meant to certify. Extract invalidate_all_fsmonitor() and call it from the existing failure branch. Clear every CE_FSMONITOR_VALID bit, revoke the untracked-token proof, disable fsmonitor use for the untracked cache, and set FSMONITOR_CHANGED only when a tracked validity bit actually changed. The new helper has an immediate production consumer. It neither issues nor closes a provider token and introduces no independent benchmark. Signed-off-by: Taylor Blau --- fsmonitor.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/fsmonitor.c b/fsmonitor.c index 4d4979770a27c6..dea229e7a8597a 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -832,6 +832,23 @@ static int apply_fsmonitor_paths(struct index_state *istate, return count; } +static void invalidate_all_fsmonitor(struct index_state *istate) +{ + unsigned int i; + int changed = 0; + + for (i = 0; i < istate->cache_nr; i++) { + if (istate->cache[i]->ce_flags & CE_FSMONITOR_VALID) + changed = 1; + istate->cache[i]->ce_flags &= ~CE_FSMONITOR_VALID; + } + istate->fsmonitor_untracked_valid = 0; + if (istate->untracked) + istate->untracked->use_fsmonitor = 0; + if (changed) + istate->cache_changed |= FSMONITOR_CHANGED; +} + void refresh_fsmonitor(struct index_state *istate) { static int warn_once = 0; @@ -1029,24 +1046,7 @@ void refresh_fsmonitor(struct index_state *istate) * we've actually changed entries, so keep track if we * actually changed entries or not. */ - int is_cache_changed = 0; - - for (i = 0; i < istate->cache_nr; i++) { - if (istate->cache[i]->ce_flags & CE_FSMONITOR_VALID) { - is_cache_changed = 1; - istate->cache[i]->ce_flags &= ~CE_FSMONITOR_VALID; - } - } - - /* - * If we're going to check every file, ensure we save - * the results. - */ - if (is_cache_changed) - istate->cache_changed |= FSMONITOR_CHANGED; - - if (istate->untracked) - istate->untracked->use_fsmonitor = 0; + invalidate_all_fsmonitor(istate); } trace2_region_leave("fsmonitor", "apply_results", istate->repo); From d18785f78398df4bc8b18d7d8f4e094e1bbfca1b Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Thu, 23 Jul 2026 23:54:41 -0500 Subject: [PATCH 044/105] preload-index: retain proven deletions from complete bulk scans An APFS bulk scan already observes which expanded-index paths are present, but its preload consumer treats an unseen tracked entry as something ordinary preload must stat again. Missing subtrees therefore trigger redundant speculative lookups. Case-folded aliases, mount crossings, unsupported vnodes, and multiply-linked entries must not be mistaken for deletions. Retain a complete scan's per-entry state through threaded preload and classify an unseen useful entry as definitively deleted only when the expanded index makes that conclusion safe. Initialize case-folded name hashes before workers start, and record explicit per-entry fallback for aliases, mount boundaries, tracked directories, and unsupported vnode types. Leave multiply-linked tracked files to ordinary lstat. Leave collapsed sparse indexes and incomplete scans on the existing ordinary path. The APFS tests compare bulk and ordinary status for missing paths, content mismatches, aliases, unsupported tracked types, and hardlinks. Their Trace2 assertions verify that proven deletions avoid speculative lstat while authoritative refresh and fallback entries retain ordinary checks. Retaining per-entry state through preload extends its temporary lifetime; release it when preload finishes. Signed-off-by: Taylor Blau --- compat/preload-index/bulk-darwin.c | 42 ++++++++++-- name-hash.c | 8 +++ name-hash.h | 2 + preload-index-bulk-index.c | 101 ++++++++++++++++++++------- preload-index-bulk-thread.c | 2 + preload-index-bulk.c | 15 +++++ preload-index-bulk.h | 14 +++- preload-index.c | 105 +++++++++++++++++++++++++---- preload-index.h | 1 + t/t7529-preload-index-apfs.sh | 58 +++++++++++++++- 10 files changed, 298 insertions(+), 50 deletions(-) diff --git a/compat/preload-index/bulk-darwin.c b/compat/preload-index/bulk-darwin.c index f5fb62af26f4d3..882cc64a41f64b 100644 --- a/compat/preload-index/bulk-darwin.c +++ b/compat/preload-index/bulk-darwin.c @@ -345,6 +345,8 @@ static int enumerate_directory(struct preload_bulk_worker *worker, if (path_name != entry.name) free((char *)path_name); + pos = preload_bulk_index_position(scan, worker->path.buf, + worker->path.len); if (entry.type == VDIR) { struct preload_bulk_dir_identity child_identity = { .stat = { @@ -357,10 +359,19 @@ static int enumerate_directory(struct preload_bulk_worker *worker, }, }; - if (!preload_bulk_index_has_tracked_descendants( + if (pos >= 0) { + preload_bulk_record_tracked_fallback( + worker, pos); + goto next_record; + } + if (!preload_bulk_index_pos_has_tracked_descendants( scan, worker->path.buf, - worker->path.len)) + worker->path.len, pos)) { + preload_bulk_record_tracked_alias_fallback( + worker, worker->path.buf, + worker->path.len); goto next_record; + } if (((entry.access & S_IFMT) && (entry.access & S_IFMT) != S_IFDIR) || (entry.access & ~(S_IFMT | 07777))) @@ -368,6 +379,9 @@ static int enumerate_directory(struct preload_bulk_worker *worker, if (entry.dev != data->root_stat.st_dev || entry.mountstatus || (entry.flags & SF_FIRMLINK)) { + preload_bulk_record_tracked_descendants_fallback( + worker, worker->path.buf, + worker->path.len); goto next_record; } preload_bulk_schedule_directory( @@ -378,17 +392,27 @@ static int enumerate_directory(struct preload_bulk_worker *worker, goto next_record; } - pos = preload_bulk_index_position(scan, worker->path.buf, - worker->path.len); - if (pos < 0) + if (pos < 0) { + preload_bulk_record_tracked_alias_fallback( + worker, worker->path.buf, + worker->path.len); goto next_record; + } if (entry.dev != data->root_stat.st_dev) { + preload_bulk_record_tracked_fallback( + worker, pos); goto next_record; } - if (entry.type != VREG && entry.type != VLNK) + if (entry.type != VREG && entry.type != VLNK) { + preload_bulk_record_tracked_fallback( + worker, pos); goto next_record; - if (entry.linkcount != 1) + } + if (entry.linkcount != 1) { + preload_bulk_record_tracked_fallback( + worker, pos); goto next_record; + } if (fill_file_stat(&st, entry.dev, entry.fileid, entry.type, entry.mtime, entry.ctime, entry.uid, entry.gid, entry.access, @@ -417,6 +441,7 @@ static int scan_directory(struct preload_bulk_worker *worker, struct preload_bulk_scan *scan = worker->scan; struct preload_bulk_dir_identity before_identity; struct stat before, after; + size_t path_len; int fd = task->fd; int ret = -1; @@ -429,6 +454,9 @@ static int scan_directory(struct preload_bulk_worker *worker, if (preload_bulk_darwin_fd_on_root_mount(scan, fd, &before)) { if (errno != EXDEV) goto out; + path_len = strlen(task->path); + preload_bulk_record_tracked_descendants_fallback( + worker, task->path, path_len); ret = 0; goto out; } diff --git a/name-hash.c b/name-hash.c index 83757db8746230..47c659d6c75374 100644 --- a/name-hash.c +++ b/name-hash.c @@ -619,6 +619,14 @@ static void lazy_init_name_hash(struct index_state *istate) trace_performance_leave("initialize name hash"); } +int prepare_index_casefolding(struct index_state *istate) +{ + if (!repo_ignore_case(istate->repo)) + return 0; + lazy_init_name_hash(istate); + return 1; +} + /* * A test routine for t/helper/ sources. * diff --git a/name-hash.h b/name-hash.h index 0cbfc4286316b2..cc7e752ab1e27d 100644 --- a/name-hash.h +++ b/name-hash.h @@ -10,6 +10,8 @@ int index_dir_find(struct index_state *istate, const char *name, int namelen, #define index_dir_exists(i, n, l) index_dir_find((i), (n), (l), NULL) +/* Prepare the name and directory hashes for concurrent case-folded lookups. */ +int prepare_index_casefolding(struct index_state *istate); void adjust_dirname_case(struct index_state *istate, char *name); struct cache_entry *index_file_exists(struct index_state *istate, const char *name, int namelen, int igncase); diff --git a/preload-index-bulk-index.c b/preload-index-bulk-index.c index 623164822b5fdb..0d129b619cf2fc 100644 --- a/preload-index-bulk-index.c +++ b/preload-index-bulk-index.c @@ -1,4 +1,5 @@ #include "git-compat-util.h" +#include "name-hash.h" #include "preload-index-bulk.h" #include "read-cache-ll.h" @@ -14,17 +15,23 @@ int preload_bulk_index_position(struct preload_bulk_scan *scan, return index_name_pos_sparse(scan->istate, path, path_len); } -int preload_bulk_index_has_tracked_descendants(struct preload_bulk_scan *scan, - const char *path, - size_t path_len) +static int first_tracked_descendant(struct index_state *istate, + const char *path, size_t path_len, + int pos) { - int pos; + while ((unsigned int)pos < istate->cache_nr) { + const struct cache_entry *ce = istate->cache[pos]; - if (path_len > INT_MAX) - return 0; - pos = index_name_pos_sparse(scan->istate, path, path_len); - return preload_bulk_index_pos_has_tracked_descendants( - scan, path, path_len, pos); + if (ce_namelen(ce) <= path_len || + memcmp(ce->name, path, path_len)) + return -1; + if (ce->name[path_len] == '/') + return pos; + if ((unsigned char)ce->name[path_len] > '/') + return -1; + pos++; + } + return -1; } int preload_bulk_index_pos_has_tracked_descendants( @@ -32,27 +39,11 @@ int preload_bulk_index_pos_has_tracked_descendants( int pos) { struct index_state *istate = scan->istate; - const struct cache_entry *ce; if (pos >= 0) return 0; pos = -pos - 1; - while ((unsigned int)pos < istate->cache_nr) { - ce = istate->cache[pos]; - if (ce_namelen(ce) < path_len || - memcmp(ce->name, path, path_len)) - return 0; - if (ce_namelen(ce) == path_len) { - pos++; - continue; - } - if (ce->name[path_len] == '/') - return 1; - if ((unsigned char)ce->name[path_len] > '/') - return 0; - pos++; - } - return 0; + return first_tracked_descendant(istate, path, path_len, pos) >= 0; } static int record_tracked_state(struct preload_bulk_worker *worker, int pos, @@ -112,3 +103,61 @@ void preload_bulk_record_tracked( PRELOAD_BULK_TRACKED_CLEAN; record_tracked_state(worker, pos, state); } + +void preload_bulk_record_tracked_fallback( + struct preload_bulk_worker *worker, int pos) +{ + if (!tracked_entry_is_eligible(worker->scan->istate->cache[pos])) + return; + record_tracked_state(worker, pos, + PRELOAD_BULK_TRACKED_FALLBACK); +} + +void preload_bulk_record_tracked_descendants_fallback( + struct preload_bulk_worker *worker, const char *path, + size_t path_len) +{ + struct index_state *istate = worker->scan->istate; + int pos = preload_bulk_index_position(worker->scan, path, path_len); + + if (pos < 0) + pos = -pos - 1; + else + pos++; + while ((pos = first_tracked_descendant(istate, path, path_len, pos)) >= 0) { + preload_bulk_record_tracked_fallback(worker, pos); + pos++; + } +} + +int preload_bulk_record_tracked_alias_fallback( + struct preload_bulk_worker *worker, const char *path, + size_t path_len) +{ + struct preload_bulk_scan *scan = worker->scan; + struct index_state *istate = scan->istate; + struct cache_entry *ce; + struct strbuf canonical = STRBUF_INIT; + int found = 0; + int pos; + + if (!scan->case_insensitive || path_len > INT_MAX) + return 0; + if (index_dir_find(istate, path, path_len, &canonical)) { + found = 1; + preload_bulk_record_tracked_descendants_fallback( + worker, canonical.buf, canonical.len); + goto out; + } + ce = index_file_exists(istate, path, path_len, 1); + if (!ce) + goto out; + found = 1; + pos = index_name_pos_sparse(istate, ce->name, ce_namelen(ce)); + if (pos >= 0) + preload_bulk_record_tracked_fallback(worker, pos); + +out: + strbuf_release(&canonical); + return found; +} diff --git a/preload-index-bulk-thread.c b/preload-index-bulk-thread.c index 8f57f143d4761b..b1a8d430d8e21e 100644 --- a/preload-index-bulk-thread.c +++ b/preload-index-bulk-thread.c @@ -77,6 +77,8 @@ void preload_bulk_schedule_directory( task->reserved_fd = 0; release_open_fd(&scan->queue); if (saved_errno == EXDEV) { + preload_bulk_record_tracked_descendants_fallback( + worker, path, path_len); free(task); return; } diff --git a/preload-index-bulk.c b/preload-index-bulk.c index 97fbd4b6ef22ff..41b2398d3913f3 100644 --- a/preload-index-bulk.c +++ b/preload-index-bulk.c @@ -1,4 +1,5 @@ #include "git-compat-util.h" +#include "name-hash.h" #include "parse.h" #include "preload-index-bulk.h" #include "read-cache-ll.h" @@ -56,6 +57,18 @@ int preload_bulk_collect(struct index_state *istate, int threads, result->reason = "backend-unavailable"; if (!backend_available(backend)) return -1; + if (istate->sparse_index == INDEX_EXPANDED) { + /* + * Workers may need case-folding lookups for names returned by + * the filesystem. Build the lazy hash before they start. + * + * A collapsed sparse index cannot expand itself concurrently + * from the worker threads. Leave its unseen entries to the + * existing preload path, which expands them on the main thread. + */ + scan.case_insensitive = prepare_index_casefolding(istate); + scan.can_skip_unseen_preload = 1; + } if (git_env_bool("GIT_TEST_PRELOAD_INDEX_BULK", 0)) { scan.test_barrier_path = getenv( @@ -101,6 +114,8 @@ int preload_bulk_collect(struct index_state *istate, int threads, if (clean) { result->tracked_state = scan.tracked_state; result->nr = istate->cache_nr; + result->can_skip_unseen_preload = + scan.can_skip_unseen_preload; scan.tracked_state = NULL; } diff --git a/preload-index-bulk.h b/preload-index-bulk.h index 61bce71a454268..fff436d23f8d4b 100644 --- a/preload-index-bulk.h +++ b/preload-index-bulk.h @@ -78,6 +78,8 @@ struct preload_bulk_scan { unsigned char *tracked_state; int root_fd; int threads; + unsigned case_insensitive : 1; + unsigned can_skip_unseen_preload : 1; }; struct preload_bulk_run_result { @@ -95,6 +97,7 @@ struct preload_bulk_result { const char *outcome; const char *reason; struct preload_bulk_run_result run; + unsigned can_skip_unseen_preload : 1; }; void preload_bulk_schedule_directory( @@ -104,14 +107,19 @@ void preload_bulk_schedule_directory( const char *name, const char *path, size_t path_len); int preload_bulk_index_position(struct preload_bulk_scan *scan, const char *path, size_t path_len); -int preload_bulk_index_has_tracked_descendants(struct preload_bulk_scan *scan, - const char *path, - size_t path_len); int preload_bulk_index_pos_has_tracked_descendants( struct preload_bulk_scan *scan, const char *path, size_t path_len, int pos); void preload_bulk_record_tracked( struct preload_bulk_worker *worker, int pos, const struct stat *st); +void preload_bulk_record_tracked_fallback( + struct preload_bulk_worker *worker, int pos); +void preload_bulk_record_tracked_descendants_fallback( + struct preload_bulk_worker *worker, const char *path, + size_t path_len); +int preload_bulk_record_tracked_alias_fallback( + struct preload_bulk_worker *worker, const char *path, + size_t path_len); int preload_bulk_run_scan(struct preload_bulk_scan *scan, struct preload_bulk_run_result *result); const struct preload_bulk_backend *preload_bulk_platform_backend(void); diff --git a/preload-index.c b/preload-index.c index ee7045b6b7a98c..5dadcee0a50471 100644 --- a/preload-index.c +++ b/preload-index.c @@ -45,6 +45,9 @@ struct thread_data { struct index_state *index; struct pathspec pathspec; struct progress_data *progress; +#ifdef HAVE_PRELOAD_INDEX_BULK + const unsigned char *bulk_state; +#endif int offset, nr; int t2_nr_lstat; }; @@ -65,6 +68,9 @@ static void *preload_thread(void *_data) struct index_state *index = p->index; struct cache_entry **cep = index->cache + p->offset; struct cache_def cache = CACHE_DEF_INIT; +#ifdef HAVE_PRELOAD_INDEX_BULK + const unsigned char *bulk_state = p->bulk_state; +#endif nr = p->nr; if (nr + p->offset > index->cache_nr) @@ -74,9 +80,18 @@ static void *preload_thread(void *_data) do { struct cache_entry *ce = *cep++; struct stat st; +#ifdef HAVE_PRELOAD_INDEX_BULK + unsigned char state = bulk_state ? + *bulk_state++ : PRELOAD_BULK_TRACKED_UNSEEN; +#endif if (!preload_entry_needs_stat(ce)) continue; +#ifdef HAVE_PRELOAD_INDEX_BULK + if (state == PRELOAD_BULK_TRACKED_CONTENT_CHECK || + state == PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED) + continue; +#endif if (p->progress && !(nr & 31)) { struct progress_data *pd = p->progress; @@ -143,9 +158,10 @@ static size_t preload_bulk_useful_candidates(struct index_state *index) return useful; } -static size_t preload_bulk_publish_clean( +static size_t preload_bulk_apply_result( struct index_state *index, - const struct preload_bulk_result *result) + struct preload_bulk_result *result, + int *has_deferred) { size_t applied = 0; @@ -153,12 +169,27 @@ static size_t preload_bulk_publish_clean( BUG("bulk preload result does not match the index"); for (size_t i = 0; i < result->nr; i++) { - struct cache_entry *ce; + struct cache_entry *ce = index->cache[i]; unsigned char state = result->tracked_state[i]; + /* + * A complete scan which did not observe a useful entry proves + * that the entry is absent. Avoid repeating the same lookup in + * speculative preload. A status consumer may use this result + * directly; other callers retain the authoritative refresh. + */ + if (result->can_skip_unseen_preload && + state == PRELOAD_BULK_TRACKED_UNSEEN && + preload_bulk_entry_is_useful(ce)) { + state = PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED; + result->tracked_state[i] = state; + } + if ((state == PRELOAD_BULK_TRACKED_CONTENT_CHECK || + state == PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED) && + preload_entry_needs_stat(ce)) + *has_deferred = 1; if (state != PRELOAD_BULK_TRACKED_CLEAN) continue; - ce = index->cache[i]; if (!preload_bulk_entry_is_useful(ce)) continue; ce_mark_uptodate(ce); @@ -192,6 +223,23 @@ static void preload_bulk_trace_result( const struct preload_bulk_result *result, size_t applied) { + uint64_t content_check = 0, definitive_deleted = 0, fallback = 0; + + for (size_t i = 0; i < result->nr; i++) { + switch (result->tracked_state[i]) { + case PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED: + definitive_deleted++; + break; + case PRELOAD_BULK_TRACKED_CONTENT_CHECK: + content_check++; + break; + case PRELOAD_BULK_TRACKED_FALLBACK: + fallback++; + break; + default: + break; + } + } trace2_data_string("index", index->repo, "preload/bulk_result", result->outcome); if (result->reason) @@ -207,13 +255,22 @@ static void preload_bulk_trace_result( result->run.bulk_calls); trace2_data_intmax("index", index->repo, "preload/bulk_workers", result->run.threads); + trace2_data_intmax("index", index->repo, + "preload/bulk_definitive_deleted", + definitive_deleted); + trace2_data_intmax("index", index->repo, + "preload/bulk_content_check", content_check); + trace2_data_intmax("index", index->repo, + "preload/bulk_fallback", fallback); } -static void preload_bulk_try(struct index_state *index) +static unsigned char *preload_bulk_try(struct index_state *index) { struct preload_bulk_result result = { 0 }; + unsigned char *tracked_state = NULL; size_t useful; size_t applied = 0; + int has_deferred = 0; int enabled = 0; int control, threads; @@ -230,21 +287,28 @@ static void preload_bulk_try(struct index_state *index) if (!enabled || fsm_settings__get_mode(index->repo) != FSMONITOR_MODE_DISABLED || !preload_bulk_available()) - return; + return NULL; useful = preload_bulk_useful_candidates(index); trace2_data_intmax("index", index->repo, "preload/bulk_useful", useful); trace2_data_intmax("index", index->repo, "preload/bulk_cache_nr", index->cache_nr); if (!useful) - return; + return NULL; threads = preload_bulk_threads(useful); trace2_region_enter("index", "preload/bulk", index->repo); - if (!preload_bulk_collect(index, threads, &result)) - applied = preload_bulk_publish_clean(index, &result); + if (!preload_bulk_collect(index, threads, &result)) { + applied = preload_bulk_apply_result(index, &result, + &has_deferred); + } preload_bulk_trace_result(index, &result, applied); + if (has_deferred) { + tracked_state = result.tracked_state; + result.tracked_state = NULL; + } trace2_region_leave("index", "preload/bulk", index->repo); preload_bulk_result_release(&result); + return tracked_state; } #endif @@ -255,6 +319,9 @@ void preload_index(struct index_state *index, int threads, i, work, offset; struct thread_data data[MAX_PARALLEL]; struct progress_data pd; +#ifdef HAVE_PRELOAD_INDEX_BULK + unsigned char *bulk_state = NULL; +#endif int t2_sum_lstat = 0; int core_preload_index = 1; @@ -265,16 +332,24 @@ void preload_index(struct index_state *index, #ifdef HAVE_PRELOAD_INDEX_BULK if (!pathspec || !pathspec->nr) - preload_bulk_try(index); + bulk_state = preload_bulk_try(index); +#endif + if (!HAVE_THREADS) { +#ifdef HAVE_PRELOAD_INDEX_BULK + free(bulk_state); #endif - if (!HAVE_THREADS) return; + } threads = index->cache_nr / THREAD_COST; if ((index->cache_nr > 1) && (threads < 2) && git_env_bool("GIT_TEST_PRELOAD_INDEX", 0)) threads = 2; - if (threads < 2) + if (threads < 2) { +#ifdef HAVE_PRELOAD_INDEX_BULK + free(bulk_state); +#endif return; + } trace2_region_enter("index", "preload", NULL); @@ -298,6 +373,9 @@ void preload_index(struct index_state *index, int err; p->index = index; +#ifdef HAVE_PRELOAD_INDEX_BULK + p->bulk_state = bulk_state ? bulk_state + offset : NULL; +#endif if (pathspec) copy_pathspec(&p->pathspec, pathspec); p->offset = offset; @@ -317,6 +395,9 @@ void preload_index(struct index_state *index, t2_sum_lstat += p->t2_nr_lstat; } stop_progress(&pd.progress); +#ifdef HAVE_PRELOAD_INDEX_BULK + free(bulk_state); +#endif if (pathspec) { /* earlier we made deep copies for each thread to work with */ diff --git a/preload-index.h b/preload-index.h index 4b21e22b6afb19..64906fe5ac74a2 100644 --- a/preload-index.h +++ b/preload-index.h @@ -8,6 +8,7 @@ struct repository; enum preload_bulk_tracked_state { PRELOAD_BULK_TRACKED_UNSEEN = 0, PRELOAD_BULK_TRACKED_CLEAN, + PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED, PRELOAD_BULK_TRACKED_CONTENT_CHECK, PRELOAD_BULK_TRACKED_FALLBACK, }; diff --git a/t/t7529-preload-index-apfs.sh b/t/t7529-preload-index-apfs.sh index a54b049e75c5b2..b616a40b42a178 100755 --- a/t/t7529-preload-index-apfs.sh +++ b/t/t7529-preload-index-apfs.sh @@ -190,6 +190,58 @@ test_expect_success 'clean entries are published without lstat' ' check_lstat_data clean.trace 0 ' +test_expect_success 'known mismatches are not restated during preload' ' + setup_repo dirty && + test_write_lines changed-content >dirty/root && + compare_status dirty dirty.trace && + test_file_not_empty actual && + check_data dirty.trace preload/bulk_applied 7 && + check_data dirty.trace preload/bulk_content_check 1 && + check_lstat_data dirty.trace 0 && + check_data dirty.trace refresh/sum_lstat 1 +' + +test_expect_success 'missing entries bypass speculative lstat' ' + setup_repo missing && + rm missing/root && + rm -rf missing/nested && + compare_status missing missing.trace && + test_line_count = 5 actual && + check_data missing.trace preload/bulk_applied 3 && + check_data missing.trace preload/bulk_definitive_deleted 5 && + check_lstat_data missing.trace 0 && + check_data missing.trace refresh/sum_lstat 5 +' + +test_expect_success PIPE \ + 'tracked directories and unsupported vnodes fall back' ' + setup_repo tracked-types && + rm tracked-types/root tracked-types/root-peer && + mkdir tracked-types/root && + mkfifo tracked-types/root-peer && + compare_status tracked-types tracked-types.trace && + test_line_count = 2 actual && + check_data tracked-types.trace preload/bulk_applied 6 && + check_data tracked-types.trace preload/bulk_fallback 2 && + check_lstat_data tracked-types.trace 2 && + check_data tracked-types.trace refresh/sum_lstat 2 +' + +test_expect_success CASE_INSENSITIVE_FS \ + 'case aliases retain parallel preload' ' + setup_repo case-alias && + mv case-alias/root case-alias/ROOT && + mv case-alias/nested case-alias/NESTED && + compare_status case-alias case-alias.trace && + test_must_be_empty actual && + check_data case-alias.trace preload/bulk_applied 3 && + check_lstat_data case-alias.trace 5 && + { + test_have_prereq !PTHREADS || + check_data case-alias.trace refresh/sum_lstat 0 + } +' + test_expect_success ULIMIT_FILE_DESCRIPTORS \ 'bulk preload reopens directories under a low descriptor limit' ' git init low-fd && @@ -300,7 +352,7 @@ test_expect_success UTF8_NFD_TO_NFC \ test_expect_success 'multiply-linked entries are left to lstat' ' setup_repo hardlink && # Mutate through a name outside the watched worktree, then restore - # mtime. The bulk scan must reject the multiply-linked entry. + # mtime. The bulk scan must leave the multiply-linked entry to lstat. ln hardlink/root hardlink-alias && test-tool chmtime -120 hardlink/root && git -C hardlink update-index --refresh && @@ -311,7 +363,9 @@ test_expect_success 'multiply-linked entries are left to lstat' ' compare_status hardlink hardlink.trace && test_file_not_empty actual && check_data hardlink.trace preload/bulk_applied 7 && - check_lstat_data hardlink.trace 1 + check_data hardlink.trace preload/bulk_fallback 1 && + check_lstat_data hardlink.trace 1 && + check_data hardlink.trace refresh/sum_lstat 1 ' test_expect_success PIPE 'queued child replacement discards observations' ' From 7cef594b52070a8eb3c8807aaacb7c2b426a4141 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 10:50:14 -0700 Subject: [PATCH 045/105] read-cache: reject out-of-bounds index extensions load_index_extensions() enters its extension loop only when a complete eight-byte header fits before the trailing checksum. It nevertheless trusts the declared payload size. An oversized payload can send an extension parser beyond the mapped extension area, while an incomplete trailing header is silently ignored. Compute the checksum boundary once and require the initial offset, each complete header, and each declared payload to fit within it. Advance only by checked header and payload sizes, reject partial trailing headers, and report framing failures as index file corruption. Add a PERL_TEST_HELPERS regression test that overwrites an FSMN payload length with 0xffffffff and checks that porcelain-v2 status fails with the existing corruption diagnostic. Signed-off-by: Taylor Blau --- read-cache.c | 42 ++++++++++++++++++++++++++++++------- t/t7519-status-fsmonitor.sh | 25 ++++++++++++++++++++++ 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/read-cache.c b/read-cache.c index 6c449f393d8d4d..76941114ba8498 100644 --- a/read-cache.c +++ b/read-cache.c @@ -1999,27 +1999,55 @@ struct load_index_extensions static void *load_index_extensions(void *_data) { struct load_index_extensions *p = _data; - unsigned long src_offset = p->src_offset; + size_t src_offset = p->src_offset; + size_t end; + int extension_error = 0; - while (src_offset <= p->mmap_size - the_hash_algo->rawsz - 8) { + if (p->mmap_size < the_hash_algo->rawsz) { + extension_error = 1; + goto done; + } + end = p->mmap_size - the_hash_algo->rawsz; + if (src_offset > end) { + extension_error = 1; + goto done; + } + + while (src_offset < end) { /* After an array of active_nr index entries, * there can be arbitrary number of extended * sections, each of which is prefixed with * extension name (4-byte) and section length * in 4-byte network byte order. */ - uint32_t extsize = get_be32(p->mmap + src_offset + 4); + uint32_t extsize; + + if (end - src_offset < 8) { + extension_error = 1; + break; + } + extsize = get_be32(p->mmap + src_offset + 4); + if (extsize > end - src_offset - 8) { + extension_error = 1; + break; + } if (read_index_extension(p->istate, p->mmap + src_offset, p->mmap + src_offset + 8, extsize) < 0) { - munmap((void *)p->mmap, p->mmap_size); - die(_("index file corrupt")); + extension_error = 1; + break; } - src_offset += 8; - src_offset += extsize; + src_offset += 8 + extsize; } + if (src_offset != end) + extension_error = 1; +done: + if (extension_error) { + munmap((void *)p->mmap, p->mmap_size); + die(_("index file corrupt")); + } return NULL; } diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 93973ed25a448b..2e90955b52c374 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -55,6 +55,31 @@ test_lazy_prereq UNTRACKED_CACHE ' test $ret -ne 1 ' +test_expect_success PERL_TEST_HELPERS \ + 'index reader rejects an out-of-bounds extension size' ' + test_when_finished "rm -rf oversized-index-extension" && + test_create_repo oversized-index-extension && + ( + cd oversized-index-extension && + test_commit base tracked && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + test_grep FSMN .git/index >/dev/null && + perl -0777 -pe " + \$pos = index(\$_, q(FSMN)); + die q(FSMN-not-found) if \$pos < 0; + substr(\$_, \$pos + 4, 4) = pack(q(N), 0xffffffff); + " .git/index >.git/index.bad && + mv .git/index.bad .git/index && + test_must_fail git status --porcelain=v2 2>err && + test_grep "index file corrupt" err + ) +' + # Test that we detect and disallow repos that are incompatible with FSMonitor. test_expect_success 'incompatible bare repo' ' test_when_finished "rm -rf ./bare-clone actual expect" && From 5a00480ba65fbf048ae835a455c0ee9b0ae4a859 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 22:16:00 -0700 Subject: [PATCH 046/105] status: assemble and test serial semantic proof candidates The descriptor root, pinned path resolver, file verifier, and range worker cannot demonstrate a complete verification result until one caller coordinates their inputs and exposes the classifications. Introduce semantic_verify_prepare() and assemble one serial proof without changing any cache entry. Initialize conversion and root attribute state on the calling thread, retain per-entry results and stat updates, count each classification, and distinguish persistable single-link files from clean hardlinks. Reject sparse indexes and unavailable anchored opens without marking any entry clean. Add test-tool semantic-verify and register the library, helper, and integration suite in both build systems. The new tests exercise raw, converted, nested, modified, deleted, multiply-linked, structural, and SHA-256 entries, as well as unsupported-platform fallback. This is the first executable consumer of the proof primitives. It does not apply results to the index, create worker threads, or enable semantic verification in a production status command. Signed-off-by: Taylor Blau --- Makefile | 2 + convert.c | 7 ++ convert.h | 6 ++ meson.build | 1 + semantic-verify-internal.h | 20 +++++ semantic-verify-root.c | 13 +++ semantic-verify-worker.c | 5 +- semantic-verify.c | 142 ++++++++++++++++++++++++++++++++ semantic-verify.h | 37 +++++++++ t/helper/meson.build | 1 + t/helper/test-semantic-verify.c | 90 ++++++++++++++++++++ t/helper/test-tool.c | 1 + t/helper/test-tool.h | 1 + t/lib-semantic-verify.sh | 9 ++ t/meson.build | 1 + t/t7531-semantic-verify.sh | 104 +++++++++++++++++++++++ 16 files changed, 439 insertions(+), 1 deletion(-) create mode 100644 semantic-verify.c create mode 100644 t/helper/test-semantic-verify.c create mode 100644 t/lib-semantic-verify.sh create mode 100755 t/t7531-semantic-verify.sh diff --git a/Makefile b/Makefile index 9e159a28fb6959..0c8545ab270abe 100644 --- a/Makefile +++ b/Makefile @@ -866,6 +866,7 @@ TEST_BUILTINS_OBJS += test-repository.o TEST_BUILTINS_OBJS += test-revision-walking.o TEST_BUILTINS_OBJS += test-run-command.o TEST_BUILTINS_OBJS += test-scrap-cache-tree.o +TEST_BUILTINS_OBJS += test-semantic-verify.o TEST_BUILTINS_OBJS += test-serve-v2.o TEST_BUILTINS_OBJS += test-sha1.o TEST_BUILTINS_OBJS += test-sha256.o @@ -1321,6 +1322,7 @@ LIB_OBJS += semantic-verify-file.o LIB_OBJS += semantic-verify-path.o LIB_OBJS += semantic-verify-root.o LIB_OBJS += semantic-verify-worker.o +LIB_OBJS += semantic-verify.o LIB_OBJS += sequencer.o LIB_OBJS += serve.o LIB_OBJS += server-info.o diff --git a/convert.c b/convert.c index 04cb3251bedfc1..adacd107d84f31 100644 --- a/convert.c +++ b/convert.c @@ -1336,6 +1336,13 @@ struct attr_check *convert_attrs_check_alloc(void) NULL); } +void convert_attrs_prepare(struct index_state *istate) +{ + convert_attrs_init(); + /* Prime default_attr_source() and the root attribute stack on main. */ + git_check_attr(istate, "", check); +} + void convert_attrs_with_check(struct index_state *istate, struct conv_attrs *ca, const char *path, struct attr_check *attr_check) diff --git a/convert.h b/convert.h index 72686753600c05..97f399f2fe3846 100644 --- a/convert.h +++ b/convert.h @@ -92,6 +92,12 @@ struct conv_attrs { void convert_attrs(struct index_state *istate, struct conv_attrs *ca, const char *path); +/* + * Prepare conversion configuration and the default attribute source on the + * main thread before using per-thread attribute checks below. + */ +void convert_attrs_prepare(struct index_state *istate); + /* Allocate the exact six-attribute check used by convert_attrs(). */ struct attr_check *convert_attrs_check_alloc(void); diff --git a/meson.build b/meson.build index 91fde4044ad768..ff471a3a092f5a 100644 --- a/meson.build +++ b/meson.build @@ -527,6 +527,7 @@ libgit_sources = [ 'semantic-verify-path.c', 'semantic-verify-root.c', 'semantic-verify-worker.c', + 'semantic-verify.c', 'serve.c', 'server-info.c', 'setup.c', diff --git a/semantic-verify-internal.h b/semantic-verify-internal.h index e8d9f01b3907f8..ef51e739a42a94 100644 --- a/semantic-verify-internal.h +++ b/semantic-verify-internal.h @@ -44,6 +44,7 @@ struct semantic_verify_root { int semantic_verify_root_init(struct repository *repo, struct semantic_verify_root **root_out); +int semantic_verify_root_stable(const struct semantic_verify_root *root); void semantic_verify_root_clear(struct semantic_verify_root *root); int semantic_verify_openat(int dirfd, const char *path, int flags); @@ -109,4 +110,23 @@ struct semantic_verify_worker { void semantic_verify_worker_run(struct semantic_verify_worker *worker); +struct semantic_verify_proof { + struct index_state *istate; + struct semantic_verify_root *root; + struct semantic_verify_result *results; + struct semantic_verify_stat_update *stat_updates; + size_t cache_nr; + size_t stat_updates_nr; + size_t bytes_hashed; + size_t raw_clean; + size_t raw_modified; + size_t sensitive; + size_t structural; + size_t skipped; + size_t unstable; + size_t errors; + size_t hardlinks; + unsigned int namespace_unstable; +}; + #endif /* SEMANTIC_VERIFY_INTERNAL_H */ diff --git a/semantic-verify-root.c b/semantic-verify-root.c index e896cbe51f7c94..fbec93ac147607 100644 --- a/semantic-verify-root.c +++ b/semantic-verify-root.c @@ -144,6 +144,19 @@ int semantic_verify_root_init(struct repository *repo UNUSED, } #endif +int semantic_verify_root_stable(const struct semantic_verify_root *root) +{ + struct stat fd_stat, path_stat; + + if (!root || root->fd < 0) + return 0; + if (fstat(root->fd, &fd_stat) || lstat(root->path, &path_stat) || + !S_ISDIR(path_stat.st_mode)) + return 0; + return path_namespace_stat_equal(&root->stat, &fd_stat) && + path_namespace_stat_equal(&fd_stat, &path_stat); +} + void semantic_verify_root_clear(struct semantic_verify_root *root) { if (!root) diff --git a/semantic-verify-worker.c b/semantic-verify-worker.c index f4102926ae9ccb..7bbfabcd7a4f5e 100644 --- a/semantic-verify-worker.c +++ b/semantic-verify-worker.c @@ -79,7 +79,9 @@ void semantic_verify_worker_run(struct semantic_verify_worker *worker) result->error = file.error > UINT16_MAX ? EIO : file.error; worker->bytes_hashed += file.bytes_hashed; if (result->kind == SEMANTIC_VERIFY_RAW_CLEAN) { - if (!file.persistable) + if (file.persistable) + result->flags |= SEMANTIC_VERIFY_PERSISTABLE; + else worker->hardlinks++; if (memcmp(&file.stat_data, &ce->ce_stat_data, sizeof(file.stat_data))) @@ -97,6 +99,7 @@ void semantic_verify_worker_run(struct semantic_verify_worker *worker) if (result->kind != SEMANTIC_VERIFY_RAW_CLEAN) continue; result->kind = SEMANTIC_VERIFY_UNSTABLE; + result->flags = 0; result->error = EAGAIN; worker->raw_clean--; worker->unstable++; diff --git a/semantic-verify.c b/semantic-verify.c new file mode 100644 index 00000000000000..d58666ae058d8a --- /dev/null +++ b/semantic-verify.c @@ -0,0 +1,142 @@ +#define USE_THE_REPOSITORY_VARIABLE + +#include "git-compat-util.h" +#include "convert.h" +#include "object.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "semantic-verify.h" +#include "semantic-verify-internal.h" +#include "trace2.h" + +static void combine_worker(struct semantic_verify_proof *proof, + struct semantic_verify_worker *worker) +{ + size_t base = proof->stat_updates_nr; + + if (worker->updates_nr) + COPY_ARRAY(proof->stat_updates + base, worker->updates, + worker->updates_nr); + proof->stat_updates_nr += worker->updates_nr; + proof->bytes_hashed += worker->bytes_hashed; + proof->raw_clean += worker->raw_clean; + proof->raw_modified += worker->raw_modified; + proof->sensitive += worker->sensitive; + proof->structural += worker->structural; + proof->skipped += worker->skipped; + proof->unstable += worker->unstable; + proof->errors += worker->errors; + proof->hardlinks += worker->hardlinks; + proof->namespace_unstable |= worker->namespace_unstable; + free(worker->updates); +} + +int semantic_verify_prepare(struct index_state *istate, + struct semantic_verify_proof **proof_out) +{ + struct semantic_verify_proof *proof; + struct semantic_verify_worker worker = { 0 }; + + if (!istate || !proof_out) + BUG("semantic_verify_prepare requires an index and output"); + + CALLOC_ARRAY(proof, 1); + proof->istate = istate; + proof->cache_nr = istate->cache_nr; + CALLOC_ARRAY(proof->results, proof->cache_nr); + *proof_out = proof; + if (!proof->cache_nr) + return 0; + if (istate->sparse_index != INDEX_EXPANDED) { + for (size_t i = 0; i < proof->cache_nr; i++) { + proof->results[i].kind = SEMANTIC_VERIFY_STRUCTURAL; + proof->structural++; + } + return 0; + } + if (semantic_verify_root_init(istate->repo, &proof->root)) { + int saved_errno = errno; + + for (size_t i = 0; i < proof->cache_nr; i++) { + proof->results[i].kind = SEMANTIC_VERIFY_ERROR; + proof->results[i].error = saved_errno > UINT16_MAX ? + EIO : saved_errno; + } + proof->errors = proof->cache_nr; + return -1; + } + + /* Initialize conversion config and default attribute state serially. */ + convert_attrs_prepare(istate); + trace2_region_enter("semantic_verify", "prepare", istate->repo); + trace2_data_intmax("semantic_verify", istate->repo, "threads", 1); + trace2_data_intmax("semantic_verify", istate->repo, + "result-bytes", sizeof(struct semantic_verify_result)); + + worker.istate = istate; + worker.root = proof->root; + worker.results = proof->results; + worker.end = proof->cache_nr; + semantic_verify_worker_run(&worker); + ALLOC_ARRAY(proof->stat_updates, worker.updates_nr); + combine_worker(proof, &worker); + + trace2_data_intmax("semantic_verify", istate->repo, + "raw-clean", proof->raw_clean); + trace2_data_intmax("semantic_verify", istate->repo, + "raw-modified", proof->raw_modified); + trace2_data_intmax("semantic_verify", istate->repo, + "sensitive", proof->sensitive); + trace2_data_intmax("semantic_verify", istate->repo, + "structural", proof->structural); + trace2_data_intmax("semantic_verify", istate->repo, + "unstable", proof->unstable); + trace2_data_intmax("semantic_verify", istate->repo, + "errors", proof->errors); + trace2_data_intmax("semantic_verify", istate->repo, + "bytes-hashed", proof->bytes_hashed); + trace2_region_leave("semantic_verify", "prepare", istate->repo); + return 0; +} + +int semantic_verify_root_is_stable(const struct semantic_verify_proof *proof) +{ + return proof && semantic_verify_root_stable(proof->root); +} + +void semantic_verify_get_stats(const struct semantic_verify_proof *proof, + struct semantic_verify_stats *stats) +{ + if (!proof || !stats) + BUG("semantic_verify_get_stats requires proof and output"); + stats->cache_nr = proof->cache_nr; + stats->stat_updates_nr = proof->stat_updates_nr; + stats->bytes_hashed = proof->bytes_hashed; + stats->raw_clean = proof->raw_clean; + stats->raw_modified = proof->raw_modified; + stats->sensitive = proof->sensitive; + stats->structural = proof->structural; + stats->skipped = proof->skipped; + stats->unstable = proof->unstable; + stats->errors = proof->errors; + stats->hardlinks = proof->hardlinks; + stats->namespace_unstable = proof->namespace_unstable; +} + +const struct semantic_verify_result *semantic_verify_result_at( + const struct semantic_verify_proof *proof, size_t cache_pos) +{ + if (!proof || cache_pos >= proof->cache_nr) + BUG("semantic verifier result position out of range"); + return &proof->results[cache_pos]; +} + +void semantic_verify_proof_clear(struct semantic_verify_proof *proof) +{ + if (!proof) + return; + semantic_verify_root_clear(proof->root); + free(proof->stat_updates); + free(proof->results); + free(proof); +} diff --git a/semantic-verify.h b/semantic-verify.h index f6b63a2c2b220b..798dad30ae5ae4 100644 --- a/semantic-verify.h +++ b/semantic-verify.h @@ -1,6 +1,9 @@ #ifndef SEMANTIC_VERIFY_H #define SEMANTIC_VERIFY_H +struct index_state; +struct semantic_verify_proof; + enum semantic_verify_kind { SEMANTIC_VERIFY_UNCHECKED = 0, SEMANTIC_VERIFY_SKIPPED, @@ -12,9 +15,43 @@ enum semantic_verify_kind { SEMANTIC_VERIFY_ERROR, }; +enum semantic_verify_result_flags { + /* The clean result may receive persistent fsmonitor validity. */ + SEMANTIC_VERIFY_PERSISTABLE = (1u << 0), +}; + struct semantic_verify_result { uint16_t error; uint8_t kind; + uint8_t flags; }; +struct semantic_verify_stats { + size_t cache_nr; + size_t stat_updates_nr; + size_t bytes_hashed; + size_t raw_clean; + size_t raw_modified; + size_t sensitive; + size_t structural; + size_t skipped; + size_t unstable; + size_t errors; + size_t hardlinks; + unsigned int namespace_unstable; +}; + +/* Build a proof candidate without changing the index. */ +int semantic_verify_prepare(struct index_state *istate, + struct semantic_verify_proof **proof_out); +int semantic_verify_root_is_stable( + const struct semantic_verify_proof *proof); +void semantic_verify_proof_clear(struct semantic_verify_proof *proof); + +/* Introspection used by the semantic verifier test helper. */ +void semantic_verify_get_stats(const struct semantic_verify_proof *proof, + struct semantic_verify_stats *stats); +const struct semantic_verify_result *semantic_verify_result_at( + const struct semantic_verify_proof *proof, size_t cache_pos); + #endif /* SEMANTIC_VERIFY_H */ diff --git a/t/helper/meson.build b/t/helper/meson.build index 3235f10ab8aae1..7c97bfb1e6ec51 100644 --- a/t/helper/meson.build +++ b/t/helper/meson.build @@ -59,6 +59,7 @@ test_tool_sources = [ 'test-rot13-filter.c', 'test-run-command.c', 'test-scrap-cache-tree.c', + 'test-semantic-verify.c', 'test-serve-v2.c', 'test-sha1.c', 'test-sha256.c', diff --git a/t/helper/test-semantic-verify.c b/t/helper/test-semantic-verify.c new file mode 100644 index 00000000000000..bb1cffe99201a1 --- /dev/null +++ b/t/helper/test-semantic-verify.c @@ -0,0 +1,90 @@ +#define USE_THE_REPOSITORY_VARIABLE + +#include "test-tool.h" +#include "config.h" +#include "parse-options.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "semantic-verify.h" +#include "setup.h" + +static const char *kind_name(enum semantic_verify_kind kind) +{ + switch (kind) { + case SEMANTIC_VERIFY_UNCHECKED: + return "unchecked"; + case SEMANTIC_VERIFY_SKIPPED: + return "skipped"; + case SEMANTIC_VERIFY_RAW_CLEAN: + return "raw-clean"; + case SEMANTIC_VERIFY_RAW_MODIFIED: + return "raw-modified"; + case SEMANTIC_VERIFY_SENSITIVE: + return "sensitive"; + case SEMANTIC_VERIFY_STRUCTURAL: + return "structural"; + case SEMANTIC_VERIFY_UNSTABLE: + return "unstable"; + case SEMANTIC_VERIFY_ERROR: + return "error"; + } + BUG("unknown semantic verification kind"); +} + +int cmd__semantic_verify(int argc, const char **argv) +{ + struct semantic_verify_proof *proof = NULL; + struct semantic_verify_stats stats; + int show_results = 0; + int ret; + const char * const usage[] = { + "test-tool semantic-verify []", + NULL + }; + struct option opts[] = { + OPT_BOOL(0, "show-results", &show_results, + "show one result per cache entry"), + OPT_END() + }; + + argc = parse_options(argc, argv, NULL, opts, usage, 0); + if (argc) + usage_with_options(usage, opts); + + setup_git_directory(the_repository); + repo_config(the_repository, git_default_config, NULL); + prepare_repo_settings(the_repository); + the_repository->settings.command_requires_full_index = 0; + if (repo_read_index(the_repository) < 0) + die("unable to read index"); + ret = semantic_verify_prepare(the_repository->index, &proof); + semantic_verify_get_stats(proof, &stats); + if (show_results) { + for (size_t i = 0; i < stats.cache_nr; i++) { + const struct semantic_verify_result *result = + semantic_verify_result_at(proof, i); + + printf("%s %s persist=%d error=%u\n", + the_repository->index->cache[i]->name, + kind_name(result->kind), + !!(result->flags & SEMANTIC_VERIFY_PERSISTABLE), + result->error); + } + } + printf("entries=%"PRIuMAX" clean=%"PRIuMAX + " modified=%"PRIuMAX" sensitive=%"PRIuMAX + " structural=%"PRIuMAX" unstable=%"PRIuMAX + " errors=%"PRIuMAX" hardlinks=%"PRIuMAX + " bytes=%"PRIuMAX" stat_updates=%"PRIuMAX + " root_stable=%d namespace_stable=%d\n", + (uintmax_t)stats.cache_nr, (uintmax_t)stats.raw_clean, + (uintmax_t)stats.raw_modified, (uintmax_t)stats.sensitive, + (uintmax_t)stats.structural, (uintmax_t)stats.unstable, + (uintmax_t)stats.errors, (uintmax_t)stats.hardlinks, + (uintmax_t)stats.bytes_hashed, + (uintmax_t)stats.stat_updates_nr, + semantic_verify_root_is_stable(proof), + !stats.namespace_unstable); + semantic_verify_proof_clear(proof); + return !!ret; +} diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index b71a22b43bbc9e..5ccf3864beb235 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -70,6 +70,7 @@ static struct test_cmd cmds[] = { { "revision-walking", cmd__revision_walking }, { "run-command", cmd__run_command }, { "scrap-cache-tree", cmd__scrap_cache_tree }, + { "semantic-verify", cmd__semantic_verify }, { "serve-v2", cmd__serve_v2 }, { "sha1", cmd__sha1 }, { "sha1-is-sha1dc", cmd__sha1_is_sha1dc }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index f2885b33d58aa8..d1044198247ae4 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -63,6 +63,7 @@ int cmd__repository(int argc, const char **argv); int cmd__revision_walking(int argc, const char **argv); int cmd__run_command(int argc, const char **argv); int cmd__scrap_cache_tree(int argc, const char **argv); +int cmd__semantic_verify(int argc, const char **argv); int cmd__serve_v2(int argc, const char **argv); int cmd__sha1(int argc, const char **argv); int cmd__sha1_is_sha1dc(int argc, const char **argv); diff --git a/t/lib-semantic-verify.sh b/t/lib-semantic-verify.sh new file mode 100644 index 00000000000000..b46fd064711daf --- /dev/null +++ b/t/lib-semantic-verify.sh @@ -0,0 +1,9 @@ +test_lazy_prereq SEMANTIC_VERIFY_ANCHORED_OPEN ' + test_create_repo semantic-anchored-open-probe && + test_commit -C semantic-anchored-open-probe base tracked && + ( + cd semantic-anchored-open-probe && + test-tool semantic-verify --show-results >actual && + test_grep "^tracked raw-clean " actual + ) +' diff --git a/t/meson.build b/t/meson.build index 6d4c7c4bed8fdf..4ecf8343a2f935 100644 --- a/t/meson.build +++ b/t/meson.build @@ -945,6 +945,7 @@ integration_tests = [ 't7526-commit-pathspec-file.sh', 't7527-builtin-fsmonitor.sh', 't7528-signed-commit-ssh.sh', + 't7531-semantic-verify.sh', 't7600-merge.sh', 't7601-merge-pull-config.sh', 't7602-merge-octopus-many.sh', diff --git a/t/t7531-semantic-verify.sh b/t/t7531-semantic-verify.sh new file mode 100755 index 00000000000000..f391b86f5005aa --- /dev/null +++ b/t/t7531-semantic-verify.sh @@ -0,0 +1,104 @@ +#!/bin/sh + +test_description='descriptor-anchored semantic verification' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-semantic-verify.sh + +verify_repo () { + repo=$1 && + shift && + ( + cd "$repo" && + test-tool semantic-verify "$@" + ) +} + +test_expect_success !SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'unsupported platforms decline semantic verification' ' + test_create_repo anchored-open-unsupported && + test_commit -C anchored-open-unsupported base tracked && + ( + cd anchored-open-unsupported && + test_must_fail test-tool semantic-verify --show-results + ) >actual && + test_grep "^tracked error persist=0 error=[1-9][0-9]*$" actual && + test_grep "^entries=1 clean=0 modified=0 sensitive=0 structural=0 " \ + actual && + test_grep " unstable=0 errors=1 hardlinks=0 bytes=0 " actual && + test_grep " stat_updates=0 root_stable=0 namespace_stable=1" \ + actual +' + +test_lazy_prereq HARDLINKS ' + rm -f hardlink-source hardlink-alias && + : >hardlink-source && + ln hardlink-source hardlink-alias +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'classifies raw, converted, nested, and modified files' ' + test_create_repo classify && + mkdir -p classify/a/b && + test_write_lines "converted text" >classify/.gitattributes && + for path in raw converted modified deleted a/b/nested + do + test_write_lines original >"classify/$path" || return 1 + done && + test-tool chmtime -120 classify/raw classify/converted \ + classify/modified classify/deleted classify/a/b/nested && + git -C classify add . && + git -C classify commit -m base && + cp -p classify/modified classify/mtime-reference && + test_write_lines replaced >classify/modified && + touch -r classify/mtime-reference classify/modified && + rm classify/deleted classify/mtime-reference && + + verify_repo classify --show-results >actual && + test_grep "^.gitattributes raw-clean persist=1" actual && + test_grep "^raw raw-clean persist=1" actual && + test_grep "^a/b/nested raw-clean persist=1" actual && + test_grep "^converted sensitive" actual && + test_grep "^modified raw-modified" actual && + test_grep "^deleted raw-modified" actual && + test_grep "entries=6 clean=3 modified=2 sensitive=1" actual +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN,HARDLINKS \ + 'clean hardlinks are not persistable' ' + test_create_repo hardlink && + test_write_lines content >hardlink/tracked && + git -C hardlink add tracked && + git -C hardlink commit -m base && + ln hardlink/tracked hardlink/alias && + + verify_repo hardlink --show-results >actual && + test_grep "^tracked raw-clean persist=0" actual && + test_grep "hardlinks=1" actual +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'classifies structural index state' ' + test_create_repo structural && + test_write_lines tracked >structural/tracked && + git -C structural add tracked && + git -C structural commit -m base && + test_write_lines intent >structural/intent && + git -C structural add -N intent && + + verify_repo structural --show-results >actual && + test_grep "^intent structural" actual && + test_grep "structural=1" actual +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'raw hashing uses the repository object format' ' + git init --object-format=sha256 sha256 && + test_write_lines sha256 >sha256/tracked && + git -C sha256 add tracked && + git -C sha256 commit -m base && + verify_repo sha256 --show-results >actual && + test_grep "^tracked raw-clean persist=1" actual +' + +test_done From 85b1cbaa63a9fea7c478a5e8e18e363096ba2d9a Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 21:52:33 -0700 Subject: [PATCH 047/105] wt-status: separate untracked traversal from result collection Untracked status used one helper both to traverse the worktree and to copy untracked and ignored entries into status output. Checking whether a traversal actually validated the repository's UNTR cache requires that directory walk without copying results or recording user-facing timing. Factor the walk into wt_status_collect_untracked_1() with an explicit collection flag. Return whether the traversal used the index's own untracked cache, and populate the result lists and advice timing only when collection is requested. Retain wt_status_collect_untracked() as the collecting wrapper. Every existing production caller still requests collection, so status output and ordinary traversal behavior remain unchanged. Token adoption and validation-only production use are not added by this preparatory patch. Signed-off-by: Taylor Blau --- wt-status.c | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/wt-status.c b/wt-status.c index da642642d4a229..fab9f1af38bea6 100644 --- a/wt-status.c +++ b/wt-status.c @@ -828,15 +828,16 @@ void wt_status_start_untracked_cache_preload(struct wt_status *s) untracked_cache_preload_start_ordinary(istate, dir_flags); } -static void wt_status_collect_untracked(struct wt_status *s) +static int wt_status_collect_untracked_1(struct wt_status *s, int collect) { int i; + int used_untracked_cache; struct dir_struct dir = DIR_INIT; uint64_t t_begin = getnanotime(); struct index_state *istate = s->repo->index; if (!s->show_untracked_files) - return; + return 0; if (s->show_untracked_files != SHOW_ALL_UNTRACKED_FILES) dir.flags |= wt_status_untracked_dir_flags(s); @@ -859,25 +860,35 @@ static void wt_status_collect_untracked(struct wt_status *s) s->untracked_cache_preloaded; fill_directory(&dir, istate, &s->pathspec); + used_untracked_cache = dir.untracked && + dir.untracked == istate->untracked; + + if (collect) { + for (i = 0; i < dir.nr; i++) { + struct dir_entry *ent = dir.entries[i]; + if (index_name_is_other(istate, ent->name, ent->len)) + string_list_append(&s->untracked, ent->name); + } + string_list_sort_u(&s->untracked, 0); - for (i = 0; i < dir.nr; i++) { - struct dir_entry *ent = dir.entries[i]; - if (index_name_is_other(istate, ent->name, ent->len)) - string_list_append(&s->untracked, ent->name); - } - string_list_sort_u(&s->untracked, 0); - - for (i = 0; i < dir.ignored_nr; i++) { - struct dir_entry *ent = dir.ignored[i]; - if (index_name_is_other(istate, ent->name, ent->len)) - string_list_append(&s->ignored, ent->name); + for (i = 0; i < dir.ignored_nr; i++) { + struct dir_entry *ent = dir.ignored[i]; + if (index_name_is_other(istate, ent->name, ent->len)) + string_list_append(&s->ignored, ent->name); + } + string_list_sort_u(&s->ignored, 0); } - string_list_sort_u(&s->ignored, 0); dir_clear(&dir); - if (advice_enabled(ADVICE_STATUS_U_OPTION)) + if (collect && advice_enabled(ADVICE_STATUS_U_OPTION)) s->untracked_in_ms = (getnanotime() - t_begin) / 1000000; + return used_untracked_cache; +} + +static int wt_status_collect_untracked(struct wt_status *s) +{ + return wt_status_collect_untracked_1(s, 1); } static int has_unmerged(struct wt_status *s) From c2df2b4237bc3704bbfad23377a4da80b82f9a02 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Thu, 23 Jul 2026 23:55:11 -0500 Subject: [PATCH 048/105] preload-index: classify definitive tracked-file size changes S11/P01 retains scan results for later processing, but treats every changed stat observation as a pending content check. A nonzero cached size that differs from the observed size already proves that a tracked file changed. Racy timestamps, zero cached sizes, type changes, and the Windows symlink-size sentinel cannot establish that conclusion. Classify an entry as definitively modified only when its mode and type remain comparable, its cached size is nonzero, and match_stat_data() reports an actual data-size difference. Pass that terminal state through the existing preload result, skip its speculative restat, and emit a separate definitive-modification Trace2 count. Preserve ordinary content verification for every ambiguous observation. Update the APFS dirty-file test to require the new terminal classification, no speculative lstat, and the still-required authoritative refresh. This verifies the new state at its first consumer without claiming that status already consumes it directly. Signed-off-by: Taylor Blau --- preload-index-bulk-index.c | 30 ++++++++++++++++++++++++++++-- preload-index.c | 11 ++++++++++- preload-index.h | 1 + t/t7529-preload-index-apfs.sh | 4 ++-- 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/preload-index-bulk-index.c b/preload-index-bulk-index.c index 0d129b619cf2fc..bd26b72ccb2986 100644 --- a/preload-index-bulk-index.c +++ b/preload-index-bulk-index.c @@ -86,6 +86,28 @@ static int tracked_entry_is_eligible(const struct cache_entry *ce) (S_ISREG(ce->ce_mode) || S_ISLNK(ce->ce_mode)); } +/* + * Match ie_modified(): a nonzero cached size mismatch is a conclusive + * content change. Zero sizes and the historical Windows symlink sentinel + * still require an ordinary content check. Recompute the stat-data match + * because CE_MATCH_RACY_IS_DIRTY may make ie_match_stat() report a data + * change without a size mismatch. + */ +static int size_change_is_definitive(const struct cache_entry *ce, + const struct stat *st, + unsigned int changed) +{ + if (changed & (MODE_CHANGED | TYPE_CHANGED)) + return 0; +#ifdef GIT_WINDOWS_NATIVE + if (S_ISLNK(st->st_mode) && ce->ce_stat_data.sd_size == MAX_PATH) + return 0; +#endif + return ce->ce_stat_data.sd_size && + (match_stat_data(&ce->ce_stat_data, (struct stat *)st) & + DATA_CHANGED); +} + void preload_bulk_record_tracked( struct preload_bulk_worker *worker, int pos, const struct stat *st) { @@ -99,8 +121,12 @@ void preload_bulk_record_tracked( changed = ie_match_stat( scan->istate, ce, (struct stat *)st, CE_MATCH_RACY_IS_DIRTY | CE_MATCH_IGNORE_FSMONITOR); - state = changed ? PRELOAD_BULK_TRACKED_CONTENT_CHECK : - PRELOAD_BULK_TRACKED_CLEAN; + if (!changed) + state = PRELOAD_BULK_TRACKED_CLEAN; + else if (size_change_is_definitive(ce, st, changed)) + state = PRELOAD_BULK_TRACKED_DEFINITIVE_MODIFIED; + else + state = PRELOAD_BULK_TRACKED_CONTENT_CHECK; record_tracked_state(worker, pos, state); } diff --git a/preload-index.c b/preload-index.c index 5dadcee0a50471..9e082af764a82d 100644 --- a/preload-index.c +++ b/preload-index.c @@ -89,6 +89,7 @@ static void *preload_thread(void *_data) continue; #ifdef HAVE_PRELOAD_INDEX_BULK if (state == PRELOAD_BULK_TRACKED_CONTENT_CHECK || + state == PRELOAD_BULK_TRACKED_DEFINITIVE_MODIFIED || state == PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED) continue; #endif @@ -185,6 +186,7 @@ static size_t preload_bulk_apply_result( result->tracked_state[i] = state; } if ((state == PRELOAD_BULK_TRACKED_CONTENT_CHECK || + state == PRELOAD_BULK_TRACKED_DEFINITIVE_MODIFIED || state == PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED) && preload_entry_needs_stat(ce)) *has_deferred = 1; @@ -223,10 +225,14 @@ static void preload_bulk_trace_result( const struct preload_bulk_result *result, size_t applied) { - uint64_t content_check = 0, definitive_deleted = 0, fallback = 0; + uint64_t content_check = 0, definitive_modified = 0; + uint64_t definitive_deleted = 0, fallback = 0; for (size_t i = 0; i < result->nr; i++) { switch (result->tracked_state[i]) { + case PRELOAD_BULK_TRACKED_DEFINITIVE_MODIFIED: + definitive_modified++; + break; case PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED: definitive_deleted++; break; @@ -255,6 +261,9 @@ static void preload_bulk_trace_result( result->run.bulk_calls); trace2_data_intmax("index", index->repo, "preload/bulk_workers", result->run.threads); + trace2_data_intmax("index", index->repo, + "preload/bulk_definitive_modified", + definitive_modified); trace2_data_intmax("index", index->repo, "preload/bulk_definitive_deleted", definitive_deleted); diff --git a/preload-index.h b/preload-index.h index 64906fe5ac74a2..01d90e06bb6b3f 100644 --- a/preload-index.h +++ b/preload-index.h @@ -8,6 +8,7 @@ struct repository; enum preload_bulk_tracked_state { PRELOAD_BULK_TRACKED_UNSEEN = 0, PRELOAD_BULK_TRACKED_CLEAN, + PRELOAD_BULK_TRACKED_DEFINITIVE_MODIFIED, PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED, PRELOAD_BULK_TRACKED_CONTENT_CHECK, PRELOAD_BULK_TRACKED_FALLBACK, diff --git a/t/t7529-preload-index-apfs.sh b/t/t7529-preload-index-apfs.sh index b616a40b42a178..1c1cc4ed0a3785 100755 --- a/t/t7529-preload-index-apfs.sh +++ b/t/t7529-preload-index-apfs.sh @@ -190,13 +190,13 @@ test_expect_success 'clean entries are published without lstat' ' check_lstat_data clean.trace 0 ' -test_expect_success 'known mismatches are not restated during preload' ' +test_expect_success 'definitive size changes are not restated' ' setup_repo dirty && test_write_lines changed-content >dirty/root && compare_status dirty dirty.trace && test_file_not_empty actual && check_data dirty.trace preload/bulk_applied 7 && - check_data dirty.trace preload/bulk_content_check 1 && + check_data dirty.trace preload/bulk_definitive_modified 1 && check_lstat_data dirty.trace 0 && check_data dirty.trace refresh/sum_lstat 1 ' From 0200a1864d99a5768ab97879784a7fadbc321837 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 13:01:33 -0500 Subject: [PATCH 049/105] cache-tree: avoid allocations and searches while reading read_one() allocates a subtree array even for leaf cache-tree nodes. It also inserts each serialized child through cache_tree_sub(), which searches children that the writer already emits in increasing order. Allocate a child array only for non-leaf nodes and append increasing child names directly. Retain subtree_nr + 2 pointer slots for each non-leaf, but allocate them without zeroing because only populated slots are inspected. Keep cache_tree_sub() as the compatibility fallback for older, unsorted input. Existing t/t0090-cache-tree.sh tests exercise ordinary cache-tree decoding. This change adds no dedicated unsorted-input regression or isolated benchmark. Signed-off-by: Taylor Blau --- cache-tree.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/cache-tree.c b/cache-tree.c index d92f5132865f13..c811e23b14705b 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -676,20 +676,34 @@ static struct cache_tree *read_one(const char **buffer, unsigned long *size_p) /* * Just a heuristic -- we do not add directories that often but * we do not want to have to extend it immediately when we do, - * hence +2. + * hence +2. Avoid a separate allocation for the common leaf case. */ - it->subtree_alloc = subtree_nr + 2; - CALLOC_ARRAY(it->down, it->subtree_alloc); + if (subtree_nr) { + it->subtree_alloc = subtree_nr + 2; + ALLOC_ARRAY(it->down, it->subtree_alloc); + } for (i = 0; i < subtree_nr; i++) { /* read each subtree */ struct cache_tree *sub; struct cache_tree_sub *subtree; const char *name = buf; + int namelen; sub = read_one(&buf, &size); if (!sub) goto free_return; - subtree = cache_tree_sub(it, name); + namelen = strlen(name); + if (!it->subtree_nr || + subtree_name_cmp(it->down[it->subtree_nr - 1]->name, + it->down[it->subtree_nr - 1]->namelen, + name, namelen) < 0) { + FLEX_ALLOC_MEM(subtree, name, name, namelen); + subtree->namelen = namelen; + it->down[it->subtree_nr++] = subtree; + } else { + /* Be liberal in what we accept from older writers. */ + subtree = cache_tree_sub(it, name); + } subtree->cache_tree = sub; } if (subtree_nr != it->subtree_nr) From 4d3b510f72b4aba20c0b4b9ec81edce4389617f8 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 13:48:56 -0500 Subject: [PATCH 050/105] read-cache: factor refreshed entry construction Once refresh_cache_ent() verifies that an entry's content still matches, it allocates and copies a replacement, fills its stat data, and preserves a caller-cleared CE_VALID bit under assume_unchanged. Keeping that sequence in one caller would require another verified refresh path to duplicate the allocation and validity handling. Extract make_refreshed_cache_entry() as a private helper and keep refresh_cache_ent() as its first consumer. Pass !ignore_valid through the existing condition so the entry name, observed stat data, and CE_VALID behavior remain unchanged. This is a behavior-preserving refactor. It introduces no new index write, configuration, test claim, or independent performance claim. Signed-off-by: Taylor Blau --- read-cache.c | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/read-cache.c b/read-cache.c index f606bf81ddaca2..088f83559a6b56 100644 --- a/read-cache.c +++ b/read-cache.c @@ -205,6 +205,23 @@ void fill_stat_cache_info(struct index_state *istate, struct cache_entry *ce, st } } +static struct cache_entry *make_refreshed_cache_entry( + struct index_state *istate, const struct cache_entry *ce, + struct stat *st, int preserve_valid) +{ + struct cache_entry *updated = + make_empty_cache_entry(istate, ce_namelen(ce)); + + copy_cache_entry(updated, ce); + memcpy(updated->name, ce->name, ce->ce_namelen + 1); + fill_stat_cache_info(istate, updated, st); + /* Do not let assume-unchanged reacquire a caller-cleared CE_VALID. */ + if (preserve_valid && assume_unchanged && + !(ce->ce_flags & CE_VALID)) + updated->ce_flags &= ~CE_VALID; + return updated; +} + static unsigned int st_mode_from_ce(const struct cache_entry *ce) { switch (ce->ce_mode & S_IFMT) { @@ -1458,19 +1475,7 @@ static struct cache_entry *refresh_cache_ent(struct index_state *istate, return NULL; } - updated = make_empty_cache_entry(istate, ce_namelen(ce)); - copy_cache_entry(updated, ce); - memcpy(updated->name, ce->name, ce->ce_namelen + 1); - fill_stat_cache_info(istate, updated, &st); - /* - * If ignore_valid is not set, we should leave CE_VALID bit - * alone. Otherwise, paths marked with --no-assume-unchanged - * (i.e. things to be edited) will reacquire CE_VALID bit - * automatically, which is not really what we want. - */ - if (!ignore_valid && assume_unchanged && - !(ce->ce_flags & CE_VALID)) - updated->ce_flags &= ~CE_VALID; + updated = make_refreshed_cache_entry(istate, ce, &st, !ignore_valid); /* istate->cache_changed is updated in the caller */ return updated; From 49fe843df2407032109a65a675c88076050d15fe Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 22:16:40 -0700 Subject: [PATCH 051/105] status: apply complete semantic proofs atomically An index entry, worktree root, or recorded verification result can become invalid between proof preparation and index mutation. Applying early results before checking later entries would leave part of the index incorrectly trusted. Snapshot each cache entry's identity and retain an eight-byte result with an explicitly indexed optional stat update. Before modifying any entry, recheck the root, reject namespaces marked unstable during verification, validate every entry and result, and require a complete one-to-one mapping for staged stat updates. Parent directories are reopened during verification, not again during proof application. Apply only verified, persistable clean entries. Invalidate matching hardlinks and detected content changes so the ordinary refresh tail cannot accidentally accept their existing stat data. Leave converted and skipped entries to their established handling. Extend the test helper and semantic-verification suite to cover successful application, nonpersistable hardlinks, structural rejection, and replacement of an index entry after proof preparation. These tests exercise the explicit proof API; they neither replace a parent after preparation nor introduce a production status caller. Signed-off-by: Taylor Blau --- semantic-verify-internal.h | 11 +++ semantic-verify.c | 131 ++++++++++++++++++++++++++++++++ semantic-verify.h | 5 ++ t/helper/test-semantic-verify.c | 47 +++++++++++- t/t7531-semantic-verify.sh | 45 +++++++---- 5 files changed, 224 insertions(+), 15 deletions(-) diff --git a/semantic-verify-internal.h b/semantic-verify-internal.h index ef51e739a42a94..14b97921bd8d9a 100644 --- a/semantic-verify-internal.h +++ b/semantic-verify-internal.h @@ -1,6 +1,7 @@ #ifndef SEMANTIC_VERIFY_INTERNAL_H #define SEMANTIC_VERIFY_INTERNAL_H +#include "hash.h" #include "statinfo.h" #ifdef __linux__ @@ -87,6 +88,15 @@ struct semantic_verify_stat_update { struct stat_data stat_data; }; +struct semantic_verify_entry_identity { + const struct cache_entry *entry; + struct object_id oid; + struct stat_data stat_data; + char *name; + unsigned int mode; + unsigned int flags; +}; + struct semantic_verify_worker { struct index_state *istate; struct semantic_verify_root *root; @@ -114,6 +124,7 @@ struct semantic_verify_proof { struct index_state *istate; struct semantic_verify_root *root; struct semantic_verify_result *results; + struct semantic_verify_entry_identity *entry_identities; struct semantic_verify_stat_update *stat_updates; size_t cache_nr; size_t stat_updates_nr; diff --git a/semantic-verify.c b/semantic-verify.c index d58666ae058d8a..4e9ec859c0d7cd 100644 --- a/semantic-verify.c +++ b/semantic-verify.c @@ -2,6 +2,7 @@ #include "git-compat-util.h" #include "convert.h" +#include "fsmonitor.h" #include "object.h" #include "read-cache-ll.h" #include "repository.h" @@ -9,6 +10,9 @@ #include "semantic-verify-internal.h" #include "trace2.h" +#define SEMANTIC_VERIFY_ENTRY_FLAGS \ + (CE_VALID | CE_STAGEMASK | CE_INTENT_TO_ADD | CE_SKIP_WORKTREE | \ + CE_UPTODATE | CE_FSMONITOR_VALID | CE_CONTENT_CHECK_REQUIRED) static void combine_worker(struct semantic_verify_proof *proof, struct semantic_verify_worker *worker) { @@ -17,6 +21,11 @@ static void combine_worker(struct semantic_verify_proof *proof, if (worker->updates_nr) COPY_ARRAY(proof->stat_updates + base, worker->updates, worker->updates_nr); + for (size_t i = 0; i < worker->updates_nr; i++) { + uint32_t cache_pos = worker->updates[i].cache_pos; + + proof->results[cache_pos].stat_update_index = base + i; + } proof->stat_updates_nr += worker->updates_nr; proof->bytes_hashed += worker->bytes_hashed; proof->raw_clean += worker->raw_clean; @@ -39,11 +48,28 @@ int semantic_verify_prepare(struct index_state *istate, if (!istate || !proof_out) BUG("semantic_verify_prepare requires an index and output"); + if (sizeof(struct semantic_verify_result) != 8) + BUG("semantic verify result unexpectedly grew to %"PRIuMAX" bytes", + (uintmax_t)sizeof(struct semantic_verify_result)); CALLOC_ARRAY(proof, 1); proof->istate = istate; proof->cache_nr = istate->cache_nr; CALLOC_ARRAY(proof->results, proof->cache_nr); + CALLOC_ARRAY(proof->entry_identities, proof->cache_nr); + for (size_t i = 0; i < proof->cache_nr; i++) { + const struct cache_entry *ce = istate->cache[i]; + struct semantic_verify_entry_identity *identity = + &proof->entry_identities[i]; + + proof->results[i].stat_update_index = UINT32_MAX; + identity->entry = ce; + oidcpy(&identity->oid, &ce->oid); + identity->stat_data = ce->ce_stat_data; + identity->name = xstrdup(ce->name); + identity->mode = ce->ce_mode; + identity->flags = ce->ce_flags & SEMANTIC_VERIFY_ENTRY_FLAGS; + } *proof_out = proof; if (!proof->cache_nr) return 0; @@ -131,11 +157,116 @@ const struct semantic_verify_result *semantic_verify_result_at( return &proof->results[cache_pos]; } +int semantic_verify_apply_after_closure( + struct index_state *istate, + const struct semantic_verify_proof *proof) +{ + int applied = 0; + int poisoned = 0; + size_t validated_updates = 0; + + if (!istate || !proof || proof->istate != istate || + proof->cache_nr != istate->cache_nr || + proof->namespace_unstable || + !semantic_verify_root_is_stable(proof)) + return -1; + + for (size_t i = 0; i < proof->cache_nr; i++) { + const struct semantic_verify_entry_identity *identity = + &proof->entry_identities[i]; + const struct cache_entry *ce = istate->cache[i]; + + if (ce != identity->entry || + !oideq(&ce->oid, &identity->oid) || + memcmp(&ce->ce_stat_data, &identity->stat_data, + sizeof(ce->ce_stat_data)) || + strcmp(ce->name, identity->name) || + ce->ce_mode != identity->mode || + (ce->ce_flags & SEMANTIC_VERIFY_ENTRY_FLAGS) != + identity->flags) + return -1; + } + + /* Validate the complete proof before changing any cache entry. */ + for (size_t i = 0; i < proof->cache_nr; i++) { + const struct semantic_verify_result *result = &proof->results[i]; + + if (result->kind > SEMANTIC_VERIFY_ERROR || + (result->flags & ~SEMANTIC_VERIFY_PERSISTABLE)) + return -1; + if (result->kind == SEMANTIC_VERIFY_UNCHECKED || + result->kind == SEMANTIC_VERIFY_STRUCTURAL || + result->kind == SEMANTIC_VERIFY_UNSTABLE || + result->kind == SEMANTIC_VERIFY_ERROR) + return -1; + if (result->kind != SEMANTIC_VERIFY_RAW_CLEAN) { + if (result->flags || + result->stat_update_index != UINT32_MAX) + return -1; + continue; + } + if (result->stat_update_index != UINT32_MAX) { + const struct semantic_verify_stat_update *update; + + if (result->stat_update_index >= proof->stat_updates_nr) + return -1; + update = &proof->stat_updates[result->stat_update_index]; + if (update->cache_pos != i) + return -1; + validated_updates++; + } + } + if (validated_updates != proof->stat_updates_nr) + return -1; + + for (size_t i = 0; i < proof->cache_nr; i++) { + const struct semantic_verify_result *result = &proof->results[i]; + struct cache_entry *ce = istate->cache[i]; + + /* Force the ordinary refresh tail to preserve mismatches. */ + if (result->kind == SEMANTIC_VERIFY_RAW_MODIFIED || + (result->kind == SEMANTIC_VERIFY_RAW_CLEAN && + !(result->flags & SEMANTIC_VERIFY_PERSISTABLE))) { + fsmonitor_invalidate_cache_entry(ce); + mark_fsmonitor_invalid(istate, ce); + ce->ce_flags |= CE_UPDATE_IN_BASE; + istate->cache_changed |= CE_ENTRY_CHANGED; + poisoned++; + if (result->kind == SEMANTIC_VERIFY_RAW_CLEAN) + applied++; + continue; + } + if (result->kind != SEMANTIC_VERIFY_RAW_CLEAN) + continue; + if (result->stat_update_index != UINT32_MAX) { + const struct semantic_verify_stat_update *update = + &proof->stat_updates[result->stat_update_index]; + + memcpy(&ce->ce_stat_data, &update->stat_data, + sizeof(ce->ce_stat_data)); + ce->ce_flags |= CE_UPDATE_IN_BASE; + istate->cache_changed |= CE_ENTRY_CHANGED; + } + ce_mark_uptodate(ce); + if (result->flags & SEMANTIC_VERIFY_PERSISTABLE) + mark_fsmonitor_valid(istate, ce); + applied++; + } + trace2_data_intmax("semantic_verify", istate->repo, + "applied", applied); + trace2_data_intmax("semantic_verify", istate->repo, + "poisoned-for-tail", poisoned); + return applied; +} + void semantic_verify_proof_clear(struct semantic_verify_proof *proof) { if (!proof) return; semantic_verify_root_clear(proof->root); + for (size_t i = 0; i < proof->cache_nr; i++) + free(proof->entry_identities[i].name); + free(proof->entry_identities); free(proof->stat_updates); free(proof->results); free(proof); diff --git a/semantic-verify.h b/semantic-verify.h index 798dad30ae5ae4..6cb5fc3b4fb105 100644 --- a/semantic-verify.h +++ b/semantic-verify.h @@ -20,7 +20,9 @@ enum semantic_verify_result_flags { SEMANTIC_VERIFY_PERSISTABLE = (1u << 0), }; +/* Exactly eight bytes per cache entry. */ struct semantic_verify_result { + uint32_t stat_update_index; uint16_t error; uint8_t kind; uint8_t flags; @@ -44,6 +46,9 @@ struct semantic_verify_stats { /* Build a proof candidate without changing the index. */ int semantic_verify_prepare(struct index_state *istate, struct semantic_verify_proof **proof_out); +int semantic_verify_apply_after_closure( + struct index_state *istate, + const struct semantic_verify_proof *proof); int semantic_verify_root_is_stable( const struct semantic_verify_proof *proof); void semantic_verify_proof_clear(struct semantic_verify_proof *proof); diff --git a/t/helper/test-semantic-verify.c b/t/helper/test-semantic-verify.c index bb1cffe99201a1..b0048dea791490 100644 --- a/t/helper/test-semantic-verify.c +++ b/t/helper/test-semantic-verify.c @@ -36,7 +36,12 @@ int cmd__semantic_verify(int argc, const char **argv) struct semantic_verify_proof *proof = NULL; struct semantic_verify_stats stats; int show_results = 0; + int apply = 0; + int applied = -2; + int before_uptodate = 0, after_uptodate = 0; + int before_valid = 0, after_valid = 0; int ret; + const char *replace_after_prepare = NULL; const char * const usage[] = { "test-tool semantic-verify []", NULL @@ -44,6 +49,9 @@ int cmd__semantic_verify(int argc, const char **argv) struct option opts[] = { OPT_BOOL(0, "show-results", &show_results, "show one result per cache entry"), + OPT_BOOL(0, "apply", &apply, "apply the completed proof"), + OPT_STRING(0, "replace-after-prepare", &replace_after_prepare, + "path", "replace an entry after preparing the proof"), OPT_END() }; @@ -59,6 +67,29 @@ int cmd__semantic_verify(int argc, const char **argv) die("unable to read index"); ret = semantic_verify_prepare(the_repository->index, &proof); semantic_verify_get_stats(proof, &stats); + if (replace_after_prepare) { + struct index_state *istate = the_repository->index; + struct cache_entry *replacement; + int pos = index_name_pos(istate, replace_after_prepare, + strlen(replace_after_prepare)); + + if (pos < 0) + die("%s not in index", replace_after_prepare); + replacement = dup_cache_entry(istate->cache[pos], istate); + replacement->oid.hash[0] ^= 1; + replacement->ce_flags &= + ~(CE_UPTODATE | CE_FSMONITOR_VALID); + if (add_index_entry(istate, replacement, + ADD_CACHE_OK_TO_REPLACE | + ADD_CACHE_KEEP_CACHE_TREE)) + die("unable to replace %s", replace_after_prepare); + } + for (size_t i = 0; i < stats.cache_nr; i++) { + struct cache_entry *ce = the_repository->index->cache[i]; + + before_uptodate += !!ce_uptodate(ce); + before_valid += !!(ce->ce_flags & CE_FSMONITOR_VALID); + } if (show_results) { for (size_t i = 0; i < stats.cache_nr; i++) { const struct semantic_verify_result *result = @@ -71,12 +102,23 @@ int cmd__semantic_verify(int argc, const char **argv) result->error); } } + if (apply) + applied = semantic_verify_apply_after_closure( + the_repository->index, proof); + for (size_t i = 0; i < stats.cache_nr; i++) { + struct cache_entry *ce = the_repository->index->cache[i]; + + after_uptodate += !!ce_uptodate(ce); + after_valid += !!(ce->ce_flags & CE_FSMONITOR_VALID); + } printf("entries=%"PRIuMAX" clean=%"PRIuMAX " modified=%"PRIuMAX" sensitive=%"PRIuMAX " structural=%"PRIuMAX" unstable=%"PRIuMAX " errors=%"PRIuMAX" hardlinks=%"PRIuMAX " bytes=%"PRIuMAX" stat_updates=%"PRIuMAX - " root_stable=%d namespace_stable=%d\n", + " root_stable=%d namespace_stable=%d applied=%d" + " before_uptodate=%d before_valid=%d" + " after_uptodate=%d after_valid=%d\n", (uintmax_t)stats.cache_nr, (uintmax_t)stats.raw_clean, (uintmax_t)stats.raw_modified, (uintmax_t)stats.sensitive, (uintmax_t)stats.structural, (uintmax_t)stats.unstable, @@ -84,7 +126,8 @@ int cmd__semantic_verify(int argc, const char **argv) (uintmax_t)stats.bytes_hashed, (uintmax_t)stats.stat_updates_nr, semantic_verify_root_is_stable(proof), - !stats.namespace_unstable); + !stats.namespace_unstable, applied, + before_uptodate, before_valid, after_uptodate, after_valid); semantic_verify_proof_clear(proof); return !!ret; } diff --git a/t/t7531-semantic-verify.sh b/t/t7531-semantic-verify.sh index f391b86f5005aa..828a1aaa9595be 100755 --- a/t/t7531-semantic-verify.sh +++ b/t/t7531-semantic-verify.sh @@ -54,41 +54,60 @@ test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ touch -r classify/mtime-reference classify/modified && rm classify/deleted classify/mtime-reference && - verify_repo classify --show-results >actual && + verify_repo classify --show-results --apply >actual && test_grep "^.gitattributes raw-clean persist=1" actual && test_grep "^raw raw-clean persist=1" actual && test_grep "^a/b/nested raw-clean persist=1" actual && test_grep "^converted sensitive" actual && test_grep "^modified raw-modified" actual && test_grep "^deleted raw-modified" actual && - test_grep "entries=6 clean=3 modified=2 sensitive=1" actual + test_grep "entries=6 clean=3 modified=2 sensitive=1" actual && + test_grep "applied=3 .* after_uptodate=3" actual ' test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN,HARDLINKS \ - 'clean hardlinks are not persistable' ' + 'clean hardlinks require the ordinary tail' ' test_create_repo hardlink && test_write_lines content >hardlink/tracked && git -C hardlink add tracked && git -C hardlink commit -m base && ln hardlink/tracked hardlink/alias && - verify_repo hardlink --show-results >actual && + verify_repo hardlink --show-results --apply >actual && test_grep "^tracked raw-clean persist=0" actual && - test_grep "hardlinks=1" actual + test_grep "hardlinks=1" actual && + test_grep "applied=1 .* after_uptodate=0 after_valid=0" actual ' test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ - 'classifies structural index state' ' + 'structural index state rejects the whole proof' ' test_create_repo structural && - test_write_lines tracked >structural/tracked && - git -C structural add tracked && + test_write_lines tracked >structural/a-tracked && + git -C structural add a-tracked && git -C structural commit -m base && - test_write_lines intent >structural/intent && - git -C structural add -N intent && + test_write_lines intent >structural/z-intent && + git -C structural add -N z-intent && - verify_repo structural --show-results >actual && - test_grep "^intent structural" actual && - test_grep "structural=1" actual + verify_repo structural --show-results --apply >actual && + test_grep "^a-tracked raw-clean persist=1" actual && + test_grep "^z-intent structural" actual && + test_grep "applied=-1 .* after_uptodate=0" actual +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'replaced index entries reject the whole proof' ' + test_create_repo replaced-entry && + test_write_lines tracked >replaced-entry/a-tracked && + test_write_lines replaced >replaced-entry/z-replaced && + git -C replaced-entry add . && + git -C replaced-entry commit -m base && + + verify_repo replaced-entry --show-results --apply \ + --replace-after-prepare=z-replaced >actual && + test_grep "^a-tracked raw-clean persist=1" actual && + test_grep "^z-replaced raw-clean persist=1" actual && + test_grep "applied=-1 before_uptodate=0 before_valid=0 " actual && + test_grep "after_uptodate=0 after_valid=0" actual ' test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ From f95e2615a3d99b309032dfc26c4e5c84cc01ef6c Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Wed, 29 Jul 2026 16:50:12 -0500 Subject: [PATCH 052/105] status: close fsmonitor tokens around complete status scans A provider token obtained before a tracked or untracked scan does not cover worktree changes racing with that scan. Publishing it as an FSMN or FSUC proof can make a later status trust an index or untracked-cache snapshot that was never valid at that boundary. Keep bootstrap tokens pending while tracked entries are refreshed and any rooted untracked cache is traversed. For builtin providers, query again after the scans, apply intervening paths, and repeat the affected scans until a clean boundary is found or three closing queries are exhausted. Treat a trivial closing reply as complete invalidation followed by another scan; accept its replacement token only after a later clean reply. Reject provider errors, incomplete cache proofs, and exhausted retries with strong invalidation and complete fallback scans. Hook providers cannot perform a closing IPC query, so accept their token only after a complete tracked and applicable untracked collection; reject failed or trivial hook replies. A matching on-disk FSUC token can now authorize replay of recursive UNTR validity established by S01. Reconstruct that validity only after the entire extension has decoded, and only for directories without a cached per-directory exclude digest. This lets a warm status prune known-empty subtrees while still rechecking a changed .gitignore, including changes made through an unwatched hardlink alias. Trust an indexed exclude's metadata alone only when its identity is reliable and it has exactly one link; otherwise retain the complete content-hash check. Route both status collection and commit index refresh through the shared closure. Preserve ordinary behavior for existing paired state, path-limited requests, and ignored-mode collection. Cover clean and changed closures, trivial replies, retry exhaustion, provider errors, on-disk FSMN/FSUC publication, warm empty-subtree pruning, descendant events, and cached exclude changes. Signed-off-by: Taylor Blau --- builtin/commit.c | 7 +- dir.c | 467 ++++++++++++++++++++++++++-- dir.h | 11 +- fsmonitor-ll.h | 18 ++ fsmonitor.c | 215 ++++++++++++- read-cache-ll.h | 4 +- read-cache.c | 1 + t/t7519-status-fsmonitor.sh | 603 ++++++++++++++++++++++++++++++++++++ wt-status.c | 247 +++++++++++++-- wt-status.h | 8 + 10 files changed, 1518 insertions(+), 63 deletions(-) diff --git a/builtin/commit.c b/builtin/commit.c index 54e41c8ba578c1..e2f4d08b347707 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1628,9 +1628,10 @@ struct repository *repo UNUSED) progress_flag = REFRESH_PROGRESS; repo_read_index(the_repository); wt_status_start_untracked_cache_preload(&s); - refresh_index(the_repository->index, - REFRESH_QUIET|REFRESH_UNMERGED|progress_flag, - &s.pathspec, NULL, NULL); + wt_status_refresh_index( + &s, REFRESH_QUIET | REFRESH_UNMERGED | progress_flag, + s.show_untracked_files != SHOW_NO_UNTRACKED_FILES && + !s.show_ignored_mode); if (use_optional_locks()) fd = repo_hold_locked_index(the_repository, &index_lock, 0); diff --git a/dir.c b/dir.c index 940f6c744ecc7e..b2a4e4b2e5adc4 100644 --- a/dir.c +++ b/dir.c @@ -28,6 +28,7 @@ #include "varint.h" #include "ewah/ewok.h" #include "fsmonitor-ll.h" +#include "fsmonitor.h" #include "read-cache-ll.h" #include "setup.h" #include "sparse-index.h" @@ -79,10 +80,16 @@ struct untracked_cache_preload_task { char *path; struct stat_data stat_data; struct object_id exclude_oid; + unsigned int exclude_mode; unsigned int was_valid : 1; unsigned int stat_checked : 1; unsigned int stat_matches : 1; unsigned int exclude_matches : 1; + unsigned int exclude_index_present : 1; + unsigned int exclude_index_candidate : 1; + unsigned int exclude_index_matches : 1; + unsigned int exclude_index_content_matches : 1; + unsigned int normalize_exclude_oid : 1; unsigned int update_stat_data : 1; }; @@ -97,9 +104,11 @@ struct untracked_cache_preload_data { struct untracked_cache_preload { struct repository *repo; + struct index_state *istate; struct untracked_cache *uc; struct untracked_cache_dir *root; struct untracked_cache_preload_task *tasks; + struct object_id *exclude_index_oids; struct untracked_cache_preload_data *data; struct cache_time index_timestamp; char *exclude_per_dir; @@ -107,10 +116,12 @@ struct untracked_cache_preload { int threads; unsigned int dir_flags; uint64_t started_at; + unsigned int fsmonitor_excludes_only : 1; }; #define UNTRACKED_CACHE_MAX_OVERLAP_THREADS 6 #define UNTRACKED_CACHE_PRELOAD_COST 1000 +#define UNTRACKED_CACHE_EXCLUDE_PRELOAD_COST 256 #define UNTRACKED_CACHE_MAX_EXCLUDE_SIZE (1024 * 1024) static void invalidate_gitignore(struct untracked_cache *uc, @@ -128,18 +139,22 @@ static void collect_untracked_cache_preload_tasks( struct strbuf *path, struct untracked_cache_preload_task **tasks, size_t *nr, - size_t *alloc) + size_t *alloc, + int fsmonitor_excludes_only) { size_t i; - ALLOC_GROW(*tasks, *nr + 1, *alloc); - memset(&(*tasks)[*nr], 0, sizeof(**tasks)); - (*tasks)[*nr].ucd = ucd; - (*tasks)[*nr].path = xstrdup(path->len ? path->buf : "."); - (*tasks)[*nr].stat_data = ucd->stat_data; - oidcpy(&(*tasks)[*nr].exclude_oid, &ucd->exclude_oid); - (*tasks)[*nr].was_valid = ucd->valid; - (*nr)++; + if (!fsmonitor_excludes_only || + !is_null_oid(&ucd->exclude_oid)) { + ALLOC_GROW(*tasks, *nr + 1, *alloc); + memset(&(*tasks)[*nr], 0, sizeof(**tasks)); + (*tasks)[*nr].ucd = ucd; + (*tasks)[*nr].path = xstrdup(path->len ? path->buf : "."); + (*tasks)[*nr].stat_data = ucd->stat_data; + oidcpy(&(*tasks)[*nr].exclude_oid, &ucd->exclude_oid); + (*tasks)[*nr].was_valid = ucd->valid; + (*nr)++; + } for (i = 0; i < ucd->dirs_nr; i++) { struct untracked_cache_dir *child = ucd->dirs[i]; @@ -149,7 +164,7 @@ static void collect_untracked_cache_preload_tasks( strbuf_addch(path, '/'); strbuf_addstr(path, child->name); collect_untracked_cache_preload_tasks(child, path, tasks, nr, - alloc); + alloc, fsmonitor_excludes_only); strbuf_setlen(path, old_len); } } @@ -194,7 +209,8 @@ static int exclude_path_matches_fd(const char *path, static int cached_exclude_file_matches( const struct git_hash_algo *algo, - const char *path, const struct object_id *cached_oid) + const char *path, const struct object_id *cached_oid, + struct object_id *raw_oid_out, unsigned int *mode_out) { struct object_id raw_oid, normalized_oid; struct stat st, st_after; @@ -218,9 +234,13 @@ static int cached_exclude_file_matches( !path_namespace_stat_equal(&st, &st_after) || !exclude_path_matches_fd(path, &st_after)) goto out; + if (mode_out) + *mode_out = st_after.st_mode; /* add_patterns() may record either the blob or its LF-normalized form. */ hash_object_file(algo, buf, size, OBJ_BLOB, &raw_oid); + if (raw_oid_out) + oidcpy(raw_oid_out, &raw_oid); if (oideq(&raw_oid, cached_oid)) { ret = 1; goto out; @@ -236,8 +256,87 @@ static int cached_exclude_file_matches( return ret; } +static int cached_exclude_file_matches_index_stat( + const struct stat_data *sd, + const struct stat *st) +{ + struct stat_data current; + struct stat st_copy = *st; + + /* + * Compare every field saved in the index, independent of the user's + * ordinary stat-match settings. Unreliable object identities and + * multiply-linked files can conceal changes through paths outside + * the monitor's watch cone, so both retain the content-hash check. + */ + if (!fstat_is_reliable() || !S_ISREG(st->st_mode) || + st->st_nlink != 1) + return 0; + fill_stat_data(¤t, &st_copy); + return sd->sd_ctime.sec == current.sd_ctime.sec && + sd->sd_ctime.nsec == current.sd_ctime.nsec && + sd->sd_mtime.sec == current.sd_mtime.sec && + sd->sd_mtime.nsec == current.sd_mtime.nsec && + sd->sd_dev == current.sd_dev && + sd->sd_ino == current.sd_ino && + sd->sd_uid == current.sd_uid && + sd->sd_gid == current.sd_gid && + sd->sd_size == current.sd_size; +} + +static void preload_fsmonitor_excludes_from_index( + struct untracked_cache_preload *preload) +{ + struct repo_config_values *cfg = + repo_config_values(preload->istate->repo); + size_t i; + int stat_candidates = cfg->trust_ctime && cfg->check_stat; + + for (i = 0; i < preload->nr; i++) { + struct untracked_cache_preload_task *task = &preload->tasks[i]; + struct strbuf exclude_path = STRBUF_INIT; + struct cache_entry *ce; + int pos; + + if (!preload->exclude_per_dir) + continue; + if (strcmp(task->path, ".")) + strbuf_addstr(&exclude_path, task->path); + if (exclude_path.len) + strbuf_addch(&exclude_path, '/'); + strbuf_addstr(&exclude_path, preload->exclude_per_dir); + pos = index_name_pos_sparse( + preload->istate, exclude_path.buf, + exclude_path.len); + if (pos < 0) + goto next; + ce = preload->istate->cache[pos]; + if (!S_ISREG(ce->ce_mode) || + !(ce->ce_flags & CE_FSMONITOR_VALID) || + ce_skip_worktree(ce) || + (ce->ce_flags & CE_REMOVE) || + ce_intent_to_add(ce)) + goto next; + oidcpy(&preload->exclude_index_oids[i], &ce->oid); + task->exclude_index_present = 1; + if (!stat_candidates || + is_racy_timestamp(preload->istate, ce) || + (ce->ce_flags & CE_VALID)) + goto next; + /* + * Snapshot before launching workers. The main thread may + * refresh cache entries while exclude checks run. + */ + task->stat_data = ce->ce_stat_data; + task->exclude_index_candidate = 1; +next: + strbuf_release(&exclude_path); + } +} + static struct untracked_cache_preload *untracked_cache_preload_start_1( - struct index_state *istate, unsigned int dir_flags, int automatic) + struct index_state *istate, unsigned int dir_flags, int automatic, + int fsmonitor_excludes_only) { struct untracked_cache *uc = istate->untracked; struct untracked_cache_preload *preload; @@ -246,22 +345,35 @@ static struct untracked_cache_preload *untracked_cache_preload_start_1( unsigned long test_threads; int threads, online, create_threads = 1; - if (!uc || !uc->root || uc->use_fsmonitor || - uc->dir_flags != dir_flags) + if (!uc || !uc->root || uc->dir_flags != dir_flags || + (fsmonitor_excludes_only ? + !uc->use_fsmonitor : + uc->use_fsmonitor)) return NULL; CALLOC_ARRAY(preload, 1); preload->repo = istate->repo; + preload->istate = istate; preload->uc = uc; preload->root = uc->root; preload->index_timestamp = istate->timestamp; preload->exclude_per_dir = xstrdup_or_null(uc->exclude_per_dir); preload->dir_flags = dir_flags; + preload->fsmonitor_excludes_only = fsmonitor_excludes_only; collect_untracked_cache_preload_tasks( - uc->root, &path, &preload->tasks, &preload->nr, &alloc); + uc->root, &path, &preload->tasks, &preload->nr, &alloc, + fsmonitor_excludes_only); strbuf_release(&path); + if (fsmonitor_excludes_only) { + CALLOC_ARRAY(preload->exclude_index_oids, preload->nr); + preload_fsmonitor_excludes_from_index(preload); + } - threads = HAVE_THREADS ? preload->nr / UNTRACKED_CACHE_PRELOAD_COST : 1; + threads = HAVE_THREADS ? + preload->nr / (fsmonitor_excludes_only ? + UNTRACKED_CACHE_EXCLUDE_PRELOAD_COST : + UNTRACKED_CACHE_PRELOAD_COST) : + 1; online = HAVE_THREADS ? online_cpus() : 1; if (threads > online * 3) threads = online * 3; @@ -273,7 +385,7 @@ static struct untracked_cache_preload *untracked_cache_preload_start_1( UNTRACKED_CACHE_MAX_OVERLAP_THREADS : test_threads; if (threads < 1) threads = 1; - if ((size_t)threads > preload->nr) + if (preload->nr && (size_t)threads > preload->nr) threads = preload->nr; preload->threads = threads; @@ -282,6 +394,9 @@ static struct untracked_cache_preload *untracked_cache_preload_start_1( "preload_untracked_cache/threads", threads); trace2_data_intmax("dir", istate->repo, "preload_untracked_cache/automatic", automatic); + trace2_data_intmax("dir", istate->repo, + "preload_untracked_cache/fsmonitor-excludes-only", + fsmonitor_excludes_only); CALLOC_ARRAY(preload->data, threads); work = DIV_ROUND_UP(preload->nr, threads); for (i = 0; i < threads; i++) { @@ -310,6 +425,14 @@ static struct untracked_cache_preload *untracked_cache_preload_start_1( return preload; } +struct untracked_cache_preload * +untracked_cache_preload_start_fsmonitor_excludes( + struct index_state *istate, unsigned int dir_flags) +{ + return untracked_cache_preload_start_1( + istate, dir_flags, 0, 1); +} + struct untracked_cache_preload *untracked_cache_preload_start_ordinary( struct index_state *istate, unsigned int dir_flags) { @@ -318,7 +441,7 @@ struct untracked_cache_preload *untracked_cache_preload_start_ordinary( if (!uc || uc->dir_flags != dir_flags || !untracked_cache_auto_preload_worthwhile(uc)) return NULL; - return untracked_cache_preload_start_1(istate, dir_flags, 1); + return untracked_cache_preload_start_1(istate, dir_flags, 1, 0); } static void *preload_untracked_cache_thread(void *_data) @@ -332,6 +455,47 @@ static void *preload_untracked_cache_thread(void *_data) struct strbuf exclude_path = STRBUF_INIT; struct stat st; + if (preload->fsmonitor_excludes_only) { + struct object_id raw_oid; + + if (!preload->exclude_per_dir) + continue; + if (strcmp(task->path, ".")) + strbuf_addstr(&exclude_path, task->path); + if (exclude_path.len) + strbuf_addch(&exclude_path, '/'); + strbuf_addstr(&exclude_path, + preload->exclude_per_dir); + if (task->exclude_index_candidate && + oideq(&preload->exclude_index_oids[i], + &task->exclude_oid) && + !lstat(exclude_path.buf, &st) && + cached_exclude_file_matches_index_stat( + &task->stat_data, &st)) { + task->exclude_mode = st.st_mode; + task->exclude_index_matches = 1; + task->exclude_index_content_matches = 1; + task->exclude_matches = 1; + strbuf_release(&exclude_path); + continue; + } + task->exclude_matches = cached_exclude_file_matches( + preload->repo->hash_algo, + exclude_path.buf, + &task->exclude_oid, &raw_oid, + &task->exclude_mode); + if (task->exclude_matches && + task->exclude_index_present && + oideq(&preload->exclude_index_oids[i], + &raw_oid)) { + task->exclude_index_content_matches = 1; + if (!oideq(&preload->exclude_index_oids[i], + &task->exclude_oid)) + task->normalize_exclude_oid = 1; + } + strbuf_release(&exclude_path); + continue; + } if (!task->was_valid) continue; task->stat_checked = 1; @@ -361,7 +525,7 @@ static void *preload_untracked_cache_thread(void *_data) strbuf_addstr(&exclude_path, preload->exclude_per_dir); task->exclude_matches = cached_exclude_file_matches( preload->repo->hash_algo, exclude_path.buf, - &task->exclude_oid); + &task->exclude_oid, NULL, NULL); strbuf_release(&exclude_path); } return NULL; @@ -415,6 +579,33 @@ static int compute_untracked_cache_valid_recursive( return valid; } +static int compute_untracked_cache_disk_valid_recursive( + struct untracked_cache_dir *ucd) +{ + size_t i; + int valid = ucd->valid && is_null_oid(&ucd->exclude_oid); + + for (i = 0; i < ucd->dirs_nr; i++) + if (!compute_untracked_cache_disk_valid_recursive(ucd->dirs[i])) + valid = 0; + ucd->valid_recursive = valid; + return valid; +} + +static int compute_untracked_cache_fsmonitor_valid_recursive( + struct untracked_cache_dir *ucd) +{ + size_t i; + int valid = ucd->valid; + + for (i = 0; i < ucd->dirs_nr; i++) + if (!compute_untracked_cache_fsmonitor_valid_recursive( + ucd->dirs[i])) + valid = 0; + ucd->valid_recursive = valid; + return valid; +} + static void untracked_cache_preload_join( struct untracked_cache_preload *preload) { @@ -448,25 +639,219 @@ static void untracked_cache_preload_free( for (i = 0; i < preload->nr; i++) free(preload->tasks[i].path); free(preload->tasks); + free(preload->exclude_index_oids); free(preload->exclude_per_dir); free(preload); } +static int converted_exclude_matches_cache_and_index( + struct untracked_cache_preload *preload, + struct untracked_cache_preload_task *task, + struct cache_entry *ce, + const char *path) +{ + struct object_id converted_oid, normalized_oid, raw_oid; + struct stat before, after; + char *buf = NULL; + size_t size; + int converted_fd, fd = -1; + int cached_matches, ret = 0; + + fd = open_nofollow(path, O_RDONLY); + if (fd < 0 || fstat(fd, &before) || !S_ISREG(before.st_mode) || + before.st_size < 0 || + before.st_size > UNTRACKED_CACHE_MAX_EXCLUDE_SIZE) + goto done; + size = xsize_t(before.st_size); + buf = xmallocz(size + 1); + if (read_in_full(fd, buf, size) != size) + goto done; + + hash_object_file(preload->repo->hash_algo, buf, size, OBJ_BLOB, + &raw_oid); + cached_matches = oideq(&raw_oid, &task->exclude_oid); + if (!cached_matches) { + buf[size] = '\n'; + hash_object_file(preload->repo->hash_algo, buf, size + 1, + OBJ_BLOB, &normalized_oid); + cached_matches = oideq(&normalized_oid, + &task->exclude_oid); + } + if (!cached_matches || lseek(fd, 0, SEEK_SET) < 0) + goto done; + + converted_fd = xdup(fd); + if (index_fd(preload->istate, &converted_oid, converted_fd, &before, + OBJ_BLOB, path, 0) || + !oideq(&converted_oid, &ce->oid)) + goto done; + + /* + * Keep the descriptor open while computing both identities, then + * prove that neither the opened file nor its pathname changed. + */ + if (fstat(fd, &after) || + !path_namespace_stat_equal(&before, &after) || + !exclude_path_matches_fd(path, &after)) + goto done; + fill_stat_data(&task->stat_data, &after); + task->exclude_mode = after.st_mode; + ret = 1; +done: + free(buf); + if (fd >= 0) + close(fd); + return ret; +} + +static int update_preloaded_exclude_index_uptodate( + struct untracked_cache_preload *preload, + struct untracked_cache_preload_task *task, + size_t task_nr, + size_t *normalized, + size_t *index_invalidated, + int *exclude_revalidated) +{ + struct strbuf path = STRBUF_INIT; + struct cache_entry *ce; + int content_matches, converts, pos, marked = 0; + + *exclude_revalidated = -1; + if (!task->exclude_index_present || !preload->exclude_per_dir) + return 0; + if (strcmp(task->path, ".")) + strbuf_addstr(&path, task->path); + if (path.len) + strbuf_addch(&path, '/'); + strbuf_addstr(&path, preload->exclude_per_dir); + pos = index_name_pos_sparse(preload->istate, path.buf, path.len); + if (pos < 0) + goto done; + ce = preload->istate->cache[pos]; + if (!ce_stage(ce) && S_ISREG(ce->ce_mode) && + oideq(&ce->oid, &preload->exclude_index_oids[task_nr])) { + if (task->exclude_index_matches) { + converts = 0; + content_matches = 1; + } else { + converts = would_convert_to_git( + preload->istate, path.buf); + content_matches = converts ? + converted_exclude_matches_cache_and_index( + preload, task, ce, path.buf) : + task->exclude_index_content_matches; + if (converts) + *exclude_revalidated = content_matches; + } + if (!converts && task->normalize_exclude_oid) { + oidcpy(&task->ucd->exclude_oid, + &preload->exclude_index_oids[task_nr]); + (*normalized)++; + } + if (content_matches && + (ce->ce_flags & CE_FSMONITOR_VALID) && + !ce_skip_worktree(ce) && + !(ce->ce_flags & CE_REMOVE) && + !ce_intent_to_add(ce) && + (!repo_trust_executable_bit(preload->istate->repo) || + !((ce->ce_mode ^ task->exclude_mode) & 0100))) { + if (converts && + memcmp(&ce->ce_stat_data, &task->stat_data, + sizeof(ce->ce_stat_data))) { + ce->ce_stat_data = task->stat_data; + ce->ce_flags |= CE_UPDATE_IN_BASE; + preload->istate->cache_changed |= + CE_ENTRY_CHANGED; + } + ce_mark_uptodate(ce); + marked = 1; + } else { + fsmonitor_invalidate_cache_entry(ce); + preload->istate->cache_changed |= FSMONITOR_CHANGED; + (*index_invalidated)++; + } + } +done: + strbuf_release(&path); + return marked; +} + int untracked_cache_preload_finish(struct untracked_cache_preload *preload, - struct index_state *istate, - unsigned int dir_flags) + struct index_state *istate, + unsigned int dir_flags, + size_t *index_invalidated) { struct untracked_cache *uc; size_t i; int applied = 0; + if (index_invalidated) + *index_invalidated = 0; if (!preload) return 0; untracked_cache_preload_join(preload); uc = istate->untracked; if (uc != preload->uc || !uc || uc->root != preload->root || - dir_flags != preload->dir_flags) + dir_flags != preload->dir_flags || + (preload->fsmonitor_excludes_only && !uc->use_fsmonitor)) + goto done; + + if (preload->fsmonitor_excludes_only) { + size_t index_matches = 0; + size_t invalidated = 0; + size_t index_uptodate = 0; + size_t normalized = 0; + + for (i = 0; i < preload->nr; i++) { + struct untracked_cache_preload_task *task = + &preload->tasks[i]; + int exclude_matches = + oideq(&task->exclude_oid, + &task->ucd->exclude_oid) && + task->exclude_matches; + int exclude_revalidated; + + if (!exclude_matches) + invalidate_gitignore(uc, task->ucd); + else { + if (task->exclude_index_matches) + index_matches++; + } + index_uptodate += + update_preloaded_exclude_index_uptodate( + preload, task, i, &normalized, + &invalidated, &exclude_revalidated); + if (exclude_matches && exclude_revalidated == 0) + invalidate_gitignore(uc, task->ucd); + } + if (normalized) + istate->cache_changed |= UNTRACKED_CHANGED; + trace2_data_intmax( + "dir", istate->repo, + "preload_untracked_cache/index-excludes", + index_matches); + trace2_data_intmax( + "dir", istate->repo, + "preload_untracked_cache/index-uptodate", + index_uptodate); + trace2_data_intmax( + "dir", istate->repo, + "preload_untracked_cache/index-invalidated", + invalidated); + trace2_data_intmax( + "dir", istate->repo, + "preload_untracked_cache/normalized-excludes", + normalized); + trace2_data_intmax( + "dir", istate->repo, + "preload_untracked_cache/valid", + compute_untracked_cache_fsmonitor_valid_recursive( + preload->root)); + if (index_invalidated) + *index_invalidated = invalidated; + applied = 1; goto done; + } for (i = 0; i < preload->nr; i++) { struct untracked_cache_preload_task *task = &preload->tasks[i]; @@ -1684,6 +2069,10 @@ static int add_patterns(const char *fname, const char *base, int baselen, (pos = index_name_pos(istate, fname, strlen(fname))) >= 0 && !ce_stage(istate->cache[pos]) && ce_uptodate(istate->cache[pos]) && + !(istate->cache[pos]->ce_flags & + (CE_VALID | CE_REMOVE)) && + !ce_skip_worktree(istate->cache[pos]) && + !ce_intent_to_add(istate->cache[pos]) && !would_convert_to_git(istate, fname)) oidcpy(&oid_stat->oid, &istate->cache[pos]->oid); @@ -2253,9 +2642,24 @@ static void prep_exclude(struct dir_struct *dir, strbuf_addbuf(&sb, &dir->internal.basebuf); strbuf_addstr(&sb, dir->exclude_per_dir); pl->src = strbuf_detach(&sb, NULL); - add_patterns(pl->src, pl->src, stk->baselen, pl, istate, - PATTERN_NOFOLLOW, - untracked ? &oid_stat : NULL); + if (add_patterns(pl->src, pl->src, stk->baselen, pl, + istate, PATTERN_NOFOLLOW, + untracked ? &oid_stat : NULL) < 0 && + untracked && is_null_oid(&oid_stat.oid)) { + struct stat st; + + /* + * Keep a non-blob sentinel for a source that is + * present but unreadable. Otherwise a valid + * untracked-cache directory cannot distinguish + * that state from an absent per-directory + * exclude file. + */ + if (!lstat(pl->src, &st) || + !is_missing_file_error(errno)) + oidcpy(&oid_stat.oid, + the_hash_algo->empty_tree); + } } /* * NEEDSWORK: when untracked cache is enabled, prep_exclude() @@ -3234,7 +3638,9 @@ static enum path_treatment read_directory_recursive(struct dir_struct *dir, struct strbuf path = STRBUF_INIT; strbuf_add(&path, base, baselen); - if (untracked && dir->internal.untracked_cache_preloaded && + if (untracked && + (dir->internal.untracked_cache_preloaded || + dir->untracked->use_fsmonitor) && untracked->valid && untracked->valid_recursive && untracked->check_only == !!check_only && !untracked->has_untracked && @@ -3428,6 +3834,8 @@ static int treat_leading_path(struct dir_struct *dir, return state == path_recurse; } +#define UNTRACKED_CACHE_IDENT_VERSION 2 + static const char *get_ident_string(void) { static struct strbuf sb = STRBUF_INIT; @@ -3437,8 +3845,9 @@ static const char *get_ident_string(void) return sb.buf; if (uname(&uts) < 0) die_errno(_("failed to get kernel name and information")); - strbuf_addf(&sb, "Location %s, system %s", repo_get_work_tree(the_repository), - uts.sysname); + strbuf_addf(&sb, "Location %s, system %s, cache version %d", + repo_get_work_tree(the_repository), uts.sysname, + UNTRACKED_CACHE_IDENT_VERSION); return sb.buf; } @@ -4502,6 +4911,8 @@ struct untracked_cache *read_untracked_extension(const void *data, unsigned long ewah_each_bit(rd.valid, read_stat, &rd); ewah_each_bit(rd.sha1_valid, read_oid, &rd); next = rd.data; + if (next == end) + compute_untracked_cache_disk_valid_recursive(uc->root); done: free(rd.ucd); diff --git a/dir.h b/dir.h index 198d7c846f8937..f6df0b54d271e9 100644 --- a/dir.h +++ b/dir.h @@ -189,7 +189,10 @@ struct untracked_cache_dir { unsigned int stat_matches : 1; unsigned int exclude_matches : 1; unsigned int valid_recursive : 1; - /* null object ID means this directory does not have .gitignore */ + /* + * A null object ID means this directory does not have .gitignore. + * The empty-tree ID records a present source that could not be read. + */ struct object_id exclude_oid; char name[FLEX_ARRAY]; }; @@ -617,10 +620,14 @@ void untracked_cache_remove_from_index(struct index_state *, const char *); void untracked_cache_add_to_index(struct index_state *, const char *); struct untracked_cache_preload; +struct untracked_cache_preload * +untracked_cache_preload_start_fsmonitor_excludes( + struct index_state *, unsigned int dir_flags); struct untracked_cache_preload *untracked_cache_preload_start_ordinary( struct index_state *, unsigned int dir_flags); int untracked_cache_preload_finish(struct untracked_cache_preload *, - struct index_state *, unsigned int dir_flags); + struct index_state *, unsigned int dir_flags, + size_t *index_invalidated); void untracked_cache_preload_release(struct untracked_cache_preload *); void free_untracked_cache(struct untracked_cache *); diff --git a/fsmonitor-ll.h b/fsmonitor-ll.h index 8591a166665bd5..7e7564e5e2c493 100644 --- a/fsmonitor-ll.h +++ b/fsmonitor-ll.h @@ -7,6 +7,14 @@ struct strbuf; /* A provider-only marker; worktree-relative paths cannot begin with '/'. */ #define FSMONITOR_PATH_GLOBAL_INVALIDATE "//" +enum fsmonitor_token_result { + FSMONITOR_TOKEN_NOT_PENDING = 0, + FSMONITOR_TOKEN_CLEAN, + FSMONITOR_TOKEN_CHANGED, + FSMONITOR_TOKEN_TRIVIAL, + FSMONITOR_TOKEN_ERROR, +}; + extern struct trace_key trace_fsmonitor; /* @@ -55,6 +63,16 @@ void refresh_fsmonitor(struct index_state *istate); int fsmonitor_invalidate_attributes_path(struct index_state *istate, const char *name); + +/* Close a provider token which was obtained before a required scan. */ +int fsmonitor_has_pending_token(const struct index_state *istate); +int fsmonitor_pending_token_from_provider(const struct index_state *istate); +enum fsmonitor_token_result fsmonitor_query_pending_token( + struct index_state *istate, int untracked_ready); +void fsmonitor_accept_pending_token(struct index_state *istate); +void fsmonitor_reject_pending_token(struct index_state *istate); +void fsmonitor_mark_untracked_cache_valid(struct index_state *istate); + /* * Does the received result contain the "trivial" response? */ diff --git a/fsmonitor.c b/fsmonitor.c index dea229e7a8597a..94ccbceba7c307 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -807,8 +807,44 @@ enum fsmonitor_query_outcome fsmonitor_parse_builtin_response( static enum fsmonitor_query_outcome query_builtin_fsmonitor( const char *since_token, struct fsmonitor_query_result *result) { + const char *test_sequence = + getenv("GIT_TEST_FSMONITOR_QUERY_SEQUENCE"); struct strbuf raw = STRBUF_INIT; + /* + * Tests may script clean, delta, trivial, and error responses with + * C, D, T, and E. A delta uses GIT_TEST_FSMONITOR_QUERY_PATH. + */ + if (test_sequence && *test_sequence) { + static size_t query_nr; + const char *path; + char outcome; + + if (query_nr >= strlen(test_sequence)) + return FSMONITOR_QUERY_ERROR; + outcome = test_sequence[query_nr++]; + if (outcome == 'E') + return FSMONITOR_QUERY_ERROR; + + strbuf_addf(&result->token, "builtin:test:%"PRIuMAX, + (uintmax_t)query_nr); + if (outcome == 'T') { + result->outcome = FSMONITOR_QUERY_TRIVIAL; + return result->outcome; + } + if (outcome == 'D') { + path = getenv("GIT_TEST_FSMONITOR_QUERY_PATH"); + if (!path || !*path) + return FSMONITOR_QUERY_ERROR; + strbuf_addstr(&result->paths, path); + strbuf_addch(&result->paths, '\0'); + } else if (outcome != 'C') { + return FSMONITOR_QUERY_ERROR; + } + result->outcome = FSMONITOR_QUERY_DELTA; + return result->outcome; + } + if (!fsmonitor_ipc__send_query(since_token, &raw)) fsmonitor_parse_builtin_response(&raw, result); strbuf_release(&raw); @@ -849,6 +885,15 @@ static void invalidate_all_fsmonitor(struct index_state *istate) istate->cache_changed |= FSMONITOR_CHANGED; } +static void invalidate_all_fsmonitor_strong(struct index_state *istate) +{ + unsigned int i; + + invalidate_all_fsmonitor(istate); + for (i = 0; i < istate->cache_nr; i++) + fsmonitor_invalidate_cache_entry(istate->cache[i]); +} + void refresh_fsmonitor(struct index_state *istate) { static int warn_once = 0; @@ -860,6 +905,8 @@ void refresh_fsmonitor(struct index_state *istate) char *buf; unsigned int i; int is_trivial = 0; + int tracked_requires_bootstrap; + int untracked_requires_bootstrap; struct repository *r = istate->repo; enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); enum fsmonitor_reason reason = fsm_settings__get_reason(r); @@ -1000,6 +1047,10 @@ void refresh_fsmonitor(struct index_state *istate) */ trace2_region_enter("fsmonitor", "apply_results", istate->repo); + tracked_requires_bootstrap = !query_success || is_trivial || + !istate->fsmonitor_token_valid; + untracked_requires_bootstrap = !istate->fsmonitor_untracked_valid; + if (query_success && !is_trivial) { /* * Mark all pathnames returned by the monitor as dirty. @@ -1027,9 +1078,14 @@ void refresh_fsmonitor(struct index_state *istate) } } + if (tracked_requires_bootstrap) + invalidate_all_fsmonitor(istate); + /* Now mark the untracked cache for fsmonitor usage */ if (istate->untracked) - istate->untracked->use_fsmonitor = 1; + istate->untracked->use_fsmonitor = + !tracked_requires_bootstrap && + !untracked_requires_bootstrap; if (count > fsmonitor_force_update_threshold) istate->cache_changed |= FSMONITOR_CHANGED; @@ -1052,9 +1108,152 @@ void refresh_fsmonitor(struct index_state *istate) strbuf_release(&query_result); - /* Now that we've updated istate, save the last_update_token */ + /* + * A token obtained before a full scan cannot describe changes which + * race with that scan. Keep it in memory until the caller closes the + * race with a second query. The last valid token remains safe because + * a query relative to it will return a superset of changes. + */ + if (tracked_requires_bootstrap) { + if (!last_update_token.len) { + if (istate->fsmonitor_last_update) + strbuf_addstr(&last_update_token, + istate->fsmonitor_last_update); + else + strbuf_addstr(&last_update_token, "builtin:fake"); + } + FREE_AND_NULL(istate->fsmonitor_last_update_pending); + istate->fsmonitor_last_update_pending = + strbuf_detach(&last_update_token, NULL); + /* + * A trivial response cannot validate prior state, but its + * returned token is still a provider-owned boundary. Use it + * to anchor the complete scan which the caller will close with + * another query. Hook providers cannot perform that closing + * query, so do not publish their trivial-response tokens. + */ + istate->fsmonitor_pending_token_from_provider = + query_success && + (fsm_mode == FSMONITOR_MODE_IPC || !is_trivial); + istate->fsmonitor_untracked_valid = 0; + } else { + FREE_AND_NULL(istate->fsmonitor_last_update); + istate->fsmonitor_last_update = + strbuf_detach(&last_update_token, NULL); + if (untracked_requires_bootstrap) { + FREE_AND_NULL(istate->fsmonitor_last_update_pending); + istate->fsmonitor_last_update_pending = + xstrdup(istate->fsmonitor_last_update); + istate->fsmonitor_pending_token_from_provider = 1; + } else { + FREE_AND_NULL(istate->fsmonitor_last_update_pending); + istate->fsmonitor_pending_token_from_provider = 0; + } + if (istate->fsmonitor_untracked_valid && istate->untracked) { + FREE_AND_NULL(istate->fsmonitor_untracked_token); + istate->fsmonitor_untracked_token = + xstrdup(istate->fsmonitor_last_update); + } + } +} + +int fsmonitor_has_pending_token(const struct index_state *istate) +{ + return !!istate->fsmonitor_last_update_pending; +} + +int fsmonitor_pending_token_from_provider(const struct index_state *istate) +{ + return istate->fsmonitor_last_update_pending && + istate->fsmonitor_pending_token_from_provider; +} + +enum fsmonitor_token_result fsmonitor_query_pending_token( + struct index_state *istate, int untracked_ready) +{ + struct fsmonitor_query_result result = FSMONITOR_QUERY_RESULT_INIT; + enum fsmonitor_token_result ret; + int count; + + if (!istate->fsmonitor_last_update_pending) + return FSMONITOR_TOKEN_NOT_PENDING; + if (fsm_settings__get_mode(istate->repo) != FSMONITOR_MODE_IPC) + return FSMONITOR_TOKEN_ERROR; + + query_builtin_fsmonitor(istate->fsmonitor_last_update_pending, &result); + if (result.outcome == FSMONITOR_QUERY_ERROR) { + istate->fsmonitor_pending_token_from_provider = 0; + ret = FSMONITOR_TOKEN_ERROR; + goto done; + } + + FREE_AND_NULL(istate->fsmonitor_last_update_pending); + istate->fsmonitor_last_update_pending = + strbuf_detach(&result.token, NULL); + istate->fsmonitor_pending_token_from_provider = 1; + if (result.outcome == FSMONITOR_QUERY_TRIVIAL) { + invalidate_all_fsmonitor_strong(istate); + trace2_data_intmax("fsmonitor", istate->repo, + "token_closure/trivial", 1); + ret = FSMONITOR_TOKEN_TRIVIAL; + goto done; + } + + count = apply_fsmonitor_paths(istate, &result.paths); + if (istate->untracked) + istate->untracked->use_fsmonitor = !!untracked_ready; + trace2_data_intmax("fsmonitor", istate->repo, + "token_closure/apply_count", count); + ret = count ? FSMONITOR_TOKEN_CHANGED : FSMONITOR_TOKEN_CLEAN; + +done: + fsmonitor_query_result_release(&result); + return ret; +} + +void fsmonitor_accept_pending_token(struct index_state *istate) +{ + if (!fsmonitor_pending_token_from_provider(istate)) + return; FREE_AND_NULL(istate->fsmonitor_last_update); - istate->fsmonitor_last_update = strbuf_detach(&last_update_token, NULL); + istate->fsmonitor_last_update = istate->fsmonitor_last_update_pending; + istate->fsmonitor_last_update_pending = NULL; + istate->fsmonitor_pending_token_from_provider = 0; + istate->fsmonitor_token_valid = 1; + istate->fsmonitor_untracked_valid = 1; + if (istate->untracked) + istate->untracked->use_fsmonitor = 1; + istate->cache_changed |= FSMONITOR_CHANGED; + FREE_AND_NULL(istate->fsmonitor_untracked_token); + istate->fsmonitor_untracked_token = + xstrdup(istate->fsmonitor_last_update); + trace2_data_intmax("fsmonitor", istate->repo, + "token_closure/accepted", 1); +} + +void fsmonitor_reject_pending_token(struct index_state *istate) +{ + FREE_AND_NULL(istate->fsmonitor_last_update_pending); + istate->fsmonitor_pending_token_from_provider = 0; + if (!istate->fsmonitor_token_valid) + FREE_AND_NULL(istate->fsmonitor_last_update); + invalidate_all_fsmonitor_strong(istate); + trace2_data_intmax("fsmonitor", istate->repo, + "token_closure/rejected", 1); +} + +void fsmonitor_mark_untracked_cache_valid(struct index_state *istate) +{ + if (istate->fsmonitor_last_update_pending || + !istate->fsmonitor_token_valid || + !istate->fsmonitor_last_update || !istate->untracked || + istate->fsmonitor_untracked_valid) + return; + istate->fsmonitor_untracked_valid = 1; + FREE_AND_NULL(istate->fsmonitor_untracked_token); + istate->fsmonitor_untracked_token = + xstrdup(istate->fsmonitor_last_update); + istate->cache_changed |= FSMONITOR_CHANGED; } /* @@ -1086,6 +1285,7 @@ static void initialize_fsmonitor_last_update(struct index_state *istate) strbuf_addf(&last_update, "%"PRIu64"", getnanotime()); istate->fsmonitor_last_update = strbuf_detach(&last_update, NULL); + istate->fsmonitor_token_valid = 0; } void add_fsmonitor(struct index_state *istate) @@ -1104,7 +1304,7 @@ void add_fsmonitor(struct index_state *istate) /* reset the untracked cache */ if (istate->untracked) { add_untracked_cache(istate); - istate->untracked->use_fsmonitor = 1; + istate->untracked->use_fsmonitor = 0; } /* Update the fsmonitor state */ @@ -1114,6 +1314,13 @@ void add_fsmonitor(struct index_state *istate) void remove_fsmonitor(struct index_state *istate) { + istate->fsmonitor_token_valid = 0; + istate->fsmonitor_untracked_valid = 0; + FREE_AND_NULL(istate->fsmonitor_last_update_pending); + istate->fsmonitor_pending_token_from_provider = 0; + FREE_AND_NULL(istate->fsmonitor_untracked_token); + if (istate->untracked) + istate->untracked->use_fsmonitor = 0; if (istate->fsmonitor_last_update) { trace_printf_key(&trace_fsmonitor, "remove fsmonitor"); istate->cache_changed |= FSMONITOR_CHANGED; diff --git a/read-cache-ll.h b/read-cache-ll.h index db5b13d06b946c..d44a946ed04d36 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -187,13 +187,15 @@ struct index_state { fsmonitor_extension_seen : 1, fsmonitor_untracked_valid : 1, fsmonitor_untracked_extension_seen : 1, - fsmonitor_untracked_extension_invalid : 1; + fsmonitor_untracked_extension_invalid : 1, + fsmonitor_pending_token_from_provider : 1; enum sparse_index_mode sparse_index; struct hashmap name_hash; struct hashmap dir_hash; struct object_id oid; struct untracked_cache *untracked; char *fsmonitor_last_update; + char *fsmonitor_last_update_pending; char *fsmonitor_untracked_token; struct ewah_bitmap *fsmonitor_dirty; struct mem_pool *ce_mem_pool; diff --git a/read-cache.c b/read-cache.c index 57cc73dec90a0e..cf3600f2a337d6 100644 --- a/read-cache.c +++ b/read-cache.c @@ -2471,6 +2471,7 @@ void release_index(struct index_state *istate) free_name_hash(istate); cache_tree_free(&(istate->cache_tree)); free(istate->fsmonitor_last_update); + free(istate->fsmonitor_last_update_pending); free(istate->fsmonitor_untracked_token); free(istate->cache); discard_split_index(istate); diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index e8cc70c428b181..6b1fdd3bcbbc6f 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -88,6 +88,609 @@ test_expect_success 'hook parser ignores empty path records' ' ) ' +test_expect_success UNTRACKED_CACHE 'trivial hook clears a paired UNTR token' ' + test_when_finished "rm -rf hook-token-pair" && + test_create_repo hook-token-pair && + ( + cd hook-token-pair && + test_commit base tracked && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token1\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + test_grep ! FSUC .git/index && + git status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual && + test_grep FSUC .git/index && + test_hook --clobber fsmonitor-test <<-\EOF && + printf "token2\0/\0" + EOF + git status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual && + test_grep ! FSUC .git/index + ) +' + +test_expect_success UNTRACKED_CACHE 'failed hook clears a paired UNTR token' ' + test_when_finished "rm -rf hook-token-error" && + test_create_repo hook-token-error && + ( + cd hook-token-error && + test_commit base tracked && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain=v2 >/dev/null && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token1\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual && + test_grep FSUC .git/index && + test_hook --clobber fsmonitor-test <<-\EOF && + exit 1 + EOF + git status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual && + test_grep ! FSUC .git/index + ) +' + +test_expect_success UNTRACKED_CACHE \ + 'paired fsmonitor cache prunes recursively valid empty subtrees' ' + test_when_finished "rm -rf fsmonitor-untracked-prune" && + test_create_repo fsmonitor-untracked-prune && + ( + cd fsmonitor-untracked-prune && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir -p cached/empty/deep && + test_write_lines tracked >cached/empty/deep/tracked && + git add cached/empty/deep/tracked && + git commit -m base && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain=v2 \ + >.git/prime && + test_must_be_empty .git/prime && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime-fsmonitor && + test_must_be_empty .git/prime-fsmonitor && + test_grep FSUC .git/index && + + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/clean.trace" \ + git status --porcelain=v2 >.git/clean && + test_must_be_empty .git/clean && + test_trace2_data read_directory directories-visited 0 \ + <.git/clean.trace && + + test_write_lines untracked >cached/empty/deep/new && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=cached/empty/deep/new \ + git status --porcelain=v2 >.git/changed && + test_grep "^? cached/empty/deep/new$" .git/changed + ) +' + +test_expect_success UNTRACKED_CACHE,HARDLINKS \ + 'fsmonitor pruning rechecks cached per-directory excludes' ' + test_when_finished "rm -rf fsmonitor-untracked-exclude" && + test_when_finished "rm -f fsmonitor-untracked-exclude-alias" && + test_create_repo fsmonitor-untracked-exclude && + ( + cd fsmonitor-untracked-exclude && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir cached cached2 cached3 && + test_write_lines ignored >cached/.gitignore && + test_write_lines hidden >cached/ignored && + test_write_lines ignored >cached2/.gitignore && + test_write_lines hidden >cached2/ignored && + test_write_lines ignored >cached3/.gitignore && + test_write_lines hidden >cached3/ignored && + git add cached/.gitignore cached2/.gitignore \ + cached3/.gitignore && + git commit -m base && + test-tool chmtime +60 cached2/.gitignore && + test-tool chmtime =-60 cached3/.gitignore && + git update-index --refresh && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain=v2 \ + >.git/prime && + test_must_be_empty .git/prime && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime-fsmonitor && + test_must_be_empty .git/prime-fsmonitor && + test_grep FSUC .git/index && + ln cached/.gitignore ../fsmonitor-untracked-exclude-alias && + + if test_have_prereq PTHREADS + then + threads=2 + else + threads=1 + fi && + if test_have_prereq MINGW || test_have_prereq CYGWIN + then + index_excludes=0 + else + index_excludes=1 + fi && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=2 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT_NESTING=10 \ + GIT_TRACE2_EVENT="$PWD/.git/clean.trace" \ + git status --porcelain >.git/clean && + test_must_be_empty .git/clean && + test_trace2_data dir \ + preload_untracked_cache/fsmonitor-excludes-only 1 \ + <.git/clean.trace && + test_trace2_data dir preload_untracked_cache/threads \ + $threads \ + <.git/clean.trace && + test_trace2_data dir preload_untracked_cache/dirs 3 \ + <.git/clean.trace && + test_trace2_data dir \ + preload_untracked_cache/index-excludes "$index_excludes" \ + <.git/clean.trace && + test_trace2_data read_directory directories-visited 0 \ + <.git/clean.trace && + + if test_have_prereq FILEMODE + then + chmod +x ../fsmonitor-untracked-exclude-alias && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=2 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/mode.trace" \ + git status --porcelain >.git/mode && + test_trace2_data dir \ + preload_untracked_cache/index-uptodate 2 \ + <.git/mode.trace && + chmod -x ../fsmonitor-untracked-exclude-alias + else + : + fi && + + mtime=$(test-tool chmtime --get cached/.gitignore) && + test_write_lines visible \ + >../fsmonitor-untracked-exclude-alias && + test-tool chmtime =$mtime cached/.gitignore && + GIT_TEST_UNTRACKED_CACHE_THREADS=2 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/changed.trace" \ + git status >.git/changed && + test_grep "modified:.*cached/.gitignore" .git/changed && + test_grep "cached/ignored" .git/changed && + test_trace2_data status \ + fsmonitor/exclude-index-invalidated 1 \ + <.git/changed.trace && + + test_write_lines ignored \ + >../fsmonitor-untracked-exclude-alias && + test-tool chmtime =$mtime cached/.gitignore && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=2 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/restored.trace" \ + git status >.git/restored && + test_grep "nothing to commit, working tree clean" \ + .git/restored && + + test_write_lines "?? cached/ignored" >.git/flagged.expect && + + git update-index --assume-unchanged cached/.gitignore && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain >.git/assume-prime && + test_must_be_empty .git/assume-prime && + mtime=$(test-tool chmtime --get cached/.gitignore) && + test_write_lines visible \ + >../fsmonitor-untracked-exclude-alias && + test-tool chmtime =$mtime cached/.gitignore && + GIT_TEST_UNTRACKED_CACHE_THREADS=2 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/assume-changed.trace" \ + git status --porcelain >.git/assume-changed && + test_cmp .git/flagged.expect .git/assume-changed && + test_write_lines ignored \ + >../fsmonitor-untracked-exclude-alias && + test-tool chmtime =$mtime cached/.gitignore && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=2 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/assume-restored.trace" \ + git status --porcelain >.git/assume-restored && + test_must_be_empty .git/assume-restored && + git update-index --no-assume-unchanged cached/.gitignore && + + git update-index --skip-worktree cached/.gitignore && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain >.git/skip-prime && + test_must_be_empty .git/skip-prime && + mtime=$(test-tool chmtime --get cached/.gitignore) && + test_write_lines visible \ + >../fsmonitor-untracked-exclude-alias && + test-tool chmtime =$mtime cached/.gitignore && + GIT_TEST_UNTRACKED_CACHE_THREADS=2 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/skip-changed.trace" \ + git status --porcelain >.git/skip-changed && + test_cmp .git/flagged.expect .git/skip-changed && + test_write_lines ignored \ + >../fsmonitor-untracked-exclude-alias && + test-tool chmtime =$mtime cached/.gitignore && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=2 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/skip-restored.trace" \ + git status --porcelain >.git/skip-restored && + test_must_be_empty .git/skip-restored && + git update-index --no-skip-worktree cached/.gitignore + ) +' + +test_expect_success UNTRACKED_CACHE \ + 'converted cached excludes retain a stable index proof' ' + test_when_finished "rm -rf fsmonitor-converted-exclude" && + test_create_repo fsmonitor-converted-exclude && + ( + cd fsmonitor-converted-exclude && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir cached && + printf "ignored\r\n" >cached/.gitignore && + test_write_lines hidden >cached/ignored && + test_write_lines "cached/.gitignore text eol=lf" \ + >.gitattributes && + git add .gitattributes cached/.gitignore && + git commit -m base && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain >.git/prime && + test_must_be_empty .git/prime && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain >.git/prime-fsmonitor && + test_must_be_empty .git/prime-fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status >.git/settle && + test_grep "nothing to commit, working tree clean" \ + .git/settle && + + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/clean.trace" \ + git status >.git/clean && + test_grep "nothing to commit, working tree clean" \ + .git/clean && + test_trace2_data dir \ + preload_untracked_cache/index-uptodate 1 \ + <.git/clean.trace && + test_trace2_data dir \ + preload_untracked_cache/index-invalidated 0 \ + <.git/clean.trace && + test_trace2_data dir \ + preload_untracked_cache/normalized-excludes 0 \ + <.git/clean.trace && + test_trace2_data read_directory directories-visited 0 \ + <.git/clean.trace + ) +' + +test_expect_success UNTRACKED_CACHE,HARDLINKS,POSIXPERM,SANITY \ + 'fsmonitor rechecks cached unreadable per-directory excludes' ' + test_when_finished "rm -rf fsmonitor-unreadable-exclude" && + test_when_finished "rm -f fsmonitor-unreadable-exclude-alias" && + test_create_repo fsmonitor-unreadable-exclude && + ( + cd fsmonitor-unreadable-exclude && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir cached && + test_write_lines hidden >cached/.gitignore && + test_write_lines untracked >cached/hidden && + test_write_lines tracked >cached/tracked && + git add cached/tracked && + git commit -m base && + chmod 000 cached/.gitignore && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain \ + >.git/prime 2>.git/prime.err && + test_grep "^?? cached/hidden$" .git/prime && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain >.git/prime-fsmonitor \ + 2>.git/prime-fsmonitor.err && + test_grep "^?? cached/hidden$" .git/prime-fsmonitor && + empty_tree=$(git mktree .git/untracked-cache && + test_grep "cached/ $empty_tree" .git/untracked-cache && + ln cached/.gitignore \ + ../fsmonitor-unreadable-exclude-alias && + + chmod 644 ../fsmonitor-unreadable-exclude-alias && + test_write_lines "?? cached/.gitignore" \ + >.git/readable.expect && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/readable.trace" \ + git status --porcelain >.git/readable && + test_cmp .git/readable.expect .git/readable && + test_trace2_data dir \ + preload_untracked_cache/dirs "[1-9]" \ + <.git/readable.trace + ) +' + +check_weak_exclude_stat () { + repo=$1 && + key=$2 && + value=$3 && + test_create_repo "$repo" && + ( + cd "$repo" && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir cached && + test_write_lines ignored >cached/.gitignore && + test_write_lines hidden >cached/ignored && + git add cached/.gitignore && + git commit -m base && + git config "$key" "$value" && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain=v2 \ + >.git/prime && + test_must_be_empty .git/prime && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime-fsmonitor && + test_must_be_empty .git/prime-fsmonitor && + + mtime=$(test-tool chmtime --get cached/.gitignore) && + test_write_lines visible >cached/.gitignore && + test-tool chmtime =$mtime cached/.gitignore && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + GIT_TRACE2_EVENT="$PWD/.git/changed.trace" \ + git status --porcelain >.git/changed && + test_grep "^?? cached/ignored$" .git/changed && + test_trace2_data dir \ + preload_untracked_cache/index-excludes 0 \ + <.git/changed.trace + ) +} + +test_expect_success UNTRACKED_CACHE \ + 'weak stat settings retain exclude content checks' ' + test_when_finished "rm -rf weak-exclude-ctime weak-exclude-stat" && + check_weak_exclude_stat weak-exclude-ctime \ + core.trustctime false && + check_weak_exclude_stat weak-exclude-stat \ + core.checkStat minimal +' + +test_expect_success UNTRACKED_CACHE \ + 'root untracked events preserve cached descendant excludes' ' + test_when_finished "rm -rf root-untracked-event" && + test_create_repo root-untracked-event && + ( + cd root-untracked-event && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir -p cached/deep && + test_write_lines "*.ignored" >.gitignore && + test_write_lines "*.ignored" >cached/.gitignore && + test_write_lines tracked >cached/deep/tracked && + test_write_lines ignored >cached/deep/junk.ignored && + git add .gitignore cached/.gitignore cached/deep/tracked && + git commit -m base && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain=v2 \ + >.git/prime && + test_must_be_empty .git/prime && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime-fsmonitor && + test_must_be_empty .git/prime-fsmonitor && + + test_write_lines visible >root-probe && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=root-probe \ + GIT_TRACE2_EVENT="$PWD/.git/created.trace" \ + git status >.git/created && + test_grep "root-probe" .git/created && + test_trace2_data read_directory directories-visited 1 \ + <.git/created.trace && + test_trace2_data read_directory gitignore-invalidation 0 \ + <.git/created.trace && + + rm root-probe && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=root-probe \ + GIT_TRACE2_EVENT="$PWD/.git/removed.trace" \ + git status >.git/removed && + test_grep "nothing to commit, working tree clean" \ + .git/removed && + test_trace2_data read_directory directories-visited 1 \ + <.git/removed.trace && + test_trace2_data read_directory gitignore-invalidation 0 \ + <.git/removed.trace + ) +' + +prepare_builtin_closure_repo () { + test_create_repo "$1" && + ( + cd "$1" && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + if test "${2-}" = untracked + then + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain=v2 \ + >.git/actual && + test_must_be_empty .git/actual && + test_grep UNTR .git/index && + test_grep ! FSUC .git/index + else + : + fi && + git config core.fsmonitor true && + test_grep ! FSMN .git/index + ) +} + +test_expect_success UNTRACKED_CACHE 'builtin clean closure publishes its proof' ' + test_when_finished "rm -rf builtin-closure-clean" && + prepare_builtin_closure_repo builtin-closure-clean untracked && + ( + cd builtin-closure-clean && + sane_unset GIT_TEST_SPLIT_INDEX && + test_write_lines visible >visible && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_grep "^? visible$" .git/actual && + test_grep \ + "\"event\":\"region_enter\".*\"category\":\"dir\",\"label\":\"read_directory\"" \ + .git/status.trace >.git/read-directory && + test_line_count = 1 .git/read-directory && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep FSMN .git/index && + test_grep FSUC .git/index + ) +' + +test_expect_success 'builtin changed closure rescans before acceptance' ' + test_when_finished "rm -rf builtin-closure-changed" && + prepare_builtin_closure_repo builtin-closure-changed && + ( + cd builtin-closure-changed && + sane_unset GIT_TEST_SPLIT_INDEX && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CDC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + GIT_TRACE_FSMONITOR="$PWD/.git/fsmonitor.trace" \ + git status --porcelain=v2 \ + --untracked-files=no >.git/actual && + test_must_be_empty .git/actual && + test_grep "fsmonitor_refresh_callback.*tracked" \ + .git/fsmonitor.trace && + test_trace2_data fsmonitor token_closure/apply_count 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace + ) +' + +test_expect_success 'builtin initial trivial response anchors a closure' ' + test_when_finished "rm -rf builtin-initial-trivial" && + prepare_builtin_closure_repo builtin-initial-trivial && + ( + cd builtin-initial-trivial && + sane_unset GIT_TEST_SPLIT_INDEX && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 \ + --untracked-files=no >.git/actual && + test_must_be_empty .git/actual && + test_trace2_data fsm_client query/trivial-response 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/status.trace && + test-tool dump-fsmonitor >.git/fsmonitor && + test_grep "^fsmonitor last update builtin:test:2" \ + .git/fsmonitor + ) +' + +test_expect_success 'builtin trivial closure can rescan and accept' ' + test_when_finished "rm -rf builtin-closure-trivial" && + prepare_builtin_closure_repo builtin-closure-trivial && + ( + cd builtin-closure-trivial && + sane_unset GIT_TEST_SPLIT_INDEX && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CTC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 \ + --untracked-files=no >.git/actual && + test_must_be_empty .git/actual && + test_trace2_data fsmonitor token_closure/trivial 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/status.trace + ) +' + +test_expect_success 'builtin closure rejects three intervening changes' ' + test_when_finished "rm -rf builtin-closure-exhausted" && + prepare_builtin_closure_repo builtin-closure-exhausted && + ( + cd builtin-closure-exhausted && + sane_unset GIT_TEST_SPLIT_INDEX && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CDDD \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 \ + --untracked-files=no >.git/actual && + test_must_be_empty .git/actual && + test_trace2_data fsmonitor token_closure/apply_count 1 \ + <.git/status.trace >.git/applied && + test_line_count = 3 .git/applied && + test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep ! FSUC .git/index + ) +' + +test_expect_success 'builtin closure query errors fall back completely' ' + test_when_finished "rm -rf builtin-closure-error" && + prepare_builtin_closure_repo builtin-closure-error untracked && + ( + cd builtin-closure-error && + sane_unset GIT_TEST_SPLIT_INDEX && + test_write_lines visible >visible && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CE \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_grep "^? visible$" .git/actual && + test_grep \ + "\"event\":\"region_enter\".*\"category\":\"dir\",\"label\":\"read_directory\"" \ + .git/status.trace >.git/read-directory && + test_line_count = 2 .git/read-directory && + test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep ! FSUC .git/index + ) +' + # Test that we detect and disallow repos that are incompatible with FSMonitor. test_expect_success 'incompatible bare repo' ' test_when_finished "rm -rf ./bare-clone actual expect" && diff --git a/wt-status.c b/wt-status.c index fab9f1af38bea6..57e2321275dc07 100644 --- a/wt-status.c +++ b/wt-status.c @@ -34,6 +34,7 @@ #include "worktree.h" #include "lockfile.h" #include "sequencer.h" +#include "fsmonitor.h" #include "fsmonitor-settings.h" #define AB_DELAY_WARNING_IN_MS (2 * 1000) @@ -814,21 +815,50 @@ void wt_status_start_untracked_cache_preload(struct wt_status *s) { struct index_state *istate = s->repo->index; unsigned int dir_flags; + int has_fsmonitor = + fsm_settings__get_mode(s->repo) > FSMONITOR_MODE_DISABLED; if (s->untracked_cache_preload) BUG("untracked-cache preload already started"); - if (fsm_settings__get_mode(s->repo) > FSMONITOR_MODE_DISABLED || - s->pathspec.nr || + if (s->pathspec.nr || s->show_untracked_files == SHOW_NO_UNTRACKED_FILES || s->show_ignored_mode) return; - dir_flags = wt_status_untracked_dir_flags(s); + if (has_fsmonitor) { + s->untracked_cache_preload = + untracked_cache_preload_start_fsmonitor_excludes( + istate, dir_flags); + return; + } + s->untracked_cache_preload = untracked_cache_preload_start_ordinary(istate, dir_flags); } -static int wt_status_collect_untracked_1(struct wt_status *s, int collect) +static void wt_status_finish_untracked_cache_preload(struct wt_status *s) +{ + struct index_state *istate = s->repo->index; + size_t index_invalidated = 0; + + if (!s->untracked_cache_preload) + return; + s->untracked_cache_preloaded = untracked_cache_preload_finish( + s->untracked_cache_preload, istate, + wt_status_untracked_dir_flags(s), &index_invalidated); + s->untracked_cache_preload = NULL; + if (!index_invalidated) + return; + + trace2_data_intmax("status", s->repo, + "fsmonitor/exclude-index-invalidated", + index_invalidated); +} + +static int wt_status_collect_untracked_1( + struct wt_status *s, + struct string_list *untracked, + struct string_list *ignored) { int i; int used_untracked_cache; @@ -851,11 +881,7 @@ static int wt_status_collect_untracked_1(struct wt_status *s, int collect) } setup_standard_excludes(&dir); - if (s->untracked_cache_preload) { - s->untracked_cache_preloaded = untracked_cache_preload_finish( - s->untracked_cache_preload, istate, dir.flags); - s->untracked_cache_preload = NULL; - } + wt_status_finish_untracked_cache_preload(s); dir.internal.untracked_cache_preloaded = s->untracked_cache_preloaded; @@ -863,32 +889,183 @@ static int wt_status_collect_untracked_1(struct wt_status *s, int collect) used_untracked_cache = dir.untracked && dir.untracked == istate->untracked; - if (collect) { - for (i = 0; i < dir.nr; i++) { - struct dir_entry *ent = dir.entries[i]; - if (index_name_is_other(istate, ent->name, ent->len)) - string_list_append(&s->untracked, ent->name); - } - string_list_sort_u(&s->untracked, 0); + for (i = 0; i < dir.nr; i++) { + struct dir_entry *ent = dir.entries[i]; + if (index_name_is_other(istate, ent->name, ent->len)) + string_list_append(untracked, ent->name); + } + string_list_sort_u(untracked, 0); - for (i = 0; i < dir.ignored_nr; i++) { - struct dir_entry *ent = dir.ignored[i]; - if (index_name_is_other(istate, ent->name, ent->len)) - string_list_append(&s->ignored, ent->name); - } - string_list_sort_u(&s->ignored, 0); + for (i = 0; i < dir.ignored_nr; i++) { + struct dir_entry *ent = dir.ignored[i]; + if (index_name_is_other(istate, ent->name, ent->len)) + string_list_append(ignored, ent->name); } + string_list_sort_u(ignored, 0); dir_clear(&dir); - if (collect && advice_enabled(ADVICE_STATUS_U_OPTION)) + if (advice_enabled(ADVICE_STATUS_U_OPTION)) s->untracked_in_ms = (getnanotime() - t_begin) / 1000000; + if (used_untracked_cache) + fsmonitor_mark_untracked_cache_valid(istate); return used_untracked_cache; } static int wt_status_collect_untracked(struct wt_status *s) { - return wt_status_collect_untracked_1(s, 1); + if (s->untracked_from_token_closure && !s->show_ignored_mode) + return 1; + return wt_status_collect_untracked_1( + s, &s->untracked, &s->ignored); +} + +#define FSMONITOR_TOKEN_MAX_QUERIES 3 + +struct wt_status_token_closure { + struct wt_status *status; + unsigned int refresh_flags; + int can_prime; + int untracked_ready; + struct string_list staged_untracked; + struct string_list staged_ignored; + int staged_untracked_ready; + int refresh_result; + int queries; +}; + +static void wt_status_discard_staged_untracked( + struct wt_status_token_closure *closure) +{ + string_list_clear(&closure->staged_untracked, 0); + string_list_clear(&closure->staged_ignored, 0); + closure->staged_untracked_ready = 0; +} + +static int wt_status_stage_untracked( + struct wt_status_token_closure *closure) +{ + wt_status_discard_staged_untracked(closure); + closure->staged_untracked_ready = + wt_status_collect_untracked_1( + closure->status, + &closure->staged_untracked, + &closure->staged_ignored); + if (!closure->staged_untracked_ready) + wt_status_discard_staged_untracked(closure); + return closure->staged_untracked_ready; +} + +static void wt_status_publish_staged_untracked( + struct wt_status_token_closure *closure) +{ + struct wt_status *s = closure->status; + + if (!closure->staged_untracked_ready) + return; + if (s->untracked.nr || s->ignored.nr) + BUG("publishing untracked results over collected status"); + SWAP(s->untracked, closure->staged_untracked); + SWAP(s->ignored, closure->staged_ignored); + s->untracked_from_token_closure = 1; + closure->staged_untracked_ready = 0; +} + +static int wt_status_close_ordinary_fsmonitor_token( + struct wt_status_token_closure *closure, + int refreshed_before_closure) +{ + struct wt_status *s = closure->status; + struct index_state *istate = s->repo->index; + + if (!refreshed_before_closure) + closure->refresh_result = refresh_index( + istate, closure->refresh_flags, &s->pathspec, + NULL, NULL); + if (!closure->untracked_ready && closure->can_prime) + closure->untracked_ready = wt_status_stage_untracked(closure); + + while (closure->queries < FSMONITOR_TOKEN_MAX_QUERIES) { + enum fsmonitor_token_result result = + fsmonitor_query_pending_token( + istate, closure->untracked_ready); + + closure->queries++; + if (result == FSMONITOR_TOKEN_CLEAN) { + if (closure->untracked_ready) { + fsmonitor_accept_pending_token(istate); + return 1; + } + break; + } + if (result == FSMONITOR_TOKEN_ERROR || + result == FSMONITOR_TOKEN_NOT_PENDING) + break; + + /* Rescan invalidations returned by the closure query. */ + closure->refresh_result |= refresh_index( + istate, closure->refresh_flags, &s->pathspec, + NULL, NULL); + if (closure->can_prime) + closure->untracked_ready = + wt_status_stage_untracked(closure); + } + return 0; +} + +static int wt_status_close_fsmonitor_token( + struct wt_status *s, unsigned int refresh_flags, + int require_untracked, int refreshed_before_closure) +{ + struct index_state *istate = s->repo->index; + struct wt_status_token_closure closure = { + .status = s, + .refresh_flags = refresh_flags, + .staged_untracked = STRING_LIST_INIT_DUP, + .staged_ignored = STRING_LIST_INIT_DUP, + }; + + refresh_fsmonitor(istate); + if (!fsmonitor_has_pending_token(istate) || s->pathspec.nr || + fsm_settings__get_mode(s->repo) != FSMONITOR_MODE_IPC) { + if (!refreshed_before_closure) + closure.refresh_result = refresh_index( + istate, refresh_flags, &s->pathspec, + NULL, NULL); + return closure.refresh_result; + } + + closure.can_prime = require_untracked && + s->show_untracked_files != SHOW_NO_UNTRACKED_FILES && + !s->show_ignored_mode; + closure.untracked_ready = !istate->untracked || + !istate->untracked->root; + if (require_untracked && !closure.can_prime && + !closure.untracked_ready) + BUG("cannot close required untracked scan"); + trace2_region_enter("status", "fsmonitor_token_closure", s->repo); + if (wt_status_close_ordinary_fsmonitor_token( + &closure, refreshed_before_closure)) + goto accepted; + + /* Keep the last valid token and fall back to complete scans. */ + wt_status_discard_staged_untracked(&closure); + fsmonitor_reject_pending_token(istate); + closure.refresh_result |= refresh_index( + istate, refresh_flags, &s->pathspec, NULL, NULL); +accepted: + wt_status_publish_staged_untracked(&closure); + wt_status_discard_staged_untracked(&closure); + trace2_region_leave("status", "fsmonitor_token_closure", s->repo); + return closure.refresh_result; +} + +int wt_status_refresh_index(struct wt_status *s, + unsigned int refresh_flags, + int require_untracked) +{ + return wt_status_close_fsmonitor_token( + s, refresh_flags, require_untracked, 0); } static int has_unmerged(struct wt_status *s) @@ -906,6 +1083,15 @@ static int has_unmerged(struct wt_status *s) void wt_status_collect(struct wt_status *s) { + int used_untracked_cache; + + if (fsm_settings__get_mode(s->repo) > FSMONITOR_MODE_DISABLED) + wt_status_finish_untracked_cache_preload(s); + wt_status_close_fsmonitor_token( + s, REFRESH_QUIET | REFRESH_UNMERGED, + s->show_untracked_files != SHOW_NO_UNTRACKED_FILES && + !s->show_ignored_mode, 1); + trace2_region_enter("status", "worktrees", s->repo); wt_status_collect_changes_worktree(s); trace2_region_leave("status", "worktrees", s->repo); @@ -921,9 +1107,20 @@ void wt_status_collect(struct wt_status *s) } trace2_region_enter("status", "untracked", s->repo); - wt_status_collect_untracked(s); + used_untracked_cache = wt_status_collect_untracked(s); trace2_region_leave("status", "untracked", s->repo); + /* Hook providers have no second query with which to close the scan. */ + if (fsmonitor_has_pending_token(s->repo->index) && !s->pathspec.nr && + fsm_settings__get_mode(s->repo) == FSMONITOR_MODE_HOOK && + (used_untracked_cache || !s->repo->index->untracked || + !s->repo->index->untracked->root)) { + if (fsmonitor_pending_token_from_provider(s->repo->index)) + fsmonitor_accept_pending_token(s->repo->index); + else + fsmonitor_reject_pending_token(s->repo->index); + } + wt_status_get_state(s->repo, &s->state, s->branch && !strcmp(s->branch, "HEAD")); if (s->state.merge_in_progress && !has_unmerged(s)) s->committable = 1; diff --git a/wt-status.h b/wt-status.h index e64eda2d9cc666..34beac22576fc9 100644 --- a/wt-status.h +++ b/wt-status.h @@ -139,6 +139,7 @@ struct wt_status { /* These are computed during processing of the individual sections */ int committable; int workdir_dirty; + unsigned untracked_from_token_closure : 1; const char *index_file; FILE *fp; const char *prefix; @@ -157,6 +158,13 @@ void wt_status_prepare(struct repository *r, struct wt_status *s); void wt_status_print(struct wt_status *s); void wt_status_collect(struct wt_status *s); void wt_status_start_untracked_cache_preload(struct wt_status *s); +/* + * Refresh tracked entries and close any provider token. When requested, + * also close a complete untracked-cache scan before accepting that token. + */ +int wt_status_refresh_index(struct wt_status *s, + unsigned int refresh_flags, + int require_untracked); /* * Collect all changes between the two trees. Changes will be displayed as if From 6ebc45d9e144f8515af6aa113c6be8bf67f38c27 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 15:24:55 -0500 Subject: [PATCH 053/105] dir: reject oversized pattern files before allocation add_patterns() rejects pattern files larger than 100 MiB only after allocating and reading their complete contents. An oversized filesystem input can therefore exhaust the memory the limit is meant to protect, or terminate Git when GIT_ALLOC_LIMIT rejects the allocation. Check the size obtained from fstat() before allocating a filesystem pattern buffer. Preserve the existing warning, close the descriptor, and return the existing failure result. Keep the later size check for index-backed fallback data, whose size is unavailable before it is read. Strengthen the existing EXPENSIVE regression by reading its 101 MiB .gitignore under GIT_ALLOC_LIMIT=1m. The old ordering dies in xmallocz(); the early rejection preserves the expected warning without attempting the oversized allocation. Signed-off-by: Taylor Blau --- dir.c | 6 ++++++ t/t0008-ignores.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/dir.c b/dir.c index 95d8a1cce90f77..8b8beb1281bf2f 100644 --- a/dir.c +++ b/dir.c @@ -1175,6 +1175,12 @@ static int add_patterns(const char *fname, const char *base, int baselen, return r; } else { size = xsize_t(st.st_size); + if (size > PATTERN_MAX_FILE_SIZE) { + warning("ignoring excessively large pattern file: %s", + fname); + close(fd); + return -1; + } if (size == 0) { if (oid_stat) { fill_stat_data(&oid_stat->stat, &st); diff --git a/t/t0008-ignores.sh b/t/t0008-ignores.sh index ed95faf3272e60..949897c36aa9b2 100755 --- a/t/t0008-ignores.sh +++ b/t/t0008-ignores.sh @@ -959,7 +959,7 @@ test_expect_success EXPENSIVE 'large exclude file ignored in tree' ' test_when_finished "rm .gitignore" && find . -name .gitignore -exec rm "{}" ";" && dd if=/dev/zero of=.gitignore bs=101M count=1 && - git ls-files -o --exclude-standard 2>err && + GIT_ALLOC_LIMIT=1m git ls-files -o --exclude-standard 2>err && echo "warning: ignoring excessively large pattern file: .gitignore" >expect && test_cmp expect err ' From 452d90d5f36a9fdacf47d7ef3cc35b6037a8410e Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Thu, 23 Jul 2026 23:55:56 -0500 Subject: [PATCH 054/105] status: consume definitive bulk changes exactly once A complete APFS preload can already prove that tracked entries are deleted or have definitive size changes. Ordinary status nevertheless refreshes those entries and later asks worktree diff to rediscover them. Skipping refresh without preserving ambiguous content checks would either duplicate work or misreport metadata-only changes. Request terminal-result deferral explicitly from porcelain status and retain a complete per-entry result on its index. Let refresh defer proven modifications and deletions while marking ambiguous entries for the ordinary content check. Insert terminal changes into the normal status change list before running worktree diff, temporarily mark only those entries up to date, and restore their flags afterward. Clear retained results before another preload and release them with the index. Other refresh callers keep their existing behavior. Extend the APFS tests to assert direct modified and deleted results and no redundant refresh stats. Add a metadata-only mismatch that must still reach worktree diff and produce clean porcelain output. Retained terminal state trades additional temporary memory for removing the second classification of proven changes. Signed-off-by: Taylor Blau --- builtin/commit.c | 3 +- preload-index.c | 25 ++++++++-- preload-index.h | 1 + read-cache-ll.h | 3 ++ read-cache.c | 17 ++++++- t/t7529-preload-index-apfs.sh | 18 ++++++- wt-status.c | 93 +++++++++++++++++++++++++++++------ 7 files changed, 139 insertions(+), 21 deletions(-) diff --git a/builtin/commit.c b/builtin/commit.c index e2f4d08b347707..fa64ba01f2a5e7 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1629,7 +1629,8 @@ struct repository *repo UNUSED) repo_read_index(the_repository); wt_status_start_untracked_cache_preload(&s); wt_status_refresh_index( - &s, REFRESH_QUIET | REFRESH_UNMERGED | progress_flag, + &s, REFRESH_QUIET | REFRESH_UNMERGED | progress_flag | + REFRESH_DEFER_BULK_DIRTY, s.show_untracked_files != SHOW_NO_UNTRACKED_FILES && !s.show_ignored_mode); diff --git a/preload-index.c b/preload-index.c index 9e082af764a82d..72d37ae93e67d0 100644 --- a/preload-index.c +++ b/preload-index.c @@ -319,8 +319,26 @@ static unsigned char *preload_bulk_try(struct index_state *index) preload_bulk_result_release(&result); return tracked_state; } + +static void preload_bulk_finish_state(struct index_state *index, + unsigned char **state, + unsigned int refresh_flags) +{ + if ((refresh_flags & REFRESH_DEFER_BULK_DIRTY) && *state) { + index->preload_bulk_tracked_state = *state; + index->preload_bulk_tracked_nr = index->cache_nr; + *state = NULL; + } + FREE_AND_NULL(*state); +} #endif +void preload_index_bulk_result_clear(struct index_state *index) +{ + FREE_AND_NULL(index->preload_bulk_tracked_state); + index->preload_bulk_tracked_nr = 0; +} + void preload_index(struct index_state *index, const struct pathspec *pathspec, unsigned int refresh_flags) @@ -334,6 +352,7 @@ void preload_index(struct index_state *index, int t2_sum_lstat = 0; int core_preload_index = 1; + preload_index_bulk_result_clear(index); repo_config_get_bool(index->repo, "core.preloadindex", &core_preload_index); if (!core_preload_index) @@ -345,7 +364,7 @@ void preload_index(struct index_state *index, #endif if (!HAVE_THREADS) { #ifdef HAVE_PRELOAD_INDEX_BULK - free(bulk_state); + preload_bulk_finish_state(index, &bulk_state, refresh_flags); #endif return; } @@ -355,7 +374,7 @@ void preload_index(struct index_state *index, threads = 2; if (threads < 2) { #ifdef HAVE_PRELOAD_INDEX_BULK - free(bulk_state); + preload_bulk_finish_state(index, &bulk_state, refresh_flags); #endif return; } @@ -405,7 +424,7 @@ void preload_index(struct index_state *index, } stop_progress(&pd.progress); #ifdef HAVE_PRELOAD_INDEX_BULK - free(bulk_state); + preload_bulk_finish_state(index, &bulk_state, refresh_flags); #endif if (pathspec) { diff --git a/preload-index.h b/preload-index.h index 01d90e06bb6b3f..bb6deb6130cc2f 100644 --- a/preload-index.h +++ b/preload-index.h @@ -20,5 +20,6 @@ void preload_index(struct index_state *index, int repo_read_index_preload(struct repository *, const struct pathspec *pathspec, unsigned refresh_flags); +void preload_index_bulk_result_clear(struct index_state *index); #endif /* PRELOAD_INDEX_H */ diff --git a/read-cache-ll.h b/read-cache-ll.h index d44a946ed04d36..99b95d8e8d5325 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -194,6 +194,8 @@ struct index_state { struct hashmap dir_hash; struct object_id oid; struct untracked_cache *untracked; + unsigned char *preload_bulk_tracked_state; + size_t preload_bulk_tracked_nr; char *fsmonitor_last_update; char *fsmonitor_last_update_pending; char *fsmonitor_untracked_token; @@ -474,6 +476,7 @@ int fake_lstat(const struct cache_entry *ce, struct stat *st); #define REFRESH_IN_PORCELAIN (1 << 5) /* user friendly output, not "needs update" */ #define REFRESH_PROGRESS (1 << 6) /* show progress bar if stderr is tty */ #define REFRESH_IGNORE_SKIP_WORKTREE (1 << 7) /* ignore skip_worktree entries */ +#define REFRESH_DEFER_BULK_DIRTY (1 << 8) /* leave bulk results to diff */ int refresh_index(struct index_state *, unsigned int flags, const struct pathspec *pathspec, char *seen, const char *header_msg); /* * Refresh the index and write it to disk. diff --git a/read-cache.c b/read-cache.c index cf3600f2a337d6..f606bf81ddaca2 100644 --- a/read-cache.c +++ b/read-cache.c @@ -1550,7 +1550,7 @@ int refresh_index(struct index_state *istate, unsigned int flags, * cache entries quickly then in the single threaded loop below, * we only have to do the special cases that are left. */ - preload_index(istate, pathspec, 0); + preload_index(istate, pathspec, flags & REFRESH_DEFER_BULK_DIRTY); trace2_region_enter("index", "refresh", NULL); for (i = 0; i < istate->cache_nr; i++) { @@ -1594,6 +1594,20 @@ int refresh_index(struct index_state *istate, unsigned int flags, if (filtered) continue; + if ((flags & REFRESH_DEFER_BULK_DIRTY) && + istate->preload_bulk_tracked_nr == istate->cache_nr) { + unsigned char state = + istate->preload_bulk_tracked_state[i]; + + if (state == PRELOAD_BULK_TRACKED_CONTENT_CHECK) { + ce->ce_flags |= CE_CONTENT_CHECK_REQUIRED; + continue; + } + if (state == PRELOAD_BULK_TRACKED_DEFINITIVE_MODIFIED || + state == PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED) + continue; + } + new_entry = refresh_cache_ent(istate, ce, options, &cache_errno, &changed, &t2_did_lstat, &t2_did_scan); @@ -2473,6 +2487,7 @@ void release_index(struct index_state *istate) free(istate->fsmonitor_last_update); free(istate->fsmonitor_last_update_pending); free(istate->fsmonitor_untracked_token); + free(istate->preload_bulk_tracked_state); free(istate->cache); discard_split_index(istate); free_untracked_cache(istate->untracked); diff --git a/t/t7529-preload-index-apfs.sh b/t/t7529-preload-index-apfs.sh index 1c1cc4ed0a3785..e5c2c5015b0c6d 100755 --- a/t/t7529-preload-index-apfs.sh +++ b/t/t7529-preload-index-apfs.sh @@ -197,8 +197,10 @@ test_expect_success 'definitive size changes are not restated' ' test_file_not_empty actual && check_data dirty.trace preload/bulk_applied 7 && check_data dirty.trace preload/bulk_definitive_modified 1 && + test_trace2_data status preload/direct_modified 1 \ + <"$TRASH_DIRECTORY/dirty.trace" && check_lstat_data dirty.trace 0 && - check_data dirty.trace refresh/sum_lstat 1 + check_data dirty.trace refresh/sum_lstat 0 ' test_expect_success 'missing entries bypass speculative lstat' ' @@ -209,8 +211,20 @@ test_expect_success 'missing entries bypass speculative lstat' ' test_line_count = 5 actual && check_data missing.trace preload/bulk_applied 3 && check_data missing.trace preload/bulk_definitive_deleted 5 && + test_trace2_data status preload/direct_deleted 5 \ + <"$TRASH_DIRECTORY/missing.trace" && check_lstat_data missing.trace 0 && - check_data missing.trace refresh/sum_lstat 5 + check_data missing.trace refresh/sum_lstat 0 +' + +test_expect_success 'metadata-only mismatches are checked by diff' ' + setup_repo metadata && + test-tool chmtime +60 metadata/root && + compare_status metadata metadata.trace && + test_must_be_empty actual && + check_data metadata.trace preload/bulk_content_check 1 && + check_lstat_data metadata.trace 0 && + check_data metadata.trace refresh/sum_lstat 0 ' test_expect_success PIPE \ diff --git a/wt-status.c b/wt-status.c index 57e2321275dc07..7ea206bdc9609b 100644 --- a/wt-status.c +++ b/wt-status.c @@ -14,6 +14,7 @@ #include "hex.h" #include "object-name.h" #include "path.h" +#include "preload-index.h" #include "revision.h" #include "diffcore.h" #include "quote.h" @@ -458,6 +459,19 @@ static char short_submodule_status(struct wt_status_change_data *d) return d->worktree_status; } +static struct wt_status_change_data *wt_status_get_change( + struct wt_status *s, const char *path) +{ + struct string_list_item *it = string_list_insert(&s->change, path); + struct wt_status_change_data *d = it->util; + + if (!d) { + CALLOC_ARRAY(d, 1); + it->util = d; + } + return d; +} + static void wt_status_collect_changed_cb(struct diff_queue_struct *q, struct diff_options *options UNUSED, void *data) @@ -470,16 +484,10 @@ static void wt_status_collect_changed_cb(struct diff_queue_struct *q, s->workdir_dirty = 1; for (i = 0; i < q->nr; i++) { struct diff_filepair *p; - struct string_list_item *it; struct wt_status_change_data *d; p = q->queue[i]; - it = string_list_insert(&s->change, p->two->path); - d = it->util; - if (!d) { - CALLOC_ARRAY(d, 1); - it->util = d; - } + d = wt_status_get_change(s, p->two->path); if (!d->worktree_status) d->worktree_status = p->status; if (S_ISGITLINK(p->two->mode)) { @@ -525,6 +533,64 @@ static void wt_status_collect_changed_cb(struct diff_queue_struct *q, } } +static struct cache_entry **wt_status_collect_preload_changes( + struct wt_status *s, size_t *direct_nr) +{ + struct index_state *istate = s->repo->index; + struct cache_entry **direct = NULL; + size_t direct_alloc = 0; + uint64_t modified = 0, deleted = 0; + + *direct_nr = 0; + if (istate->preload_bulk_tracked_nr != istate->cache_nr) + goto clear; + for (size_t i = 0; i < istate->cache_nr; i++) { + struct cache_entry *ce = istate->cache[i]; + struct wt_status_change_data *d; + unsigned char state = + istate->preload_bulk_tracked_state[i]; + int status; + + if (state == PRELOAD_BULK_TRACKED_DEFINITIVE_MODIFIED) { + status = DIFF_STATUS_MODIFIED; + modified++; + } else if (state == PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED) { + status = DIFF_STATUS_DELETED; + deleted++; + } else { + continue; + } + + d = wt_status_get_change(s, ce->name); + if (!d->worktree_status) + d->worktree_status = status; + d->mode_index = ce->ce_mode; + d->mode_worktree = status == DIFF_STATUS_MODIFIED ? + ce->ce_mode : 0; + oidcpy(&d->oid_index, &ce->oid); + ce_mark_uptodate(ce); + ALLOC_GROW(direct, *direct_nr + 1, direct_alloc); + direct[(*direct_nr)++] = ce; + s->workdir_dirty = 1; + } + trace2_data_intmax("status", s->repo, "preload/direct_modified", + modified); + trace2_data_intmax("status", s->repo, "preload/direct_deleted", + deleted); + +clear: + preload_index_bulk_result_clear(istate); + return direct; +} + +static void wt_status_release_preload_changes( + struct cache_entry **direct, size_t direct_nr) +{ + for (size_t i = 0; i < direct_nr; i++) + direct[i]->ce_flags &= ~CE_UPTODATE; + free(direct); +} + static int unmerged_mask(struct index_state *istate, const char *path) { int pos, mask; @@ -554,16 +620,10 @@ static void wt_status_collect_updated_cb(struct diff_queue_struct *q, for (i = 0; i < q->nr; i++) { struct diff_filepair *p; - struct string_list_item *it; struct wt_status_change_data *d; p = q->queue[i]; - it = string_list_insert(&s->change, p->two->path); - d = it->util; - if (!d) { - CALLOC_ARRAY(d, 1); - it->util = d; - } + d = wt_status_get_change(s, p->two->path); if (!d->index_status) d->index_status = p->status; switch (p->status) { @@ -639,8 +699,11 @@ void wt_status_collect_changes_trees(struct wt_status *s, static void wt_status_collect_changes_worktree(struct wt_status *s) { + struct cache_entry **direct; + size_t direct_nr; struct rev_info rev; + direct = wt_status_collect_preload_changes(s, &direct_nr); repo_init_revisions(s->repo, &rev, NULL); setup_revisions(0, NULL, &rev, NULL); rev.diffopt.output_format |= DIFF_FORMAT_CALLBACK; @@ -661,6 +724,7 @@ static void wt_status_collect_changes_worktree(struct wt_status *s) rev.diffopt.rename_score = s->rename_score >= 0 ? s->rename_score : rev.diffopt.rename_score; copy_pathspec(&rev.prune_data, &s->pathspec); run_diff_files(&rev, 0); + wt_status_release_preload_changes(direct, direct_nr); release_revisions(&rev); } @@ -850,6 +914,7 @@ static void wt_status_finish_untracked_cache_preload(struct wt_status *s) if (!index_invalidated) return; + preload_index_bulk_result_clear(istate); trace2_data_intmax("status", s->repo, "fsmonitor/exclude-index-invalidated", index_invalidated); From f47e1cf9aeec9c0f82ad8a5bd4874f0ff1b76114 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 13:01:53 -0500 Subject: [PATCH 055/105] read-cache: decode bounded TREE and UNTR extensions in parallel The index extension worker decodes TREE and UNTR serially even though their parsers read the same immutable mapping and publish to different index_state fields. An unconditional additional worker would consume cache-entry workers and interfere with split-index assembly. Use the bounded framing from S02/P01 to select exactly one TREE and one UNTR extension. Require extension-offset metadata and at least four index workers; start an additional TREE worker only when both payloads reach 1 MiB. Leave at least two cache-entry workers available and join the TREE worker before unmapping the index. Keep LINK, duplicate or missing extensions, insufficient workers, small payloads, and auxiliary-worker creation failures on the existing serial path. Malformed framing still reports index file corruption. Allow GIT_TEST_PARALLEL_INDEX_EXTENSIONS to bypass only the payload threshold. Add a PTHREADS, UNTRACKED_CACHE, and SHA1 regression that compares parallel and serial status, cache-tree, and untracked-cache results and checks the extension/parallel/tree-untracked Trace2 marker. The regression unsets GIT_TEST_SPLIT_INDEX because split indexes intentionally remain on the serial path. The eligible path adds one auxiliary worker and its stack. The benchmark covers the complete series, not this patch in isolation. Signed-off-by: Taylor Blau --- read-cache.c | 117 +++++++++++++++++++++++++++++++++--- t/t7519-status-fsmonitor.sh | 39 ++++++++++++ 2 files changed, 148 insertions(+), 8 deletions(-) diff --git a/read-cache.c b/read-cache.c index 76941114ba8498..5558c83b01e092 100644 --- a/read-cache.c +++ b/read-cache.c @@ -1994,23 +1994,104 @@ struct load_index_extensions const char *mmap; size_t mmap_size; unsigned long src_offset; + int allow_parallel; + int force_parallel; }; +struct load_index_extension { + pthread_t pthread; + struct index_state *istate; + const char *ext; + const char *data; + unsigned long size; + int result; +}; + +#define PARALLEL_INDEX_EXTENSION_THRESHOLD (1024 * 1024) + +static void *load_one_index_extension(void *_data) +{ + struct load_index_extension *p = _data; + + trace2_thread_start("index-extension"); + trace2_data_intmax("index", p->istate->repo, + "extension/parallel/tree-untracked", 1); + p->result = read_index_extension(p->istate, p->ext, p->data, p->size); + trace2_thread_exit(); + return NULL; +} + +/* + * TREE and UNTR are usually the two largest index extensions. They read the + * same immutable mmap but publish to separate index_state fields, so they can + * be decoded concurrently. Keep split indexes on the established serial + * path because LINK changes how the completed index is assembled. + */ +static int find_parallel_index_extensions(struct load_index_extensions *p, + struct load_index_extension *tree) +{ + size_t offset = p->src_offset; + size_t end = p->mmap_size - the_hash_algo->rawsz; + int tree_nr = 0, untracked_nr = 0, link_nr = 0; + uint32_t untracked_size = 0; + + if (!p->allow_parallel) + return 0; + + while (offset <= end - 8) { + const char *ext = p->mmap + offset; + uint32_t size = get_be32(ext + 4); + + if (size > end - offset - 8) + return 0; + + switch (CACHE_EXT(ext)) { + case CACHE_EXT_TREE: + tree_nr++; + tree->istate = p->istate; + tree->ext = ext; + tree->data = ext + 8; + tree->size = size; + break; + case CACHE_EXT_UNTRACKED: + untracked_nr++; + untracked_size = size; + break; + case CACHE_EXT_LINK: + link_nr++; + break; + } + + offset += 8 + size; + } + + return offset == end && tree_nr == 1 && untracked_nr == 1 && !link_nr && + (p->force_parallel || + (tree->size >= PARALLEL_INDEX_EXTENSION_THRESHOLD && + untracked_size >= PARALLEL_INDEX_EXTENSION_THRESHOLD)); +} + static void *load_index_extensions(void *_data) { struct load_index_extensions *p = _data; size_t src_offset = p->src_offset; size_t end; + struct load_index_extension tree = { 0 }; + int tree_thread = 0; int extension_error = 0; if (p->mmap_size < the_hash_algo->rawsz) { extension_error = 1; - goto done; + goto join_tree; } end = p->mmap_size - the_hash_algo->rawsz; if (src_offset > end) { extension_error = 1; - goto done; + goto join_tree; + } + if (find_parallel_index_extensions(p, &tree) && + !pthread_create(&tree.pthread, NULL, load_one_index_extension, &tree)) { + tree_thread = 1; } while (src_offset < end) { @@ -2021,20 +2102,19 @@ static void *load_index_extensions(void *_data) * in 4-byte network byte order. */ uint32_t extsize; + const char *ext = p->mmap + src_offset; if (end - src_offset < 8) { extension_error = 1; break; } - extsize = get_be32(p->mmap + src_offset + 4); + extsize = get_be32(ext + 4); if (extsize > end - src_offset - 8) { extension_error = 1; break; } - if (read_index_extension(p->istate, - p->mmap + src_offset, - p->mmap + src_offset + 8, - extsize) < 0) { + if ((!tree_thread || CACHE_EXT(ext) != CACHE_EXT_TREE) && + read_index_extension(p->istate, ext, ext + 8, extsize) < 0) { extension_error = 1; break; } @@ -2043,11 +2123,22 @@ static void *load_index_extensions(void *_data) if (src_offset != end) extension_error = 1; -done: +join_tree: + if (tree_thread) { + int err = pthread_join(tree.pthread, NULL); + + if (err) + die(_("unable to join load_index_extension thread: %s"), + strerror(err)); + if (tree.result < 0) + extension_error = 1; + } + if (extension_error) { munmap((void *)p->mmap, p->mmap_size); die(_("index file corrupt")); } + return NULL; } @@ -2279,6 +2370,8 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) p.istate = istate; p.mmap = mmap; p.mmap_size = mmap_size; + p.allow_parallel = 0; + p.force_parallel = git_env_bool("GIT_TEST_PARALLEL_INDEX_EXTENSIONS", 0); src_offset = sizeof(*hdr); @@ -2300,13 +2393,21 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) extension_offset = read_eoie_extension(mmap, mmap_size); if (extension_offset) { int err; + struct load_index_extension tree = { 0 }; p.src_offset = extension_offset; + /* Keep at least two workers available for cache entries. */ + p.allow_parallel = nr_threads > 3; + if (p.allow_parallel) + p.allow_parallel = + find_parallel_index_extensions(&p, &tree); err = pthread_create(&p.pthread, NULL, load_index_extensions, &p); if (err) die(_("unable to create load_index_extensions thread: %s"), strerror(err)); nr_threads--; + if (p.allow_parallel) + nr_threads--; } } diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 2e90955b52c374..273cda0a0f6bc8 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -502,4 +502,43 @@ test_expect_success 'status succeeds with sparse index' ' ) ' +test_expect_success PTHREADS,UNTRACKED_CACHE,SHA1 'load cache-tree and untracked-cache extensions in parallel' ' + test_create_repo parallel-extensions && + ( + cd parallel-extensions && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir dir && + echo tracked >dir/tracked && + git config index.threads 4 && + git add dir/tracked && + git commit -m initial && + git config core.untrackedCache true && + git status --porcelain >/dev/null && + echo modified >>dir/tracked && + echo untracked >dir/untracked && + GIT_TEST_INDEX_THREADS=1 \ + git --no-optional-locks status --porcelain >"$TRASH_DIRECTORY/parallel-serial.status" && + GIT_TEST_INDEX_THREADS=1 \ + test-tool dump-cache-tree >"$TRASH_DIRECTORY/parallel-serial.tree" && + GIT_TEST_INDEX_THREADS=1 \ + test-tool dump-untracked-cache >"$TRASH_DIRECTORY/parallel-serial.untracked" && + GIT_TEST_INDEX_THREADS=4 \ + GIT_TEST_PARALLEL_INDEX_EXTENSIONS=1 \ + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/parallel-extensions.trace" \ + git --no-optional-locks status --porcelain >"$TRASH_DIRECTORY/parallel-parallel.status" && + GIT_TEST_INDEX_THREADS=4 GIT_TEST_PARALLEL_INDEX_EXTENSIONS=1 \ + test-tool dump-cache-tree >"$TRASH_DIRECTORY/parallel-parallel.tree" && + GIT_TEST_INDEX_THREADS=4 GIT_TEST_PARALLEL_INDEX_EXTENSIONS=1 \ + test-tool dump-untracked-cache >"$TRASH_DIRECTORY/parallel-parallel.untracked" && + test_grep "extension/parallel/tree-untracked" \ + "$TRASH_DIRECTORY/parallel-extensions.trace" && + test_cmp "$TRASH_DIRECTORY/parallel-serial.status" \ + "$TRASH_DIRECTORY/parallel-parallel.status" && + test_cmp "$TRASH_DIRECTORY/parallel-serial.tree" \ + "$TRASH_DIRECTORY/parallel-parallel.tree" && + test_cmp "$TRASH_DIRECTORY/parallel-serial.untracked" \ + "$TRASH_DIRECTORY/parallel-parallel.untracked" + ) +' + test_done From 1bdde0504818f159973d11ec74cc7e31e90d5488 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 13:51:15 -0500 Subject: [PATCH 056/105] status: refresh verified writable bulk-preload entries Bulk preload defers metadata-mismatched entries to run_diff_files() for a content check. When writable status confirms that such an entry is clean, it still leaves old stat data in the index. The next status must therefore repeat a content check already known to match. Request DIFF_UPDATE_INDEX_STAT only when status holds the index lock and bulk preload covers every indexed entry. After a real stat and a successful content and mode check, refresh only entries marked CE_CONTENT_CHECK_REQUIRED. Build the replacement with the helper from S15/P01 and install it with replace_index_entry(), preserving existing CE_VALID and index-change handling. Read-only status, incomplete bulk scans, dirty entries, and ordinary diff callers keep their existing behavior. The APFS regression compares both status output and the written index with ordinary status, and requires one bulk content check with no refresh-time lstat. Signed-off-by: Taylor Blau --- builtin/commit.c | 4 ++++ diff-lib.c | 10 ++++++++-- diff.h | 2 ++ read-cache-ll.h | 3 +++ read-cache.c | 7 +++++++ t/t7529-preload-index-apfs.sh | 30 ++++++++++++++++++++++++++++++ wt-status.c | 3 ++- wt-status.h | 1 + 8 files changed, 57 insertions(+), 3 deletions(-) diff --git a/builtin/commit.c b/builtin/commit.c index fa64ba01f2a5e7..be04f9a6943590 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1638,6 +1638,10 @@ struct repository *repo UNUSED) fd = repo_hold_locked_index(the_repository, &index_lock, 0); else fd = -1; + s.bulk_update_index_stat = + 0 <= fd && + the_repository->index->preload_bulk_tracked_nr == + the_repository->index->cache_nr; s.is_initial = repo_get_oid(the_repository, s.reference, &oid) ? 1 : 0; if (!s.is_initial) diff --git a/diff-lib.c b/diff-lib.c index 33b4233738cac3..8f8aba3038a8a6 100644 --- a/diff-lib.c +++ b/diff-lib.c @@ -132,6 +132,8 @@ void run_diff_files(struct rev_info *revs, unsigned int option) unsigned int oldmode, newmode; int fsmonitor_valid = 0; struct cache_entry *ce = istate->cache[i]; + struct stat st; + int has_stat = 0; int changed; unsigned dirty_submodule = 0; const struct object_id *old_oid, *new_oid; @@ -253,8 +255,6 @@ void run_diff_files(struct rev_info *revs, unsigned int option) fsmonitor_valid = !!(ce->ce_flags & CE_FSMONITOR_VALID); } else { - struct stat st; - changed = check_removed(ce, &st); if (changed) { if (changed < 0) { @@ -276,11 +276,17 @@ void run_diff_files(struct rev_info *revs, unsigned int option) changed = match_stat_with_submodule(&revs->diffopt, ce, &st, ce_option, &dirty_submodule); + has_stat = 1; newmode = ce_mode_from_stat(revs->repo, ce, st.st_mode); fsmonitor_valid = fsmonitor_stat_can_be_valid(&st); } if (!changed && !dirty_submodule) { + if ((option & DIFF_UPDATE_INDEX_STAT) && has_stat && + (ce->ce_flags & CE_CONTENT_CHECK_REQUIRED)) { + refresh_index_entry_stat(istate, i, &st); + ce = istate->cache[i]; + } ce_mark_uptodate(ce); if (fsmonitor_valid) mark_fsmonitor_valid(istate, ce); diff --git a/diff.h b/diff.h index bb5cddaf3499e9..eb81289415f8f3 100644 --- a/diff.h +++ b/diff.h @@ -698,6 +698,8 @@ void diff_get_merge_base(const struct rev_info *revs, struct object_id *mb); #define DIFF_SILENT_ON_REMOVED 01 /* report racily-clean paths as modified */ #define DIFF_RACY_IS_MODIFIED 02 +/* update index stat data for content-checked entries */ +#define DIFF_UPDATE_INDEX_STAT 04 void run_diff_files(struct rev_info *revs, unsigned int option); #define DIFF_INDEX_CACHED 01 diff --git a/read-cache-ll.h b/read-cache-ll.h index 99b95d8e8d5325..84d5d1a015b2b9 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -497,6 +497,9 @@ int repo_refresh_and_write_index(struct repository*, unsigned int refresh_flags, struct cache_entry *refresh_cache_entry(struct index_state *, struct cache_entry *, unsigned int); +/* The caller must first verify the entry's content and mode against st. */ +void refresh_index_entry_stat(struct index_state *, int, struct stat *); + void set_alternate_index_output(const char *); extern int verify_index_checksum; diff --git a/read-cache.c b/read-cache.c index 088f83559a6b56..8c61bc7b7253e9 100644 --- a/read-cache.c +++ b/read-cache.c @@ -222,6 +222,13 @@ static struct cache_entry *make_refreshed_cache_entry( return updated; } +void refresh_index_entry_stat(struct index_state *istate, int nr, + struct stat *st) +{ + replace_index_entry(istate, nr, make_refreshed_cache_entry( + istate, istate->cache[nr], st, 1)); +} + static unsigned int st_mode_from_ce(const struct cache_entry *ce) { switch (ce->ce_mode & S_IFMT) { diff --git a/t/t7529-preload-index-apfs.sh b/t/t7529-preload-index-apfs.sh index e5c2c5015b0c6d..957002c565b2cf 100755 --- a/t/t7529-preload-index-apfs.sh +++ b/t/t7529-preload-index-apfs.sh @@ -59,6 +59,22 @@ bulk_status () { git -C "$repo" status --porcelain=v2 >"$output" } +writable_ordinary_status () { + git -C "$1" -c core.preloadIndex=false \ + status --porcelain=v2 >"$2" +} + +writable_bulk_status () { + repo=$1 && + output=$2 && + writable_trace=$TRASH_DIRECTORY/$3 && + rm -f "$writable_trace" && + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TRACE2_EVENT="$writable_trace" \ + git -C "$repo" status --porcelain=v2 >"$output" +} + check_data () { test_trace2_data index "$2" "$3" <"$TRASH_DIRECTORY/$1" } @@ -241,6 +257,20 @@ test_expect_success PIPE \ check_data tracked-types.trace refresh/sum_lstat 2 ' +test_expect_success 'writable status retains refreshed stat data' ' + setup_repo writable-stat && + test-tool chmtime +60 writable-stat/root && + cp writable-stat/.git/index before.index && + writable_ordinary_status writable-stat expect && + cp writable-stat/.git/index ordinary.index && + cp before.index writable-stat/.git/index && + writable_bulk_status writable-stat actual writable-stat.trace && + test_cmp expect actual && + test_cmp ordinary.index writable-stat/.git/index && + check_data writable-stat.trace preload/bulk_content_check 1 && + check_data writable-stat.trace refresh/sum_lstat 0 +' + test_expect_success CASE_INSENSITIVE_FS \ 'case aliases retain parallel preload' ' setup_repo case-alias && diff --git a/wt-status.c b/wt-status.c index 7ea206bdc9609b..f9734d63c75494 100644 --- a/wt-status.c +++ b/wt-status.c @@ -723,7 +723,8 @@ static void wt_status_collect_changes_worktree(struct wt_status *s) rev.diffopt.rename_limit = s->rename_limit >= 0 ? s->rename_limit : rev.diffopt.rename_limit; rev.diffopt.rename_score = s->rename_score >= 0 ? s->rename_score : rev.diffopt.rename_score; copy_pathspec(&rev.prune_data, &s->pathspec); - run_diff_files(&rev, 0); + run_diff_files(&rev, s->bulk_update_index_stat ? + DIFF_UPDATE_INDEX_STAT : 0); wt_status_release_preload_changes(direct, direct_nr); release_revisions(&rev); } diff --git a/wt-status.h b/wt-status.h index 34beac22576fc9..e5cdc803f885e4 100644 --- a/wt-status.h +++ b/wt-status.h @@ -140,6 +140,7 @@ struct wt_status { int committable; int workdir_dirty; unsigned untracked_from_token_closure : 1; + unsigned bulk_update_index_stat : 1; const char *index_file; FILE *fp; const char *prefix; From 63f24e712e1c9a4d7d12329f2fcb11a2b8a71f60 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 22:24:52 -0700 Subject: [PATCH 057/105] status: verify semantic proof ranges in parallel Serial proof preparation hashes every eligible indexed file on one worker. Independent index ranges can instead use separate pinned directory state, conversion checks, buffers, and result counters without allowing workers to mutate shared cache entries. Partition the index into at most 32 bounded ranges after preparing conversion state on the calling thread. Start one worker per range, join started workers, and merge their stat updates and counters in index order. Without thread support, use one worker; if thread creation fails, finish unstarted ranges synchronously. Extend test-tool semantic-verify with an explicit thread count. Add a regression that compares the complete results for one and four workers over nested directories with distinct attribute files. Each worker requires its own 256 KiB hash buffer and attribute check. The regression establishes deterministic classifications, not a timed speedup, and no production status command enables this verifier. Signed-off-by: Taylor Blau --- semantic-verify-internal.h | 3 ++ semantic-verify.c | 90 +++++++++++++++++++++++++++++---- semantic-verify.h | 7 +++ t/helper/test-semantic-verify.c | 9 +++- t/t7531-semantic-verify.sh | 20 ++++++++ 5 files changed, 118 insertions(+), 11 deletions(-) diff --git a/semantic-verify-internal.h b/semantic-verify-internal.h index 14b97921bd8d9a..44d7fa49bee013 100644 --- a/semantic-verify-internal.h +++ b/semantic-verify-internal.h @@ -3,6 +3,7 @@ #include "hash.h" #include "statinfo.h" +#include "thread-utils.h" #ifdef __linux__ #include @@ -98,6 +99,8 @@ struct semantic_verify_entry_identity { }; struct semantic_verify_worker { + pthread_t pthread; + unsigned int started; struct index_state *istate; struct semantic_verify_root *root; struct semantic_verify_result *results; diff --git a/semantic-verify.c b/semantic-verify.c index 4e9ec859c0d7cd..0604ae6cde7d4a 100644 --- a/semantic-verify.c +++ b/semantic-verify.c @@ -13,6 +13,37 @@ #define SEMANTIC_VERIFY_ENTRY_FLAGS \ (CE_VALID | CE_STAGEMASK | CE_INTENT_TO_ADD | CE_SKIP_WORKTREE | \ CE_UPTODATE | CE_FSMONITOR_VALID | CE_CONTENT_CHECK_REQUIRED) +#define SEMANTIC_VERIFY_MAX_THREADS 32 + +static void *run_worker(void *data) +{ + semantic_verify_worker_run(data); + return NULL; +} + +static unsigned int select_thread_count( + size_t cache_nr, + const struct semantic_verify_options *options) +{ + unsigned int nr; + + if (!HAVE_THREADS) + return 1; + if (options && options->nr_threads) { + nr = options->nr_threads; + } else { + unsigned int cpus = online_cpus(); + + nr = cpus > SEMANTIC_VERIFY_MAX_THREADS / 2 ? + SEMANTIC_VERIFY_MAX_THREADS : cpus * 2; + } + if (nr > SEMANTIC_VERIFY_MAX_THREADS) + nr = SEMANTIC_VERIFY_MAX_THREADS; + if (nr > cache_nr && cache_nr) + nr = cache_nr; + return nr; +} + static void combine_worker(struct semantic_verify_proof *proof, struct semantic_verify_worker *worker) { @@ -41,17 +72,20 @@ static void combine_worker(struct semantic_verify_proof *proof, } int semantic_verify_prepare(struct index_state *istate, + const struct semantic_verify_options *options, struct semantic_verify_proof **proof_out) { struct semantic_verify_proof *proof; - struct semantic_verify_worker worker = { 0 }; + struct semantic_verify_worker *workers; + unsigned int nr_threads; + size_t updates_nr = 0; + int create_threads = 1; if (!istate || !proof_out) BUG("semantic_verify_prepare requires an index and output"); if (sizeof(struct semantic_verify_result) != 8) BUG("semantic verify result unexpectedly grew to %"PRIuMAX" bytes", (uintmax_t)sizeof(struct semantic_verify_result)); - CALLOC_ARRAY(proof, 1); proof->istate = istate; proof->cache_nr = istate->cache_nr; @@ -94,18 +128,54 @@ int semantic_verify_prepare(struct index_state *istate, /* Initialize conversion config and default attribute state serially. */ convert_attrs_prepare(istate); + nr_threads = select_thread_count(proof->cache_nr, options); + CALLOC_ARRAY(workers, nr_threads); trace2_region_enter("semantic_verify", "prepare", istate->repo); - trace2_data_intmax("semantic_verify", istate->repo, "threads", 1); + trace2_data_intmax("semantic_verify", istate->repo, + "threads", nr_threads); trace2_data_intmax("semantic_verify", istate->repo, "result-bytes", sizeof(struct semantic_verify_result)); - worker.istate = istate; - worker.root = proof->root; - worker.results = proof->results; - worker.end = proof->cache_nr; - semantic_verify_worker_run(&worker); - ALLOC_ARRAY(proof->stat_updates, worker.updates_nr); - combine_worker(proof, &worker); + for (unsigned int i = 0; i < nr_threads; i++) { + struct semantic_verify_worker *worker = &workers[i]; + int err; + + worker->istate = istate; + worker->root = proof->root; + worker->results = proof->results; + worker->start = st_mult(proof->cache_nr, i) / nr_threads; + worker->end = st_mult(proof->cache_nr, i + 1) / nr_threads; + if (nr_threads == 1 || !create_threads) { + semantic_verify_worker_run(worker); + continue; + } + err = pthread_create(&worker->pthread, NULL, run_worker, worker); + if (!err) { + worker->started = 1; + continue; + } + create_threads = 0; + trace2_data_intmax("semantic_verify", istate->repo, + "thread-failure", err); + semantic_verify_worker_run(worker); + } + for (unsigned int i = 0; i < nr_threads; i++) { + int err; + + if (!workers[i].started) + continue; + err = pthread_join(workers[i].pthread, NULL); + if (err) + die("could not join semantic verifier thread: %s", + strerror(err)); + } + + for (unsigned int i = 0; i < nr_threads; i++) + updates_nr += workers[i].updates_nr; + ALLOC_ARRAY(proof->stat_updates, updates_nr); + for (unsigned int i = 0; i < nr_threads; i++) + combine_worker(proof, &workers[i]); + free(workers); trace2_data_intmax("semantic_verify", istate->repo, "raw-clean", proof->raw_clean); diff --git a/semantic-verify.h b/semantic-verify.h index 6cb5fc3b4fb105..3c67de0580b6dd 100644 --- a/semantic-verify.h +++ b/semantic-verify.h @@ -4,6 +4,12 @@ struct index_state; struct semantic_verify_proof; +struct semantic_verify_options { + unsigned int nr_threads; +}; + +#define SEMANTIC_VERIFY_OPTIONS_INIT { 0 } + enum semantic_verify_kind { SEMANTIC_VERIFY_UNCHECKED = 0, SEMANTIC_VERIFY_SKIPPED, @@ -45,6 +51,7 @@ struct semantic_verify_stats { /* Build a proof candidate without changing the index. */ int semantic_verify_prepare(struct index_state *istate, + const struct semantic_verify_options *options, struct semantic_verify_proof **proof_out); int semantic_verify_apply_after_closure( struct index_state *istate, diff --git a/t/helper/test-semantic-verify.c b/t/helper/test-semantic-verify.c index b0048dea791490..9e0926633607c0 100644 --- a/t/helper/test-semantic-verify.c +++ b/t/helper/test-semantic-verify.c @@ -33,8 +33,10 @@ static const char *kind_name(enum semantic_verify_kind kind) int cmd__semantic_verify(int argc, const char **argv) { + struct semantic_verify_options options = SEMANTIC_VERIFY_OPTIONS_INIT; struct semantic_verify_proof *proof = NULL; struct semantic_verify_stats stats; + int thread_count = 0; int show_results = 0; int apply = 0; int applied = -2; @@ -47,6 +49,8 @@ int cmd__semantic_verify(int argc, const char **argv) NULL }; struct option opts[] = { + OPT_INTEGER(0, "threads", &thread_count, + "number of verifier threads"), OPT_BOOL(0, "show-results", &show_results, "show one result per cache entry"), OPT_BOOL(0, "apply", &apply, "apply the completed proof"), @@ -58,6 +62,9 @@ int cmd__semantic_verify(int argc, const char **argv) argc = parse_options(argc, argv, NULL, opts, usage, 0); if (argc) usage_with_options(usage, opts); + if (thread_count < 0) + die("negative semantic verifier thread count"); + options.nr_threads = thread_count; setup_git_directory(the_repository); repo_config(the_repository, git_default_config, NULL); @@ -65,7 +72,7 @@ int cmd__semantic_verify(int argc, const char **argv) the_repository->settings.command_requires_full_index = 0; if (repo_read_index(the_repository) < 0) die("unable to read index"); - ret = semantic_verify_prepare(the_repository->index, &proof); + ret = semantic_verify_prepare(the_repository->index, &options, &proof); semantic_verify_get_stats(proof, &stats); if (replace_after_prepare) { struct index_state *istate = the_repository->index; diff --git a/t/t7531-semantic-verify.sh b/t/t7531-semantic-verify.sh index 828a1aaa9595be..15d427fdc5eddf 100755 --- a/t/t7531-semantic-verify.sh +++ b/t/t7531-semantic-verify.sh @@ -110,6 +110,26 @@ test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ test_grep "after_uptodate=0 after_valid=0" actual ' +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN,PTHREADS \ + 'parallel verification preserves index-order results' ' + test_create_repo parallel && + i=0 && + while test $i -lt 16 + do + mkdir "parallel/d$i" && + printf "*.txt -text attr_%s=value\n" "$i" \ + >"parallel/d$i/.gitattributes" && + printf "content %s\n" "$i" >"parallel/d$i/file.txt" && + i=$((i + 1)) || return 1 + done && + git -C parallel add . && + git -C parallel commit -m base && + + verify_repo parallel --threads=1 --show-results >expect && + verify_repo parallel --threads=4 --show-results >actual && + test_cmp expect actual +' + test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ 'raw hashing uses the repository object format' ' git init --object-format=sha256 sha256 && From 8a6156f639b7e6d0453eff8a03c6acdb53bc4897 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 22:29:12 -0700 Subject: [PATCH 058/105] status: fingerprint configuration for semantic clean proofs A clean result cannot be reused after configuration changes that alter status or the conversion of tracked worktree bytes. Treating every configuration-origin change as a conversion change would also discard semantic proofs whose effective conversion rules remain identical. Add independently length-framed full and semantic configuration digests using a caller-selected Git object hash algorithm. Bind each full entry to its key, optional value, scope, origin type, and source file. Include effective line-ending and round-trip encoding settings, plus clean, process, and required filters, in the narrower semantic stream. Mark configured clean-side filters unsafe for direct raw verification. Register the configuration implementation and its unit suite in both Make and Meson. The tests distinguish ordinary status changes from conversion changes, confirm that an origin-only change affects only the full digest, and distinguish clean filters from smudge-only rules. This patch exposes and tests digest construction. Neither semantic proof preparation nor proof application consumes these digests, and no production status or index-read path uses them at this boundary. Signed-off-by: Taylor Blau --- Makefile | 2 + clean-status-config.c | 80 ++++++++++++++++++++++++++++ clean-status-config.h | 26 +++++++++ hash-framing.h | 30 +++++++++++ meson.build | 1 + t/meson.build | 1 + t/unit-tests/u-clean-status-config.c | 72 +++++++++++++++++++++++++ 7 files changed, 212 insertions(+) create mode 100644 clean-status-config.c create mode 100644 clean-status-config.h create mode 100644 hash-framing.h create mode 100644 t/unit-tests/u-clean-status-config.c diff --git a/Makefile b/Makefile index 0c8545ab270abe..a4720dc87aaaea 100644 --- a/Makefile +++ b/Makefile @@ -1123,6 +1123,7 @@ LIB_OBJS += cbtree.o LIB_OBJS += chdir-notify.o LIB_OBJS += checkout.o LIB_OBJS += chunk-format.o +LIB_OBJS += clean-status-config.o LIB_OBJS += color.o LIB_OBJS += column.o LIB_OBJS += combine-diff.o @@ -1542,6 +1543,7 @@ THIRD_PARTY_SOURCES += sha1dc/% THIRD_PARTY_SOURCES += $(UNIT_TEST_DIR)/clar/% THIRD_PARTY_SOURCES += $(UNIT_TEST_DIR)/clar/clar/% +CLAR_TEST_SUITES += u-clean-status-config CLAR_TEST_SUITES += u-ctype CLAR_TEST_SUITES += u-dir CLAR_TEST_SUITES += u-example-decorate diff --git a/clean-status-config.c b/clean-status-config.c new file mode 100644 index 00000000000000..96037d61ad1104 --- /dev/null +++ b/clean-status-config.c @@ -0,0 +1,80 @@ +#include "git-compat-util.h" +#include "clean-status-config.h" +#include "config.h" +#include "hash-framing.h" +#include "strbuf.h" + +void clean_status_config_init(struct clean_status_config_digest *digest, + const struct git_hash_algo *algo) +{ + if (!algo) + BUG("clean-status config digest requires a hash algorithm"); + memset(digest, 0, sizeof(*digest)); + git_hash_init(&digest->ctx, algo); + git_hash_init(&digest->semantic_ctx, algo); + /* Invalidate proofs written before multiply-linked files stayed dirty. */ + hash_optional_cstring(&digest->ctx, + "clean-status-config-hardlink-v1"); + digest->initialized = 1; +} + +static void hash_config_entry(struct git_hash_ctx *ctx, + const char *key, const char *value, + const struct config_context *config_ctx) +{ + uint32_t metadata[2] = { 0 }; + + hash_optional_cstring(ctx, key); + hash_optional_cstring(ctx, value); + if (config_ctx && config_ctx->kvi) { + put_be32(&metadata[0], config_ctx->kvi->scope); + put_be32(&metadata[1], config_ctx->kvi->origin_type); + hash_length_delimited(ctx, metadata, sizeof(metadata)); + hash_optional_cstring(ctx, config_ctx->kvi->filename); + } else { + hash_length_delimited(ctx, metadata, sizeof(metadata)); + hash_optional_cstring(ctx, NULL); + } +} + +static void hash_effective_config_entry(struct git_hash_ctx *ctx, + const char *key, + const char *value) +{ + hash_optional_cstring(ctx, key); + hash_optional_cstring(ctx, value); +} + +void clean_status_config_add(struct clean_status_config_digest *digest, + const char *key, const char *value, + const struct config_context *ctx) +{ + const char *suffix; + int semantic; + + if (!digest->initialized || digest->finalized) + BUG("invalid clean-status config digest state"); + hash_config_entry(&digest->ctx, key, value, ctx); + semantic = !strcmp(key, "core.autocrlf") || + !strcmp(key, "core.eol") || + !strcmp(key, "core.checkroundtripencoding"); + if (skip_prefix(key, "filter.", &suffix) && + (ends_with(suffix, ".clean") || ends_with(suffix, ".process") || + ends_with(suffix, ".required"))) { + digest->unsafe_filter = 1; + semantic = 1; + } + if (semantic) { + hash_effective_config_entry(&digest->semantic_ctx, key, value); + digest->semantic_config_explicit = 1; + } +} + +void clean_status_config_final(struct clean_status_config_digest *digest) +{ + if (!digest->initialized || digest->finalized) + BUG("invalid clean-status config digest state"); + git_hash_final(digest->hash, &digest->ctx); + git_hash_final(digest->semantic_hash, &digest->semantic_ctx); + digest->finalized = 1; +} diff --git a/clean-status-config.h b/clean-status-config.h new file mode 100644 index 00000000000000..dd4f0647420858 --- /dev/null +++ b/clean-status-config.h @@ -0,0 +1,26 @@ +#ifndef CLEAN_STATUS_CONFIG_H +#define CLEAN_STATUS_CONFIG_H + +#include "hash.h" + +struct config_context; + +struct clean_status_config_digest { + struct git_hash_ctx ctx; + struct git_hash_ctx semantic_ctx; + unsigned char hash[GIT_MAX_RAWSZ]; + unsigned char semantic_hash[GIT_MAX_RAWSZ]; + unsigned initialized : 1; + unsigned finalized : 1; + unsigned unsafe_filter : 1; + unsigned semantic_config_explicit : 1; +}; + +void clean_status_config_init(struct clean_status_config_digest *digest, + const struct git_hash_algo *algo); +void clean_status_config_add(struct clean_status_config_digest *digest, + const char *key, const char *value, + const struct config_context *ctx); +void clean_status_config_final(struct clean_status_config_digest *digest); + +#endif /* CLEAN_STATUS_CONFIG_H */ diff --git a/hash-framing.h b/hash-framing.h new file mode 100644 index 00000000000000..b15294b684a90d --- /dev/null +++ b/hash-framing.h @@ -0,0 +1,30 @@ +#ifndef HASH_FRAMING_H +#define HASH_FRAMING_H + +#include "hash.h" + +static inline void hash_length_delimited(struct git_hash_ctx *ctx, + const void *data, size_t len) +{ + uint32_t size; + + if (len > UINT32_MAX) + BUG("length-delimited hash input too long"); + put_be32(&size, len); + git_hash_update(ctx, &size, sizeof(size)); + if (len) + git_hash_update(ctx, data, len); +} + +static inline void hash_optional_cstring(struct git_hash_ctx *ctx, + const char *value) +{ + static const unsigned char missing = 0; + + if (value) + hash_length_delimited(ctx, value, strlen(value)); + else + hash_length_delimited(ctx, &missing, sizeof(missing)); +} + +#endif /* HASH_FRAMING_H */ diff --git a/meson.build b/meson.build index ff471a3a092f5a..962f4e1dd1283c 100644 --- a/meson.build +++ b/meson.build @@ -331,6 +331,7 @@ libgit_sources = [ 'chdir-notify.c', 'checkout.c', 'chunk-format.c', + 'clean-status-config.c', 'color.c', 'column.c', 'combine-diff.c', diff --git a/t/meson.build b/t/meson.build index 4ecf8343a2f935..639db8354a46cb 100644 --- a/t/meson.build +++ b/t/meson.build @@ -1,4 +1,5 @@ clar_test_suites = [ + 'unit-tests/u-clean-status-config.c', 'unit-tests/u-ctype.c', 'unit-tests/u-dir.c', 'unit-tests/u-example-decorate.c', diff --git a/t/unit-tests/u-clean-status-config.c b/t/unit-tests/u-clean-status-config.c new file mode 100644 index 00000000000000..e922970c1abe4e --- /dev/null +++ b/t/unit-tests/u-clean-status-config.c @@ -0,0 +1,72 @@ +#include "unit-test.h" +#include "clean-status-config.h" +#include "config.h" + +static void digest_one(struct clean_status_config_digest *digest, + const char *key, const char *value, + const struct config_context *ctx) +{ + clean_status_config_init(digest, &hash_algos[GIT_HASH_SHA1]); + clean_status_config_add(digest, key, value, ctx); + clean_status_config_final(digest); +} + +static int hashes_equal(const unsigned char *a, const unsigned char *b) +{ + return hasheq(a, b, &hash_algos[GIT_HASH_SHA1]); +} + +void test_clean_status_config__non_semantic_values_only_change_full_hash(void) +{ + struct clean_status_config_digest a, b; + + digest_one(&a, "status.showuntrackedfiles", "normal", NULL); + digest_one(&b, "status.showuntrackedfiles", "all", NULL); + cl_assert(!hashes_equal(a.hash, b.hash)); + cl_assert(hashes_equal(a.semantic_hash, b.semantic_hash)); + cl_assert(!a.semantic_config_explicit); + cl_assert(!b.semantic_config_explicit); +} + +void test_clean_status_config__semantic_values_change_semantic_hash(void) +{ + struct clean_status_config_digest a, b; + + digest_one(&a, "core.autocrlf", "true", NULL); + digest_one(&b, "core.autocrlf", "false", NULL); + cl_assert(!hashes_equal(a.semantic_hash, b.semantic_hash)); + cl_assert(a.semantic_config_explicit); + cl_assert(b.semantic_config_explicit); +} + +void test_clean_status_config__origin_only_affects_full_hash(void) +{ + struct key_value_info global_kvi = KVI_INIT; + struct key_value_info local_kvi = KVI_INIT; + struct config_context global_ctx = { .kvi = &global_kvi }; + struct config_context local_ctx = { .kvi = &local_kvi }; + struct clean_status_config_digest global, local; + + global_kvi.scope = CONFIG_SCOPE_GLOBAL; + global_kvi.origin_type = CONFIG_ORIGIN_FILE; + global_kvi.filename = "/global"; + local_kvi.scope = CONFIG_SCOPE_LOCAL; + local_kvi.origin_type = CONFIG_ORIGIN_FILE; + local_kvi.filename = "/local"; + digest_one(&global, "core.eol", "lf", &global_ctx); + digest_one(&local, "core.eol", "lf", &local_ctx); + cl_assert(!hashes_equal(global.hash, local.hash)); + cl_assert(hashes_equal(global.semantic_hash, local.semantic_hash)); +} + +void test_clean_status_config__clean_filters_are_unsafe(void) +{ + struct clean_status_config_digest clean, smudge; + + digest_one(&clean, "filter.demo.clean", "command", NULL); + digest_one(&smudge, "filter.demo.smudge", "command", NULL); + cl_assert(clean.unsafe_filter); + cl_assert(clean.semantic_config_explicit); + cl_assert(!smudge.unsafe_filter); + cl_assert(!smudge.semantic_config_explicit); +} From e0b98c4a7e8457ed6917119dc0aba4091592a784 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 22:30:22 -0700 Subject: [PATCH 059/105] status: write bounded attribute manifests A reusable clean-status proof must identify the .gitattributes source that governs conversion in every tracked directory. An unordered list of paths and hashes cannot distinguish duplicate records, ambiguous paths, or a worktree source from its indexed counterpart. Add a length-delimited manifest writer with an entry count, repository-relative .gitattributes paths, explicit source kinds, and object-format-sized hashes. Reject invalid paths, unknown source kinds, overflows, duplicate records, and nonincreasing path order before appending an entry. Register the new library and Clar suite in both Make and Meson. Focused tests cover ordered records and reject malformed, duplicate, and out-of-order paths. This defines a tested encoding; it does not enable a status fast path. Signed-off-by: Taylor Blau --- Makefile | 2 + attr-manifest.c | 94 ++++++++++++++++++++++++++++++++++ attr-manifest.h | 35 +++++++++++++ meson.build | 1 + t/meson.build | 1 + t/unit-tests/u-attr-manifest.c | 53 +++++++++++++++++++ 6 files changed, 186 insertions(+) create mode 100644 attr-manifest.c create mode 100644 attr-manifest.h create mode 100644 t/unit-tests/u-attr-manifest.c diff --git a/Makefile b/Makefile index a4720dc87aaaea..67916eeab63f72 100644 --- a/Makefile +++ b/Makefile @@ -1110,6 +1110,7 @@ LIB_OBJS += archive-tar.o LIB_OBJS += archive-zip.o LIB_OBJS += archive.o LIB_OBJS += attr.o +LIB_OBJS += attr-manifest.o LIB_OBJS += base85.o LIB_OBJS += bisect.o LIB_OBJS += blame.o @@ -1543,6 +1544,7 @@ THIRD_PARTY_SOURCES += sha1dc/% THIRD_PARTY_SOURCES += $(UNIT_TEST_DIR)/clar/% THIRD_PARTY_SOURCES += $(UNIT_TEST_DIR)/clar/clar/% +CLAR_TEST_SUITES += u-attr-manifest CLAR_TEST_SUITES += u-clean-status-config CLAR_TEST_SUITES += u-ctype CLAR_TEST_SUITES += u-dir diff --git a/attr-manifest.c b/attr-manifest.c new file mode 100644 index 00000000000000..41220073ff59c7 --- /dev/null +++ b/attr-manifest.c @@ -0,0 +1,94 @@ +#include "git-compat-util.h" +#include "attr-manifest.h" +#include "environment.h" +#include "read-cache-ll.h" +#include "strbuf.h" + +/* + * A manifest begins with a 32-bit entry count. Each entry contains a 32-bit + * path length, four bytes of source metadata, an object-format hash, and the + * unterminated path. Paths are strictly increasing. + */ +static int attr_manifest_path_valid(const unsigned char *path, size_t len) +{ + const char *base; + char *copy; + int valid; + + if (!len || path[0] == '/' || memchr(path, '\0', len)) + return 0; + copy = xmemdupz(path, len); + base = strrchr(copy, '/'); + base = base ? base + 1 : copy; + valid = !strcmp(base, GITATTRIBUTES_FILE) && + verify_path(copy, S_IFREG | 0644); + free(copy); + return valid; +} + +static int attr_manifest_entry_cmp(const struct attr_manifest_entry *a, + const struct attr_manifest_entry *b) +{ + size_t common = a->path_len < b->path_len ? a->path_len : b->path_len; + int cmp = memcmp(a->path, b->path, common); + + if (cmp) + return cmp; + return a->path_len < b->path_len ? -1 : a->path_len > b->path_len; +} + +void attr_manifest_writer_init(struct attr_manifest_writer *writer, + struct strbuf *buf, + const struct git_hash_algo *algo) +{ + uint32_t count; + + if (!algo) + BUG("attribute manifest requires a hash algorithm"); + memset(writer, 0, sizeof(*writer)); + writer->buf = buf; + writer->algo = algo; + strbuf_reset(buf); + put_be32(&count, 0); + strbuf_add(buf, &count, sizeof(count)); +} + +int attr_manifest_writer_add(struct attr_manifest_writer *writer, + const char *path, + enum attr_manifest_source source, + const unsigned char *hash) +{ + struct attr_manifest_entry previous, current; + unsigned char metadata[4] = { source, 0, 0, 0 }; + uint32_t path_len_be; + size_t entry_offset, path_len = strlen(path); + + if (!writer->buf || !writer->algo || !hash || !path_len || + path_len > UINT32_MAX || writer->nr == UINT32_MAX || + (source != ATTR_MANIFEST_WORKTREE && + source != ATTR_MANIFEST_INDEX) || + !attr_manifest_path_valid((const unsigned char *)path, path_len)) + return -1; + + current.path = (const unsigned char *)path; + current.path_len = path_len; + if (writer->nr) { + previous.path = (const unsigned char *)writer->buf->buf + + writer->last_path_offset; + previous.path_len = writer->last_path_len; + if (attr_manifest_entry_cmp(&previous, ¤t) >= 0) + return -1; + } + + entry_offset = writer->buf->len; + put_be32(&path_len_be, path_len); + strbuf_add(writer->buf, &path_len_be, sizeof(path_len_be)); + strbuf_add(writer->buf, metadata, sizeof(metadata)); + strbuf_add(writer->buf, hash, writer->algo->rawsz); + strbuf_add(writer->buf, path, path_len); + writer->last_path_offset = entry_offset + sizeof(path_len_be) + + sizeof(metadata) + writer->algo->rawsz; + writer->last_path_len = path_len; + put_be32(writer->buf->buf, ++writer->nr); + return 0; +} diff --git a/attr-manifest.h b/attr-manifest.h new file mode 100644 index 00000000000000..75296bae8f1f0e --- /dev/null +++ b/attr-manifest.h @@ -0,0 +1,35 @@ +#ifndef ATTR_MANIFEST_H +#define ATTR_MANIFEST_H + +#include "hash.h" + +struct strbuf; + +enum attr_manifest_source { + ATTR_MANIFEST_WORKTREE = 1, + ATTR_MANIFEST_INDEX = 2, +}; + +struct attr_manifest_entry { + const unsigned char *path; + uint32_t path_len; + enum attr_manifest_source source; + const unsigned char *hash; +}; + +struct attr_manifest_writer { + struct strbuf *buf; + const struct git_hash_algo *algo; + size_t last_path_offset; + uint32_t last_path_len; + uint32_t nr; +}; + +void attr_manifest_writer_init(struct attr_manifest_writer *writer, + struct strbuf *buf, + const struct git_hash_algo *algo); +int attr_manifest_writer_add(struct attr_manifest_writer *writer, + const char *path, + enum attr_manifest_source source, + const unsigned char *hash); +#endif /* ATTR_MANIFEST_H */ diff --git a/meson.build b/meson.build index 962f4e1dd1283c..56b903ab289d75 100644 --- a/meson.build +++ b/meson.build @@ -317,6 +317,7 @@ libgit_sources = [ 'archive-tar.c', 'archive-zip.c', 'archive.c', + 'attr-manifest.c', 'attr.c', 'base85.c', 'bisect.c', diff --git a/t/meson.build b/t/meson.build index 639db8354a46cb..d38d0f5be3cb08 100644 --- a/t/meson.build +++ b/t/meson.build @@ -1,4 +1,5 @@ clar_test_suites = [ + 'unit-tests/u-attr-manifest.c', 'unit-tests/u-clean-status-config.c', 'unit-tests/u-ctype.c', 'unit-tests/u-dir.c', diff --git a/t/unit-tests/u-attr-manifest.c b/t/unit-tests/u-attr-manifest.c new file mode 100644 index 00000000000000..277e3101d042e7 --- /dev/null +++ b/t/unit-tests/u-attr-manifest.c @@ -0,0 +1,53 @@ +#include "unit-test.h" +#include "attr-manifest.h" +#include "strbuf.h" + +static void fill_hash(unsigned char *hash, unsigned char value, + const struct git_hash_algo *algo) +{ + memset(hash, value, algo->rawsz); +} + +static void add_entry(struct attr_manifest_writer *writer, const char *path, + enum attr_manifest_source source, unsigned char value) +{ + unsigned char hash[GIT_MAX_RAWSZ]; + + fill_hash(hash, value, writer->algo); + cl_assert_equal_i(attr_manifest_writer_add(writer, path, source, hash), 0); +} + +void test_attr_manifest__writer_serializes_sorted_entries(void) +{ + struct attr_manifest_writer writer; + struct strbuf manifest = STRBUF_INIT; + + attr_manifest_writer_init(&writer, &manifest, &hash_algos[GIT_HASH_SHA256]); + add_entry(&writer, ".gitattributes", ATTR_MANIFEST_INDEX, 1); + add_entry(&writer, "a/.gitattributes", ATTR_MANIFEST_WORKTREE, 2); + cl_assert_equal_i(get_be32(manifest.buf), 2); + cl_assert_equal_i(writer.nr, 2); + strbuf_release(&manifest); +} + +void test_attr_manifest__writer_rejects_invalid_or_unsorted_paths(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct attr_manifest_writer writer; + struct strbuf manifest = STRBUF_INIT; + unsigned char hash[GIT_MAX_RAWSZ]; + + fill_hash(hash, 1, algo); + attr_manifest_writer_init(&writer, &manifest, algo); + add_entry(&writer, "b/.gitattributes", ATTR_MANIFEST_INDEX, 1); + cl_assert_equal_i(attr_manifest_writer_add(&writer, "a/.gitattributes", + ATTR_MANIFEST_INDEX, hash), -1); + cl_assert_equal_i(attr_manifest_writer_add(&writer, "b/.gitattributes", + ATTR_MANIFEST_INDEX, hash), -1); + cl_assert_equal_i(attr_manifest_writer_add(&writer, "/.gitattributes", + ATTR_MANIFEST_INDEX, hash), -1); + cl_assert_equal_i(attr_manifest_writer_add(&writer, "b/not-attributes", + ATTR_MANIFEST_INDEX, hash), -1); + cl_assert_equal_i(writer.nr, 1); + strbuf_release(&manifest); +} From a8e66a77293b44e6cf74132cafcd7d8dca27d888 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 22:31:01 -0700 Subject: [PATCH 060/105] status: validate attribute manifest records A persisted attribute manifest is untrusted even when its writer was careful. A truncated record, forged entry count, invalid source, or trailing byte could otherwise make a later reader accept incomplete or ambiguous conversion history. Add a bounded cursor for the format from S07/P01. Check the declared count against the minimum record size, enforce valid paths, known source kinds, zero reserved bytes, object-format-sized hashes, strict path ordering, and exact consumption of the input. Test round trips, an empty manifest, truncation, trailing data, nonzero reserved bytes, and an overstated entry count. Decoding only exposes records; it does not publish or consume status history. Signed-off-by: Taylor Blau --- attr-manifest.c | 79 ++++++++++++++++++++++++++++++++++ attr-manifest.h | 17 ++++++++ t/unit-tests/u-attr-manifest.c | 59 +++++++++++++++++++++++++ 3 files changed, 155 insertions(+) diff --git a/attr-manifest.c b/attr-manifest.c index 41220073ff59c7..694c787d45fd82 100644 --- a/attr-manifest.c +++ b/attr-manifest.c @@ -92,3 +92,82 @@ int attr_manifest_writer_add(struct attr_manifest_writer *writer, put_be32(writer->buf->buf, ++writer->nr); return 0; } + +int attr_manifest_cursor_init(struct attr_manifest_cursor *cursor, + const void *data, size_t len, + const struct git_hash_algo *algo) +{ + const unsigned char *bytes = data; + size_t minimum_entry_size; + + if (!algo) + BUG("attribute manifest requires a hash algorithm"); + if (len < sizeof(uint32_t)) + return -1; + minimum_entry_size = sizeof(uint32_t) + 4 + algo->rawsz + 1; + cursor->p = bytes + sizeof(uint32_t); + cursor->end = bytes + len; + cursor->last_path = NULL; + cursor->algo = algo; + cursor->last_path_len = 0; + cursor->remaining = get_be32(bytes); + if (cursor->remaining > + (len - sizeof(uint32_t)) / minimum_entry_size) + return -1; + return 0; +} + +int attr_manifest_cursor_next(struct attr_manifest_cursor *cursor, + struct attr_manifest_entry *entry) +{ + struct attr_manifest_entry previous; + uint32_t path_len; + size_t available; + + if (!cursor->remaining) + return cursor->p == cursor->end ? 0 : -1; + available = cursor->end - cursor->p; + if (available < sizeof(uint32_t) + 4 + cursor->algo->rawsz) + return -1; + path_len = get_be32(cursor->p); + cursor->p += sizeof(uint32_t); + entry->source = cursor->p[0]; + if ((entry->source != ATTR_MANIFEST_WORKTREE && + entry->source != ATTR_MANIFEST_INDEX) || + cursor->p[1] || cursor->p[2] || cursor->p[3]) + return -1; + cursor->p += 4; + entry->hash = cursor->p; + cursor->p += cursor->algo->rawsz; + available = cursor->end - cursor->p; + if (!path_len || available < path_len || + !attr_manifest_path_valid(cursor->p, path_len)) + return -1; + entry->path = cursor->p; + entry->path_len = path_len; + if (cursor->last_path) { + previous.path = cursor->last_path; + previous.path_len = cursor->last_path_len; + if (attr_manifest_entry_cmp(&previous, entry) >= 0) + return -1; + } + cursor->last_path = entry->path; + cursor->last_path_len = entry->path_len; + cursor->p += path_len; + cursor->remaining--; + return 1; +} + +int attr_manifest_valid(const void *data, size_t len, + const struct git_hash_algo *algo) +{ + struct attr_manifest_cursor cursor; + struct attr_manifest_entry entry; + int ret; + + if (attr_manifest_cursor_init(&cursor, data, len, algo)) + return 0; + while ((ret = attr_manifest_cursor_next(&cursor, &entry)) > 0) + ; + return !ret; +} diff --git a/attr-manifest.h b/attr-manifest.h index 75296bae8f1f0e..a3e6de4b556c33 100644 --- a/attr-manifest.h +++ b/attr-manifest.h @@ -17,6 +17,15 @@ struct attr_manifest_entry { const unsigned char *hash; }; +struct attr_manifest_cursor { + const unsigned char *p; + const unsigned char *end; + const unsigned char *last_path; + const struct git_hash_algo *algo; + uint32_t last_path_len; + uint32_t remaining; +}; + struct attr_manifest_writer { struct strbuf *buf; const struct git_hash_algo *algo; @@ -32,4 +41,12 @@ int attr_manifest_writer_add(struct attr_manifest_writer *writer, const char *path, enum attr_manifest_source source, const unsigned char *hash); +int attr_manifest_cursor_init(struct attr_manifest_cursor *cursor, + const void *data, size_t len, + const struct git_hash_algo *algo); +int attr_manifest_cursor_next(struct attr_manifest_cursor *cursor, + struct attr_manifest_entry *entry); +int attr_manifest_valid(const void *data, size_t len, + const struct git_hash_algo *algo); + #endif /* ATTR_MANIFEST_H */ diff --git a/t/unit-tests/u-attr-manifest.c b/t/unit-tests/u-attr-manifest.c index 277e3101d042e7..d03cc41c273da5 100644 --- a/t/unit-tests/u-attr-manifest.c +++ b/t/unit-tests/u-attr-manifest.c @@ -51,3 +51,62 @@ void test_attr_manifest__writer_rejects_invalid_or_unsorted_paths(void) cl_assert_equal_i(writer.nr, 1); strbuf_release(&manifest); } + +void test_attr_manifest__reader_round_trips_entries(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA256]; + struct attr_manifest_writer writer; + struct attr_manifest_cursor cursor; + struct attr_manifest_entry entry; + struct strbuf manifest = STRBUF_INIT; + + attr_manifest_writer_init(&writer, &manifest, algo); + add_entry(&writer, ".gitattributes", ATTR_MANIFEST_INDEX, 1); + add_entry(&writer, "a/.gitattributes", ATTR_MANIFEST_WORKTREE, 2); + cl_assert(attr_manifest_valid(manifest.buf, manifest.len, algo)); + cl_assert_equal_i(attr_manifest_cursor_init(&cursor, manifest.buf, + manifest.len, algo), 0); + cl_assert_equal_i(attr_manifest_cursor_next(&cursor, &entry), 1); + cl_assert_equal_i(entry.source, ATTR_MANIFEST_INDEX); + cl_assert_equal_i(entry.hash[0], 1); + cl_assert_equal_i(attr_manifest_cursor_next(&cursor, &entry), 1); + cl_assert_equal_i(entry.source, ATTR_MANIFEST_WORKTREE); + cl_assert_equal_i(entry.hash[0], 2); + cl_assert_equal_i(attr_manifest_cursor_next(&cursor, &entry), 0); + strbuf_release(&manifest); +} + +void test_attr_manifest__reader_rejects_corrupt_encoding(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct attr_manifest_writer writer; + struct strbuf manifest = STRBUF_INIT; + size_t metadata_offset = 2 * sizeof(uint32_t); + unsigned char saved; + + attr_manifest_writer_init(&writer, &manifest, algo); + add_entry(&writer, ".gitattributes", ATTR_MANIFEST_INDEX, 1); + cl_assert(!attr_manifest_valid(manifest.buf, manifest.len - 1, algo)); + strbuf_addch(&manifest, 0); + cl_assert(!attr_manifest_valid(manifest.buf, manifest.len, algo)); + strbuf_setlen(&manifest, manifest.len - 1); + + saved = manifest.buf[metadata_offset + 1]; + manifest.buf[metadata_offset + 1] = 1; + cl_assert(!attr_manifest_valid(manifest.buf, manifest.len, algo)); + manifest.buf[metadata_offset + 1] = saved; + put_be32(manifest.buf, 2); + cl_assert(!attr_manifest_valid(manifest.buf, manifest.len, algo)); + strbuf_release(&manifest); +} + +void test_attr_manifest__reader_accepts_empty_manifest(void) +{ + struct attr_manifest_writer writer; + struct strbuf manifest = STRBUF_INIT; + + attr_manifest_writer_init(&writer, &manifest, &hash_algos[GIT_HASH_SHA1]); + cl_assert(attr_manifest_valid(manifest.buf, manifest.len, + &hash_algos[GIT_HASH_SHA1])); + strbuf_release(&manifest); +} From 4338534e40c8a9ce2d9343e94d8db04ff4a04f79 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 22:31:54 -0700 Subject: [PATCH 061/105] status: iterate changed attribute manifest entries Refreshing conversion history must invalidate added, removed, and changed .gitattributes sources without reporting unchanged paths. Acting on a partially decoded stream would be worse: a corrupt trailing record could leave some paths invalidated before the failure is known. Validate both complete manifests with S07/P02 before invoking a callback. Merge their ordered cursors without copying entries, report each added or removed path once, and treat a source-kind or hash change as a modification. Do not invoke the callback for records whose path, source, and hash all match. Up-front validation deliberately reads each entire manifest before merging. The additional pass prevents partial callback effects without materializing another collection of paths. Unit tests cover additions, removals, source changes, identical manifests, and a malformed tail that must produce no callbacks. Signed-off-by: Taylor Blau --- attr-manifest.c | 59 ++++++++++++++++++++++++++ attr-manifest.h | 7 ++++ t/unit-tests/u-attr-manifest.c | 77 ++++++++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+) diff --git a/attr-manifest.c b/attr-manifest.c index 694c787d45fd82..46aed49a430050 100644 --- a/attr-manifest.c +++ b/attr-manifest.c @@ -37,6 +37,14 @@ static int attr_manifest_entry_cmp(const struct attr_manifest_entry *a, return a->path_len < b->path_len ? -1 : a->path_len > b->path_len; } +static int attr_manifest_entry_equal(const struct attr_manifest_entry *a, + const struct attr_manifest_entry *b, + const struct git_hash_algo *algo) +{ + return !attr_manifest_entry_cmp(a, b) && a->source == b->source && + !memcmp(a->hash, b->hash, algo->rawsz); +} + void attr_manifest_writer_init(struct attr_manifest_writer *writer, struct strbuf *buf, const struct git_hash_algo *algo) @@ -171,3 +179,54 @@ int attr_manifest_valid(const void *data, size_t len, ; return !ret; } + +int attr_manifest_for_each_changed(const void *old_data, size_t old_len, + const void *new_data, size_t new_len, + const struct git_hash_algo *algo, + attr_manifest_change_fn fn, void *data) +{ + struct attr_manifest_cursor old_cursor, new_cursor; + struct attr_manifest_entry old_entry, new_entry; + int old_ret, new_ret; + + /* + * Callers use this as a transactional change set. Validate both + * streams before allowing the callback to observe any entry. + */ + if (!attr_manifest_valid(old_data, old_len, algo) || + !attr_manifest_valid(new_data, new_len, algo)) + return -1; + if (attr_manifest_cursor_init(&old_cursor, old_data, old_len, algo) || + attr_manifest_cursor_init(&new_cursor, new_data, new_len, algo)) + return -1; + old_ret = attr_manifest_cursor_next(&old_cursor, &old_entry); + new_ret = attr_manifest_cursor_next(&new_cursor, &new_entry); + while (old_ret > 0 || new_ret > 0) { + struct attr_manifest_entry changed; + int has_changed = 1; + int cmp; + + if (old_ret <= 0) + cmp = 1; + else if (new_ret <= 0) + cmp = -1; + else + cmp = attr_manifest_entry_cmp(&old_entry, &new_entry); + if (cmp < 0) { + changed = old_entry; + old_ret = attr_manifest_cursor_next(&old_cursor, &old_entry); + } else if (cmp > 0) { + changed = new_entry; + new_ret = attr_manifest_cursor_next(&new_cursor, &new_entry); + } else { + changed = new_entry; + has_changed = !attr_manifest_entry_equal( + &old_entry, &new_entry, algo); + old_ret = attr_manifest_cursor_next(&old_cursor, &old_entry); + new_ret = attr_manifest_cursor_next(&new_cursor, &new_entry); + } + if (has_changed && fn(&changed, data)) + return -1; + } + return old_ret < 0 || new_ret < 0 ? -1 : 0; +} diff --git a/attr-manifest.h b/attr-manifest.h index a3e6de4b556c33..a38acccc224832 100644 --- a/attr-manifest.h +++ b/attr-manifest.h @@ -34,6 +34,9 @@ struct attr_manifest_writer { uint32_t nr; }; +typedef int (*attr_manifest_change_fn)(const struct attr_manifest_entry *entry, + void *data); + void attr_manifest_writer_init(struct attr_manifest_writer *writer, struct strbuf *buf, const struct git_hash_algo *algo); @@ -48,5 +51,9 @@ int attr_manifest_cursor_next(struct attr_manifest_cursor *cursor, struct attr_manifest_entry *entry); int attr_manifest_valid(const void *data, size_t len, const struct git_hash_algo *algo); +int attr_manifest_for_each_changed(const void *old_data, size_t old_len, + const void *new_data, size_t new_len, + const struct git_hash_algo *algo, + attr_manifest_change_fn fn, void *data); #endif /* ATTR_MANIFEST_H */ diff --git a/t/unit-tests/u-attr-manifest.c b/t/unit-tests/u-attr-manifest.c index d03cc41c273da5..41f1d606889606 100644 --- a/t/unit-tests/u-attr-manifest.c +++ b/t/unit-tests/u-attr-manifest.c @@ -110,3 +110,80 @@ void test_attr_manifest__reader_accepts_empty_manifest(void) &hash_algos[GIT_HASH_SHA1])); strbuf_release(&manifest); } + +static int record_changed_path(const struct attr_manifest_entry *entry, + void *data) +{ + struct strbuf *paths = data; + + if (paths->len) + strbuf_addch(paths, ' '); + strbuf_add(paths, entry->path, entry->path_len); + return 0; +} + +void test_attr_manifest__iterates_added_removed_and_modified_entries(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct attr_manifest_writer old_writer, new_writer; + struct strbuf old = STRBUF_INIT, new = STRBUF_INIT, changed = STRBUF_INIT; + + attr_manifest_writer_init(&old_writer, &old, algo); + add_entry(&old_writer, ".gitattributes", ATTR_MANIFEST_INDEX, 1); + add_entry(&old_writer, "a/.gitattributes", ATTR_MANIFEST_INDEX, 2); + add_entry(&old_writer, "c/.gitattributes", ATTR_MANIFEST_INDEX, 3); + attr_manifest_writer_init(&new_writer, &new, algo); + add_entry(&new_writer, ".gitattributes", ATTR_MANIFEST_INDEX, 1); + add_entry(&new_writer, "a/.gitattributes", ATTR_MANIFEST_WORKTREE, 2); + add_entry(&new_writer, "b/.gitattributes", ATTR_MANIFEST_INDEX, 4); + + cl_assert_equal_i(attr_manifest_for_each_changed( + old.buf, old.len, new.buf, new.len, algo, + record_changed_path, &changed), 0); + cl_assert_equal_s(changed.buf, + "a/.gitattributes b/.gitattributes c/.gitattributes"); + strbuf_release(&changed); + strbuf_release(&new); + strbuf_release(&old); +} + +void test_attr_manifest__does_not_report_identical_entries(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct attr_manifest_writer old_writer, new_writer; + struct strbuf old = STRBUF_INIT, new = STRBUF_INIT, changed = STRBUF_INIT; + + attr_manifest_writer_init(&old_writer, &old, algo); + add_entry(&old_writer, ".gitattributes", ATTR_MANIFEST_INDEX, 1); + attr_manifest_writer_init(&new_writer, &new, algo); + add_entry(&new_writer, ".gitattributes", ATTR_MANIFEST_INDEX, 1); + cl_assert_equal_i(attr_manifest_for_each_changed( + old.buf, old.len, new.buf, new.len, algo, + record_changed_path, &changed), 0); + cl_assert_equal_i(changed.len, 0); + strbuf_release(&changed); + strbuf_release(&new); + strbuf_release(&old); +} + +void test_attr_manifest__rejects_malformed_tail_before_callbacks(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct attr_manifest_writer old_writer, new_writer; + struct strbuf old = STRBUF_INIT, new = STRBUF_INIT, changed = STRBUF_INIT; + + attr_manifest_writer_init(&old_writer, &old, algo); + add_entry(&old_writer, ".gitattributes", ATTR_MANIFEST_INDEX, 1); + attr_manifest_writer_init(&new_writer, &new, algo); + add_entry(&new_writer, ".gitattributes", ATTR_MANIFEST_INDEX, 2); + strbuf_addch(&new, 0); + + cl_assert_equal_i(attr_manifest_for_each_changed( + old.buf, old.len, new.buf, new.len, algo, + record_changed_path, &changed), -1); + cl_assert_equal_i(changed.len, 0); + + strbuf_release(&changed); + strbuf_release(&new); + strbuf_release(&old); +} From fc7923bef1ae79d597c323d830994e51d99bd0ee Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 22:33:17 -0700 Subject: [PATCH 062/105] status: read attribute sources through pinned paths Hashing .gitattributes through an absolute pathname does not establish that the file stayed inside the original worktree. Replacing an ancestor can redirect the read, while replacing or changing the source during the read can make a pathname recheck certify different bytes. Resolve each source beneath the root and parent descriptors introduced by S06/P01 and S06/P02. Read a regular, singly linked file in bounded chunks; compare descriptor and pathname identities before and after reading, reject truncation and appended data, and use the reopen check from S06/P03 for the final component. A missing, nonregular, or oversized worktree source remains eligible for indexed fallback. A hard link, unstable parent, read error, or unsupported anchored-open platform instead rejects the observation. Register the source library and Clar suite in both Make and Meson. Tests cover a file larger than one read buffer, missing and nonregular sources, hard links, and a replaced cached parent. The tested reader does not change ordinary status. Signed-off-by: Taylor Blau --- Makefile | 2 + meson.build | 1 + t/meson.build | 1 + t/unit-tests/u-worktree-attr-source.c | 193 ++++++++++++++++++++++++++ worktree-attr-source.c | 93 +++++++++++++ worktree-attr-source.h | 12 ++ 6 files changed, 302 insertions(+) create mode 100644 t/unit-tests/u-worktree-attr-source.c create mode 100644 worktree-attr-source.c create mode 100644 worktree-attr-source.h diff --git a/Makefile b/Makefile index 67916eeab63f72..9faa8a6587171b 100644 --- a/Makefile +++ b/Makefile @@ -1383,6 +1383,7 @@ LIB_OBJS += versioncmp.o LIB_OBJS += walker.o LIB_OBJS += wildmatch.o LIB_OBJS += worktree.o +LIB_OBJS += worktree-attr-source.o LIB_OBJS += wrapper.o LIB_OBJS += write-or-die.o LIB_OBJS += ws.o @@ -1575,6 +1576,7 @@ CLAR_TEST_SUITES += u-strvec CLAR_TEST_SUITES += u-trailer CLAR_TEST_SUITES += u-urlmatch-normalization CLAR_TEST_SUITES += u-utf8-width +CLAR_TEST_SUITES += u-worktree-attr-source CLAR_TEST_PROG = $(UNIT_TEST_BIN)/unit-tests$(X) CLAR_TEST_OBJS = $(patsubst %,$(UNIT_TEST_DIR)/%.o,$(CLAR_TEST_SUITES)) CLAR_TEST_OBJS += $(UNIT_TEST_DIR)/clar/clar.o diff --git a/meson.build b/meson.build index 56b903ab289d75..912e7522aa1ace 100644 --- a/meson.build +++ b/meson.build @@ -584,6 +584,7 @@ libgit_sources = [ 'walker.c', 'wildmatch.c', 'worktree.c', + 'worktree-attr-source.c', 'wrapper.c', 'write-or-die.c', 'ws.c', diff --git a/t/meson.build b/t/meson.build index d38d0f5be3cb08..73372cd547f84c 100644 --- a/t/meson.build +++ b/t/meson.build @@ -31,6 +31,7 @@ clar_test_suites = [ 'unit-tests/u-trailer.c', 'unit-tests/u-urlmatch-normalization.c', 'unit-tests/u-utf8-width.c', + 'unit-tests/u-worktree-attr-source.c', ] clar_sources = [ diff --git a/t/unit-tests/u-worktree-attr-source.c b/t/unit-tests/u-worktree-attr-source.c new file mode 100644 index 00000000000000..59bddc563639a1 --- /dev/null +++ b/t/unit-tests/u-worktree-attr-source.c @@ -0,0 +1,193 @@ +#include "unit-test.h" + +#include "dir.h" +#include "hash.h" +#include "repository.h" +#include "semantic-verify-internal.h" +#include "strbuf.h" +#include "worktree-attr-source.h" +#include "wrapper.h" + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +struct worktree_attr_source_fixture { + char *worktree; + struct repository repo; + struct semantic_verify_root *root; + struct semantic_verify_path *path; +}; + +static void source_fixture_init(struct worktree_attr_source_fixture *fixture) +{ + const char *tmp = getenv("TMPDIR"); + + memset(fixture, 0, sizeof(*fixture)); + fixture->worktree = xstrfmt( + "%s/worktree-attr-source.XXXXXX", tmp ? tmp : "/tmp"); + cl_assert(mkdtemp(fixture->worktree) != NULL); + fixture->repo.worktree = fixture->worktree; + fixture->repo.hash_algo = &hash_algos[GIT_HASH_SHA1]; + cl_must_pass(semantic_verify_root_init( + &fixture->repo, &fixture->root)); + fixture->path = semantic_verify_path_new(fixture->root); + cl_assert(fixture->path != NULL); +} + +static void source_fixture_release( + struct worktree_attr_source_fixture *fixture, + unsigned int *namespace_unstable, + size_t *namespace_unstable_from) +{ + struct strbuf worktree = STRBUF_INIT; + + semantic_verify_path_free( + fixture->path, namespace_unstable, namespace_unstable_from); + semantic_verify_root_clear(fixture->root); + strbuf_addstr(&worktree, fixture->worktree); + cl_must_pass(remove_dir_recursively(&worktree, 0)); + strbuf_release(&worktree); + free(fixture->worktree); +} + +static void make_directory(struct worktree_attr_source_fixture *fixture, + const char *name) +{ + struct strbuf path = STRBUF_INIT; + + strbuf_addf(&path, "%s/%s", fixture->worktree, name); + cl_must_pass(mkdir(path.buf, 0777)); + strbuf_release(&path); +} +#endif + +void test_worktree_attr_source__hashes_large_regular_file(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + struct worktree_attr_source_fixture fixture; + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct git_hash_ctx ctx; + struct strbuf contents = STRBUF_INIT; + struct strbuf source = STRBUF_INIT; + unsigned char actual[GIT_MAX_RAWSZ], expected[GIT_MAX_RAWSZ]; + unsigned int namespace_unstable; + int found; + + source_fixture_init(&fixture); + make_directory(&fixture, "a"); + strbuf_addchars(&contents, 'x', 64 * 1024 + 17); + strbuf_addf(&source, "%s/a/.gitattributes", fixture.worktree); + write_file_buf(source.buf, contents.buf, contents.len); + git_hash_init(&ctx, algo); + git_hash_update(&ctx, contents.buf, contents.len); + git_hash_final(expected, &ctx); + git_hash_discard(&ctx); + + cl_must_pass(worktree_attr_source_read( + fixture.path, "a/.gitattributes", 7, algo, actual, &found)); + cl_assert_equal_i(found, 1); + cl_assert(!memcmp(actual, expected, algo->rawsz)); + + source_fixture_release(&fixture, &namespace_unstable, NULL); + cl_assert_equal_i(namespace_unstable, 0); + strbuf_release(&source); + strbuf_release(&contents); +#endif +} + +void test_worktree_attr_source__reports_missing_and_non_regular_files(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + struct worktree_attr_source_fixture fixture; + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + unsigned char hash[GIT_MAX_RAWSZ]; + unsigned int namespace_unstable; + int found; + + source_fixture_init(&fixture); + cl_must_pass(worktree_attr_source_read( + fixture.path, ".gitattributes", 0, algo, hash, &found)); + cl_assert_equal_i(found, 0); + cl_must_pass(worktree_attr_source_read( + fixture.path, "missing/.gitattributes", 1, + algo, hash, &found)); + cl_assert_equal_i(found, 0); + + make_directory(&fixture, "attributes-directory"); + cl_must_pass(worktree_attr_source_read( + fixture.path, "attributes-directory", 2, algo, hash, &found)); + cl_assert_equal_i(found, 0); + + source_fixture_release(&fixture, &namespace_unstable, NULL); + cl_assert_equal_i(namespace_unstable, 0); +#endif +} + +void test_worktree_attr_source__rejects_hardlinked_file(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + struct worktree_attr_source_fixture fixture; + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct strbuf alias = STRBUF_INIT, source = STRBUF_INIT; + unsigned char hash[GIT_MAX_RAWSZ]; + unsigned int namespace_unstable; + int found; + + source_fixture_init(&fixture); + make_directory(&fixture, "a"); + strbuf_addf(&source, "%s/a/.gitattributes", fixture.worktree); + strbuf_addf(&alias, "%s/attributes-alias", fixture.worktree); + write_file(source.buf, "*.dat text\n"); + cl_must_pass(link(source.buf, alias.buf)); + + cl_assert_equal_i(worktree_attr_source_read( + fixture.path, "a/.gitattributes", 3, algo, hash, &found), -1); + cl_assert_equal_i(found, 0); + + source_fixture_release(&fixture, &namespace_unstable, NULL); + cl_assert_equal_i(namespace_unstable, 0); + strbuf_release(&source); + strbuf_release(&alias); +#endif +} + +void test_worktree_attr_source__detects_replaced_cached_parent(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + struct worktree_attr_source_fixture fixture; + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct strbuf old_parent = STRBUF_INIT, parent = STRBUF_INIT; + struct strbuf source = STRBUF_INIT; + unsigned char hash[GIT_MAX_RAWSZ]; + size_t namespace_unstable_from; + unsigned int namespace_unstable; + int found; + + source_fixture_init(&fixture); + make_directory(&fixture, "a"); + strbuf_addf(&parent, "%s/a", fixture.worktree); + strbuf_addf(&old_parent, "%s/a-old", fixture.worktree); + strbuf_addf(&source, "%s/.gitattributes", parent.buf); + write_file(source.buf, "*.dat text\n"); + cl_must_pass(worktree_attr_source_read( + fixture.path, "a/.gitattributes", 17, algo, hash, &found)); + cl_assert_equal_i(found, 1); + + cl_must_pass(rename(parent.buf, old_parent.buf)); + cl_must_pass(mkdir(parent.buf, 0777)); + source_fixture_release( + &fixture, &namespace_unstable, &namespace_unstable_from); + cl_assert_equal_i(namespace_unstable, 1); + cl_assert_equal_i(namespace_unstable_from, 17); + + strbuf_release(&source); + strbuf_release(&old_parent); + strbuf_release(&parent); +#endif +} diff --git a/worktree-attr-source.c b/worktree-attr-source.c new file mode 100644 index 00000000000000..d56043eba7a576 --- /dev/null +++ b/worktree-attr-source.c @@ -0,0 +1,93 @@ +#include "git-compat-util.h" +#include "attr.h" +#include "hash.h" +#include "path-namespace.h" +#include "semantic-verify-internal.h" +#include "worktree-attr-source.h" + +#define WORKTREE_ATTR_HASH_BUFFER_SIZE (64 * 1024) + +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + +int worktree_attr_source_read( + struct semantic_verify_path *path UNUSED, + const char *name UNUSED, size_t position UNUSED, + const struct git_hash_algo *algo UNUSED, + unsigned char *hash UNUSED, int *found) +{ + *found = 0; + return -1; +} + +#else + +int worktree_attr_source_read(struct semantic_verify_path *path, + const char *name, size_t position, + const struct git_hash_algo *algo, + unsigned char *hash, int *found) +{ + struct git_hash_ctx ctx = { 0 }; + struct stat before, after, named; + unsigned char buffer[WORKTREE_ATTR_HASH_BUFFER_SIZE]; + const char *basename; + ssize_t got; + size_t remaining, size; + int parent_fd, fd = -1, ret = -1; + char extra; + + *found = 0; + if (semantic_verify_resolve_parent(path, name, position, + &parent_fd, &basename)) { + if (errno == ENOENT || errno == ENOTDIR || errno == ELOOP || + errno == EXDEV) + return 0; + return -1; + } + if (fstatat(parent_fd, basename, &before, AT_SYMLINK_NOFOLLOW)) + return errno == ENOENT || errno == ENOTDIR ? 0 : -1; + if (!S_ISREG(before.st_mode) || + before.st_size < 0 || before.st_size >= ATTR_MAX_FILE_SIZE) + return 0; + if (before.st_nlink != 1) + return -1; + + fd = semantic_verify_openat(parent_fd, basename, + O_RDONLY | O_NONBLOCK | O_NOFOLLOW); + if (fd < 0) + return -1; + if (fstat(fd, &after) || + !path_namespace_stat_equal(&before, &after)) + goto done; + size = xsize_t(before.st_size); + remaining = size; + git_hash_init(&ctx, algo); + while (remaining) { + size_t want = remaining < sizeof(buffer) ? + remaining : sizeof(buffer); + + got = xread(fd, buffer, want); + if (got <= 0) + goto done; + git_hash_update(&ctx, buffer, got); + remaining -= got; + } + got = xread(fd, &extra, 1); + if (got != 0 || + fstat(fd, &after) || + fstatat(parent_fd, basename, &named, AT_SYMLINK_NOFOLLOW) || + !path_namespace_stat_equal(&before, &after) || + !path_namespace_stat_equal(&after, &named) || + path_namespace_reopen_component( + parent_fd, basename, O_RDONLY | O_NONBLOCK | O_NOFOLLOW, + semantic_verify_openat, &after)) + goto done; + git_hash_final(hash, &ctx); + *found = 1; + ret = 0; +done: + git_hash_discard(&ctx); + close(fd); + return ret; +} + +#endif /* SEMANTIC_VERIFY_HAS_ANCHORED_OPEN */ diff --git a/worktree-attr-source.h b/worktree-attr-source.h new file mode 100644 index 00000000000000..2db26c68a71119 --- /dev/null +++ b/worktree-attr-source.h @@ -0,0 +1,12 @@ +#ifndef WORKTREE_ATTR_SOURCE_H +#define WORKTREE_ATTR_SOURCE_H + +struct git_hash_algo; +struct semantic_verify_path; + +int worktree_attr_source_read(struct semantic_verify_path *path, + const char *name, size_t position, + const struct git_hash_algo *algo, + unsigned char *hash, int *found); + +#endif /* WORKTREE_ATTR_SOURCE_H */ From 199a31821fa1aabbeff9864a0aa737d82d426716 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 22:34:40 -0700 Subject: [PATCH 063/105] status: build worktree attribute manifests Checking only tracked .gitattributes entries misses worktree attribute files in ancestor directories. Conversely, silently substituting an indexed source for a hard-linked or unstable worktree file would certify conversion rules that Git did not safely observe. Collect the root and every directory scope containing a tracked entry. Read each candidate with S07/P04, prefer a stable worktree source, and use an available indexed object only when the worktree source is absent, nonregular, or oversized. Serialize existing sources in strict path order and hash the complete validated manifest with the repository's object algorithm. Reject unmerged or sparse indexes, missing indexed objects, source-read errors, hard links, replaced ancestors, and an unstable worktree root. Reset the output on failure so callers cannot retain a partial proof. Register the new library in both build systems. Unit tests cover tracked scopes, indexed fallback, a hard-linked worktree source despite an available indexed object, missing objects, and structural indexes. The builder is independently testable but does not run from status. Signed-off-by: Taylor Blau --- Makefile | 1 + hash-framing.h | 11 ++ meson.build | 1 + t/unit-tests/u-attr-manifest.c | 284 +++++++++++++++++++++++++++++++++ worktree-attr-manifest.c | 210 ++++++++++++++++++++++++ worktree-attr-manifest.h | 19 +++ 6 files changed, 526 insertions(+) create mode 100644 worktree-attr-manifest.c create mode 100644 worktree-attr-manifest.h diff --git a/Makefile b/Makefile index 9faa8a6587171b..2eb2b731f6a1d3 100644 --- a/Makefile +++ b/Makefile @@ -1383,6 +1383,7 @@ LIB_OBJS += versioncmp.o LIB_OBJS += walker.o LIB_OBJS += wildmatch.o LIB_OBJS += worktree.o +LIB_OBJS += worktree-attr-manifest.o LIB_OBJS += worktree-attr-source.o LIB_OBJS += wrapper.o LIB_OBJS += write-or-die.o diff --git a/hash-framing.h b/hash-framing.h index b15294b684a90d..f20b455e590f87 100644 --- a/hash-framing.h +++ b/hash-framing.h @@ -27,4 +27,15 @@ static inline void hash_optional_cstring(struct git_hash_ctx *ctx, hash_length_delimited(ctx, &missing, sizeof(missing)); } +static inline void hash_buffer_digest(const struct git_hash_algo *algo, + const void *data, size_t len, + unsigned char *hash) +{ + struct git_hash_ctx ctx; + + git_hash_init(&ctx, algo); + git_hash_update(&ctx, data, len); + git_hash_final(hash, &ctx); +} + #endif /* HASH_FRAMING_H */ diff --git a/meson.build b/meson.build index 912e7522aa1ace..98a692d05c1bb3 100644 --- a/meson.build +++ b/meson.build @@ -584,6 +584,7 @@ libgit_sources = [ 'walker.c', 'wildmatch.c', 'worktree.c', + 'worktree-attr-manifest.c', 'worktree-attr-source.c', 'wrapper.c', 'write-or-die.c', diff --git a/t/unit-tests/u-attr-manifest.c b/t/unit-tests/u-attr-manifest.c index 41f1d606889606..9f00a7d6f22b04 100644 --- a/t/unit-tests/u-attr-manifest.c +++ b/t/unit-tests/u-attr-manifest.c @@ -1,6 +1,15 @@ #include "unit-test.h" #include "attr-manifest.h" +#include "dir.h" +#include "hash.h" +#include "odb.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "semantic-verify-internal.h" +#include "setup.h" #include "strbuf.h" +#include "worktree-attr-manifest.h" +#include "wrapper.h" static void fill_hash(unsigned char *hash, unsigned char value, const struct git_hash_algo *algo) @@ -187,3 +196,278 @@ void test_attr_manifest__rejects_malformed_tail_before_callbacks(void) strbuf_release(&new); strbuf_release(&old); } + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +static char *create_worktree(void) +{ + const char *tmp = getenv("TMPDIR"); + char *path = xstrfmt("%s/attr-manifest.XXXXXX", tmp ? tmp : "/tmp"); + + cl_assert(mkdtemp(path) != NULL); + return path; +} + +static void remove_worktree(char *worktree) +{ + struct strbuf path = STRBUF_INIT; + + strbuf_addstr(&path, worktree); + cl_assert_equal_i(remove_dir_recursively(&path, 0), 0); + strbuf_release(&path); + free(worktree); +} + +static struct cache_entry *add_index_path(struct index_state *istate, + size_t pos, const char *path, + unsigned int stage) +{ + size_t len = strlen(path); + struct cache_entry *ce = make_empty_cache_entry(istate, len); + + ce->ce_mode = S_IFREG | 0644; + ce->ce_flags = create_ce_flags(stage); + ce->ce_namelen = len; + memcpy(ce->name, path, len + 1); + istate->cache[pos] = ce; + return ce; +} + +static void init_object_store(struct repository *repo, const char *worktree) +{ + struct strbuf object_dir = STRBUF_INIT; + + strbuf_addf(&object_dir, "%s/objects", worktree); + repo->objects = odb_new(repo, object_dir.buf, ""); + strbuf_release(&object_dir); +} +#endif + +void test_attr_manifest__builds_sources_for_tracked_scopes(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *worktree = create_worktree(); + struct repository repo = { .worktree = worktree, .hash_algo = algo }; + struct index_state istate = INDEX_STATE_INIT(&repo); + struct worktree_attr_manifest_stats stats; + struct attr_manifest_cursor cursor; + struct attr_manifest_entry entry; + struct git_hash_ctx ctx; + struct strbuf path = STRBUF_INIT, manifest = STRBUF_INIT; + char root_source[] = "*.root text\n"; + unsigned char expected[GIT_MAX_RAWSZ], hash[GIT_MAX_RAWSZ]; + + strbuf_addf(&path, "%s/a", worktree); + cl_assert_equal_i(mkdir(path.buf, 0777), 0); + strbuf_reset(&path); + strbuf_addf(&path, "%s/b", worktree); + cl_assert_equal_i(mkdir(path.buf, 0777), 0); + strbuf_reset(&path); + strbuf_addf(&path, "%s/.gitattributes", worktree); + write_file_buf(path.buf, root_source, strlen(root_source)); + git_hash_init(&ctx, algo); + git_hash_update(&ctx, root_source, strlen(root_source)); + git_hash_final(expected, &ctx); + strbuf_reset(&path); + strbuf_addf(&path, "%s/a/.gitattributes", worktree); + write_file(path.buf, "*.dat -text\n"); + + CALLOC_ARRAY(istate.cache, 2); + istate.cache_alloc = istate.cache_nr = 2; + add_index_path(&istate, 0, "a/file", 0); + add_index_path(&istate, 1, "b/file", 0); + cl_assert_equal_i(worktree_attr_manifest_build( + &istate, &manifest, hash, &stats), 0); + cl_assert_equal_i(stats.candidates, 3); + cl_assert_equal_i(stats.worktree_sources, 2); + cl_assert_equal_i(stats.index_sources, 0); + cl_assert(attr_manifest_valid(manifest.buf, manifest.len, algo)); + + cl_assert_equal_i(attr_manifest_cursor_init( + &cursor, manifest.buf, manifest.len, algo), 0); + cl_assert_equal_i(attr_manifest_cursor_next(&cursor, &entry), 1); + cl_assert_equal_i(entry.source, ATTR_MANIFEST_WORKTREE); + cl_assert_equal_i(entry.path_len, strlen(GITATTRIBUTES_FILE)); + cl_assert(!memcmp(entry.path, GITATTRIBUTES_FILE, entry.path_len)); + cl_assert(!memcmp(entry.hash, expected, algo->rawsz)); + cl_assert_equal_i(attr_manifest_cursor_next(&cursor, &entry), 1); + cl_assert_equal_i(entry.source, ATTR_MANIFEST_WORKTREE); + cl_assert_equal_i(attr_manifest_cursor_next(&cursor, &entry), 0); + + strbuf_release(&manifest); + strbuf_release(&path); + release_index(&istate); + remove_worktree(worktree); +#endif +} + +void test_attr_manifest__falls_back_to_index_source(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *worktree = create_worktree(); + char source[] = "*.dat text\n"; + struct repository repo = { .worktree = worktree, .hash_algo = algo }; + struct index_state istate = INDEX_STATE_INIT(&repo); + struct worktree_attr_manifest_stats stats; + struct attr_manifest_cursor cursor; + struct attr_manifest_entry entry; + struct cache_entry *attributes; + struct strbuf manifest = STRBUF_INIT; + unsigned char hash[GIT_MAX_RAWSZ]; + int have_repository, ret; + + init_object_store(&repo, worktree); + CALLOC_ARRAY(istate.cache, 1); + istate.cache_alloc = istate.cache_nr = 1; + attributes = add_index_path(&istate, 0, GITATTRIBUTES_FILE, 0); + cl_must_pass(odb_pretend_object( + repo.objects, source, strlen(source), OBJ_BLOB, + &attributes->oid)); + + have_repository = startup_info->have_repository; + startup_info->have_repository = 1; + ret = worktree_attr_manifest_build( + &istate, &manifest, hash, &stats); + startup_info->have_repository = have_repository; + cl_assert_equal_i(ret, 0); + cl_assert_equal_i(stats.candidates, 1); + cl_assert_equal_i(stats.worktree_sources, 0); + cl_assert_equal_i(stats.index_sources, 1); + cl_assert_equal_i(attr_manifest_cursor_init( + &cursor, manifest.buf, manifest.len, algo), 0); + cl_assert_equal_i(attr_manifest_cursor_next(&cursor, &entry), 1); + cl_assert_equal_i(entry.source, ATTR_MANIFEST_INDEX); + cl_assert_equal_i(entry.path_len, strlen(GITATTRIBUTES_FILE)); + cl_assert(!memcmp(entry.path, GITATTRIBUTES_FILE, entry.path_len)); + cl_assert(!memcmp(entry.hash, attributes->oid.hash, algo->rawsz)); + cl_assert_equal_i(attr_manifest_cursor_next(&cursor, &entry), 0); + + strbuf_release(&manifest); + release_index(&istate); + odb_free(repo.objects); + remove_worktree(worktree); +#endif +} + +void test_attr_manifest__rejects_hardlinked_source_over_index(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *worktree = create_worktree(); + char indexed_source[] = "*.dat text\n"; + struct repository repo = { .worktree = worktree, .hash_algo = algo }; + struct index_state istate = INDEX_STATE_INIT(&repo); + struct worktree_attr_manifest_stats stats; + struct cache_entry *attributes; + struct strbuf source = STRBUF_INIT, alias = STRBUF_INIT; + struct strbuf manifest = STRBUF_INIT; + unsigned char hash[GIT_MAX_RAWSZ]; + int have_repository, ret; + + init_object_store(&repo, worktree); + strbuf_addf(&source, "%s/a", worktree); + cl_assert_equal_i(mkdir(source.buf, 0777), 0); + strbuf_addstr(&source, "/" GITATTRIBUTES_FILE); + write_file(source.buf, "*.dat -text\n"); + strbuf_addf(&alias, "%s/attributes-alias", worktree); + cl_assert_equal_i(link(source.buf, alias.buf), 0); + + CALLOC_ARRAY(istate.cache, 2); + istate.cache_alloc = istate.cache_nr = 2; + attributes = add_index_path( + &istate, 0, "a/" GITATTRIBUTES_FILE, 0); + add_index_path(&istate, 1, "a/file", 0); + cl_must_pass(odb_pretend_object( + repo.objects, indexed_source, strlen(indexed_source), OBJ_BLOB, + &attributes->oid)); + + have_repository = startup_info->have_repository; + startup_info->have_repository = 1; + ret = worktree_attr_manifest_build( + &istate, &manifest, hash, &stats); + startup_info->have_repository = have_repository; + cl_assert_equal_i(ret, -1); + cl_assert_equal_i(manifest.len, 0); + cl_assert_equal_i(stats.worktree_sources, 0); + cl_assert_equal_i(stats.index_sources, 0); + + strbuf_release(&manifest); + strbuf_release(&alias); + strbuf_release(&source); + release_index(&istate); + odb_free(repo.objects); + remove_worktree(worktree); +#endif +} + +void test_attr_manifest__rejects_missing_index_source(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *worktree = create_worktree(); + struct repository repo = { .worktree = worktree, .hash_algo = algo }; + struct index_state istate = INDEX_STATE_INIT(&repo); + struct worktree_attr_manifest_stats stats; + struct cache_entry *attributes; + struct strbuf manifest = STRBUF_INIT; + unsigned char hash[GIT_MAX_RAWSZ]; + unsigned char missing[GIT_MAX_RAWSZ]; + + init_object_store(&repo, worktree); + CALLOC_ARRAY(istate.cache, 1); + istate.cache_alloc = istate.cache_nr = 1; + attributes = add_index_path(&istate, 0, GITATTRIBUTES_FILE, 0); + fill_hash(missing, 0x42, algo); + oidread(&attributes->oid, missing, algo); + strbuf_addstr(&manifest, "discard me"); + + cl_assert_equal_i(worktree_attr_manifest_build( + &istate, &manifest, hash, &stats), -1); + cl_assert_equal_i(manifest.len, 0); + + strbuf_release(&manifest); + release_index(&istate); + odb_free(repo.objects); + remove_worktree(worktree); +#endif +} + +void test_attr_manifest__builder_rejects_structural_indexes(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *worktree = create_worktree(); + struct repository repo = { .worktree = worktree, .hash_algo = algo }; + struct index_state istate = INDEX_STATE_INIT(&repo); + struct worktree_attr_manifest_stats stats; + struct strbuf manifest = STRBUF_INIT; + unsigned char hash[GIT_MAX_RAWSZ]; + + CALLOC_ARRAY(istate.cache, 1); + istate.cache_alloc = istate.cache_nr = 1; + add_index_path(&istate, 0, "file", 1); + cl_assert_equal_i(worktree_attr_manifest_build( + &istate, &manifest, hash, &stats), -1); + cl_assert_equal_i(manifest.len, 0); + istate.cache[0]->ce_flags = create_ce_flags(0); + istate.sparse_index = INDEX_COLLAPSED; + cl_assert_equal_i(worktree_attr_manifest_build( + &istate, &manifest, hash, &stats), -1); + cl_assert_equal_i(manifest.len, 0); + + strbuf_release(&manifest); + release_index(&istate); + remove_worktree(worktree); +#endif +} diff --git a/worktree-attr-manifest.c b/worktree-attr-manifest.c new file mode 100644 index 00000000000000..f30757677068cb --- /dev/null +++ b/worktree-attr-manifest.c @@ -0,0 +1,210 @@ +#include "git-compat-util.h" +#include "attr-manifest.h" +#include "dir.h" +#include "environment.h" +#include "hash-framing.h" +#include "object.h" +#include "odb.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "semantic-verify-internal.h" +#include "string-list.h" +#include "strbuf.h" +#include "worktree-attr-manifest.h" +#include "worktree-attr-source.h" + +struct attr_manifest_candidate { + unsigned char worktree_hash[GIT_MAX_RAWSZ]; + unsigned char index_hash[GIT_MAX_RAWSZ]; + unsigned int index_present : 1; + unsigned int worktree_present : 1; + unsigned int error : 1; +}; + +struct attr_manifest_probe_data { + struct string_list *candidates; + struct semantic_verify_root *root; + const struct git_hash_algo *algo; + size_t start; + size_t end; + unsigned int namespace_unstable; +}; + +static int collect_candidates(struct index_state *istate, + struct string_list *candidates) +{ + struct strbuf candidate = STRBUF_INIT; + const char *previous = NULL; + size_t previous_len = 0; + unsigned int i; + int ret = -1; + + string_list_append(candidates, GITATTRIBUTES_FILE); + for (i = 0; i < istate->cache_nr; i++) { + const struct cache_entry *ce = istate->cache[i]; + const char *slash = ce->name; + + if (ce_stage(ce) || S_ISSPARSEDIR(ce->ce_mode)) + goto done; + while ((slash = strchr(slash, '/')) != NULL) { + size_t len = slash - ce->name; + + if (!previous || previous_len <= len || + !is_dir_sep(previous[len]) || + fspathncmp(previous, ce->name, len)) { + strbuf_reset(&candidate); + strbuf_add(&candidate, ce->name, len + 1); + strbuf_addstr(&candidate, GITATTRIBUTES_FILE); + string_list_append(candidates, candidate.buf); + } + slash++; + } + previous = ce->name; + previous_len = ce->ce_namelen; + } + string_list_sort(candidates); + string_list_remove_duplicates(candidates, 0); + ret = candidates->nr <= UINT32_MAX ? 0 : -1; +done: + strbuf_release(&candidate); + return ret; +} + +static int collect_index_sources(struct index_state *istate, + struct string_list *candidates) +{ + struct strbuf candidate = STRBUF_INIT; + unsigned int i; + int ret = 0; + + for (i = 0; i < candidates->nr; i++) { + struct attr_manifest_candidate *state; + + CALLOC_ARRAY(state, 1); + candidates->items[i].util = state; + } + for (i = 0; i < istate->cache_nr; i++) { + const struct cache_entry *ce = istate->cache[i]; + const char *base = strrchr(ce->name, '/'); + struct string_list_item *item; + struct attr_manifest_candidate *state; + + base = base ? base + 1 : ce->name; + if (fspathcmp(base, GITATTRIBUTES_FILE)) + continue; + strbuf_reset(&candidate); + if (base != ce->name) + strbuf_add(&candidate, ce->name, base - ce->name); + strbuf_addstr(&candidate, GITATTRIBUTES_FILE); + item = string_list_lookup(candidates, candidate.buf); + if (!item) + BUG("tracked attribute source lacks manifest candidate"); + state = item->util; + if ((S_ISREG(ce->ce_mode) || S_ISLNK(ce->ce_mode)) && + odb_has_object(istate->repo->objects, &ce->oid, 0)) { + state->index_present = 1; + memcpy(state->index_hash, ce->oid.hash, + istate->repo->hash_algo->rawsz); + } else if (S_ISREG(ce->ce_mode) || S_ISLNK(ce->ce_mode)) { + ret = -1; + break; + } + } + strbuf_release(&candidate); + return ret; +} + +static void probe_attr_manifest_candidates( + struct attr_manifest_probe_data *data) +{ + struct semantic_verify_path *path = + semantic_verify_path_new(data->root); + size_t i; + + for (i = data->start; i < data->end; i++) { + struct string_list_item *item = &data->candidates->items[i]; + struct attr_manifest_candidate *candidate = item->util; + int found; + + if (worktree_attr_source_read(path, item->string, i, data->algo, + candidate->worktree_hash, &found)) + candidate->error = 1; + else + candidate->worktree_present = found; + } + semantic_verify_path_free(path, &data->namespace_unstable, NULL); +} + +static int probe_candidates(struct string_list *candidates, + struct semantic_verify_root *root, + const struct git_hash_algo *algo) +{ + struct attr_manifest_probe_data data = { + .candidates = candidates, + .root = root, + .algo = algo, + .end = candidates->nr, + }; + + probe_attr_manifest_candidates(&data); + return data.namespace_unstable ? -1 : 0; +} + +int worktree_attr_manifest_build( + struct index_state *istate, + struct strbuf *manifest, + unsigned char *manifest_hash, + struct worktree_attr_manifest_stats *stats) +{ + struct string_list candidates = STRING_LIST_INIT_DUP; + struct semantic_verify_root *root = NULL; + struct attr_manifest_writer writer; + const struct git_hash_algo *algo = istate->repo->hash_algo; + unsigned int i; + int ret = -1; + + memset(stats, 0, sizeof(*stats)); + if (istate->sparse_index != INDEX_EXPANDED || + semantic_verify_root_init(istate->repo, &root) || + collect_candidates(istate, &candidates) || + collect_index_sources(istate, &candidates)) + goto done; + stats->candidates = candidates.nr; + if (probe_candidates(&candidates, root, algo)) + goto done; + attr_manifest_writer_init(&writer, manifest, algo); + for (i = 0; i < candidates.nr; i++) { + const char *name = candidates.items[i].string; + struct attr_manifest_candidate *state = candidates.items[i].util; + enum attr_manifest_source source; + const unsigned char *hash; + + if (state->error) + goto done; + if (state->worktree_present) { + source = ATTR_MANIFEST_WORKTREE; + hash = state->worktree_hash; + stats->worktree_sources++; + } else if (state->index_present) { + source = ATTR_MANIFEST_INDEX; + hash = state->index_hash; + stats->index_sources++; + } else { + continue; + } + if (attr_manifest_writer_add(&writer, name, source, hash)) + goto done; + } + if (!semantic_verify_root_stable(root)) + goto done; + if (!attr_manifest_valid(manifest->buf, manifest->len, algo)) + BUG("newly built attribute manifest is invalid"); + hash_buffer_digest(algo, manifest->buf, manifest->len, manifest_hash); + ret = 0; +done: + semantic_verify_root_clear(root); + string_list_clear(&candidates, 1); + if (ret) + strbuf_reset(manifest); + return ret; +} diff --git a/worktree-attr-manifest.h b/worktree-attr-manifest.h new file mode 100644 index 00000000000000..4c3e8dc4ba762c --- /dev/null +++ b/worktree-attr-manifest.h @@ -0,0 +1,19 @@ +#ifndef WORKTREE_ATTR_MANIFEST_H +#define WORKTREE_ATTR_MANIFEST_H + +struct index_state; +struct strbuf; + +struct worktree_attr_manifest_stats { + size_t candidates; + size_t worktree_sources; + size_t index_sources; +}; + +int worktree_attr_manifest_build( + struct index_state *istate, + struct strbuf *manifest, + unsigned char *manifest_hash, + struct worktree_attr_manifest_stats *stats); + +#endif /* WORKTREE_ATTR_MANIFEST_H */ From ed40439dc46fd57eaf12fdf3ffe0cf3a636aff39 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 22:40:39 -0700 Subject: [PATCH 064/105] status: probe attribute sources in parallel The manifest builder from S07/P05 probes every tracked directory scope, including scopes without a worktree .gitattributes file. Those independent filesystem observations can run concurrently; parallel probing does not eliminate or reduce the source probes. Divide the sorted candidate list into contiguous ranges and give each worker its own anchored path state. Limit the worker count, preserve one-worker execution on builds without threads, and keep serialization on the calling thread so scheduling cannot change manifest order. If creating a worker fails, finish that range and all unstarted ranges on the calling thread. Join started workers and reject observed namespace instability before emitting the manifest. Unit tests compare serial and parallel manifest bytes and hashes and inject a worker-creation failure to verify complete, identical fallback output. Worker and descriptor state is bounded; no timing or memory result is claimed. Signed-off-by: Taylor Blau --- t/unit-tests/u-attr-manifest.c | 141 +++++++++++++++++++++++++++++++++ worktree-attr-manifest.c | 99 ++++++++++++++++++++--- worktree-attr-manifest.h | 2 + 3 files changed, 229 insertions(+), 13 deletions(-) diff --git a/t/unit-tests/u-attr-manifest.c b/t/unit-tests/u-attr-manifest.c index 9f00a7d6f22b04..a41eb2a7975fac 100644 --- a/t/unit-tests/u-attr-manifest.c +++ b/t/unit-tests/u-attr-manifest.c @@ -8,9 +8,15 @@ #include "semantic-verify-internal.h" #include "setup.h" #include "strbuf.h" +#include "thread-utils.h" #include "worktree-attr-manifest.h" #include "wrapper.h" +#define ATTR_MANIFEST_TEST_THREADS "GIT_TEST_ATTR_MANIFEST_THREADS" +#define ATTR_MANIFEST_TEST_THREAD_FAIL_AT \ + "GIT_TEST_ATTR_MANIFEST_THREAD_FAIL_AT" +#define ATTR_MANIFEST_TEST_SOURCE_NR 257 + static void fill_hash(unsigned char *hash, unsigned char value, const struct git_hash_algo *algo) { @@ -441,6 +447,141 @@ void test_attr_manifest__rejects_missing_index_source(void) #endif } +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +struct many_sources_fixture { + const struct git_hash_algo *algo; + char *worktree; + struct repository repo; + struct index_state istate; +}; + +static void many_sources_fixture_init(struct many_sources_fixture *fixture) +{ + struct strbuf path = STRBUF_INIT; + size_t i; + + memset(fixture, 0, sizeof(*fixture)); + fixture->algo = &hash_algos[GIT_HASH_SHA1]; + fixture->worktree = create_worktree(); + fixture->repo.worktree = fixture->worktree; + fixture->repo.hash_algo = fixture->algo; + index_state_init(&fixture->istate, &fixture->repo); + CALLOC_ARRAY(fixture->istate.cache, ATTR_MANIFEST_TEST_SOURCE_NR); + fixture->istate.cache_alloc = fixture->istate.cache_nr = + ATTR_MANIFEST_TEST_SOURCE_NR; + + for (i = 0; i < ATTR_MANIFEST_TEST_SOURCE_NR; i++) { + strbuf_reset(&path); + strbuf_addf(&path, "%s/d%03" PRIuMAX, + fixture->worktree, (uintmax_t)i); + cl_assert_equal_i(mkdir(path.buf, 0777), 0); + strbuf_addstr(&path, "/" GITATTRIBUTES_FILE); + write_file(path.buf, "source %" PRIuMAX "\n", (uintmax_t)i); + strbuf_reset(&path); + strbuf_addf(&path, "d%03" PRIuMAX "/file", (uintmax_t)i); + add_index_path(&fixture->istate, i, path.buf, 0); + } + strbuf_release(&path); +} + +static void many_sources_fixture_release(struct many_sources_fixture *fixture) +{ + release_index(&fixture->istate); + remove_worktree(fixture->worktree); +} + +static void clear_attr_manifest_thread_env(void *unused UNUSED) +{ + unsetenv(ATTR_MANIFEST_TEST_THREADS); + unsetenv(ATTR_MANIFEST_TEST_THREAD_FAIL_AT); +} +#endif + +void test_attr_manifest__parallel_probes_match_serial_output(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + struct many_sources_fixture fixture; + struct worktree_attr_manifest_stats serial_stats, parallel_stats; + struct strbuf serial = STRBUF_INIT, parallel = STRBUF_INIT; + unsigned char serial_hash[GIT_MAX_RAWSZ]; + unsigned char parallel_hash[GIT_MAX_RAWSZ]; + + if (!HAVE_THREADS) + return; + cl_set_cleanup(clear_attr_manifest_thread_env, NULL); + many_sources_fixture_init(&fixture); + + xsetenv(ATTR_MANIFEST_TEST_THREADS, "1", 1); + cl_assert_equal_i(worktree_attr_manifest_build( + &fixture.istate, &serial, serial_hash, &serial_stats), 0); + cl_assert_equal_i(serial_stats.candidates, + ATTR_MANIFEST_TEST_SOURCE_NR + 1); + cl_assert_equal_i(serial_stats.threads, 1); + cl_assert_equal_i(serial_stats.worktree_sources, + ATTR_MANIFEST_TEST_SOURCE_NR); + + xsetenv(ATTR_MANIFEST_TEST_THREADS, "2", 1); + cl_assert_equal_i(worktree_attr_manifest_build( + &fixture.istate, ¶llel, parallel_hash, ¶llel_stats), 0); + cl_assert_equal_i(parallel_stats.candidates, + ATTR_MANIFEST_TEST_SOURCE_NR + 1); + cl_assert_equal_i(parallel_stats.threads, 2); + cl_assert_equal_i(parallel_stats.thread_failures, 0); + cl_assert_equal_i(parallel_stats.worktree_sources, + ATTR_MANIFEST_TEST_SOURCE_NR); + cl_assert_equal_i(serial.len, parallel.len); + cl_assert(!memcmp(serial.buf, parallel.buf, serial.len)); + cl_assert(!memcmp(serial_hash, parallel_hash, fixture.algo->rawsz)); + + strbuf_release(¶llel); + strbuf_release(&serial); + many_sources_fixture_release(&fixture); + clear_attr_manifest_thread_env(NULL); +#endif +} + +void test_attr_manifest__thread_failure_completes_remaining_ranges(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + struct many_sources_fixture fixture; + struct worktree_attr_manifest_stats serial_stats, fallback_stats; + struct strbuf serial = STRBUF_INIT, fallback = STRBUF_INIT; + unsigned char serial_hash[GIT_MAX_RAWSZ]; + unsigned char fallback_hash[GIT_MAX_RAWSZ]; + + if (!HAVE_THREADS) + return; + cl_set_cleanup(clear_attr_manifest_thread_env, NULL); + many_sources_fixture_init(&fixture); + + xsetenv(ATTR_MANIFEST_TEST_THREADS, "1", 1); + cl_assert_equal_i(worktree_attr_manifest_build( + &fixture.istate, &serial, serial_hash, &serial_stats), 0); + xsetenv(ATTR_MANIFEST_TEST_THREADS, "2", 1); + xsetenv(ATTR_MANIFEST_TEST_THREAD_FAIL_AT, "1", 1); + cl_assert_equal_i(worktree_attr_manifest_build( + &fixture.istate, &fallback, fallback_hash, &fallback_stats), 0); + cl_assert_equal_i(fallback_stats.candidates, + ATTR_MANIFEST_TEST_SOURCE_NR + 1); + cl_assert_equal_i(fallback_stats.threads, 2); + cl_assert_equal_i(fallback_stats.thread_failures, 1); + cl_assert_equal_i(fallback_stats.worktree_sources, + ATTR_MANIFEST_TEST_SOURCE_NR); + cl_assert_equal_i(serial.len, fallback.len); + cl_assert(!memcmp(serial.buf, fallback.buf, serial.len)); + cl_assert(!memcmp(serial_hash, fallback_hash, fixture.algo->rawsz)); + + strbuf_release(&fallback); + strbuf_release(&serial); + many_sources_fixture_release(&fixture); + clear_attr_manifest_thread_env(NULL); +#endif +} + void test_attr_manifest__builder_rejects_structural_indexes(void) { #if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN diff --git a/worktree-attr-manifest.c b/worktree-attr-manifest.c index f30757677068cb..1a0234360c67cb 100644 --- a/worktree-attr-manifest.c +++ b/worktree-attr-manifest.c @@ -5,14 +5,19 @@ #include "hash-framing.h" #include "object.h" #include "odb.h" +#include "parse.h" #include "read-cache-ll.h" #include "repository.h" #include "semantic-verify-internal.h" #include "string-list.h" #include "strbuf.h" +#include "thread-utils.h" #include "worktree-attr-manifest.h" #include "worktree-attr-source.h" +#define ATTR_MANIFEST_FILES_PER_THREAD 256 +#define ATTR_MANIFEST_MAX_THREADS 32 + struct attr_manifest_candidate { unsigned char worktree_hash[GIT_MAX_RAWSZ]; unsigned char index_hash[GIT_MAX_RAWSZ]; @@ -30,6 +35,12 @@ struct attr_manifest_probe_data { unsigned int namespace_unstable; }; +struct attr_manifest_thread { + struct attr_manifest_probe_data probe; + pthread_t pthread; + unsigned int started : 1; +}; + static int collect_candidates(struct index_state *istate, struct string_list *candidates) { @@ -114,9 +125,9 @@ static int collect_index_sources(struct index_state *istate, return ret; } -static void probe_attr_manifest_candidates( - struct attr_manifest_probe_data *data) +static void *probe_attr_manifest_candidates(void *cb_data) { + struct attr_manifest_probe_data *data = cb_data; struct semantic_verify_path *path = semantic_verify_path_new(data->root); size_t i; @@ -133,21 +144,83 @@ static void probe_attr_manifest_candidates( candidate->worktree_present = found; } semantic_verify_path_free(path, &data->namespace_unstable, NULL); + return NULL; +} + +static size_t select_thread_count(size_t candidates) +{ + size_t cpus, test_threads, threads; + + if (!HAVE_THREADS) + return 1; + threads = DIV_ROUND_UP(candidates, ATTR_MANIFEST_FILES_PER_THREAD); + cpus = online_cpus(); + if (threads > cpus * 2) + threads = cpus * 2; + test_threads = git_env_ulong("GIT_TEST_ATTR_MANIFEST_THREADS", 0); + if (test_threads) + threads = test_threads; + if (threads > ATTR_MANIFEST_MAX_THREADS) + threads = ATTR_MANIFEST_MAX_THREADS; + if (threads > candidates) + threads = candidates; + return threads ? threads : 1; +} + +static int create_probe_thread(struct attr_manifest_thread *worker, + size_t thread_id) +{ + if (git_env_ulong("GIT_TEST_ATTR_MANIFEST_THREAD_FAIL_AT", + ULONG_MAX) == thread_id) + return EAGAIN; + return pthread_create(&worker->pthread, NULL, + probe_attr_manifest_candidates, &worker->probe); } static int probe_candidates(struct string_list *candidates, struct semantic_verify_root *root, - const struct git_hash_algo *algo) + const struct git_hash_algo *algo, + struct worktree_attr_manifest_stats *stats) { - struct attr_manifest_probe_data data = { - .candidates = candidates, - .root = root, - .algo = algo, - .end = candidates->nr, - }; - - probe_attr_manifest_candidates(&data); - return data.namespace_unstable ? -1 : 0; + struct attr_manifest_thread *workers; + size_t thread_id, threads = select_thread_count(candidates->nr); + int create_threads = HAVE_THREADS; + int ret = 0; + + CALLOC_ARRAY(workers, threads); + for (thread_id = 0; thread_id < threads; thread_id++) { + struct attr_manifest_thread *worker = &workers[thread_id]; + struct attr_manifest_probe_data *data = &worker->probe; + int err; + + data->candidates = candidates; + data->root = root; + data->algo = algo; + data->start = st_mult(candidates->nr, thread_id) / threads; + data->end = st_mult(candidates->nr, thread_id + 1) / threads; + if (threads == 1 || !create_threads) { + probe_attr_manifest_candidates(data); + continue; + } + err = create_probe_thread(worker, thread_id); + if (!err) { + worker->started = 1; + continue; + } + stats->thread_failures++; + create_threads = 0; + probe_attr_manifest_candidates(data); + } + for (thread_id = 0; thread_id < threads; thread_id++) { + struct attr_manifest_thread *worker = &workers[thread_id]; + + if (worker->started && pthread_join(worker->pthread, NULL)) + die("unable to join attribute manifest thread"); + ret |= worker->probe.namespace_unstable; + } + stats->threads = threads; + free(workers); + return ret ? -1 : 0; } int worktree_attr_manifest_build( @@ -170,7 +243,7 @@ int worktree_attr_manifest_build( collect_index_sources(istate, &candidates)) goto done; stats->candidates = candidates.nr; - if (probe_candidates(&candidates, root, algo)) + if (probe_candidates(&candidates, root, algo, stats)) goto done; attr_manifest_writer_init(&writer, manifest, algo); for (i = 0; i < candidates.nr; i++) { diff --git a/worktree-attr-manifest.h b/worktree-attr-manifest.h index 4c3e8dc4ba762c..4b3e17c26abd35 100644 --- a/worktree-attr-manifest.h +++ b/worktree-attr-manifest.h @@ -6,8 +6,10 @@ struct strbuf; struct worktree_attr_manifest_stats { size_t candidates; + size_t threads; size_t worktree_sources; size_t index_sources; + size_t thread_failures; }; int worktree_attr_manifest_build( From 42e25d346f064439b258c244b2df70ae87634b6e Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 14:30:48 -0700 Subject: [PATCH 065/105] fsmonitor: add a bounded clean-proof record format A provider token cannot establish clean status unless its configuration, conversion semantics, attribute sources, and manifest are recorded as one verifiable observation. Parsing a malformed record directly into index state could also publish part of an invalid proof. Define a versioned, length-delimited clean-proof codec with distinct magic, bounded flags, a bounded token, object-format-sized configuration and attribute hashes, the validated attribute manifest, and a trailing checksum. Parse into temporary state and publish the decoded view only after all lengths, flags, token bytes, manifest records, and the checksum agree. Allow a generic writer to retain validated history while clearing its token and stat bindings instead of asserting a fresh provider epoch. Register the library and Clar suite in both Make and Meson. Tests cover both object formats, corrupt and truncated records, invalid flags, embedded token NULs, checksum changes, and selective clearing of epoch bindings. The codec does not attach an extension to an index or enable an early status answer. Signed-off-by: Taylor Blau --- Makefile | 2 + fsmonitor-clean-proof.c | 116 ++++++++++++++++++++ fsmonitor-clean-proof.h | 42 ++++++++ hash-framing.h | 10 ++ meson.build | 1 + t/meson.build | 1 + t/unit-tests/u-fsmonitor-clean-proof.c | 141 +++++++++++++++++++++++++ 7 files changed, 313 insertions(+) create mode 100644 fsmonitor-clean-proof.c create mode 100644 fsmonitor-clean-proof.h create mode 100644 t/unit-tests/u-fsmonitor-clean-proof.c diff --git a/Makefile b/Makefile index 2eb2b731f6a1d3..f2fe60decb2d5b 100644 --- a/Makefile +++ b/Makefile @@ -1176,6 +1176,7 @@ LIB_OBJS += fetch-object-info.o LIB_OBJS += fetch-pack.o LIB_OBJS += fmt-merge-msg.o LIB_OBJS += fsck.o +LIB_OBJS += fsmonitor-clean-proof.o LIB_OBJS += fsmonitor.o LIB_OBJS += fsmonitor-ipc.o LIB_OBJS += fsmonitor-settings.o @@ -1552,6 +1553,7 @@ CLAR_TEST_SUITES += u-ctype CLAR_TEST_SUITES += u-dir CLAR_TEST_SUITES += u-example-decorate CLAR_TEST_SUITES += u-fsmonitor-attributes +CLAR_TEST_SUITES += u-fsmonitor-clean-proof CLAR_TEST_SUITES += u-hash CLAR_TEST_SUITES += u-hashmap CLAR_TEST_SUITES += u-list-objects-filter-options diff --git a/fsmonitor-clean-proof.c b/fsmonitor-clean-proof.c new file mode 100644 index 00000000000000..3c45c014469737 --- /dev/null +++ b/fsmonitor-clean-proof.c @@ -0,0 +1,116 @@ +#include "git-compat-util.h" +#include "attr-manifest.h" +#include "fsmonitor-clean-proof.h" +#include "hash-framing.h" +#include "strbuf.h" + +#define FSMONITOR_CLEAN_PROOF_MAGIC 0x46534331 /* "FSC1" */ +#define FSMONITOR_CLEAN_PROOF_HEADER_WORDS 5 + +int fsmonitor_clean_proof_parse(struct fsmonitor_clean_proof *proof, + const void *data, size_t len, + const struct git_hash_algo *algo) +{ + struct fsmonitor_clean_proof parsed = { 0 }; + const unsigned char *p = data; + const unsigned char *end = p + len; + unsigned char checksum[GIT_MAX_RAWSZ]; + size_t hashes_len = 4 * algo->rawsz; + uint32_t token_len, manifest_len; + + memset(proof, 0, sizeof(*proof)); + if (len < FSMONITOR_CLEAN_PROOF_HEADER_WORDS * sizeof(uint32_t) + + hashes_len + 1) + return -1; + if (get_be32(p) != FSMONITOR_CLEAN_PROOF_VERSION) + return -1; + p += sizeof(uint32_t); + if (get_be32(p) != FSMONITOR_CLEAN_PROOF_MAGIC) + return -1; + p += sizeof(uint32_t); + parsed.flags = get_be32(p); + p += sizeof(uint32_t); + token_len = get_be32(p); + p += sizeof(uint32_t); + manifest_len = get_be32(p); + p += sizeof(uint32_t); + if (parsed.flags & ~FSMONITOR_CLEAN_PROOF_ALL || !token_len || + token_len > FSMONITOR_CLEAN_PROOF_TOKEN_MAX || + manifest_len < sizeof(uint32_t) || + (size_t)(end - p) < token_len || memchr(p, '\0', token_len)) + return -1; + parsed.token = p; + parsed.token_len = token_len; + p += token_len; + if ((size_t)(end - p) < hashes_len || + (size_t)(end - p) - hashes_len != manifest_len) + return -1; + parsed.config_hash = p; + p += algo->rawsz; + parsed.semantic_hash = p; + p += algo->rawsz; + parsed.attr_hash = p; + p += algo->rawsz; + parsed.attr_manifest = p; + parsed.attr_manifest_len = manifest_len; + p += manifest_len; + if (!attr_manifest_valid(parsed.attr_manifest, + parsed.attr_manifest_len, algo)) + return -1; + hash_buffer_digest(algo, data, len - algo->rawsz, checksum); + if (memcmp(checksum, p, algo->rawsz)) + return -1; + *proof = parsed; + return 0; +} + +int fsmonitor_clean_proof_write(struct strbuf *out, + const struct fsmonitor_clean_proof *proof, + const struct git_hash_algo *algo) +{ + uint32_t value; + + strbuf_reset(out); + if (!proof->token || !proof->token_len || + proof->token_len > FSMONITOR_CLEAN_PROOF_TOKEN_MAX || + proof->token_len > UINT32_MAX || + memchr(proof->token, '\0', proof->token_len) || + proof->flags & ~FSMONITOR_CLEAN_PROOF_ALL || + !proof->config_hash || !proof->semantic_hash || !proof->attr_hash || + !proof->attr_manifest || proof->attr_manifest_len > UINT32_MAX || + !attr_manifest_valid(proof->attr_manifest, + proof->attr_manifest_len, algo)) + return -1; + + put_be32(&value, FSMONITOR_CLEAN_PROOF_VERSION); + strbuf_add(out, &value, sizeof(value)); + put_be32(&value, FSMONITOR_CLEAN_PROOF_MAGIC); + strbuf_add(out, &value, sizeof(value)); + put_be32(&value, proof->flags); + strbuf_add(out, &value, sizeof(value)); + put_be32(&value, proof->token_len); + strbuf_add(out, &value, sizeof(value)); + put_be32(&value, proof->attr_manifest_len); + strbuf_add(out, &value, sizeof(value)); + strbuf_add(out, proof->token, proof->token_len); + strbuf_add(out, proof->config_hash, algo->rawsz); + strbuf_add(out, proof->semantic_hash, algo->rawsz); + strbuf_add(out, proof->attr_hash, algo->rawsz); + strbuf_add(out, proof->attr_manifest, proof->attr_manifest_len); + hash_append_checksum(out, algo); + return 0; +} + +int fsmonitor_clean_proof_copy_without_bindings( + struct strbuf *out, const void *data, size_t len, + const struct git_hash_algo *algo) +{ + struct fsmonitor_clean_proof proof; + + strbuf_reset(out); + if (fsmonitor_clean_proof_parse(&proof, data, len, algo)) + return -1; + proof.flags &= ~(FSMONITOR_CLEAN_PROOF_TOKEN_BOUND | + FSMONITOR_CLEAN_PROOF_STAT_BOUND); + return fsmonitor_clean_proof_write(out, &proof, algo); +} diff --git a/fsmonitor-clean-proof.h b/fsmonitor-clean-proof.h new file mode 100644 index 00000000000000..0d4da4cd725803 --- /dev/null +++ b/fsmonitor-clean-proof.h @@ -0,0 +1,42 @@ +#ifndef FSMONITOR_CLEAN_PROOF_H +#define FSMONITOR_CLEAN_PROOF_H + +#include "hash.h" + +struct strbuf; + +#define FSMONITOR_CLEAN_PROOF_VERSION 1 +#define FSMONITOR_CLEAN_PROOF_TOKEN_MAX 4096 + +#define FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE (1u << 0) +#define FSMONITOR_CLEAN_PROOF_TOKEN_BOUND (1u << 1) +#define FSMONITOR_CLEAN_PROOF_STAT_BOUND (1u << 2) +#define FSMONITOR_CLEAN_PROOF_FULL_INDEX (1u << 3) +#define FSMONITOR_CLEAN_PROOF_ALL \ + (FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | \ + FSMONITOR_CLEAN_PROOF_TOKEN_BOUND | \ + FSMONITOR_CLEAN_PROOF_STAT_BOUND | \ + FSMONITOR_CLEAN_PROOF_FULL_INDEX) + +struct fsmonitor_clean_proof { + uint32_t flags; + const unsigned char *token; + size_t token_len; + const unsigned char *config_hash; + const unsigned char *semantic_hash; + const unsigned char *attr_hash; + const unsigned char *attr_manifest; + size_t attr_manifest_len; +}; + +int fsmonitor_clean_proof_parse(struct fsmonitor_clean_proof *proof, + const void *data, size_t len, + const struct git_hash_algo *algo); +int fsmonitor_clean_proof_write(struct strbuf *out, + const struct fsmonitor_clean_proof *proof, + const struct git_hash_algo *algo); +int fsmonitor_clean_proof_copy_without_bindings( + struct strbuf *out, const void *data, size_t len, + const struct git_hash_algo *algo); + +#endif /* FSMONITOR_CLEAN_PROOF_H */ diff --git a/hash-framing.h b/hash-framing.h index f20b455e590f87..6808cc288faa04 100644 --- a/hash-framing.h +++ b/hash-framing.h @@ -2,6 +2,7 @@ #define HASH_FRAMING_H #include "hash.h" +#include "strbuf.h" static inline void hash_length_delimited(struct git_hash_ctx *ctx, const void *data, size_t len) @@ -38,4 +39,13 @@ static inline void hash_buffer_digest(const struct git_hash_algo *algo, git_hash_final(hash, &ctx); } +static inline void hash_append_checksum(struct strbuf *out, + const struct git_hash_algo *algo) +{ + unsigned char hash[GIT_MAX_RAWSZ]; + + hash_buffer_digest(algo, out->buf, out->len, hash); + strbuf_add(out, hash, algo->rawsz); +} + #endif /* HASH_FRAMING_H */ diff --git a/meson.build b/meson.build index 98a692d05c1bb3..c81db56fb078b4 100644 --- a/meson.build +++ b/meson.build @@ -380,6 +380,7 @@ libgit_sources = [ 'fetch-pack.c', 'fmt-merge-msg.c', 'fsck.c', + 'fsmonitor-clean-proof.c', 'fsmonitor.c', 'fsmonitor-ipc.c', 'fsmonitor-settings.c', diff --git a/t/meson.build b/t/meson.build index 73372cd547f84c..812a11c51d18d7 100644 --- a/t/meson.build +++ b/t/meson.build @@ -5,6 +5,7 @@ clar_test_suites = [ 'unit-tests/u-dir.c', 'unit-tests/u-example-decorate.c', 'unit-tests/u-fsmonitor-attributes.c', + 'unit-tests/u-fsmonitor-clean-proof.c', 'unit-tests/u-hash.c', 'unit-tests/u-hashmap.c', 'unit-tests/u-list-objects-filter-options.c', diff --git a/t/unit-tests/u-fsmonitor-clean-proof.c b/t/unit-tests/u-fsmonitor-clean-proof.c new file mode 100644 index 00000000000000..b4691221c75f49 --- /dev/null +++ b/t/unit-tests/u-fsmonitor-clean-proof.c @@ -0,0 +1,141 @@ +#include "unit-test.h" +#include "attr-manifest.h" +#include "fsmonitor-clean-proof.h" +#include "strbuf.h" + +struct proof_fixture { + struct strbuf manifest; + struct strbuf encoded; + unsigned char config_hash[GIT_MAX_RAWSZ]; + unsigned char semantic_hash[GIT_MAX_RAWSZ]; + unsigned char attr_hash[GIT_MAX_RAWSZ]; + struct fsmonitor_clean_proof proof; +}; + +static void fixture_init(struct proof_fixture *fixture, + const struct git_hash_algo *algo) +{ + struct attr_manifest_writer writer; + unsigned char hash[GIT_MAX_RAWSZ]; + static const unsigned char token[] = "builtin:1:2"; + + memset(fixture, 0, sizeof(*fixture)); + fixture->manifest = (struct strbuf)STRBUF_INIT; + fixture->encoded = (struct strbuf)STRBUF_INIT; + memset(hash, 1, algo->rawsz); + memset(fixture->config_hash, 2, algo->rawsz); + memset(fixture->semantic_hash, 3, algo->rawsz); + memset(fixture->attr_hash, 4, algo->rawsz); + attr_manifest_writer_init(&writer, &fixture->manifest, algo); + cl_assert_equal_i(attr_manifest_writer_add( + &writer, ".gitattributes", ATTR_MANIFEST_INDEX, hash), 0); + fixture->proof.flags = FSMONITOR_CLEAN_PROOF_ALL; + fixture->proof.token = token; + fixture->proof.token_len = sizeof(token) - 1; + fixture->proof.config_hash = fixture->config_hash; + fixture->proof.semantic_hash = fixture->semantic_hash; + fixture->proof.attr_hash = fixture->attr_hash; + fixture->proof.attr_manifest = + (const unsigned char *)fixture->manifest.buf; + fixture->proof.attr_manifest_len = fixture->manifest.len; +} + +static void fixture_release(struct proof_fixture *fixture) +{ + strbuf_release(&fixture->encoded); + strbuf_release(&fixture->manifest); +} + +static void assert_round_trip(const struct git_hash_algo *algo) +{ + struct proof_fixture fixture; + struct fsmonitor_clean_proof parsed; + + fixture_init(&fixture, algo); + cl_assert_equal_i(fsmonitor_clean_proof_write( + &fixture.encoded, &fixture.proof, algo), 0); + cl_assert_equal_i(fsmonitor_clean_proof_parse( + &parsed, fixture.encoded.buf, fixture.encoded.len, algo), 0); + cl_assert_equal_i(parsed.flags, fixture.proof.flags); + cl_assert_equal_i(parsed.token_len, fixture.proof.token_len); + cl_assert(!memcmp(parsed.token, fixture.proof.token, parsed.token_len)); + cl_assert(!memcmp(parsed.config_hash, fixture.config_hash, algo->rawsz)); + cl_assert(!memcmp(parsed.attr_manifest, fixture.manifest.buf, + parsed.attr_manifest_len)); + fixture_release(&fixture); +} + +static void assert_rejected(struct fsmonitor_clean_proof *parsed, + const struct strbuf *encoded, + const struct git_hash_algo *algo) +{ + memset(parsed, 0xff, sizeof(*parsed)); + cl_assert_equal_i(fsmonitor_clean_proof_parse( + parsed, encoded->buf, encoded->len, algo), -1); + cl_assert_equal_i(parsed->flags, 0); + cl_assert_equal_p(parsed->token, NULL); + cl_assert_equal_i(parsed->token_len, 0); + cl_assert_equal_p(parsed->config_hash, NULL); + cl_assert_equal_p(parsed->semantic_hash, NULL); + cl_assert_equal_p(parsed->attr_hash, NULL); + cl_assert_equal_p(parsed->attr_manifest, NULL); + cl_assert_equal_i(parsed->attr_manifest_len, 0); +} + +void test_fsmonitor_clean_proof__round_trips_both_object_formats(void) +{ + assert_round_trip(&hash_algos[GIT_HASH_SHA1]); + assert_round_trip(&hash_algos[GIT_HASH_SHA256]); +} + +void test_fsmonitor_clean_proof__rejects_corrupt_records(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct proof_fixture fixture; + struct fsmonitor_clean_proof parsed; + size_t token_offset = 5 * sizeof(uint32_t); + uint32_t saved; + unsigned char byte; + + fixture_init(&fixture, algo); + cl_assert_equal_i(fsmonitor_clean_proof_write( + &fixture.encoded, &fixture.proof, algo), 0); + fixture.encoded.len--; + assert_rejected(&parsed, &fixture.encoded, algo); + fixture.encoded.len++; + saved = get_be32(fixture.encoded.buf + 2 * sizeof(uint32_t)); + put_be32(fixture.encoded.buf + 2 * sizeof(uint32_t), 1u << 31); + assert_rejected(&parsed, &fixture.encoded, algo); + put_be32(fixture.encoded.buf + 2 * sizeof(uint32_t), saved); + byte = fixture.encoded.buf[token_offset]; + fixture.encoded.buf[token_offset] = '\0'; + assert_rejected(&parsed, &fixture.encoded, algo); + fixture.encoded.buf[token_offset] = byte; + fixture.encoded.buf[fixture.encoded.len - 1] ^= 1; + assert_rejected(&parsed, &fixture.encoded, algo); + fixture_release(&fixture); +} + +void test_fsmonitor_clean_proof__clears_only_epoch_bindings(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA256]; + struct proof_fixture fixture; + struct fsmonitor_clean_proof parsed; + struct strbuf unbound = STRBUF_INIT; + + fixture_init(&fixture, algo); + cl_assert_equal_i(fsmonitor_clean_proof_write( + &fixture.encoded, &fixture.proof, algo), 0); + cl_assert_equal_i(fsmonitor_clean_proof_copy_without_bindings( + &unbound, fixture.encoded.buf, fixture.encoded.len, algo), 0); + cl_assert_equal_i(fsmonitor_clean_proof_parse( + &parsed, unbound.buf, unbound.len, algo), 0); + cl_assert_equal_i(parsed.flags, + FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | + FSMONITOR_CLEAN_PROOF_FULL_INDEX); + cl_assert_equal_i(parsed.token_len, fixture.proof.token_len); + cl_assert(!memcmp(parsed.attr_manifest, fixture.manifest.buf, + fixture.manifest.len)); + strbuf_release(&unbound); + fixture_release(&fixture); +} From fc309a20394c72a5805d584db2d2d972ddceaf8c Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 14:42:56 -0700 Subject: [PATCH 066/105] status: retain validated worktree attribute manifests A persisted attribute manifest must not become current merely because its bytes were present in an index. Invalid records or unknown proof flags could otherwise leave half-loaded history available to a later clean-status decision. Introduce a state object that owns separate persisted and current manifest buffers, their hashes, proof flags, and validity bits. Validate incoming bytes with S07/P02 and accept only the flags defined by S07/P07. Clear the persisted view before loading; copy it into the current view only through explicit adoption. Register the state library and Clar suite in both Make and Meson. Tests cover valid loading and adoption, explicit current-state invalidation, and removal of a previously valid persisted view after malformed input. This state does not read an index extension, scan the worktree, invalidate attribute paths, or activate a status optimization. Signed-off-by: Taylor Blau --- Makefile | 2 + clean-status-manifest.c | 56 +++++++++++++++++++++ clean-status-manifest.h | 29 +++++++++++ meson.build | 1 + t/meson.build | 1 + t/unit-tests/u-clean-status-manifest.c | 70 ++++++++++++++++++++++++++ 6 files changed, 159 insertions(+) create mode 100644 clean-status-manifest.c create mode 100644 clean-status-manifest.h create mode 100644 t/unit-tests/u-clean-status-manifest.c diff --git a/Makefile b/Makefile index f2fe60decb2d5b..8bdaab2391f7b2 100644 --- a/Makefile +++ b/Makefile @@ -1125,6 +1125,7 @@ LIB_OBJS += chdir-notify.o LIB_OBJS += checkout.o LIB_OBJS += chunk-format.o LIB_OBJS += clean-status-config.o +LIB_OBJS += clean-status-manifest.o LIB_OBJS += color.o LIB_OBJS += column.o LIB_OBJS += combine-diff.o @@ -1549,6 +1550,7 @@ THIRD_PARTY_SOURCES += $(UNIT_TEST_DIR)/clar/clar/% CLAR_TEST_SUITES += u-attr-manifest CLAR_TEST_SUITES += u-clean-status-config +CLAR_TEST_SUITES += u-clean-status-manifest CLAR_TEST_SUITES += u-ctype CLAR_TEST_SUITES += u-dir CLAR_TEST_SUITES += u-example-decorate diff --git a/clean-status-manifest.c b/clean-status-manifest.c new file mode 100644 index 00000000000000..713d8bd4d5e104 --- /dev/null +++ b/clean-status-manifest.c @@ -0,0 +1,56 @@ +#include "git-compat-util.h" +#include "attr-manifest.h" +#include "clean-status-manifest.h" +#include "fsmonitor-clean-proof.h" +#include "hash-framing.h" + +void clean_status_manifest_init(struct clean_status_manifest_state *state) +{ + memset(state, 0, sizeof(*state)); + strbuf_init(&state->disk, 0); + strbuf_init(&state->current, 0); +} + +void clean_status_manifest_release(struct clean_status_manifest_state *state) +{ + strbuf_release(&state->disk); + strbuf_release(&state->current); +} + +int clean_status_manifest_load(struct clean_status_manifest_state *state, + const void *data, size_t len, uint32_t flags, + const struct git_hash_algo *algo) +{ + state->disk_valid = 0; + state->disk_flags = 0; + strbuf_reset(&state->disk); + if (flags & ~FSMONITOR_CLEAN_PROOF_ALL || + !attr_manifest_valid(data, len, algo)) + return -1; + strbuf_add(&state->disk, data, len); + hash_buffer_digest(algo, data, len, state->disk_hash); + state->disk_flags = flags; + state->disk_valid = 1; + return 0; +} + +void clean_status_manifest_adopt_disk( + struct clean_status_manifest_state *state) +{ + if (!state->disk_valid) + BUG("cannot adopt an invalid clean-status manifest"); + strbuf_reset(&state->current); + strbuf_addbuf(&state->current, &state->disk); + memcpy(state->current_hash, state->disk_hash, + sizeof(state->current_hash)); + state->current_flags = state->disk_flags; + state->current_valid = 1; + state->checked = 1; +} + +void clean_status_manifest_invalidate( + struct clean_status_manifest_state *state) +{ + state->current_valid = 0; + state->current_flags = 0; +} diff --git a/clean-status-manifest.h b/clean-status-manifest.h new file mode 100644 index 00000000000000..e924ba9fade2fe --- /dev/null +++ b/clean-status-manifest.h @@ -0,0 +1,29 @@ +#ifndef CLEAN_STATUS_MANIFEST_H +#define CLEAN_STATUS_MANIFEST_H + +#include "hash.h" +#include "strbuf.h" + +struct clean_status_manifest_state { + struct strbuf disk; + struct strbuf current; + unsigned char disk_hash[GIT_MAX_RAWSZ]; + unsigned char current_hash[GIT_MAX_RAWSZ]; + uint32_t disk_flags; + uint32_t current_flags; + unsigned disk_valid : 1; + unsigned current_valid : 1; + unsigned checked : 1; +}; + +void clean_status_manifest_init(struct clean_status_manifest_state *state); +void clean_status_manifest_release(struct clean_status_manifest_state *state); +int clean_status_manifest_load(struct clean_status_manifest_state *state, + const void *data, size_t len, uint32_t flags, + const struct git_hash_algo *algo); +void clean_status_manifest_adopt_disk( + struct clean_status_manifest_state *state); +void clean_status_manifest_invalidate( + struct clean_status_manifest_state *state); + +#endif /* CLEAN_STATUS_MANIFEST_H */ diff --git a/meson.build b/meson.build index c81db56fb078b4..abc61a3c8bd6d4 100644 --- a/meson.build +++ b/meson.build @@ -333,6 +333,7 @@ libgit_sources = [ 'checkout.c', 'chunk-format.c', 'clean-status-config.c', + 'clean-status-manifest.c', 'color.c', 'column.c', 'combine-diff.c', diff --git a/t/meson.build b/t/meson.build index 812a11c51d18d7..d0ed8bb9461d07 100644 --- a/t/meson.build +++ b/t/meson.build @@ -1,6 +1,7 @@ clar_test_suites = [ 'unit-tests/u-attr-manifest.c', 'unit-tests/u-clean-status-config.c', + 'unit-tests/u-clean-status-manifest.c', 'unit-tests/u-ctype.c', 'unit-tests/u-dir.c', 'unit-tests/u-example-decorate.c', diff --git a/t/unit-tests/u-clean-status-manifest.c b/t/unit-tests/u-clean-status-manifest.c new file mode 100644 index 00000000000000..e6d83c564a8a6b --- /dev/null +++ b/t/unit-tests/u-clean-status-manifest.c @@ -0,0 +1,70 @@ +#include "unit-test.h" +#include "attr-manifest.h" +#include "clean-status-manifest.h" +#include "fsmonitor-clean-proof.h" + +static void make_manifest(struct strbuf *manifest, + const struct git_hash_algo *algo) +{ + struct attr_manifest_writer writer; + unsigned char hash[GIT_MAX_RAWSZ]; + + memset(hash, 1, algo->rawsz); + attr_manifest_writer_init(&writer, manifest, algo); + cl_assert_equal_i(attr_manifest_writer_add( + &writer, ".gitattributes", ATTR_MANIFEST_INDEX, hash), 0); +} + +void test_clean_status_manifest__loads_and_adopts_valid_history(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct clean_status_manifest_state state; + struct strbuf manifest = STRBUF_INIT; + + clean_status_manifest_init(&state); + make_manifest(&manifest, algo); + cl_assert_equal_i(clean_status_manifest_load( + &state, manifest.buf, manifest.len, + FSMONITOR_CLEAN_PROOF_ALL, algo), 0); + cl_assert(state.disk_valid); + clean_status_manifest_adopt_disk(&state); + cl_assert(state.current_valid); + cl_assert(state.checked); + cl_assert_equal_i(state.current_flags, FSMONITOR_CLEAN_PROOF_ALL); + cl_assert_equal_i(state.current.len, manifest.len); + cl_assert(!memcmp(state.current.buf, manifest.buf, manifest.len)); + clean_status_manifest_invalidate(&state); + cl_assert(!state.current_valid); + cl_assert_equal_i(state.current_flags, 0); + clean_status_manifest_release(&state); + strbuf_release(&manifest); +} + +void test_clean_status_manifest__rejects_invalid_history(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct clean_status_manifest_state state; + struct strbuf manifest = STRBUF_INIT; + size_t valid_len; + + clean_status_manifest_init(&state); + make_manifest(&manifest, algo); + valid_len = manifest.len; + cl_assert_equal_i(clean_status_manifest_load( + &state, manifest.buf, manifest.len, + FSMONITOR_CLEAN_PROOF_ALL, algo), 0); + cl_assert(state.disk_valid); + cl_assert_equal_i(state.disk_flags, FSMONITOR_CLEAN_PROOF_ALL); + cl_assert_equal_i(state.disk.len, valid_len); + cl_assert(!memcmp(state.disk.buf, manifest.buf, valid_len)); + + strbuf_addch(&manifest, 0); + cl_assert_equal_i(clean_status_manifest_load( + &state, manifest.buf, manifest.len, + FSMONITOR_CLEAN_PROOF_ALL, algo), -1); + cl_assert(!state.disk_valid); + cl_assert_equal_i(state.disk_flags, 0); + cl_assert_equal_i(state.disk.len, 0); + clean_status_manifest_release(&state); + strbuf_release(&manifest); +} From 5d2748cae42f7d94818ceef11df345c3b5fb6f6b Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 11 Jul 2026 10:32:32 -0700 Subject: [PATCH 067/105] path: snapshot external source namespaces An external attributes file can be missing, reached through a symbolic link, or redirected when an ancestor is replaced. Hashing its contents alone cannot distinguish stable absence from a changed containing namespace, or identical bytes reached through a different path. Extend the filesystem-identity primitives from S06/P03 to capture the lstat identity or absence of every component of an absolute path. Compare snapshots component by component and hash their explicit states and canonical identity fields with length-delimited framing. Reject capture with EAGAIN when the platform cannot report reliable object identity instead of hashing fabricated identity fields. Expose whether the final component exists and release all snapshot storage explicitly. Tests verify equal snapshots and hashes, a missing target that subsequently appears, and an ancestor replacement that changes the namespace even when the replacement has identical content. The snapshot is independently testable. It does not read an external attribute source or establish a status speedup. Signed-off-by: Taylor Blau --- path-namespace.c | 155 +++++++++++++++++++++++++++++ path-namespace.h | 11 ++ t/unit-tests/u-path-namespace.c | 171 ++++++++++++++++++++++++++++++++ 3 files changed, 337 insertions(+) diff --git a/path-namespace.c b/path-namespace.c index 151634b886b663..14cc149c8ef188 100644 --- a/path-namespace.c +++ b/path-namespace.c @@ -1,5 +1,25 @@ #include "git-compat-util.h" +#include "abspath.h" +#include "hash.h" +#include "hash-framing.h" #include "path-namespace.h" +#include "strbuf.h" + +enum namespace_entry_state { + NAMESPACE_ENTRY_MISSING = 0, + NAMESPACE_ENTRY_PRESENT = 1, +}; + +struct stat_fingerprint { + struct path_stat_identity identity; + unsigned int state; +}; + +struct path_namespace_snapshot { + struct stat_fingerprint *entries; + size_t nr; + size_t alloc; +}; void path_stat_identity_init(struct path_stat_identity *identity, const struct stat *st) @@ -37,6 +57,133 @@ int path_stat_identity_equal(const struct path_stat_identity *a, return !memcmp(a, b, sizeof(*a)); } +static void stat_fingerprint_init(struct stat_fingerprint *fingerprint, + const struct stat *st) +{ + memset(fingerprint, 0, sizeof(*fingerprint)); + fingerprint->state = NAMESPACE_ENTRY_PRESENT; + path_stat_identity_init(&fingerprint->identity, st); + if (S_ISDIR(st->st_mode)) { + /* Unrelated children do not change which object a path names. */ + fingerprint->identity.fields[3] = 0; + fingerprint->identity.fields[6] = 0; + for (size_t i = 7; i <= 10; i++) + fingerprint->identity.fields[i] = 0; + } +} + +static int stat_fingerprint_equal(const struct stat_fingerprint *a, + const struct stat_fingerprint *b) +{ + return a->state == b->state && + path_stat_identity_equal(&a->identity, &b->identity); +} + +static int capture_entry(const char *path, + struct path_namespace_snapshot *snapshot) +{ + struct stat st; + struct stat_fingerprint *entry; + + ALLOC_GROW(snapshot->entries, snapshot->nr + 1, snapshot->alloc); + entry = &snapshot->entries[snapshot->nr++]; + memset(entry, 0, sizeof(*entry)); + if (!lstat(path, &st)) { + stat_fingerprint_init(entry, &st); + return 0; + } + if (errno == ENOENT || errno == ENOTDIR) { + entry->state = NAMESPACE_ENTRY_MISSING; + return 0; + } + return -1; +} + +int path_namespace_capture(const char *path, + struct path_namespace_snapshot **snapshot_out) +{ + struct path_namespace_snapshot *snapshot; + struct strbuf prefix = STRBUF_INIT; + size_t root_len, pos; + int ret = -1; + + if (!fstat_is_reliable()) { + errno = EAGAIN; + return -1; + } + + CALLOC_ARRAY(snapshot, 1); + root_len = offset_1st_component(path); + if (!root_len) + goto done; + strbuf_add(&prefix, path, root_len); + if (capture_entry(prefix.buf, snapshot)) + goto done; + pos = root_len; + while (path[pos]) { + size_t start, end; + + while (path[pos] && is_dir_sep(path[pos])) + pos++; + if (!path[pos]) + break; + start = pos; + while (path[pos] && !is_dir_sep(path[pos])) + pos++; + end = pos; + strbuf_complete(&prefix, '/'); + strbuf_add(&prefix, path + start, end - start); + if (capture_entry(prefix.buf, snapshot)) + goto done; + } + *snapshot_out = snapshot; + snapshot = NULL; + ret = 0; +done: + path_namespace_clear(snapshot); + strbuf_release(&prefix); + return ret; +} + +int path_namespace_equal(const struct path_namespace_snapshot *a, + const struct path_namespace_snapshot *b) +{ + if (a->nr != b->nr) + return 0; + for (size_t i = 0; i < a->nr; i++) + if (!stat_fingerprint_equal(&a->entries[i], &b->entries[i])) + return 0; + return 1; +} + +int path_namespace_target_present( + const struct path_namespace_snapshot *snapshot) +{ + return snapshot->nr && + snapshot->entries[snapshot->nr - 1].state == + NAMESPACE_ENTRY_PRESENT; +} + +void path_namespace_hash(struct git_hash_ctx *ctx, + const struct path_namespace_snapshot *snapshot) +{ + uint32_t value; + uint64_t field; + + put_be32(&value, snapshot->nr); + hash_length_delimited(ctx, &value, sizeof(value)); + for (size_t i = 0; i < snapshot->nr; i++) { + put_be32(&value, snapshot->entries[i].state); + hash_length_delimited(ctx, &value, sizeof(value)); + for (size_t j = 0; + j < ARRAY_SIZE(snapshot->entries[i].identity.fields); j++) { + put_be64(&field, + snapshot->entries[i].identity.fields[j]); + hash_length_delimited(ctx, &field, sizeof(field)); + } + } +} + int path_namespace_stat_equal(const struct stat *a, const struct stat *b) { struct path_stat_identity first, second; @@ -87,3 +234,11 @@ int path_namespace_reopen_component( errno = saved_errno; return -1; } + +void path_namespace_clear(struct path_namespace_snapshot *snapshot) +{ + if (!snapshot) + return; + free(snapshot->entries); + free(snapshot); +} diff --git a/path-namespace.h b/path-namespace.h index c26f4f12aebd48..16a607f94118f8 100644 --- a/path-namespace.h +++ b/path-namespace.h @@ -1,6 +1,8 @@ #ifndef PATH_NAMESPACE_H #define PATH_NAMESPACE_H +struct git_hash_ctx; +struct path_namespace_snapshot; struct stat; typedef int (*path_namespace_open_fn)(int dirfd, const char *path, int flags); @@ -15,9 +17,18 @@ void path_stat_identity_init(struct path_stat_identity *identity, const struct stat *st); int path_stat_identity_equal(const struct path_stat_identity *a, const struct path_stat_identity *b); +int path_namespace_capture(const char *path, + struct path_namespace_snapshot **snapshot_out); +int path_namespace_equal(const struct path_namespace_snapshot *a, + const struct path_namespace_snapshot *b); +int path_namespace_target_present( + const struct path_namespace_snapshot *snapshot); +void path_namespace_hash(struct git_hash_ctx *ctx, + const struct path_namespace_snapshot *snapshot); int path_namespace_stat_equal(const struct stat *a, const struct stat *b); int path_namespace_reopen_component( int parent_fd, const char *component, int flags, path_namespace_open_fn open_fn, const struct stat *expected); +void path_namespace_clear(struct path_namespace_snapshot *snapshot); #endif /* PATH_NAMESPACE_H */ diff --git a/t/unit-tests/u-path-namespace.c b/t/unit-tests/u-path-namespace.c index 4e0d9dfab24d5d..3c80140a8bbf72 100644 --- a/t/unit-tests/u-path-namespace.c +++ b/t/unit-tests/u-path-namespace.c @@ -1,7 +1,11 @@ #include "unit-test.h" +#include "dir.h" +#include "hash.h" #include "path-namespace.h" +#include "strbuf.h" #include "tempfile.h" +#include "wrapper.h" #define ASSERT_STAT_FIELD_MATTERS(base, changed, field) do { \ (changed) = (base); \ @@ -115,3 +119,170 @@ void test_path_namespace__reopen_component(void) cl_must_pass(delete_tempfile(&first)); cl_must_pass(delete_tempfile(&second)); } + +static char *create_namespace(void) +{ + const char *tmp = getenv("TMPDIR"); + char *path = xstrfmt("%s/path-namespace.XXXXXX", tmp ? tmp : "/tmp"); + + cl_assert(mkdtemp(path) != NULL); + return path; +} + +static void remove_namespace(char *path) +{ + struct strbuf root = STRBUF_INIT; + + strbuf_addstr(&root, path); + cl_must_pass(remove_dir_recursively(&root, 0)); + strbuf_release(&root); + free(path); +} + +static void hash_namespace(const struct path_namespace_snapshot *snapshot, + unsigned char *hash) +{ + struct git_hash_ctx ctx; + + git_hash_init(&ctx, &hash_algos[GIT_HASH_SHA1]); + path_namespace_hash(&ctx, snapshot); + git_hash_final(hash, &ctx); + git_hash_discard(&ctx); +} + +void test_path_namespace__equal_snapshots_have_equal_hashes(void) +{ + struct path_namespace_snapshot *first = NULL, *second = NULL; + struct strbuf directory = STRBUF_INIT, target = STRBUF_INIT; + unsigned char first_hash[GIT_MAX_RAWSZ], second_hash[GIT_MAX_RAWSZ]; + char *root; + + if (!fstat_is_reliable()) + cl_skip(); + root = create_namespace(); + + strbuf_addf(&directory, "%s/a", root); + cl_must_pass(mkdir(directory.buf, 0777)); + strbuf_addf(&target, "%s/target", directory.buf); + write_file(target.buf, "contents\n"); + + cl_must_pass(path_namespace_capture(target.buf, &first)); + cl_must_pass(path_namespace_capture(target.buf, &second)); + cl_assert(path_namespace_target_present(first)); + cl_assert(path_namespace_target_present(second)); + cl_assert(path_namespace_equal(first, second)); + hash_namespace(first, first_hash); + hash_namespace(second, second_hash); + cl_assert(!memcmp(first_hash, second_hash, + hash_algos[GIT_HASH_SHA1].rawsz)); + + path_namespace_clear(second); + path_namespace_clear(first); + strbuf_release(&target); + strbuf_release(&directory); + remove_namespace(root); +} + +void test_path_namespace__captures_missing_and_replaced_components(void) +{ + struct path_namespace_snapshot *missing = NULL, *created = NULL; + struct path_namespace_snapshot *replaced = NULL; + struct strbuf directory = STRBUF_INIT, old_directory = STRBUF_INIT; + struct strbuf target = STRBUF_INIT; + unsigned char missing_hash[GIT_MAX_RAWSZ], created_hash[GIT_MAX_RAWSZ]; + unsigned char replaced_hash[GIT_MAX_RAWSZ]; + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *root; + + if (!fstat_is_reliable()) { + cl_assert(path_namespace_capture(".", &missing) < 0); + cl_assert_equal_i(errno, EAGAIN); + return; + } + root = create_namespace(); + + strbuf_addf(&directory, "%s/a", root); + strbuf_addf(&old_directory, "%s/a-old", root); + cl_must_pass(mkdir(directory.buf, 0777)); + strbuf_addf(&target, "%s/target", directory.buf); + + cl_must_pass(path_namespace_capture(target.buf, &missing)); + cl_assert(!path_namespace_target_present(missing)); + hash_namespace(missing, missing_hash); + + write_file(target.buf, "contents\n"); + cl_must_pass(path_namespace_capture(target.buf, &created)); + cl_assert(path_namespace_target_present(created)); + cl_assert(!path_namespace_equal(missing, created)); + hash_namespace(created, created_hash); + cl_assert(memcmp(missing_hash, created_hash, algo->rawsz)); + + cl_must_pass(rename(directory.buf, old_directory.buf)); + cl_must_pass(mkdir(directory.buf, 0777)); + write_file(target.buf, "contents\n"); + cl_must_pass(path_namespace_capture(target.buf, &replaced)); + cl_assert(path_namespace_target_present(replaced)); + cl_assert(!path_namespace_equal(created, replaced)); + hash_namespace(replaced, replaced_hash); + cl_assert(memcmp(created_hash, replaced_hash, algo->rawsz)); + + path_namespace_clear(replaced); + path_namespace_clear(created); + path_namespace_clear(missing); + strbuf_release(&target); + strbuf_release(&old_directory); + strbuf_release(&directory); + remove_namespace(root); +} + +void test_path_namespace__unrelated_ancestor_entries_leave_target_unchanged(void) +{ + struct path_namespace_snapshot *first = NULL, *second = NULL; + struct path_namespace_snapshot *modified = NULL, *permissions = NULL; + struct strbuf directory = STRBUF_INIT, target = STRBUF_INIT; + struct strbuf unrelated = STRBUF_INIT; + unsigned char first_hash[GIT_MAX_RAWSZ], second_hash[GIT_MAX_RAWSZ]; + unsigned char modified_hash[GIT_MAX_RAWSZ]; + struct stat st; + char *root; + + if (!fstat_is_reliable()) + cl_skip(); + root = create_namespace(); + + strbuf_addf(&directory, "%s/a", root); + cl_must_pass(mkdir(directory.buf, 0777)); + strbuf_addf(&target, "%s/target", directory.buf); + write_file(target.buf, "contents\n"); + cl_must_pass(path_namespace_capture(target.buf, &first)); + hash_namespace(first, first_hash); + + strbuf_addf(&unrelated, "%s/unrelated", root); + write_file(unrelated.buf, "unrelated\n"); + cl_must_pass(path_namespace_capture(target.buf, &second)); + hash_namespace(second, second_hash); + cl_assert(path_namespace_equal(first, second)); + cl_assert(!memcmp(first_hash, second_hash, + hash_algos[GIT_HASH_SHA1].rawsz)); + + write_file(target.buf, "changed contents\n"); + cl_must_pass(path_namespace_capture(target.buf, &modified)); + hash_namespace(modified, modified_hash); + cl_assert(!path_namespace_equal(second, modified)); + cl_assert(memcmp(second_hash, modified_hash, + hash_algos[GIT_HASH_SHA1].rawsz)); + + cl_must_pass(stat(directory.buf, &st)); + cl_must_pass(chmod(directory.buf, st.st_mode ^ S_IXGRP)); + cl_must_pass(path_namespace_capture(target.buf, &permissions)); + cl_assert(!path_namespace_equal(modified, permissions)); + + path_namespace_clear(permissions); + path_namespace_clear(modified); + path_namespace_clear(second); + path_namespace_clear(first); + strbuf_release(&unrelated); + strbuf_release(&target); + strbuf_release(&directory); + remove_namespace(root); +} From 89f57435c7e88b3d83d5e5480631c331ee57f7ed Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 22:47:10 -0700 Subject: [PATCH 068/105] attr: fingerprint external sources in stable namespaces An external attributes file can change conversion without changing a worktree attribute manifest. Content alone is also insufficient: an ancestor or linked target can be replaced, and a missing source is safe to reuse only while its containing namespace remains stable. Capture the normalized absolute-path namespace with S07/P09 before and after observing each enabled source. For a present source, require nonblocking-open support, a regular singly linked file below the attribute-file limit, and matching descriptor, pathname, and target identities. Read the entire file into one allocation. Record source configuration and contents in one framed digest, and component and target identities in a separate namespace digest. Recheck the complete namespace for stable missing sources. Enabled sources inherit the namespace capture's fail-closed identity check; disabled sources remain unobserved and safely digestible. Reject instability rather than publishing an incomplete fingerprint. Register the fingerprint library and Clar suite in both Make and Meson. Tests separate content from metadata changes, detect an altered ancestor of a missing source, preserve disabled-source digests, and exercise both object formats. This does not select repository attribute sources or integrate fingerprints into status. Signed-off-by: Taylor Blau --- Makefile | 2 + attr-fingerprint.c | 132 ++++++++++++++++++++++++++++++ attr-fingerprint.h | 21 +++++ meson.build | 1 + path-namespace.c | 12 +++ path-namespace.h | 2 + t/meson.build | 1 + t/unit-tests/u-attr-fingerprint.c | 127 ++++++++++++++++++++++++++++ 8 files changed, 298 insertions(+) create mode 100644 attr-fingerprint.c create mode 100644 attr-fingerprint.h create mode 100644 t/unit-tests/u-attr-fingerprint.c diff --git a/Makefile b/Makefile index 8bdaab2391f7b2..d3a4ee8b399294 100644 --- a/Makefile +++ b/Makefile @@ -1110,6 +1110,7 @@ LIB_OBJS += archive-tar.o LIB_OBJS += archive-zip.o LIB_OBJS += archive.o LIB_OBJS += attr.o +LIB_OBJS += attr-fingerprint.o LIB_OBJS += attr-manifest.o LIB_OBJS += base85.o LIB_OBJS += bisect.o @@ -1548,6 +1549,7 @@ THIRD_PARTY_SOURCES += sha1dc/% THIRD_PARTY_SOURCES += $(UNIT_TEST_DIR)/clar/% THIRD_PARTY_SOURCES += $(UNIT_TEST_DIR)/clar/clar/% +CLAR_TEST_SUITES += u-attr-fingerprint CLAR_TEST_SUITES += u-attr-manifest CLAR_TEST_SUITES += u-clean-status-config CLAR_TEST_SUITES += u-clean-status-manifest diff --git a/attr-fingerprint.c b/attr-fingerprint.c new file mode 100644 index 00000000000000..ce21f0510032e9 --- /dev/null +++ b/attr-fingerprint.c @@ -0,0 +1,132 @@ +#include "git-compat-util.h" +#include "abspath.h" +#include "attr-fingerprint.h" +#include "attr.h" +#include "hash-framing.h" +#include "path-namespace.h" +#include "strbuf.h" +#include "wrapper.h" + +static int open_attr_source(const char *path) +{ +#ifdef O_NONBLOCK + return git_open_cloexec(path, O_RDONLY | O_NONBLOCK); +#else + (void)path; + errno = ENOSYS; + return -1; +#endif +} + +static int hash_source(struct git_hash_ctx *content_ctx, + struct git_hash_ctx *namespace_ctx, + const struct attr_fingerprint_source *source, + int *present) +{ + struct path_namespace_snapshot *before = NULL, *after = NULL; + struct stat opened_before, opened_after, named; + struct strbuf normalized = STRBUF_INIT; + char *absolute = NULL; + char *buf = NULL; + ssize_t got; + size_t size; + uint32_t state; + int fd = -1, ret = -1; + char extra; + + hash_optional_cstring(content_ctx, source->path); + hash_optional_cstring(namespace_ctx, source->path); + put_be32(&state, source->enabled); + hash_length_delimited(content_ctx, &state, sizeof(state)); + hash_length_delimited(namespace_ctx, &state, sizeof(state)); + *present = 0; + if (!source->enabled || !source->path) + return 0; + + absolute = absolute_pathdup(source->path); + strbuf_addstr(&normalized, absolute); + if (strbuf_normalize_path(&normalized) || + path_namespace_capture(normalized.buf, &before)) + goto done; + *present = path_namespace_target_present(before); + if (!*present) { + if (path_namespace_capture(normalized.buf, &after) || + !path_namespace_equal(before, after)) + goto done; + state = 0; + hash_length_delimited(content_ctx, &state, sizeof(state)); + path_namespace_hash(namespace_ctx, before); + ret = 0; + goto done; + } + + fd = open_attr_source(normalized.buf); + if (fd < 0 || fstat(fd, &opened_before) || + !S_ISREG(opened_before.st_mode) || opened_before.st_nlink != 1 || + opened_before.st_size < 0 || + opened_before.st_size >= ATTR_MAX_FILE_SIZE) + goto done; + size = xsize_t(opened_before.st_size); + buf = xmalloc(size ? size : 1); + got = read_in_full(fd, buf, size); + if (got < 0 || (size_t)got != size || read(fd, &extra, 1) != 0 || + fstat(fd, &opened_after) || stat(normalized.buf, &named) || + !path_namespace_stat_equal(&opened_before, &opened_after) || + !path_namespace_stat_equal(&opened_after, &named) || + path_namespace_capture(normalized.buf, &after) || + !path_namespace_equal(before, after)) + goto done; + state = 1; + hash_length_delimited(content_ctx, &state, sizeof(state)); + path_namespace_hash(namespace_ctx, before); + path_namespace_hash_stat(namespace_ctx, &opened_after); + hash_length_delimited(content_ctx, buf, size); + ret = 0; +done: + if (fd >= 0) + close(fd); + free(buf); + free(absolute); + path_namespace_clear(before); + path_namespace_clear(after); + strbuf_release(&normalized); + return ret; +} + +static int fingerprint_sources( + const struct attr_fingerprint_source *sources, size_t nr, + const struct git_hash_algo *algo, struct attr_fingerprint *result) +{ + struct git_hash_ctx content_ctx, namespace_ctx; + uint32_t count; + + memset(result, 0, sizeof(*result)); + git_hash_init(&content_ctx, algo); + git_hash_init(&namespace_ctx, algo); + hash_optional_cstring(&content_ctx, "attribute-source-content-v1"); + hash_optional_cstring(&namespace_ctx, + "attribute-source-namespace-v1"); + if (nr > UINT32_MAX) + return -1; + put_be32(&count, nr); + hash_length_delimited(&content_ctx, &count, sizeof(count)); + hash_length_delimited(&namespace_ctx, &count, sizeof(count)); + for (size_t i = 0; i < nr; i++) { + int present; + + if (hash_source(&content_ctx, &namespace_ctx, &sources[i], + &present)) + return -1; + result->sources_present |= present; + } + git_hash_final(result->content_hash, &content_ctx); + git_hash_final(result->namespace_hash, &namespace_ctx); + return 0; +} + +int attr_fingerprint_sources( + const struct attr_fingerprint_source *sources, size_t nr, + const struct git_hash_algo *algo, struct attr_fingerprint *result) +{ + return fingerprint_sources(sources, nr, algo, result); +} diff --git a/attr-fingerprint.h b/attr-fingerprint.h new file mode 100644 index 00000000000000..7f3d4f0b7c1688 --- /dev/null +++ b/attr-fingerprint.h @@ -0,0 +1,21 @@ +#ifndef ATTR_FINGERPRINT_H +#define ATTR_FINGERPRINT_H + +#include "hash.h" + +struct attr_fingerprint_source { + const char *path; + unsigned int enabled : 1; +}; + +struct attr_fingerprint { + unsigned char content_hash[GIT_MAX_RAWSZ]; + unsigned char namespace_hash[GIT_MAX_RAWSZ]; + unsigned int sources_present : 1; +}; + +int attr_fingerprint_sources( + const struct attr_fingerprint_source *sources, size_t nr, + const struct git_hash_algo *algo, struct attr_fingerprint *result); + +#endif /* ATTR_FINGERPRINT_H */ diff --git a/meson.build b/meson.build index abc61a3c8bd6d4..ee646f1a998466 100644 --- a/meson.build +++ b/meson.build @@ -317,6 +317,7 @@ libgit_sources = [ 'archive-tar.c', 'archive-zip.c', 'archive.c', + 'attr-fingerprint.c', 'attr-manifest.c', 'attr.c', 'base85.c', diff --git a/path-namespace.c b/path-namespace.c index 14cc149c8ef188..533c8b53262899 100644 --- a/path-namespace.c +++ b/path-namespace.c @@ -184,6 +184,18 @@ void path_namespace_hash(struct git_hash_ctx *ctx, } } +void path_namespace_hash_stat(struct git_hash_ctx *ctx, const struct stat *st) +{ + struct path_stat_identity identity; + uint64_t field; + + path_stat_identity_init(&identity, st); + for (size_t i = 0; i < ARRAY_SIZE(identity.fields); i++) { + put_be64(&field, identity.fields[i]); + hash_length_delimited(ctx, &field, sizeof(field)); + } +} + int path_namespace_stat_equal(const struct stat *a, const struct stat *b) { struct path_stat_identity first, second; diff --git a/path-namespace.h b/path-namespace.h index 16a607f94118f8..d702b2570aae73 100644 --- a/path-namespace.h +++ b/path-namespace.h @@ -25,6 +25,8 @@ int path_namespace_target_present( const struct path_namespace_snapshot *snapshot); void path_namespace_hash(struct git_hash_ctx *ctx, const struct path_namespace_snapshot *snapshot); +void path_namespace_hash_stat(struct git_hash_ctx *ctx, + const struct stat *st); int path_namespace_stat_equal(const struct stat *a, const struct stat *b); int path_namespace_reopen_component( int parent_fd, const char *component, int flags, diff --git a/t/meson.build b/t/meson.build index d0ed8bb9461d07..0e733642e2c8a5 100644 --- a/t/meson.build +++ b/t/meson.build @@ -1,4 +1,5 @@ clar_test_suites = [ + 'unit-tests/u-attr-fingerprint.c', 'unit-tests/u-attr-manifest.c', 'unit-tests/u-clean-status-config.c', 'unit-tests/u-clean-status-manifest.c', diff --git a/t/unit-tests/u-attr-fingerprint.c b/t/unit-tests/u-attr-fingerprint.c new file mode 100644 index 00000000000000..e7b61b687c788f --- /dev/null +++ b/t/unit-tests/u-attr-fingerprint.c @@ -0,0 +1,127 @@ +#include "unit-test.h" +#include "attr-fingerprint.h" +#include "dir.h" +#include "strbuf.h" +#include "wrapper.h" + +static char *create_directory(void) +{ + const char *tmp = getenv("TMPDIR"); + char *path = xstrfmt("%s/attr-fingerprint.XXXXXX", + tmp ? tmp : "/tmp"); + + cl_assert(mkdtemp(path) != NULL); + return path; +} + +static void remove_directory(char *path) +{ + struct strbuf cleanup = STRBUF_INIT; + + strbuf_addstr(&cleanup, path); + cl_assert_equal_i(remove_dir_recursively(&cleanup, 0), 0); + strbuf_release(&cleanup); + free(path); +} + +static void fingerprint(const char *path, int enabled, + const struct git_hash_algo *algo, + struct attr_fingerprint *result) +{ + struct attr_fingerprint_source source = { + .path = path, + .enabled = enabled, + }; + + cl_assert_equal_i(attr_fingerprint_sources( + &source, 1, algo, result), 0); +} + +void test_attr_fingerprint__separates_contents_from_namespace(void) +{ +#ifndef O_NONBLOCK + cl_skip(); +#else + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *directory = create_directory(); + struct strbuf path = STRBUF_INIT; + struct attr_fingerprint initial, metadata, changed; + struct stat st; + + strbuf_addf(&path, "%s/attributes", directory); + write_file(path.buf, "*.txt text\n"); + fingerprint(path.buf, 1, algo, &initial); + cl_assert(initial.sources_present); + cl_assert_equal_i(stat(path.buf, &st), 0); + cl_assert_equal_i(chmod(path.buf, st.st_mode ^ S_IXUSR), 0); + fingerprint(path.buf, 1, algo, &metadata); + cl_assert(!memcmp(initial.content_hash, metadata.content_hash, + algo->rawsz)); + cl_assert(memcmp(initial.namespace_hash, metadata.namespace_hash, + algo->rawsz)); + write_file(path.buf, "*.txt -text\n"); + fingerprint(path.buf, 1, algo, &changed); + cl_assert(memcmp(metadata.content_hash, changed.content_hash, + algo->rawsz)); + + strbuf_release(&path); + remove_directory(directory); +#endif +} + +void test_attr_fingerprint__records_missing_parent_namespaces(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA256]; + char *directory; + struct strbuf path = STRBUF_INIT; + struct attr_fingerprint before, after; + struct stat st; + + if (!fstat_is_reliable()) { + struct attr_fingerprint_source source = { + .path = "missing/attributes", + .enabled = 1, + }; + + cl_assert(attr_fingerprint_sources( + &source, 1, algo, &before) < 0); + cl_assert_equal_i(errno, EAGAIN); + return; + } + directory = create_directory(); + + strbuf_addf(&path, "%s/missing/attributes", directory); + fingerprint(path.buf, 1, algo, &before); + cl_assert(!before.sources_present); + cl_assert_equal_i(stat(directory, &st), 0); + cl_assert_equal_i(chmod(directory, st.st_mode ^ S_IXGRP), 0); + fingerprint(path.buf, 1, algo, &after); + cl_assert(!after.sources_present); + cl_assert(!memcmp(before.content_hash, after.content_hash, algo->rawsz)); + cl_assert(memcmp(before.namespace_hash, after.namespace_hash, + algo->rawsz)); + + strbuf_release(&path); + remove_directory(directory); +} + +void test_attr_fingerprint__does_not_observe_disabled_sources(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *directory = create_directory(); + struct strbuf path = STRBUF_INIT; + struct attr_fingerprint before, after; + + strbuf_addf(&path, "%s/attributes", directory); + fingerprint(path.buf, 0, algo, &before); + write_file(path.buf, "*.txt text\n"); + fingerprint(path.buf, 0, algo, &after); + cl_assert(!before.sources_present); + cl_assert(!after.sources_present); + cl_assert(!memcmp(before.content_hash, after.content_hash, algo->rawsz)); + cl_assert(!memcmp(before.namespace_hash, after.namespace_hash, + algo->rawsz)); + + strbuf_release(&path); + remove_directory(directory); +} From 5a099ee99902b60c9a4182cce0c45fdf51385a33 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 25 Jul 2026 21:55:32 -0500 Subject: [PATCH 069/105] status: bind semantic configuration to its index A clean-status configuration digest cannot establish which index it describes while it remains detached from the repository and index that will consume it. External attribute content and namespace must also be recorded before an index can reuse conversion-dependent history. Attach a finalized, repository-bound digest at the beginning of do_read_index(), fingerprint the system, global, and info attribute sources, and store the resulting state on the index. Ignore an unfinalized digest, another repository's digest, and a second attachment. Release the state with release_index(). Extend the existing clean-status configuration unit suite to exercise repository binding, one-shot attachment, semantic and attribute hashes, unsafe-filter state, and index-lifetime cleanup. Register the new production object with both Make and Meson. Signed-off-by: Taylor Blau --- Makefile | 1 + attr-fingerprint.c | 35 ++++++++ attr-fingerprint.h | 4 + clean-status-internal.h | 24 ++++++ clean-status.c | 72 ++++++++++++++++ clean-status.h | 15 ++++ meson.build | 1 + read-cache-ll.h | 2 + read-cache.c | 3 + t/unit-tests/u-clean-status-config.c | 121 +++++++++++++++++++++++++++ 10 files changed, 278 insertions(+) create mode 100644 clean-status-internal.h create mode 100644 clean-status.c create mode 100644 clean-status.h diff --git a/Makefile b/Makefile index 9ddaf95ee566e8..b1d8379fdfaa41 100644 --- a/Makefile +++ b/Makefile @@ -1125,6 +1125,7 @@ LIB_OBJS += cbtree.o LIB_OBJS += chdir-notify.o LIB_OBJS += checkout.o LIB_OBJS += chunk-format.o +LIB_OBJS += clean-status.o LIB_OBJS += clean-status-config.o LIB_OBJS += clean-status-manifest.o LIB_OBJS += color.o diff --git a/attr-fingerprint.c b/attr-fingerprint.c index ce21f0510032e9..6a9cde2821614c 100644 --- a/attr-fingerprint.c +++ b/attr-fingerprint.c @@ -2,8 +2,11 @@ #include "abspath.h" #include "attr-fingerprint.h" #include "attr.h" +#include "environment.h" #include "hash-framing.h" +#include "path.h" #include "path-namespace.h" +#include "repository.h" #include "strbuf.h" #include "wrapper.h" @@ -130,3 +133,35 @@ int attr_fingerprint_sources( { return fingerprint_sources(sources, nr, algo, result); } + +static int repository_sources(struct repository *repo, + struct attr_fingerprint_source *sources, + char **info_attributes) +{ + if (getenv(GIT_ATTR_SOURCE_ENVIRONMENT)) + return -1; + sources[0].path = git_attr_system_file(); + sources[0].enabled = git_attr_system_is_enabled(); + sources[1].path = git_attr_global_file(); + sources[1].enabled = 1; + *info_attributes = repo_git_path(repo, INFOATTRIBUTES_FILE); + sources[2].path = *info_attributes; + sources[2].enabled = 1; + return 0; +} + +int attr_fingerprint_repository(struct repository *repo, + struct attr_fingerprint *result) +{ + struct attr_fingerprint_source sources[3]; + char *info_attributes = NULL; + int ret; + + memset(result, 0, sizeof(*result)); + if (repository_sources(repo, sources, &info_attributes)) + return -1; + ret = attr_fingerprint_sources(sources, ARRAY_SIZE(sources), + repo->hash_algo, result); + free(info_attributes); + return ret; +} diff --git a/attr-fingerprint.h b/attr-fingerprint.h index 7f3d4f0b7c1688..a159aa0697468c 100644 --- a/attr-fingerprint.h +++ b/attr-fingerprint.h @@ -3,6 +3,8 @@ #include "hash.h" +struct repository; + struct attr_fingerprint_source { const char *path; unsigned int enabled : 1; @@ -17,5 +19,7 @@ struct attr_fingerprint { int attr_fingerprint_sources( const struct attr_fingerprint_source *sources, size_t nr, const struct git_hash_algo *algo, struct attr_fingerprint *result); +int attr_fingerprint_repository(struct repository *repo, + struct attr_fingerprint *result); #endif /* ATTR_FINGERPRINT_H */ diff --git a/clean-status-internal.h b/clean-status-internal.h new file mode 100644 index 00000000000000..998173a39b008c --- /dev/null +++ b/clean-status-internal.h @@ -0,0 +1,24 @@ +#ifndef CLEAN_STATUS_INTERNAL_H +#define CLEAN_STATUS_INTERNAL_H + +#include "hash.h" + +struct index_state; + +struct clean_status_state { + unsigned char current_config_hash[GIT_MAX_RAWSZ]; + unsigned char current_semantic_hash[GIT_MAX_RAWSZ]; + unsigned char current_attr_hash[GIT_MAX_RAWSZ]; + unsigned char current_attr_namespace_hash[GIT_MAX_RAWSZ]; + unsigned current_config_valid : 1; + unsigned current_semantic_valid : 1; + unsigned current_attr_valid : 1; + unsigned current_semantic_explicit : 1; + unsigned current_attr_sources_present : 1; + unsigned config_enforced : 1; + unsigned unsafe_filter : 1; +}; + +struct clean_status_state *clean_status_get_state(struct index_state *istate); + +#endif /* CLEAN_STATUS_INTERNAL_H */ diff --git a/clean-status.c b/clean-status.c new file mode 100644 index 00000000000000..22ab0b25fe5b94 --- /dev/null +++ b/clean-status.c @@ -0,0 +1,72 @@ +#include "git-compat-util.h" +#include "attr-fingerprint.h" +#include "clean-status.h" +#include "clean-status-internal.h" +#include "read-cache-ll.h" +#include "repository.h" + +static struct repository *configured_repo; +static unsigned char configured_hash[GIT_MAX_RAWSZ]; +static unsigned char configured_semantic_hash[GIT_MAX_RAWSZ]; +static int configured_hash_valid; +static int configured_unsafe_filter; +static int configured_semantic_explicit; + +struct clean_status_state *clean_status_get_state(struct index_state *istate) +{ + if (!istate->clean_status) + CALLOC_ARRAY(istate->clean_status, 1); + return istate->clean_status; +} + +void clean_status_set_config_digest( + struct repository *repo, + const struct clean_status_config_digest *digest) +{ + configured_repo = repo; + configured_hash_valid = digest && digest->finalized; + configured_unsafe_filter = configured_hash_valid && digest->unsafe_filter; + configured_semantic_explicit = configured_hash_valid && + digest->semantic_config_explicit; + if (!configured_hash_valid) + return; + memcpy(configured_hash, digest->hash, repo->hash_algo->rawsz); + memcpy(configured_semantic_hash, digest->semantic_hash, + repo->hash_algo->rawsz); +} + +void clean_status_attach_config(struct index_state *istate) +{ + struct clean_status_state *state = istate->clean_status; + struct attr_fingerprint attrs; + + if (state && state->current_config_valid) + return; + if (!configured_hash_valid || configured_repo != istate->repo) + return; + state = clean_status_get_state(istate); + memcpy(state->current_config_hash, configured_hash, + istate->repo->hash_algo->rawsz); + memcpy(state->current_semantic_hash, configured_semantic_hash, + istate->repo->hash_algo->rawsz); + state->current_config_valid = 1; + state->current_semantic_valid = 1; + state->current_semantic_explicit = configured_semantic_explicit; + state->config_enforced = 1; + state->unsafe_filter = configured_unsafe_filter; + if (!attr_fingerprint_repository(istate->repo, &attrs)) { + memcpy(state->current_attr_hash, attrs.content_hash, + istate->repo->hash_algo->rawsz); + memcpy(state->current_attr_namespace_hash, attrs.namespace_hash, + istate->repo->hash_algo->rawsz); + state->current_attr_valid = 1; + state->current_attr_sources_present = attrs.sources_present; + } +} + +void clean_status_release(struct index_state *istate) +{ + if (!istate->clean_status) + return; + FREE_AND_NULL(istate->clean_status); +} diff --git a/clean-status.h b/clean-status.h new file mode 100644 index 00000000000000..6749e9c5329016 --- /dev/null +++ b/clean-status.h @@ -0,0 +1,15 @@ +#ifndef CLEAN_STATUS_H +#define CLEAN_STATUS_H + +#include "clean-status-config.h" + +struct index_state; +struct repository; + +void clean_status_set_config_digest( + struct repository *repo, + const struct clean_status_config_digest *digest); +void clean_status_attach_config(struct index_state *istate); +void clean_status_release(struct index_state *istate); + +#endif /* CLEAN_STATUS_H */ diff --git a/meson.build b/meson.build index ee646f1a998466..d3b433856cb165 100644 --- a/meson.build +++ b/meson.build @@ -333,6 +333,7 @@ libgit_sources = [ 'chdir-notify.c', 'checkout.c', 'chunk-format.c', + 'clean-status.c', 'clean-status-config.c', 'clean-status-manifest.c', 'color.c', diff --git a/read-cache-ll.h b/read-cache-ll.h index d44a946ed04d36..943ccc296a71d6 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -142,6 +142,7 @@ static inline unsigned create_ce_flags(unsigned stage) #define FSMONITOR_CHANGED (1 << 8) struct split_index; +struct clean_status_state; struct untracked_cache; struct progress; struct pattern_list; @@ -202,6 +203,7 @@ struct index_state { struct progress *progress; struct repository *repo; struct pattern_list *sparse_checkout_patterns; + struct clean_status_state *clean_status; }; /** diff --git a/read-cache.c b/read-cache.c index cf3600f2a337d6..2b2493b45aee97 100644 --- a/read-cache.c +++ b/read-cache.c @@ -16,6 +16,7 @@ #include "tempfile.h" #include "lockfile.h" #include "cache-tree.h" +#include "clean-status.h" #include "refs.h" #include "dir.h" #include "object-file.h" @@ -2241,6 +2242,7 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) int nr_threads, cpus; struct index_entry_offset_table *ieot = NULL; + clean_status_attach_config(istate); if (istate->initialized) return istate->cache_nr; @@ -2473,6 +2475,7 @@ void release_index(struct index_state *istate) free(istate->fsmonitor_last_update); free(istate->fsmonitor_last_update_pending); free(istate->fsmonitor_untracked_token); + clean_status_release(istate); free(istate->cache); discard_split_index(istate); free_untracked_cache(istate->untracked); diff --git a/t/unit-tests/u-clean-status-config.c b/t/unit-tests/u-clean-status-config.c index e922970c1abe4e..4d836e10d15315 100644 --- a/t/unit-tests/u-clean-status-config.c +++ b/t/unit-tests/u-clean-status-config.c @@ -1,6 +1,14 @@ #include "unit-test.h" +#include "attr-fingerprint.h" +#include "clean-status.h" #include "clean-status-config.h" +#include "clean-status-internal.h" #include "config.h" +#include "dir.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "strbuf.h" +#include "wrapper.h" static void digest_one(struct clean_status_config_digest *digest, const char *key, const char *value, @@ -70,3 +78,116 @@ void test_clean_status_config__clean_filters_are_unsafe(void) cl_assert(!smudge.unsafe_filter); cl_assert(!smudge.semantic_config_explicit); } + +#if defined(O_NONBLOCK) && !defined(GIT_WINDOWS_NATIVE) +static char *create_gitdir(int with_attributes) +{ + const char *tmp = getenv("TMPDIR"); + char *gitdir = xstrfmt("%s/clean-status-config.XXXXXX", + tmp ? tmp : "/tmp"); + struct strbuf path = STRBUF_INIT; + + cl_assert(mkdtemp(gitdir) != NULL); + if (with_attributes) { + strbuf_addf(&path, "%s/info", gitdir); + cl_assert_equal_i(mkdir(path.buf, 0777), 0); + strbuf_addstr(&path, "/attributes"); + write_file(path.buf, "*.txt text\n"); + } + strbuf_release(&path); + return gitdir; +} + +static void remove_gitdir(char *gitdir) +{ + struct strbuf path = STRBUF_INIT; + + strbuf_addstr(&path, gitdir); + cl_assert_equal_i(remove_dir_recursively(&path, 0), 0); + strbuf_release(&path); + free(gitdir); +} + +static void clear_staged_config(void *unused UNUSED) +{ + clean_status_set_config_digest(NULL, NULL); +} +#endif + +void test_clean_status_config__attaches_only_to_the_staged_repository(void) +{ +#if !defined(O_NONBLOCK) || defined(GIT_WINDOWS_NATIVE) + cl_skip(); +#else + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *gitdir_a = create_gitdir(1); + char *gitdir_b = create_gitdir(0); + struct repository repo_a = { + .gitdir = gitdir_a, + .commondir = gitdir_a, + .hash_algo = algo, + }; + struct repository repo_b = { + .gitdir = gitdir_b, + .commondir = gitdir_b, + .hash_algo = algo, + }; + struct index_state istate_a = INDEX_STATE_INIT(&repo_a); + struct index_state istate_b = INDEX_STATE_INIT(&repo_b); + struct clean_status_config_digest digest, replacement; + struct clean_status_state *state; + struct attr_fingerprint attrs; + + cl_set_cleanup(clear_staged_config, NULL); + digest_one(&digest, "filter.demo.clean", "cat", NULL); + digest_one(&replacement, "core.autocrlf", "false", NULL); + cl_assert_equal_i(attr_fingerprint_repository(&repo_a, &attrs), 0); + cl_assert(attrs.sources_present); + + clean_status_set_config_digest(&repo_a, &digest); + clean_status_attach_config(&istate_b); + cl_assert_equal_p(istate_b.clean_status, NULL); + clean_status_attach_config(&istate_a); + state = istate_a.clean_status; + cl_assert(state != NULL); + cl_assert(state->current_config_valid); + cl_assert(state->current_semantic_valid); + cl_assert(state->current_attr_valid); + cl_assert(state->config_enforced); + cl_assert(state->unsafe_filter); + cl_assert(state->current_semantic_explicit); + cl_assert_equal_i(state->current_attr_sources_present, + attrs.sources_present); + cl_assert(hashes_equal(state->current_config_hash, digest.hash)); + cl_assert(hashes_equal(state->current_semantic_hash, + digest.semantic_hash)); + cl_assert(!memcmp(state->current_attr_hash, attrs.content_hash, + algo->rawsz)); + cl_assert(!memcmp(state->current_attr_namespace_hash, + attrs.namespace_hash, algo->rawsz)); + + clean_status_set_config_digest(&repo_a, &replacement); + clean_status_attach_config(&istate_a); + cl_assert(hashes_equal(state->current_config_hash, digest.hash)); + cl_assert(hashes_equal(state->current_semantic_hash, + digest.semantic_hash)); + cl_assert(state->unsafe_filter); + + release_index(&istate_a); + cl_assert_equal_p(istate_a.clean_status, NULL); + release_index(&istate_b); + clear_staged_config(NULL); + if (repo_a.config) { + git_configset_clear(repo_a.config); + FREE_AND_NULL(repo_a.config); + } + if (repo_b.config) { + git_configset_clear(repo_b.config); + FREE_AND_NULL(repo_b.config); + } + repo_settings_clear(&repo_a); + repo_settings_clear(&repo_b); + remove_gitdir(gitdir_b); + remove_gitdir(gitdir_a); +#endif +} From 5139eb94d89ff0c06f7c1fabc5dd0d6f8ebcf148 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Sat, 25 Jul 2026 21:55:36 -0500 Subject: [PATCH 070/105] commit: stage semantic configuration before reading the index Index attachment cannot recover the configuration seen by git status or git commit if their callbacks finish without recording it. A separate configuration pass could also bind a different stream from the one that established the commands' existing behavior. Wrap each existing status or commit callback so the original callback and clean-status digest consume the same key, value, and context. Finalize and stage the digest after the existing configuration pass and before either command reads its index. Preserve determine_whence(), advice_enabled(), the original callback, configuration order, and option handling. The index-owned attachment and its existing configuration unit coverage are supplied by S08/P01; this patch adds no command-specific regression. Signed-off-by: Taylor Blau --- builtin/commit.c | 42 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/builtin/commit.c b/builtin/commit.c index e2f4d08b347707..29f339f89a2254 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -13,6 +13,7 @@ #include "config.h" #include "lockfile.h" #include "cache-tree.h" +#include "clean-status.h" #include "color.h" #include "dir.h" #include "editor.h" @@ -205,11 +206,34 @@ static void determine_whence(struct wt_status *s) s->whence = whence; } -static void status_init_config(struct wt_status *s, config_fn_t fn) +struct status_config_callback_data { + struct wt_status *status; + config_fn_t fn; + struct clean_status_config_digest *clean_digest; +}; + +static int status_config_callback(const char *key, const char *value, + const struct config_context *ctx, void *cb) +{ + struct status_config_callback_data *data = cb; + + clean_status_config_add(data->clean_digest, key, value, ctx); + return data->fn(key, value, ctx, data->status); +} + +static void status_init_config_with_clean_digest( + struct wt_status *s, config_fn_t fn, + struct clean_status_config_digest *clean_digest) { + struct status_config_callback_data data = { + .status = s, + .fn = fn, + .clean_digest = clean_digest, + }; + wt_status_prepare(the_repository, s); init_diff_ui_defaults(); - repo_config(the_repository, fn, s); + repo_config(the_repository, status_config_callback, &data); determine_whence(s); s->hints = advice_enabled(ADVICE_STATUS_HINTS); /* must come after repo_config() */ } @@ -1542,6 +1566,7 @@ struct repository *repo UNUSED) static int no_renames = -1; static const char *rename_score_arg = (const char *)-1; static struct wt_status s; + struct clean_status_config_digest clean_digest; unsigned int progress_flag = 0; int fd; struct object_id oid; @@ -1605,7 +1630,11 @@ struct repository *repo UNUSED) prepare_repo_settings(the_repository); the_repository->settings.command_requires_full_index = 0; - status_init_config(&s, git_status_config); + clean_status_config_init(&clean_digest, the_repository->hash_algo); + status_init_config_with_clean_digest( + &s, git_status_config, &clean_digest); + clean_status_config_final(&clean_digest); + clean_status_set_config_digest(the_repository, &clean_digest); argc = parse_options(argc, argv, prefix, builtin_status_options, builtin_status_usage, 0); @@ -1703,6 +1732,7 @@ int cmd_commit(int argc, struct repository *repo UNUSED) { static struct wt_status s; + struct clean_status_config_digest clean_digest; static const char *cleanup_arg = NULL; static struct option builtin_commit_options[] = { OPT__QUIET(&quiet, N_("suppress summary after successful commit")), @@ -1807,7 +1837,11 @@ int cmd_commit(int argc, prepare_repo_settings(the_repository); the_repository->settings.command_requires_full_index = 0; - status_init_config(&s, git_commit_config); + clean_status_config_init(&clean_digest, the_repository->hash_algo); + status_init_config_with_clean_digest( + &s, git_commit_config, &clean_digest); + clean_status_config_final(&clean_digest); + clean_status_set_config_digest(the_repository, &clean_digest); s.commit_template = 1; status_format = STATUS_FORMAT_NONE; /* Ignore status.short */ s.colopts = 0; From b91d3153614237d07a2f78166893cf981739f807 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 00:46:34 -0500 Subject: [PATCH 071/105] status: classify durable index source identities An index with a null trailing checksum cannot be bound to the file that was actually read unless the platform supplies a durable file identity. Treating a directory, multiply linked file, or unsupported platform as equivalent would turn identity comparison into an unwarranted correctness guarantee. Add clean_status_identity_from_stat() for single-link regular files and make clean_status_identity_is_durable() return true only on Apple platforms. Keep unsupported platforms explicitly ineligible instead of inferring durability from stat fields alone. Register the identity object and its unit suite with Make and Meson. The tests reject directories and multiply linked files, accept a single-link regular file, and check the appropriate platform result. Actual null-checksum index verification remains a separate change. Signed-off-by: Taylor Blau --- Makefile | 2 ++ clean-status-identity.c | 21 +++++++++++++++++++++ clean-status-identity.h | 16 ++++++++++++++++ meson.build | 1 + t/meson.build | 1 + t/unit-tests/u-clean-status-identity.c | 26 ++++++++++++++++++++++++++ 6 files changed, 67 insertions(+) create mode 100644 clean-status-identity.c create mode 100644 clean-status-identity.h create mode 100644 t/unit-tests/u-clean-status-identity.c diff --git a/Makefile b/Makefile index b1d8379fdfaa41..ce5d750fb0dd09 100644 --- a/Makefile +++ b/Makefile @@ -1127,6 +1127,7 @@ LIB_OBJS += checkout.o LIB_OBJS += chunk-format.o LIB_OBJS += clean-status.o LIB_OBJS += clean-status-config.o +LIB_OBJS += clean-status-identity.o LIB_OBJS += clean-status-manifest.o LIB_OBJS += color.o LIB_OBJS += column.o @@ -1553,6 +1554,7 @@ THIRD_PARTY_SOURCES += $(UNIT_TEST_DIR)/clar/clar/% CLAR_TEST_SUITES += u-attr-fingerprint CLAR_TEST_SUITES += u-attr-manifest CLAR_TEST_SUITES += u-clean-status-config +CLAR_TEST_SUITES += u-clean-status-identity CLAR_TEST_SUITES += u-clean-status-manifest CLAR_TEST_SUITES += u-ctype CLAR_TEST_SUITES += u-dir diff --git a/clean-status-identity.c b/clean-status-identity.c new file mode 100644 index 00000000000000..1d4c71116d6600 --- /dev/null +++ b/clean-status-identity.c @@ -0,0 +1,21 @@ +#include "git-compat-util.h" +#include "clean-status-identity.h" + +int clean_status_identity_from_stat(struct clean_status_identity *identity, + const struct stat *st) +{ + memset(identity, 0, sizeof(*identity)); + if (!S_ISREG(st->st_mode) || st->st_nlink != 1) + return -1; + path_stat_identity_init(&identity->stat, st); + return 0; +} + +int clean_status_identity_is_durable(void) +{ +#ifdef __APPLE__ + return 1; +#else + return 0; +#endif +} diff --git a/clean-status-identity.h b/clean-status-identity.h new file mode 100644 index 00000000000000..b0453effcdcb91 --- /dev/null +++ b/clean-status-identity.h @@ -0,0 +1,16 @@ +#ifndef CLEAN_STATUS_IDENTITY_H +#define CLEAN_STATUS_IDENTITY_H + +#include "path-namespace.h" + +struct stat; + +struct clean_status_identity { + struct path_stat_identity stat; +}; + +int clean_status_identity_from_stat(struct clean_status_identity *identity, + const struct stat *st); +int clean_status_identity_is_durable(void); + +#endif /* CLEAN_STATUS_IDENTITY_H */ diff --git a/meson.build b/meson.build index d3b433856cb165..8f34aafa0fe276 100644 --- a/meson.build +++ b/meson.build @@ -335,6 +335,7 @@ libgit_sources = [ 'chunk-format.c', 'clean-status.c', 'clean-status-config.c', + 'clean-status-identity.c', 'clean-status-manifest.c', 'color.c', 'column.c', diff --git a/t/meson.build b/t/meson.build index 7b764958d084d4..832001d876b22a 100644 --- a/t/meson.build +++ b/t/meson.build @@ -2,6 +2,7 @@ clar_test_suites = [ 'unit-tests/u-attr-fingerprint.c', 'unit-tests/u-attr-manifest.c', 'unit-tests/u-clean-status-config.c', + 'unit-tests/u-clean-status-identity.c', 'unit-tests/u-clean-status-manifest.c', 'unit-tests/u-ctype.c', 'unit-tests/u-dir.c', diff --git a/t/unit-tests/u-clean-status-identity.c b/t/unit-tests/u-clean-status-identity.c new file mode 100644 index 00000000000000..33e7b80fcfc7e7 --- /dev/null +++ b/t/unit-tests/u-clean-status-identity.c @@ -0,0 +1,26 @@ +#include "unit-test.h" +#include "clean-status-identity.h" + +void test_clean_status_identity__requires_a_single_link_regular_file(void) +{ + struct clean_status_identity identity; + struct stat st = { 0 }; + + st.st_mode = S_IFDIR | 0755; + st.st_nlink = 1; + cl_assert_equal_i(clean_status_identity_from_stat(&identity, &st), -1); + st.st_mode = S_IFREG | 0644; + st.st_nlink = 2; + cl_assert_equal_i(clean_status_identity_from_stat(&identity, &st), -1); + st.st_nlink = 1; + cl_assert_equal_i(clean_status_identity_from_stat(&identity, &st), 0); +} + +void test_clean_status_identity__durability_is_platform_specific(void) +{ +#ifdef __APPLE__ + cl_assert(clean_status_identity_is_durable()); +#else + cl_assert(!clean_status_identity_is_durable()); +#endif +} From 41e30034b6d8d83e1ca9b1c5d16edce650b41e3e Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 23:15:36 -0700 Subject: [PATCH 072/105] read-cache: bind null-checksum indexes to their source file With index.skipHash enabled, a null trailing checksum cannot prove that verify_index_from() reopened the index that do_read_index() parsed. Replacing the pathname between those operations can otherwise make an unread index appear valid. Record the identity from the index reader's existing fstat() result. When verifying a null-checksum index on an Apple platform, compare it with the identity from the verifier's existing file observation. Reject an absent, nonregular, multiply linked, or replaced identity. Leave checksummed indexes and platforms without durable identities on their existing paths. Reuse the identity classification from S08/P03 without adding an index-read system call. Register the new object and unit suite with Make and Meson; the unit test replaces the index pathname and checks the unsupported fallback. Signed-off-by: Taylor Blau --- Makefile | 2 ++ clean-status-identity.c | 6 ++++ clean-status-identity.h | 2 ++ clean-status-index.c | 29 +++++++++++++++++++ clean-status-internal.h | 3 ++ clean-status.h | 5 ++++ meson.build | 1 + read-cache.c | 4 +++ t/meson.build | 1 + t/unit-tests/u-clean-status-index.c | 43 +++++++++++++++++++++++++++++ 10 files changed, 96 insertions(+) create mode 100644 clean-status-index.c create mode 100644 t/unit-tests/u-clean-status-index.c diff --git a/Makefile b/Makefile index ce5d750fb0dd09..400b9168333827 100644 --- a/Makefile +++ b/Makefile @@ -1128,6 +1128,7 @@ LIB_OBJS += chunk-format.o LIB_OBJS += clean-status.o LIB_OBJS += clean-status-config.o LIB_OBJS += clean-status-identity.o +LIB_OBJS += clean-status-index.o LIB_OBJS += clean-status-manifest.o LIB_OBJS += color.o LIB_OBJS += column.o @@ -1555,6 +1556,7 @@ CLAR_TEST_SUITES += u-attr-fingerprint CLAR_TEST_SUITES += u-attr-manifest CLAR_TEST_SUITES += u-clean-status-config CLAR_TEST_SUITES += u-clean-status-identity +CLAR_TEST_SUITES += u-clean-status-index CLAR_TEST_SUITES += u-clean-status-manifest CLAR_TEST_SUITES += u-ctype CLAR_TEST_SUITES += u-dir diff --git a/clean-status-identity.c b/clean-status-identity.c index 1d4c71116d6600..415ecab19a64f2 100644 --- a/clean-status-identity.c +++ b/clean-status-identity.c @@ -19,3 +19,9 @@ int clean_status_identity_is_durable(void) return 0; #endif } + +int clean_status_identity_equal(const struct clean_status_identity *a, + const struct clean_status_identity *b) +{ + return path_stat_identity_equal(&a->stat, &b->stat); +} diff --git a/clean-status-identity.h b/clean-status-identity.h index b0453effcdcb91..68e459a0349a1f 100644 --- a/clean-status-identity.h +++ b/clean-status-identity.h @@ -12,5 +12,7 @@ struct clean_status_identity { int clean_status_identity_from_stat(struct clean_status_identity *identity, const struct stat *st); int clean_status_identity_is_durable(void); +int clean_status_identity_equal(const struct clean_status_identity *a, + const struct clean_status_identity *b); #endif /* CLEAN_STATUS_IDENTITY_H */ diff --git a/clean-status-index.c b/clean-status-index.c new file mode 100644 index 00000000000000..4733e53e3a9fcc --- /dev/null +++ b/clean-status-index.c @@ -0,0 +1,29 @@ +#include "git-compat-util.h" +#include "clean-status.h" +#include "clean-status-internal.h" +#include "read-cache-ll.h" + +void clean_status_record_source_identity(struct index_state *istate, + const struct stat *st) +{ + struct clean_status_state *state = istate->clean_status; + + if (!state || state->source_identity_valid || + !clean_status_identity_is_durable()) + return; + if (!clean_status_identity_from_stat(&state->source_identity, st)) + state->source_identity_valid = 1; +} + +int clean_status_verify_null_index(const struct index_state *istate, + const struct stat *st) +{ + const struct clean_status_state *state = istate->clean_status; + struct clean_status_identity identity; + + if (!state || !clean_status_identity_is_durable()) + return 1; + return state->source_identity_valid && + !clean_status_identity_from_stat(&identity, st) && + clean_status_identity_equal(&identity, &state->source_identity); +} diff --git a/clean-status-internal.h b/clean-status-internal.h index 998173a39b008c..8df534f6e75689 100644 --- a/clean-status-internal.h +++ b/clean-status-internal.h @@ -1,11 +1,13 @@ #ifndef CLEAN_STATUS_INTERNAL_H #define CLEAN_STATUS_INTERNAL_H +#include "clean-status-identity.h" #include "hash.h" struct index_state; struct clean_status_state { + struct clean_status_identity source_identity; unsigned char current_config_hash[GIT_MAX_RAWSZ]; unsigned char current_semantic_hash[GIT_MAX_RAWSZ]; unsigned char current_attr_hash[GIT_MAX_RAWSZ]; @@ -17,6 +19,7 @@ struct clean_status_state { unsigned current_attr_sources_present : 1; unsigned config_enforced : 1; unsigned unsafe_filter : 1; + unsigned source_identity_valid : 1; }; struct clean_status_state *clean_status_get_state(struct index_state *istate); diff --git a/clean-status.h b/clean-status.h index 6749e9c5329016..421945a917def2 100644 --- a/clean-status.h +++ b/clean-status.h @@ -5,11 +5,16 @@ struct index_state; struct repository; +struct stat; void clean_status_set_config_digest( struct repository *repo, const struct clean_status_config_digest *digest); void clean_status_attach_config(struct index_state *istate); +void clean_status_record_source_identity(struct index_state *istate, + const struct stat *st); +int clean_status_verify_null_index(const struct index_state *istate, + const struct stat *st); void clean_status_release(struct index_state *istate); #endif /* CLEAN_STATUS_H */ diff --git a/meson.build b/meson.build index 8f34aafa0fe276..ca2c44295b3e5a 100644 --- a/meson.build +++ b/meson.build @@ -336,6 +336,7 @@ libgit_sources = [ 'clean-status.c', 'clean-status-config.c', 'clean-status-identity.c', + 'clean-status-index.c', 'clean-status-manifest.c', 'color.c', 'column.c', diff --git a/read-cache.c b/read-cache.c index 2b2493b45aee97..e58b29bdd142d4 100644 --- a/read-cache.c +++ b/read-cache.c @@ -2260,6 +2260,7 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) if (fstat(fd, &st)) die_errno(_("%s: cannot stat the open index"), path); + clean_status_record_source_identity(istate, &st); mmap_size = xsize_t(st.st_size); if (mmap_size < sizeof(struct cache_header) + the_hash_algo->rawsz) @@ -2749,6 +2750,9 @@ static int verify_index_from(const struct index_state *istate, const char *path) if (st.st_size < sizeof(struct cache_header) + the_hash_algo->rawsz) goto out; + if (is_null_oid(&istate->oid) && + !clean_status_verify_null_index(istate, &st)) + goto out; n = pread_in_full(fd, hash, the_hash_algo->rawsz, st.st_size - the_hash_algo->rawsz); if (n != the_hash_algo->rawsz) diff --git a/t/meson.build b/t/meson.build index 832001d876b22a..45243e9949d383 100644 --- a/t/meson.build +++ b/t/meson.build @@ -3,6 +3,7 @@ clar_test_suites = [ 'unit-tests/u-attr-manifest.c', 'unit-tests/u-clean-status-config.c', 'unit-tests/u-clean-status-identity.c', + 'unit-tests/u-clean-status-index.c', 'unit-tests/u-clean-status-manifest.c', 'unit-tests/u-ctype.c', 'unit-tests/u-dir.c', diff --git a/t/unit-tests/u-clean-status-index.c b/t/unit-tests/u-clean-status-index.c new file mode 100644 index 00000000000000..5d769692eea894 --- /dev/null +++ b/t/unit-tests/u-clean-status-index.c @@ -0,0 +1,43 @@ +#include "unit-test.h" +#include "clean-status.h" +#include "clean-status-internal.h" +#include "dir.h" +#include "read-cache-ll.h" +#include "strbuf.h" +#include "wrapper.h" + +void test_clean_status_index__binds_the_parsed_source(void) +{ + const char *tmp = getenv("TMPDIR"); + char *worktree = xstrfmt("%s/status-source.XXXXXX", + tmp ? tmp : "/tmp"); + struct index_state istate = { 0 }; + struct strbuf path = STRBUF_INIT, replacement = STRBUF_INIT; + struct strbuf cleanup = STRBUF_INIT; + struct stat original, current; + + cl_assert(mkdtemp(worktree) != NULL); + strbuf_addf(&path, "%s/index", worktree); + strbuf_addf(&replacement, "%s/replacement", worktree); + write_file(path.buf, "original"); + write_file(replacement.buf, "replacement"); + cl_assert_equal_i(stat(path.buf, &original), 0); + clean_status_get_state(&istate); + clean_status_record_source_identity(&istate, &original); + cl_assert(clean_status_verify_null_index(&istate, &original)); + + cl_assert_equal_i(rename(replacement.buf, path.buf), 0); + cl_assert_equal_i(stat(path.buf, ¤t), 0); + if (clean_status_identity_is_durable()) + cl_assert(!clean_status_verify_null_index(&istate, ¤t)); + else + cl_assert(clean_status_verify_null_index(&istate, ¤t)); + + clean_status_release(&istate); + strbuf_addstr(&cleanup, worktree); + cl_assert_equal_i(remove_dir_recursively(&cleanup, 0), 0); + strbuf_release(&cleanup); + strbuf_release(&replacement); + strbuf_release(&path); + free(worktree); +} From 3b91385d9697c5ac57b9b82cc8e2a643794b7fe7 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 00:47:32 -0500 Subject: [PATCH 073/105] read-cache: validate persisted fsmonitor semantic history A filesystem-monitor token does not establish that saved configuration, conversion rules, attribute inputs, or their complete manifest still describe the current index. Accepting duplicate, stale, or partially bound history could let status trust cached worktree state under different semantics. Recognize the FSCF index extension and delegate malformed-record rejection to the bounded clean-proof parser from S07/P07. Publish its token, configuration and semantic hashes, attribute hash, and manifest only after the complete record validates. Reject duplicate records, and adopt a manifest only when the current token, hashes, complete proof flags, and filter policy all agree. Record stronger semantic mismatches and withhold incoherent history. Integrate validation into post_read_index_from(), release all owned record and manifest storage with the index, and document the extension layout. Register the history object and unit suite with Make and Meson. A SHA-1 fixture rejects duplicate records; a SHA-256 fixture accepts coherent history and detects a changed semantic hash. Signed-off-by: Taylor Blau --- Documentation/gitformat-index.adoc | 30 +++++++ Makefile | 2 + clean-status-history.c | 112 ++++++++++++++++++++++++ clean-status-internal.h | 18 +++- clean-status.c | 9 +- clean-status.h | 6 ++ meson.build | 1 + read-cache.c | 5 ++ t/meson.build | 1 + t/unit-tests/u-clean-status-history.c | 120 ++++++++++++++++++++++++++ 10 files changed, 302 insertions(+), 2 deletions(-) create mode 100644 clean-status-history.c create mode 100644 t/unit-tests/u-clean-status-history.c diff --git a/Documentation/gitformat-index.adoc b/Documentation/gitformat-index.adoc index aaa9c29b4653b8..047310ec26d105 100644 --- a/Documentation/gitformat-index.adoc +++ b/Documentation/gitformat-index.adoc @@ -379,6 +379,36 @@ The remaining data of each directory block is grouped by type: - A NUL-terminated string containing the opaque file system monitor token associated with the untracked-cache data. +== File System Monitor semantic proof + + The file system monitor semantic proof records the configuration and + attribute inputs for a completed worktree-content verification. Its + signature is { 'F', 'S', 'C', 'F' }. + + The extension consists of: + + - 32-bit version number (currently 1). + + - 32-bit magic number identifying version 1 records (`FSC1`). + + - 32-bit flags. The low four bits respectively indicate a complete + attribute manifest, a provider-token binding, a stat-data binding, and + coverage of the full index. All other bits must be zero. + + - 32-bit length of the provider token. + + - 32-bit length of the attribute manifest. + + - The provider token, without a terminating NUL. + + - Three hashes, using the index hash algorithm, over the relevant Git + configuration, semantic-conversion configuration, and attribute state. + + - The attribute manifest described by its length above. + + - A hash over all preceding bytes in this extension, using the index hash + algorithm. + == End of Index Entry The End of Index Entry (EOIE) is used to locate the end of the variable diff --git a/Makefile b/Makefile index 400b9168333827..cde58e8bd4728f 100644 --- a/Makefile +++ b/Makefile @@ -1127,6 +1127,7 @@ LIB_OBJS += checkout.o LIB_OBJS += chunk-format.o LIB_OBJS += clean-status.o LIB_OBJS += clean-status-config.o +LIB_OBJS += clean-status-history.o LIB_OBJS += clean-status-identity.o LIB_OBJS += clean-status-index.o LIB_OBJS += clean-status-manifest.o @@ -1555,6 +1556,7 @@ THIRD_PARTY_SOURCES += $(UNIT_TEST_DIR)/clar/clar/% CLAR_TEST_SUITES += u-attr-fingerprint CLAR_TEST_SUITES += u-attr-manifest CLAR_TEST_SUITES += u-clean-status-config +CLAR_TEST_SUITES += u-clean-status-history CLAR_TEST_SUITES += u-clean-status-identity CLAR_TEST_SUITES += u-clean-status-index CLAR_TEST_SUITES += u-clean-status-manifest diff --git a/clean-status-history.c b/clean-status-history.c new file mode 100644 index 00000000000000..6caa8006916cf0 --- /dev/null +++ b/clean-status-history.c @@ -0,0 +1,112 @@ +#include "git-compat-util.h" +#include "clean-status.h" +#include "clean-status-internal.h" +#include "fsmonitor-clean-proof.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "strbuf.h" +#include "trace2.h" + +static void invalidate_disk_history(struct clean_status_state *state) +{ + state->disk_config_seen = 1; + state->disk_config_invalid = 1; + state->disk_config_valid = 0; + state->disk_semantic_valid = 0; + state->disk_attr_valid = 0; + FREE_AND_NULL(state->disk_config_token); + strbuf_reset(&state->disk_config_raw); + state->manifest.disk_valid = 0; + state->manifest.disk_flags = 0; + strbuf_reset(&state->manifest.disk); +} + +int clean_status_read_fsmonitor_config(struct index_state *istate, + const void *data, unsigned long size) +{ + struct clean_status_state *state = clean_status_get_state(istate); + struct fsmonitor_clean_proof proof; + + if (state->disk_config_seen || + fsmonitor_clean_proof_parse(&proof, data, size, + istate->repo->hash_algo) || + clean_status_manifest_load(&state->manifest, + proof.attr_manifest, + proof.attr_manifest_len, + proof.flags, + istate->repo->hash_algo)) { + invalidate_disk_history(state); + trace2_data_intmax("fsmonitor", istate->repo, + "config/invalid-extension", 1); + return 0; + } + + state->disk_config_seen = 1; + state->disk_config_token = xmemdupz(proof.token, proof.token_len); + memcpy(state->disk_config_hash, proof.config_hash, + istate->repo->hash_algo->rawsz); + memcpy(state->disk_semantic_hash, proof.semantic_hash, + istate->repo->hash_algo->rawsz); + memcpy(state->disk_attr_hash, proof.attr_hash, + istate->repo->hash_algo->rawsz); + strbuf_add(&state->disk_config_raw, data, size); + state->disk_config_valid = 1; + state->disk_semantic_valid = 1; + state->disk_attr_valid = 1; + return 0; +} + +void clean_status_prepare_fsmonitor_config(struct index_state *istate) +{ + struct clean_status_state *state = istate->clean_status; + const struct git_hash_algo *algo = istate->repo->hash_algo; + int token_coherent, config_coherent, semantic_changed, attr_changed; + int coherent; + + if (!state || !state->current_config_valid) + return; + token_coherent = state->disk_config_valid && + !state->disk_config_invalid && istate->fsmonitor_token_valid && + istate->fsmonitor_last_update && state->disk_config_token && + !strcmp(state->disk_config_token, istate->fsmonitor_last_update); + config_coherent = state->disk_config_valid && + !memcmp(state->disk_config_hash, state->current_config_hash, + algo->rawsz); + semantic_changed = state->disk_semantic_valid && + state->current_semantic_valid && + memcmp(state->disk_semantic_hash, state->current_semantic_hash, + algo->rawsz); + attr_changed = (state->disk_attr_valid && !state->current_attr_valid) || + (state->disk_attr_valid && state->current_attr_valid && + memcmp(state->disk_attr_hash, state->current_attr_hash, + algo->rawsz)); + coherent = token_coherent && config_coherent && + state->disk_semantic_valid && state->current_semantic_valid && + !semantic_changed && state->disk_attr_valid && + state->current_attr_valid && !attr_changed && + state->manifest.disk_valid && + (state->manifest.disk_flags & FSMONITOR_CLEAN_PROOF_ALL) == + FSMONITOR_CLEAN_PROOF_ALL && + !state->unsafe_filter; + state->config_revalidated = coherent; + state->initial_coherent = coherent; + FREE_AND_NULL(state->config_revalidated_token); + if (coherent) { + state->config_revalidated_token = + xstrdup(istate->fsmonitor_last_update); + clean_status_manifest_adopt_disk(&state->manifest); + } + state->config_mismatch = state->config_enforced && !coherent; + state->strong_mismatch = state->config_enforced && + (state->disk_config_invalid || state->unsafe_filter || + semantic_changed || attr_changed || + (state->disk_config_valid && !state->current_attr_valid) || + (!state->disk_semantic_valid && + state->current_semantic_explicit) || + (!state->disk_attr_valid && + state->current_attr_sources_present)); + trace2_data_intmax("fsmonitor", istate->repo, + "config/coherent", coherent); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/initial-mismatch", state->strong_mismatch); +} diff --git a/clean-status-internal.h b/clean-status-internal.h index 8df534f6e75689..e6b1078ea334a2 100644 --- a/clean-status-internal.h +++ b/clean-status-internal.h @@ -2,16 +2,23 @@ #define CLEAN_STATUS_INTERNAL_H #include "clean-status-identity.h" -#include "hash.h" +#include "clean-status-manifest.h" struct index_state; struct clean_status_state { struct clean_status_identity source_identity; + struct clean_status_manifest_state manifest; + struct strbuf disk_config_raw; + char *disk_config_token; + char *config_revalidated_token; unsigned char current_config_hash[GIT_MAX_RAWSZ]; + unsigned char disk_config_hash[GIT_MAX_RAWSZ]; unsigned char current_semantic_hash[GIT_MAX_RAWSZ]; + unsigned char disk_semantic_hash[GIT_MAX_RAWSZ]; unsigned char current_attr_hash[GIT_MAX_RAWSZ]; unsigned char current_attr_namespace_hash[GIT_MAX_RAWSZ]; + unsigned char disk_attr_hash[GIT_MAX_RAWSZ]; unsigned current_config_valid : 1; unsigned current_semantic_valid : 1; unsigned current_attr_valid : 1; @@ -19,7 +26,16 @@ struct clean_status_state { unsigned current_attr_sources_present : 1; unsigned config_enforced : 1; unsigned unsafe_filter : 1; + unsigned config_mismatch : 1; + unsigned strong_mismatch : 1; + unsigned config_revalidated : 1; + unsigned initial_coherent : 1; unsigned source_identity_valid : 1; + unsigned disk_config_valid : 1; + unsigned disk_semantic_valid : 1; + unsigned disk_attr_valid : 1; + unsigned disk_config_seen : 1; + unsigned disk_config_invalid : 1; }; struct clean_status_state *clean_status_get_state(struct index_state *istate); diff --git a/clean-status.c b/clean-status.c index 22ab0b25fe5b94..84674c0c45ef76 100644 --- a/clean-status.c +++ b/clean-status.c @@ -14,8 +14,11 @@ static int configured_semantic_explicit; struct clean_status_state *clean_status_get_state(struct index_state *istate) { - if (!istate->clean_status) + if (!istate->clean_status) { CALLOC_ARRAY(istate->clean_status, 1); + clean_status_manifest_init(&istate->clean_status->manifest); + strbuf_init(&istate->clean_status->disk_config_raw, 0); + } return istate->clean_status; } @@ -68,5 +71,9 @@ void clean_status_release(struct index_state *istate) { if (!istate->clean_status) return; + clean_status_manifest_release(&istate->clean_status->manifest); + strbuf_release(&istate->clean_status->disk_config_raw); + free(istate->clean_status->disk_config_token); + free(istate->clean_status->config_revalidated_token); FREE_AND_NULL(istate->clean_status); } diff --git a/clean-status.h b/clean-status.h index 421945a917def2..6a23fb03be112a 100644 --- a/clean-status.h +++ b/clean-status.h @@ -11,10 +11,16 @@ void clean_status_set_config_digest( struct repository *repo, const struct clean_status_config_digest *digest); void clean_status_attach_config(struct index_state *istate); + void clean_status_record_source_identity(struct index_state *istate, const struct stat *st); int clean_status_verify_null_index(const struct index_state *istate, const struct stat *st); + +int clean_status_read_fsmonitor_config(struct index_state *istate, + const void *data, unsigned long size); +void clean_status_prepare_fsmonitor_config(struct index_state *istate); + void clean_status_release(struct index_state *istate); #endif /* CLEAN_STATUS_H */ diff --git a/meson.build b/meson.build index ca2c44295b3e5a..59b7097fca049d 100644 --- a/meson.build +++ b/meson.build @@ -335,6 +335,7 @@ libgit_sources = [ 'chunk-format.c', 'clean-status.c', 'clean-status-config.c', + 'clean-status-history.c', 'clean-status-identity.c', 'clean-status-index.c', 'clean-status-manifest.c', diff --git a/read-cache.c b/read-cache.c index e58b29bdd142d4..7e769d3e4ddb5f 100644 --- a/read-cache.c +++ b/read-cache.c @@ -72,6 +72,7 @@ #define CACHE_EXT_LINK 0x6c696e6b /* "link" */ #define CACHE_EXT_UNTRACKED 0x554E5452 /* "UNTR" */ #define CACHE_EXT_FSMONITOR 0x46534D4E /* "FSMN" */ +#define CACHE_EXT_FSMONITOR_CONFIG 0x46534346 /* "FSCF" */ #define CACHE_EXT_FSMONITOR_UNTRACKED 0x46535543 /* "FSUC" */ #define CACHE_EXT_ENDOFINDEXENTRIES 0x454F4945 /* "EOIE" */ #define CACHE_EXT_INDEXENTRYOFFSETTABLE 0x49454F54 /* "IEOT" */ @@ -1779,6 +1780,9 @@ static int read_index_extension(struct index_state *istate, case CACHE_EXT_FSMONITOR: read_fsmonitor_extension(istate, data, sz); break; + case CACHE_EXT_FSMONITOR_CONFIG: + clean_status_read_fsmonitor_config(istate, data, sz); + break; case CACHE_EXT_FSMONITOR_UNTRACKED: read_fsmonitor_untracked_extension(istate, data, sz); break; @@ -1984,6 +1988,7 @@ static void post_read_index_from(struct index_state *istate) tweak_untracked_cache(istate); tweak_split_index(istate); prepare_fsmonitor_untracked(istate); + clean_status_prepare_fsmonitor_config(istate); tweak_fsmonitor(istate); } diff --git a/t/meson.build b/t/meson.build index 45243e9949d383..e4131eac00a60e 100644 --- a/t/meson.build +++ b/t/meson.build @@ -2,6 +2,7 @@ clar_test_suites = [ 'unit-tests/u-attr-fingerprint.c', 'unit-tests/u-attr-manifest.c', 'unit-tests/u-clean-status-config.c', + 'unit-tests/u-clean-status-history.c', 'unit-tests/u-clean-status-identity.c', 'unit-tests/u-clean-status-index.c', 'unit-tests/u-clean-status-manifest.c', diff --git a/t/unit-tests/u-clean-status-history.c b/t/unit-tests/u-clean-status-history.c new file mode 100644 index 00000000000000..49795b05545ecf --- /dev/null +++ b/t/unit-tests/u-clean-status-history.c @@ -0,0 +1,120 @@ +#include "unit-test.h" +#include "attr-manifest.h" +#include "clean-status.h" +#include "clean-status-internal.h" +#include "fsmonitor-clean-proof.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "strbuf.h" + +struct history_fixture { + struct repository repo; + struct index_state istate; + struct strbuf manifest; + struct strbuf encoded; + unsigned char config_hash[GIT_MAX_RAWSZ]; + unsigned char semantic_hash[GIT_MAX_RAWSZ]; + unsigned char attr_hash[GIT_MAX_RAWSZ]; +}; + +static void fixture_init(struct history_fixture *fixture, + const struct git_hash_algo *algo) +{ + static const unsigned char token[] = "builtin:1:2"; + struct attr_manifest_writer writer; + struct fsmonitor_clean_proof proof; + unsigned char hash[GIT_MAX_RAWSZ]; + + memset(fixture, 0, sizeof(*fixture)); + fixture->repo.hash_algo = algo; + index_state_init(&fixture->istate, &fixture->repo); + fixture->manifest = (struct strbuf)STRBUF_INIT; + fixture->encoded = (struct strbuf)STRBUF_INIT; + memset(hash, 1, algo->rawsz); + memset(fixture->config_hash, 2, algo->rawsz); + memset(fixture->semantic_hash, 3, algo->rawsz); + memset(fixture->attr_hash, 4, algo->rawsz); + attr_manifest_writer_init(&writer, &fixture->manifest, algo); + cl_assert_equal_i(attr_manifest_writer_add( + &writer, ".gitattributes", ATTR_MANIFEST_INDEX, hash), 0); + memset(&proof, 0, sizeof(proof)); + proof.flags = FSMONITOR_CLEAN_PROOF_ALL; + proof.token = token; + proof.token_len = sizeof(token) - 1; + proof.config_hash = fixture->config_hash; + proof.semantic_hash = fixture->semantic_hash; + proof.attr_hash = fixture->attr_hash; + proof.attr_manifest = (const unsigned char *)fixture->manifest.buf; + proof.attr_manifest_len = fixture->manifest.len; + cl_assert_equal_i(fsmonitor_clean_proof_write( + &fixture->encoded, &proof, algo), 0); +} + +static void fixture_release(struct history_fixture *fixture) +{ + clean_status_release(&fixture->istate); + free(fixture->istate.fsmonitor_last_update); + strbuf_release(&fixture->encoded); + strbuf_release(&fixture->manifest); +} + +static struct clean_status_state *install_current( + struct history_fixture *fixture) +{ + struct clean_status_state *state = + clean_status_get_state(&fixture->istate); + const struct git_hash_algo *algo = fixture->repo.hash_algo; + + memcpy(state->current_config_hash, fixture->config_hash, algo->rawsz); + memcpy(state->current_semantic_hash, fixture->semantic_hash, algo->rawsz); + memcpy(state->current_attr_hash, fixture->attr_hash, algo->rawsz); + state->current_config_valid = 1; + state->current_semantic_valid = 1; + state->current_attr_valid = 1; + state->config_enforced = 1; + fixture->istate.fsmonitor_last_update = xstrdup("builtin:1:2"); + fixture->istate.fsmonitor_token_valid = 1; + return state; +} + +void test_clean_status_history__reads_valid_history_once(void) +{ + struct history_fixture fixture; + struct clean_status_state *state; + + fixture_init(&fixture, &hash_algos[GIT_HASH_SHA1]); + cl_assert_equal_i(clean_status_read_fsmonitor_config( + &fixture.istate, fixture.encoded.buf, fixture.encoded.len), 0); + state = fixture.istate.clean_status; + cl_assert(state->disk_config_valid); + cl_assert(state->manifest.disk_valid); + cl_assert_equal_s(state->disk_config_token, "builtin:1:2"); + + cl_assert_equal_i(clean_status_read_fsmonitor_config( + &fixture.istate, fixture.encoded.buf, fixture.encoded.len), 0); + cl_assert(state->disk_config_invalid); + cl_assert(!state->disk_config_valid); + cl_assert(!state->manifest.disk_valid); + fixture_release(&fixture); +} + +void test_clean_status_history__adopts_only_coherent_proofs(void) +{ + struct history_fixture fixture; + struct clean_status_state *state; + + fixture_init(&fixture, &hash_algos[GIT_HASH_SHA256]); + clean_status_read_fsmonitor_config( + &fixture.istate, fixture.encoded.buf, fixture.encoded.len); + state = install_current(&fixture); + clean_status_prepare_fsmonitor_config(&fixture.istate); + cl_assert(state->initial_coherent); + cl_assert(state->manifest.current_valid); + cl_assert_equal_i(state->manifest.current.len, fixture.manifest.len); + + state->current_semantic_hash[0] ^= 1; + clean_status_prepare_fsmonitor_config(&fixture.istate); + cl_assert(state->strong_mismatch); + cl_assert(!state->initial_coherent); + fixture_release(&fixture); +} From adfbf38307bec8ce513b08c38bf53afc162d6c6b Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 00:47:47 -0500 Subject: [PATCH 074/105] read-cache: write only token-closed fsmonitor history Reading a validated FSCF record is not enough to preserve it during a generic index rewrite. Writing fresh token or stat bindings before the current provider token is revalidated would claim a semantic proof that the index has not established. Write a newly bound FSCF extension only when configuration, attributes, the complete manifest, the valid provider token, and its revalidated token all agree. Otherwise preserve an existing validated record with its token and stat bindings cleared; never serialize malformed or missing history. Add the extension to the existing index writer. Extend the history unit tests to distinguish closed proofs from preserved unbound manifests. Add a test-tool round trip and t7519 coverage that read, write, and reread a coherent FSCF record through a real index. Signed-off-by: Taylor Blau --- clean-status-history.c | 65 ++++++++++++++ clean-status-internal.h | 2 + clean-status.c | 19 ++++ clean-status.h | 8 ++ fsmonitor.c | 23 ++++- read-cache.c | 14 +++ t/helper/test-read-cache.c | 120 ++++++++++++++++++++++++++ t/t7519-status-fsmonitor.sh | 13 +++ t/t7527-builtin-fsmonitor.sh | 62 +++++++++++++ t/unit-tests/u-clean-status-history.c | 81 +++++++++++++++++ 10 files changed, 405 insertions(+), 2 deletions(-) diff --git a/clean-status-history.c b/clean-status-history.c index 6caa8006916cf0..2e1f1e29660799 100644 --- a/clean-status-history.c +++ b/clean-status-history.c @@ -110,3 +110,68 @@ void clean_status_prepare_fsmonitor_config(struct index_state *istate) trace2_data_intmax("fsmonitor", istate->repo, "semantic/initial-mismatch", state->strong_mismatch); } + +static int current_proof_is_writable(const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + + return state && istate->fsmonitor_token_valid && + istate->fsmonitor_last_update && + state->config_enforced && state->current_config_valid && + state->current_semantic_valid && state->current_attr_valid && + state->manifest.current_valid && + (state->manifest.current_flags & FSMONITOR_CLEAN_PROOF_ALL) == + FSMONITOR_CLEAN_PROOF_ALL && + clean_status_revalidated_token_matches(istate); +} + +void clean_status_advance_fsmonitor_config_token( + struct index_state *istate, const char *next_token) +{ + struct clean_status_state *state = istate->clean_status; + + if (!next_token || !current_proof_is_writable(istate)) + return; + FREE_AND_NULL(state->config_revalidated_token); + state->config_revalidated_token = xstrdup(next_token); + trace2_data_intmax("fsmonitor", istate->repo, + "config/token-advanced", 1); +} + +int clean_status_should_write_fsmonitor_config( + const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + + return current_proof_is_writable(istate) || + (state && state->disk_config_valid && + !state->disk_config_invalid && state->disk_config_raw.len); +} + +void clean_status_write_fsmonitor_config(struct strbuf *out, + const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + const struct git_hash_algo *algo = istate->repo->hash_algo; + + if (current_proof_is_writable(istate)) { + struct fsmonitor_clean_proof proof = { + .flags = state->manifest.current_flags, + .token = (const unsigned char *)istate->fsmonitor_last_update, + .token_len = strlen(istate->fsmonitor_last_update), + .config_hash = state->current_config_hash, + .semantic_hash = state->current_semantic_hash, + .attr_hash = state->current_attr_hash, + .attr_manifest = + (const unsigned char *)state->manifest.current.buf, + .attr_manifest_len = state->manifest.current.len, + }; + + if (fsmonitor_clean_proof_write(out, &proof, algo)) + BUG("cannot serialize validated fsmonitor clean proof"); + return; + } + if (fsmonitor_clean_proof_copy_without_bindings( + out, state->disk_config_raw.buf, state->disk_config_raw.len, algo)) + BUG("cannot preserve validated fsmonitor clean proof"); +} diff --git a/clean-status-internal.h b/clean-status-internal.h index e6b1078ea334a2..755d9ea059daec 100644 --- a/clean-status-internal.h +++ b/clean-status-internal.h @@ -39,5 +39,7 @@ struct clean_status_state { }; struct clean_status_state *clean_status_get_state(struct index_state *istate); +int clean_status_revalidated_token_matches( + const struct index_state *istate); #endif /* CLEAN_STATUS_INTERNAL_H */ diff --git a/clean-status.c b/clean-status.c index 84674c0c45ef76..98ae9a1490b2a3 100644 --- a/clean-status.c +++ b/clean-status.c @@ -67,6 +67,25 @@ void clean_status_attach_config(struct index_state *istate) } } +int clean_status_revalidated_token_matches(const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + + return state && state->config_revalidated && + state->config_revalidated_token && + istate->fsmonitor_last_update && + !strcmp(state->config_revalidated_token, + istate->fsmonitor_last_update); +} + +void clean_status_invalidate_current_proof(struct index_state *istate) +{ + if (!istate->clean_status) + return; + istate->clean_status->config_revalidated = 0; + istate->clean_status->initial_coherent = 0; +} + void clean_status_release(struct index_state *istate) { if (!istate->clean_status) diff --git a/clean-status.h b/clean-status.h index 6a23fb03be112a..229bc75f0f4043 100644 --- a/clean-status.h +++ b/clean-status.h @@ -6,6 +6,7 @@ struct index_state; struct repository; struct stat; +struct strbuf; void clean_status_set_config_digest( struct repository *repo, @@ -20,6 +21,13 @@ int clean_status_verify_null_index(const struct index_state *istate, int clean_status_read_fsmonitor_config(struct index_state *istate, const void *data, unsigned long size); void clean_status_prepare_fsmonitor_config(struct index_state *istate); +void clean_status_invalidate_current_proof(struct index_state *istate); +void clean_status_advance_fsmonitor_config_token( + struct index_state *istate, const char *next_token); +int clean_status_should_write_fsmonitor_config( + const struct index_state *istate); +void clean_status_write_fsmonitor_config(struct strbuf *out, + const struct index_state *istate); void clean_status_release(struct index_state *istate); diff --git a/fsmonitor.c b/fsmonitor.c index 94ccbceba7c307..9e90d158402fdd 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -3,6 +3,7 @@ #include "git-compat-util.h" #include "attr.h" +#include "clean-status.h" #include "config.h" #include "dir.h" #include "environment.h" @@ -636,6 +637,7 @@ static void fsmonitor_refresh_callback(struct index_state *istate, char *name) { int len = strlen(name); int pos; + int attributes_may_have_changed; size_t nr_in_cone; trace_printf_key(&trace_fsmonitor, @@ -645,6 +647,7 @@ static void fsmonitor_refresh_callback(struct index_state *istate, char *name) if (!strcmp(name, FSMONITOR_PATH_GLOBAL_INVALIDATE)) { unsigned int i; + clean_status_invalidate_current_proof(istate); git_attr_invalidate_all(); untracked_cache_invalidate_all(istate); for (i = 0; i < istate->cache_nr; i++) @@ -655,12 +658,15 @@ static void fsmonitor_refresh_callback(struct index_state *istate, char *name) return; } pos = index_name_pos(istate, name, len); - fsmonitor_invalidate_attributes_path(istate, name); + attributes_may_have_changed = + fsmonitor_invalidate_attributes_path(istate, name); if (name[len - 1] == '/') nr_in_cone = handle_path_with_trailing_slash(istate, name, pos); else nr_in_cone = handle_path_without_trailing_slash(istate, name, pos); + if (pos < 0 && nr_in_cone) + attributes_may_have_changed = 1; /* * If we did not find an exact match for this pathname or any @@ -670,10 +676,15 @@ static void fsmonitor_refresh_callback(struct index_state *istate, char *name) */ if (!nr_in_cone && repo_ignore_case(the_repository)) { nr_in_cone = handle_using_name_hash_icase(istate, name); - if (!nr_in_cone) + if (!nr_in_cone) { nr_in_cone = handle_using_dir_name_hash_icase( istate, name); + if (nr_in_cone) + attributes_may_have_changed = 1; + } } + if (attributes_may_have_changed) + clean_status_invalidate_current_proof(istate); if (nr_in_cone) trace_printf_key(&trace_fsmonitor, @@ -1137,6 +1148,14 @@ void refresh_fsmonitor(struct index_state *istate) (fsm_mode == FSMONITOR_MODE_IPC || !is_trivial); istate->fsmonitor_untracked_valid = 0; } else { + /* + * The applied delta carries an existing proof forward: + * tracked paths are now invalid in FSMN, while semantic + * events have already expired the proof itself. + */ + if (fsm_mode == FSMONITOR_MODE_IPC) + clean_status_advance_fsmonitor_config_token( + istate, last_update_token.buf); FREE_AND_NULL(istate->fsmonitor_last_update); istate->fsmonitor_last_update = strbuf_detach(&last_update_token, NULL); diff --git a/read-cache.c b/read-cache.c index 7e769d3e4ddb5f..2a7cca8d3e636f 100644 --- a/read-cache.c +++ b/read-cache.c @@ -2840,6 +2840,7 @@ enum write_extensions { WRITE_RESOLVE_UNDO_EXTENSION = 1<<2, WRITE_UNTRACKED_CACHE_EXTENSION = 1<<3, WRITE_FSMONITOR_EXTENSION = 1<<4, + WRITE_FSCF_EXTENSION = 1<<5, }; #define WRITE_ALL_EXTENSIONS ((enum write_extensions)-1) @@ -3107,6 +3108,19 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, goto out; } } + if (write_extensions & WRITE_FSCF_EXTENSION && + clean_status_should_write_fsmonitor_config(istate)) { + strbuf_reset(&sb); + clean_status_write_fsmonitor_config(&sb, istate); + err = write_index_ext_header(f, eoie_c, + CACHE_EXT_FSMONITOR_CONFIG, + sb.len) < 0; + hashwrite(f, sb.buf, sb.len); + if (err) { + ret = -1; + goto out; + } + } if (istate->sparse_index) { if (write_index_ext_header(f, eoie_c, CACHE_EXT_SPARSE_DIRECTORIES, 0) < 0) { ret = -1; diff --git a/t/helper/test-read-cache.c b/t/helper/test-read-cache.c index 372b55b419d6b4..8c2e97056b7e01 100644 --- a/t/helper/test-read-cache.c +++ b/t/helper/test-read-cache.c @@ -2,13 +2,19 @@ #include "test-tool.h" #include "attr.h" +#include "attr-fingerprint.h" +#include "attr-manifest.h" +#include "clean-status.h" +#include "clean-status-internal.h" #include "config.h" #include "dir.h" #include "environment.h" #include "ewah/ewok.h" #include "ewah/ewok_rlw.h" #include "fsmonitor.h" +#include "fsmonitor-clean-proof.h" #include "fsmonitor-ll.h" +#include "lockfile.h" #include "read-cache-ll.h" #include "repository.h" #include "setup.h" @@ -237,6 +243,116 @@ static int test_fsmn_parser(void) return 0; } +static int write_test_index(void) +{ + struct lock_file index_lock = LOCK_INIT; + + repo_hold_locked_index(the_repository, &index_lock, LOCK_DIE_ON_ERROR); + if (write_locked_index(the_repository->index, &index_lock, COMMIT_LOCK)) + return error("unable to write test index"); + return 0; +} + +static int test_fscf_history_is_coherent(const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + + return state && state->disk_config_valid && + !state->disk_config_invalid && state->disk_semantic_valid && + state->disk_attr_valid && state->manifest.disk_valid && + (state->manifest.disk_flags & FSMONITOR_CLEAN_PROOF_ALL) == + FSMONITOR_CLEAN_PROOF_ALL && + state->disk_config_raw.len && state->initial_coherent; +} + +static int test_fscf_config(const char *key, const char *value, + const struct config_context *ctx, void *cb) +{ + struct clean_status_config_digest *config = cb; + + clean_status_config_add(config, key, value, ctx); + return git_default_config(key, value, ctx, NULL); +} + +static int test_fscf_history(int seed_existing_token) +{ + struct clean_status_config_digest config; + struct attr_fingerprint attrs; + struct attr_manifest_writer writer; + struct strbuf manifest = STRBUF_INIT; + struct strbuf encoded = STRBUF_INIT; + unsigned char index_hash[GIT_MAX_RAWSZ] = { 0 }; + const char *token; + struct fsmonitor_clean_proof proof = { + .flags = FSMONITOR_CLEAN_PROOF_ALL, + }; + const struct git_hash_algo *algo; + int ret = 1; + + setup_git_directory(the_repository); + algo = the_repository->hash_algo; + clean_status_config_init(&config, algo); + repo_config(the_repository, test_fscf_config, &config); + clean_status_config_final(&config); + clean_status_set_config_digest(the_repository, &config); + if (repo_read_index(the_repository) < 0) + return error("unable to read test index"); + if (seed_existing_token) { + if (!the_repository->index->fsmonitor_token_valid || + !the_repository->index->fsmonitor_last_update) + return error("existing fsmonitor token was not valid"); + the_repository->index->fsmonitor_has_run_once = 1; + token = the_repository->index->fsmonitor_last_update; + } else { + token = "fscf-test-token"; + } + if (attr_fingerprint_repository(the_repository, &attrs)) + return error("unable to fingerprint attribute sources"); + + attr_manifest_writer_init(&writer, &manifest, algo); + if (attr_manifest_writer_add(&writer, ".gitattributes", + ATTR_MANIFEST_INDEX, index_hash)) + return error("unable to write test attribute manifest"); + proof.config_hash = config.hash; + proof.semantic_hash = config.semantic_hash; + proof.attr_hash = attrs.content_hash; + proof.token = (const unsigned char *)token; + proof.token_len = strlen(token); + proof.attr_manifest = (const unsigned char *)manifest.buf; + proof.attr_manifest_len = manifest.len; + if (fsmonitor_clean_proof_write(&encoded, &proof, algo)) + return error("unable to write test clean proof"); + + if (!seed_existing_token) { + FREE_AND_NULL(the_repository->index->fsmonitor_last_update); + the_repository->index->fsmonitor_last_update = xstrdup(token); + the_repository->index->fsmonitor_token_valid = 1; + } + clean_status_read_fsmonitor_config(the_repository->index, + encoded.buf, encoded.len); + clean_status_prepare_fsmonitor_config(the_repository->index); + if (!test_fscf_history_is_coherent(the_repository->index)) + return error("test clean proof was not coherent"); + if (write_test_index()) + goto done; + if (seed_existing_token) { + ret = 0; + goto done; + } + + discard_index(the_repository->index); + if (repo_read_index(the_repository) < 0) + return error("unable to reread test index"); + if (!test_fscf_history_is_coherent(the_repository->index)) + return error("FSCF did not survive an index round trip"); + ret = 0; + +done: + strbuf_release(&encoded); + strbuf_release(&manifest); + return ret; +} + static int test_fsmonitor_directory_attributes(void) { struct attr_check *check; @@ -288,6 +404,10 @@ int cmd__read_cache(int argc, const char **argv) return test_fsuc_parser(); if (argc == 2 && !strcmp(argv[1], "--test-fsmn-parser")) return test_fsmn_parser(); + if (argc == 2 && !strcmp(argv[1], "--test-fscf-round-trip")) + return test_fscf_history(0); + if (argc == 2 && !strcmp(argv[1], "--test-fscf-seed")) + return test_fscf_history(1); if (argc == 2 && !strcmp(argv[1], "--test-fsmonitor-directory-attributes")) return test_fsmonitor_directory_attributes(); diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 6b1fdd3bcbbc6f..90228af9d007d2 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -3,6 +3,7 @@ test_description='git status with file system watcher' . ./test-lib.sh +. "$TEST_DIRECTORY"/lib-semantic-verify.sh # Note, after "git reset --hard HEAD" no extensions exist other than 'TREE' # "git update-index --fsmonitor" can be used to get the extension written @@ -68,6 +69,18 @@ test_expect_success 'FSUC parser fails closed' ' test-tool read-cache --test-fsuc-parser ' +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'FSCF survives index I/O and generic rewrites' ' + test_when_finished "rm -rf fscf-round-trip" && + test_create_repo fscf-round-trip && + ( + cd fscf-round-trip && + test_commit base tracked && + test-tool read-cache --test-fscf-round-trip && + test_grep FSCF .git/index + ) +' + test_expect_success 'hook parser ignores empty path records' ' test_when_finished "rm -rf empty-hook-record" && test_create_repo empty-hook-record && diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index dd9badbff281a7..dec4a493d4eb8a 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -3,6 +3,7 @@ test_description='built-in file system watcher' . ./test-lib.sh +. "$TEST_DIRECTORY"/lib-semantic-verify.sh if ! test_have_prereq FSMONITOR_DAEMON then @@ -1622,4 +1623,65 @@ test_expect_success MACOS 'worktree binding rejects same-gitdir aliases' ' git -C binding-a fsmonitor--daemon stop ' +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'ordinary deltas advance only attribute-stable proofs' ' + test_when_finished "rm -rf token-carry" && + test_create_repo token-carry && + ( + cd token-carry && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test_write_lines "*.txt text" >.gitattributes && + git add .gitattributes && + git commit -m attributes && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain=v2 \ + >.git/initial && + test_must_be_empty .git/initial && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSUC .git/index && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + test-tool read-cache --test-fscf-seed && + test_grep FSUC .git/index && + test_grep FSCF .git/index && + + touch x && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=x \ + GIT_TRACE2_EVENT="$PWD/.git/created.trace" \ + git status --porcelain=v2 >.git/created && + test_grep "^? x$" .git/created && + test_trace2_data fsmonitor config/token-advanced 1 \ + <.git/created.trace && + + rm x && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=x \ + GIT_TRACE2_EVENT="$PWD/.git/deleted.trace" \ + git status --porcelain=v2 >.git/deleted && + test_must_be_empty .git/deleted && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/deleted.trace && + test_trace2_data fsmonitor config/token-advanced 1 \ + <.git/deleted.trace && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=.gitattributes \ + GIT_TRACE2_EVENT="$PWD/.git/attributes.trace" \ + git status --porcelain=v2 >.git/attributes && + test_must_be_empty .git/attributes && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/attributes.trace && + test_trace2_data fsmonitor apply_count 1 \ + <.git/attributes.trace && + ! test_trace2_data fsmonitor config/token-advanced 1 \ + <.git/attributes.trace + ) +' + test_done diff --git a/t/unit-tests/u-clean-status-history.c b/t/unit-tests/u-clean-status-history.c index 49795b05545ecf..1f3072b2a239cf 100644 --- a/t/unit-tests/u-clean-status-history.c +++ b/t/unit-tests/u-clean-status-history.c @@ -118,3 +118,84 @@ void test_clean_status_history__adopts_only_coherent_proofs(void) cl_assert(!state->initial_coherent); fixture_release(&fixture); } + +void test_clean_status_history__preserves_unbound_manifests(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct history_fixture fixture; + struct clean_status_state *state; + struct fsmonitor_clean_proof parsed; + struct strbuf rewritten = STRBUF_INIT; + + fixture_init(&fixture, algo); + clean_status_read_fsmonitor_config( + &fixture.istate, fixture.encoded.buf, fixture.encoded.len); + state = install_current(&fixture); + clean_status_prepare_fsmonitor_config(&fixture.istate); + cl_assert(clean_status_should_write_fsmonitor_config(&fixture.istate)); + clean_status_write_fsmonitor_config(&rewritten, &fixture.istate); + cl_assert_equal_i(fsmonitor_clean_proof_parse( + &parsed, rewritten.buf, rewritten.len, algo), 0); + cl_assert_equal_i(parsed.flags, FSMONITOR_CLEAN_PROOF_ALL); + + FREE_AND_NULL(fixture.istate.fsmonitor_last_update); + fixture.istate.fsmonitor_last_update = xstrdup("builtin:1:3"); + clean_status_write_fsmonitor_config(&rewritten, &fixture.istate); + cl_assert_equal_i(fsmonitor_clean_proof_parse( + &parsed, rewritten.buf, rewritten.len, algo), 0); + cl_assert_equal_i(parsed.flags, + FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | + FSMONITOR_CLEAN_PROOF_FULL_INDEX); + + FREE_AND_NULL(fixture.istate.fsmonitor_last_update); + fixture.istate.fsmonitor_last_update = xstrdup("builtin:1:2"); + state->current_config_valid = 0; + clean_status_write_fsmonitor_config(&rewritten, &fixture.istate); + cl_assert_equal_i(fsmonitor_clean_proof_parse( + &parsed, rewritten.buf, rewritten.len, algo), 0); + cl_assert_equal_i(parsed.flags, + FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | + FSMONITOR_CLEAN_PROOF_FULL_INDEX); + strbuf_release(&rewritten); + fixture_release(&fixture); +} + +void test_clean_status_history__advances_only_current_proofs(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct history_fixture fixture; + struct clean_status_state *state; + struct fsmonitor_clean_proof parsed; + struct strbuf rewritten = STRBUF_INIT; + + fixture_init(&fixture, algo); + clean_status_read_fsmonitor_config( + &fixture.istate, fixture.encoded.buf, fixture.encoded.len); + state = install_current(&fixture); + clean_status_prepare_fsmonitor_config(&fixture.istate); + + clean_status_advance_fsmonitor_config_token( + &fixture.istate, "builtin:1:3"); + cl_assert_equal_s(state->config_revalidated_token, "builtin:1:3"); + FREE_AND_NULL(fixture.istate.fsmonitor_last_update); + fixture.istate.fsmonitor_last_update = xstrdup("builtin:1:3"); + cl_assert(clean_status_should_write_fsmonitor_config(&fixture.istate)); + + clean_status_invalidate_current_proof(&fixture.istate); + cl_assert(!state->config_revalidated); + cl_assert(!state->initial_coherent); + clean_status_advance_fsmonitor_config_token( + &fixture.istate, "builtin:1:4"); + cl_assert_equal_s(state->config_revalidated_token, "builtin:1:3"); + FREE_AND_NULL(fixture.istate.fsmonitor_last_update); + fixture.istate.fsmonitor_last_update = xstrdup("builtin:1:4"); + clean_status_write_fsmonitor_config(&rewritten, &fixture.istate); + cl_assert_equal_i(fsmonitor_clean_proof_parse( + &parsed, rewritten.buf, rewritten.len, algo), 0); + cl_assert_equal_i(parsed.flags, + FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | + FSMONITOR_CLEAN_PROOF_FULL_INDEX); + + strbuf_release(&rewritten); + fixture_release(&fixture); +} From 16e0ff232f3eef616df9bbdf616aa2e9ecd672ab Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 10 Jul 2026 22:53:53 -0700 Subject: [PATCH 075/105] read-cache: preserve validated fsmonitor history across indexes move_index_extensions() transfers extensions to a replacement index, but index-owned FSCF history would otherwise remain on the old state. A generic rewrite could silently discard a validated manifest, while sharing its storage would create a lifetime hazard. Copy only a parsed, valid serialized record into independently owned destination storage. Reload the saved manifest through its validated parser, copy the existing token and hashes, and leave an absent or invalid source untouched. Invoke the transfer from move_index_extensions() so ordinary index release owns each copy. Extend the existing history unit suite with a real extension transfer. Verify the copied record and manifest, invalidate the source, reject a second transfer from that source, and confirm that the independent first destination remains valid. Signed-off-by: Taylor Blau --- clean-status-history.c | 34 +++++++++++++++++++++++ clean-status.h | 2 ++ read-cache.c | 1 + t/unit-tests/u-clean-status-history.c | 40 ++++++++++++++++++++++++++- 4 files changed, 76 insertions(+), 1 deletion(-) diff --git a/clean-status-history.c b/clean-status-history.c index 2e1f1e29660799..61fc2f02b550c1 100644 --- a/clean-status-history.c +++ b/clean-status-history.c @@ -175,3 +175,37 @@ void clean_status_write_fsmonitor_config(struct strbuf *out, out, state->disk_config_raw.buf, state->disk_config_raw.len, algo)) BUG("cannot preserve validated fsmonitor clean proof"); } + +void clean_status_copy_fsmonitor_history(struct index_state *dst, + const struct index_state *src) +{ + const struct clean_status_state *src_state = src->clean_status; + struct clean_status_state *dst_state; + + if (!src_state || !src_state->disk_config_valid || + src_state->disk_config_invalid || !src_state->disk_config_raw.len) + return; + dst_state = clean_status_get_state(dst); + FREE_AND_NULL(dst_state->disk_config_token); + strbuf_reset(&dst_state->disk_config_raw); + dst_state->disk_config_token = + xstrdup_or_null(src_state->disk_config_token); + strbuf_addbuf(&dst_state->disk_config_raw, + &src_state->disk_config_raw); + memcpy(dst_state->disk_config_hash, src_state->disk_config_hash, + dst->repo->hash_algo->rawsz); + memcpy(dst_state->disk_semantic_hash, src_state->disk_semantic_hash, + dst->repo->hash_algo->rawsz); + memcpy(dst_state->disk_attr_hash, src_state->disk_attr_hash, + dst->repo->hash_algo->rawsz); + if (clean_status_manifest_load( + &dst_state->manifest, src_state->manifest.disk.buf, + src_state->manifest.disk.len, src_state->manifest.disk_flags, + dst->repo->hash_algo)) + BUG("cannot copy validated clean-status manifest"); + dst_state->disk_config_seen = 1; + dst_state->disk_config_valid = 1; + dst_state->disk_semantic_valid = src_state->disk_semantic_valid; + dst_state->disk_attr_valid = src_state->disk_attr_valid; + dst_state->disk_config_invalid = 0; +} diff --git a/clean-status.h b/clean-status.h index 229bc75f0f4043..926c5d037997ce 100644 --- a/clean-status.h +++ b/clean-status.h @@ -28,6 +28,8 @@ int clean_status_should_write_fsmonitor_config( const struct index_state *istate); void clean_status_write_fsmonitor_config(struct strbuf *out, const struct index_state *istate); +void clean_status_copy_fsmonitor_history(struct index_state *dst, + const struct index_state *src); void clean_status_release(struct index_state *istate); diff --git a/read-cache.c b/read-cache.c index 2a7cca8d3e636f..43cbfc19240862 100644 --- a/read-cache.c +++ b/read-cache.c @@ -3575,6 +3575,7 @@ void *read_blob_data_from_index(struct index_state *istate, void move_index_extensions(struct index_state *dst, struct index_state *src) { + clean_status_copy_fsmonitor_history(dst, src); dst->untracked = src->untracked; src->untracked = NULL; dst->cache_tree = src->cache_tree; diff --git a/t/unit-tests/u-clean-status-history.c b/t/unit-tests/u-clean-status-history.c index 1f3072b2a239cf..4a39bfa8568811 100644 --- a/t/unit-tests/u-clean-status-history.c +++ b/t/unit-tests/u-clean-status-history.c @@ -159,7 +159,6 @@ void test_clean_status_history__preserves_unbound_manifests(void) strbuf_release(&rewritten); fixture_release(&fixture); } - void test_clean_status_history__advances_only_current_proofs(void) { const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; @@ -199,3 +198,42 @@ void test_clean_status_history__advances_only_current_proofs(void) strbuf_release(&rewritten); fixture_release(&fixture); } + +void test_clean_status_history__copies_validated_history(void) +{ + struct history_fixture fixture; + struct repository dst_repo = { + .hash_algo = &hash_algos[GIT_HASH_SHA1], + }; + struct index_state dst = INDEX_STATE_INIT(&dst_repo); + struct index_state invalid_dst = INDEX_STATE_INIT(&dst_repo); + struct clean_status_state *dst_state; + + fixture_init(&fixture, &hash_algos[GIT_HASH_SHA1]); + clean_status_read_fsmonitor_config( + &fixture.istate, fixture.encoded.buf, fixture.encoded.len); + move_index_extensions(&dst, &fixture.istate); + dst_state = dst.clean_status; + cl_assert(dst_state != NULL); + cl_assert(dst_state->disk_config_valid); + cl_assert(!dst_state->disk_config_invalid); + cl_assert(dst_state->disk_semantic_valid); + cl_assert(dst_state->disk_attr_valid); + cl_assert(dst_state->manifest.disk_valid); + cl_assert_equal_i(dst_state->manifest.disk_flags, + FSMONITOR_CLEAN_PROOF_ALL); + cl_assert_equal_i(dst_state->disk_config_raw.len, + fixture.encoded.len); + + clean_status_read_fsmonitor_config( + &fixture.istate, fixture.encoded.buf, fixture.encoded.len); + move_index_extensions(&invalid_dst, &fixture.istate); + cl_assert(!invalid_dst.clean_status); + cl_assert(dst_state->disk_config_valid); + cl_assert_equal_i(dst_state->disk_config_raw.len, + fixture.encoded.len); + + release_index(&invalid_dst); + release_index(&dst); + fixture_release(&fixture); +} From e8e671b5c7f9d450e95c5c4f8b29cb5057382476 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 20 Jul 2026 20:42:14 -0500 Subject: [PATCH 076/105] status: hold external attributes stable across refresh Fingerprinting an external attribute file while reading the index does not prevent the attribute parser from reopening a replaced file during preload or status collection. Cached stat data could then be evaluated with conversion rules that the original fingerprint did not cover. Capture the system, global, and info attribute bytes and namespace once and keep the immutable snapshot active from untracked-cache preload through collection. Parse snapshot lines with the ordinary attribute rules, including byte-order marks, embedded NULs, and line endings. End the snapshot and release its bounded source buffers with status. Make a failed capture or changed attribute content sticky and invalidate fsmonitor validity and the untracked cache before ordinary refresh. Preserve hook-provider behavior when semantic history is absent or only the namespace changes: hooks have no closing query and retain their reported-path contract. An observed content change still invalidates hook-derived state. Add t7531 integration coverage for file-parser parity, missing attribute history, an observed hook-time attribute change, and the hook missing-history exception. Update the existing history unit test to exercise the public strong-mismatch predicate. The namespace-only hook branch has no dedicated regression in this patch. Signed-off-by: Taylor Blau --- attr-fingerprint.c | 93 ++++++++++++++++++++-- attr-fingerprint.h | 18 +++++ attr.c | 89 +++++++++++++++++++-- attr.h | 12 +++ clean-status.c | 62 +++++++++++++++ clean-status.h | 10 +++ fsmonitor.c | 9 +++ fsmonitor.h | 2 + t/t7531-semantic-verify.sh | 110 ++++++++++++++++++++++++++ t/unit-tests/u-clean-status-history.c | 2 +- wt-status.c | 50 ++++++++++++ wt-status.h | 3 + 12 files changed, 446 insertions(+), 14 deletions(-) diff --git a/attr-fingerprint.c b/attr-fingerprint.c index 6a9cde2821614c..d7fdc1870dd2c3 100644 --- a/attr-fingerprint.c +++ b/attr-fingerprint.c @@ -10,6 +10,17 @@ #include "strbuf.h" #include "wrapper.h" +struct attr_source_snapshot_entry { + char *path; + char *buf; + size_t len; +}; + +struct attr_source_snapshot { + struct attr_fingerprint fingerprint; + struct attr_source_snapshot_entry sources[ATTR_SOURCE_SNAPSHOT_NR]; +}; + static int open_attr_source(const char *path) { #ifdef O_NONBLOCK @@ -24,7 +35,8 @@ static int open_attr_source(const char *path) static int hash_source(struct git_hash_ctx *content_ctx, struct git_hash_ctx *namespace_ctx, const struct attr_fingerprint_source *source, - int *present) + int *present, + struct attr_source_snapshot_entry *snapshot) { struct path_namespace_snapshot *before = NULL, *after = NULL; struct stat opened_before, opened_after, named; @@ -84,6 +96,12 @@ static int hash_source(struct git_hash_ctx *content_ctx, path_namespace_hash(namespace_ctx, before); path_namespace_hash_stat(namespace_ctx, &opened_after); hash_length_delimited(content_ctx, buf, size); + if (snapshot) { + snapshot->path = xstrdup(source->path); + snapshot->buf = buf; + snapshot->len = size; + buf = NULL; + } ret = 0; done: if (fd >= 0) @@ -98,11 +116,14 @@ static int hash_source(struct git_hash_ctx *content_ctx, static int fingerprint_sources( const struct attr_fingerprint_source *sources, size_t nr, - const struct git_hash_algo *algo, struct attr_fingerprint *result) + const struct git_hash_algo *algo, struct attr_fingerprint *result, + struct attr_source_snapshot *snapshot) { struct git_hash_ctx content_ctx, namespace_ctx; uint32_t count; + if (snapshot && nr != ARRAY_SIZE(snapshot->sources)) + BUG("attribute snapshot source count mismatch"); memset(result, 0, sizeof(*result)); git_hash_init(&content_ctx, algo); git_hash_init(&namespace_ctx, algo); @@ -116,9 +137,11 @@ static int fingerprint_sources( hash_length_delimited(&namespace_ctx, &count, sizeof(count)); for (size_t i = 0; i < nr; i++) { int present; + struct attr_source_snapshot_entry *entry = + snapshot ? &snapshot->sources[i] : NULL; if (hash_source(&content_ctx, &namespace_ctx, &sources[i], - &present)) + &present, entry)) return -1; result->sources_present |= present; } @@ -131,7 +154,7 @@ int attr_fingerprint_sources( const struct attr_fingerprint_source *sources, size_t nr, const struct git_hash_algo *algo, struct attr_fingerprint *result) { - return fingerprint_sources(sources, nr, algo, result); + return fingerprint_sources(sources, nr, algo, result, NULL); } static int repository_sources(struct repository *repo, @@ -153,7 +176,7 @@ static int repository_sources(struct repository *repo, int attr_fingerprint_repository(struct repository *repo, struct attr_fingerprint *result) { - struct attr_fingerprint_source sources[3]; + struct attr_fingerprint_source sources[ATTR_SOURCE_SNAPSHOT_NR]; char *info_attributes = NULL; int ret; @@ -165,3 +188,63 @@ int attr_fingerprint_repository(struct repository *repo, free(info_attributes); return ret; } + +int attr_source_snapshot_repository(struct repository *repo, + struct attr_source_snapshot **result) +{ + struct attr_fingerprint_source sources[ATTR_SOURCE_SNAPSHOT_NR]; + struct attr_source_snapshot *snapshot; + char *info_attributes = NULL; + + if (!result) + BUG("attr_source_snapshot_repository requires an output"); + *result = NULL; + if (repository_sources(repo, sources, &info_attributes)) + return -1; + CALLOC_ARRAY(snapshot, 1); + if (fingerprint_sources(sources, ARRAY_SIZE(sources), repo->hash_algo, + &snapshot->fingerprint, snapshot)) { + attr_source_snapshot_free(snapshot); + free(info_attributes); + return -1; + } + free(info_attributes); + *result = snapshot; + return 0; +} + +const struct attr_fingerprint *attr_source_snapshot_fingerprint( + const struct attr_source_snapshot *snapshot) +{ + return snapshot ? &snapshot->fingerprint : NULL; +} + +int attr_source_snapshot_read( + const struct attr_source_snapshot *snapshot, + enum attr_source_snapshot_kind kind, + const char **path, const char **buf, size_t *len) +{ + const struct attr_source_snapshot_entry *source; + + if (!snapshot || kind >= ATTR_SOURCE_SNAPSHOT_NR || + !path || !buf || !len) + BUG("invalid attribute snapshot read"); + source = &snapshot->sources[kind]; + if (!source->buf) + return 0; + *path = source->path; + *buf = source->buf; + *len = source->len; + return 1; +} + +void attr_source_snapshot_free(struct attr_source_snapshot *snapshot) +{ + if (!snapshot) + return; + for (size_t i = 0; i < ARRAY_SIZE(snapshot->sources); i++) { + free(snapshot->sources[i].path); + free(snapshot->sources[i].buf); + } + free(snapshot); +} diff --git a/attr-fingerprint.h b/attr-fingerprint.h index a159aa0697468c..6d15646fcd1975 100644 --- a/attr-fingerprint.h +++ b/attr-fingerprint.h @@ -16,10 +16,28 @@ struct attr_fingerprint { unsigned int sources_present : 1; }; +enum attr_source_snapshot_kind { + ATTR_SOURCE_SNAPSHOT_SYSTEM, + ATTR_SOURCE_SNAPSHOT_GLOBAL, + ATTR_SOURCE_SNAPSHOT_INFO, + ATTR_SOURCE_SNAPSHOT_NR, +}; + +struct attr_source_snapshot; + int attr_fingerprint_sources( const struct attr_fingerprint_source *sources, size_t nr, const struct git_hash_algo *algo, struct attr_fingerprint *result); int attr_fingerprint_repository(struct repository *repo, struct attr_fingerprint *result); +int attr_source_snapshot_repository(struct repository *repo, + struct attr_source_snapshot **result); +const struct attr_fingerprint *attr_source_snapshot_fingerprint( + const struct attr_source_snapshot *snapshot); +int attr_source_snapshot_read( + const struct attr_source_snapshot *snapshot, + enum attr_source_snapshot_kind kind, + const char **path, const char **buf, size_t *len); +void attr_source_snapshot_free(struct attr_source_snapshot *snapshot); #endif /* ATTR_FINGERPRINT_H */ diff --git a/attr.c b/attr.c index 87808ba3755d04..04f28e119f2361 100644 --- a/attr.c +++ b/attr.c @@ -14,6 +14,7 @@ #include "environment.h" #include "exec-cmd.h" #include "attr.h" +#include "attr-fingerprint.h" #include "dir.h" #include "gettext.h" #include "path.h" @@ -477,6 +478,8 @@ static struct check_vector { pthread_mutex_t mutex; } check_vector; +static const struct attr_source_snapshot *source_snapshot; + static inline void vector_lock(void) { pthread_mutex_lock(&check_vector.mutex); @@ -541,6 +544,26 @@ void git_attr_invalidate_all(void) drop_all_attr_stacks(); } +void git_attr_source_snapshot_begin( + const struct attr_source_snapshot *snapshot) +{ + if (!snapshot) + BUG("cannot begin a NULL attribute source snapshot"); + if (source_snapshot) + BUG("attribute source snapshots cannot be nested"); + drop_all_attr_stacks(); + source_snapshot = snapshot; +} + +void git_attr_source_snapshot_end( + const struct attr_source_snapshot *snapshot) +{ + if (!snapshot || source_snapshot != snapshot) + BUG("ending an inactive attribute source snapshot"); + drop_all_attr_stacks(); + source_snapshot = NULL; +} + struct attr_check *attr_check_alloc(void) { struct attr_check *c = xcalloc(1, sizeof(struct attr_check)); @@ -673,6 +696,15 @@ static struct attr_stack *read_attr_from_array(const char **list) return res; } +static void handle_attr_line_buf(struct attr_stack *res, + struct strbuf *line, const char *path, + int *lineno, unsigned flags) +{ + if (!*lineno && starts_with(line->buf, utf8_bom)) + strbuf_remove(line, 0, strlen(utf8_bom)); + handle_attr_line(res, line->buf, path, ++*lineno, flags); +} + /* * Callers into the attribute system assume there is a single, system-wide * global state where attributes are read from and when the state is flipped by @@ -726,17 +758,48 @@ static struct attr_stack *read_attr_from_file(const char *path, unsigned flags) } CALLOC_ARRAY(res, 1); - while (strbuf_getline(&buf, fp) != EOF) { - if (!lineno && starts_with(buf.buf, utf8_bom)) - strbuf_remove(&buf, 0, strlen(utf8_bom)); - handle_attr_line(res, buf.buf, path, ++lineno, flags); - } + while (strbuf_getline(&buf, fp) != EOF) + handle_attr_line_buf(res, &buf, path, &lineno, flags); fclose(fp); strbuf_release(&buf); return res; } +static struct attr_stack *read_attr_from_snapshot( + enum attr_source_snapshot_kind kind, unsigned flags) +{ + struct attr_stack *res; + struct strbuf line = STRBUF_INIT; + const char *path, *buf; + size_t length; + size_t offset = 0; + int lineno = 0; + + if (!source_snapshot) + BUG("attribute source snapshot is not set"); + if (!attr_source_snapshot_read(source_snapshot, kind, + &path, &buf, &length)) + return NULL; + + CALLOC_ARRAY(res, 1); + while (offset < length) { + const char *start = buf + offset; + const char *newline = memchr(start, '\n', length - offset); + size_t len = newline ? (size_t)(newline - start) : + length - offset; + + if (newline && len && start[len - 1] == '\r') + len--; + strbuf_reset(&line); + strbuf_add(&line, start, len); + handle_attr_line_buf(res, &line, path, &lineno, flags); + offset = newline ? (size_t)(newline - buf) + 1 : length; + } + strbuf_release(&line); + return res; +} + static struct attr_stack *read_attr_from_buf(char *buf, size_t length, const char *path, unsigned flags) { @@ -927,13 +990,21 @@ static void bootstrap_attr_stack(struct index_state *istate, push_stack(stack, e, NULL, 0); /* system-wide frame */ - if (git_attr_system_is_enabled()) { + if (source_snapshot) { + e = read_attr_from_snapshot( + ATTR_SOURCE_SNAPSHOT_SYSTEM, flags); + push_stack(stack, e, NULL, 0); + } else if (git_attr_system_is_enabled()) { e = read_attr_from_file(git_attr_system_file(), flags); push_stack(stack, e, NULL, 0); } /* home directory */ - if (git_attr_global_file()) { + if (source_snapshot) { + e = read_attr_from_snapshot( + ATTR_SOURCE_SNAPSHOT_GLOBAL, flags); + push_stack(stack, e, NULL, 0); + } else if (git_attr_global_file()) { e = read_attr_from_file(git_attr_global_file(), flags); push_stack(stack, e, NULL, 0); } @@ -943,7 +1014,9 @@ static void bootstrap_attr_stack(struct index_state *istate, push_stack(stack, e, xstrdup(""), 0); /* info frame */ - if (startup_info->have_repository) + if (source_snapshot) + e = read_attr_from_snapshot(ATTR_SOURCE_SNAPSHOT_INFO, flags); + else if (startup_info->have_repository) e = read_attr_from_file(git_path_info_attributes(), flags); else e = NULL; diff --git a/attr.h b/attr.h index cca94379362f10..af2ae096d6642a 100644 --- a/attr.h +++ b/attr.h @@ -129,6 +129,7 @@ struct index_state; * `git_attr_name()`. */ struct git_attr; +struct attr_source_snapshot; /* opaque structures used internally for attribute collection */ struct all_attrs_item; @@ -230,6 +231,17 @@ void git_attr_set_direction(enum git_attr_direction new_direction); /* Discard cached attributes after a provider-wide invalidation. */ void git_attr_invalidate_all(void); +/* + * Read system, global, and info attributes from an immutable snapshot. + * begin() and end() must be strictly paired, cannot nest, and the caller must + * keep the snapshot alive until end(). Readers may run concurrently while a + * snapshot is active, but begin() and end() require that there are no readers. + */ +void git_attr_source_snapshot_begin( + const struct attr_source_snapshot *snapshot); +void git_attr_source_snapshot_end( + const struct attr_source_snapshot *snapshot); + void attr_start(void); /* Return the system gitattributes file. */ diff --git a/clean-status.c b/clean-status.c index 98ae9a1490b2a3..f4d123c0dab190 100644 --- a/clean-status.c +++ b/clean-status.c @@ -4,6 +4,7 @@ #include "clean-status-internal.h" #include "read-cache-ll.h" #include "repository.h" +#include "trace2.h" static struct repository *configured_repo; static unsigned char configured_hash[GIT_MAX_RAWSZ]; @@ -86,6 +87,67 @@ void clean_status_invalidate_current_proof(struct index_state *istate) istate->clean_status->initial_coherent = 0; } +int clean_status_capture_attr_snapshot( + struct index_state *istate, + struct attr_source_snapshot **snapshot) +{ + struct clean_status_state *state = istate->clean_status; + struct attr_source_snapshot *captured = NULL; + const struct attr_fingerprint *attrs; + int valid, changed = 0; + + if (!snapshot) + BUG("clean_status_capture_attr_snapshot requires an output"); + *snapshot = NULL; + if (!fstat_is_reliable() || !state || !state->current_config_valid || + !state->config_enforced) + return 0; + valid = !attr_source_snapshot_repository(istate->repo, &captured); + attrs = attr_source_snapshot_fingerprint(captured); + if (!valid || !state->current_attr_valid) { + changed = CLEAN_STATUS_ATTR_CONTENT_CHANGED | + CLEAN_STATUS_ATTR_NAMESPACE_CHANGED; + } else { + if (memcmp(attrs->content_hash, state->current_attr_hash, + istate->repo->hash_algo->rawsz)) + changed |= CLEAN_STATUS_ATTR_CONTENT_CHANGED; + if (memcmp(attrs->namespace_hash, + state->current_attr_namespace_hash, + istate->repo->hash_algo->rawsz)) + changed |= CLEAN_STATUS_ATTR_NAMESPACE_CHANGED; + } + if (valid) { + memcpy(state->current_attr_hash, attrs->content_hash, + istate->repo->hash_algo->rawsz); + memcpy(state->current_attr_namespace_hash, attrs->namespace_hash, + istate->repo->hash_algo->rawsz); + state->current_attr_valid = 1; + state->current_attr_sources_present = attrs->sources_present; + } else { + state->current_attr_valid = 0; + } + if (changed || state->unsafe_filter) { + state->config_mismatch = 1; + state->strong_mismatch = 1; + state->config_revalidated = 0; + } + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/rechecked", 1); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/mismatch", state->strong_mismatch); + if (!valid) + return -1; + *snapshot = captured; + return changed; +} + +int clean_status_fsmonitor_strong_mismatch(const struct index_state *istate) +{ + return istate->clean_status && + istate->clean_status->current_config_valid && + istate->clean_status->strong_mismatch; +} + void clean_status_release(struct index_state *istate) { if (!istate->clean_status) diff --git a/clean-status.h b/clean-status.h index 926c5d037997ce..b7ba330760024e 100644 --- a/clean-status.h +++ b/clean-status.h @@ -4,14 +4,24 @@ #include "clean-status-config.h" struct index_state; +struct attr_source_snapshot; struct repository; struct stat; struct strbuf; +enum clean_status_attr_change { + CLEAN_STATUS_ATTR_CONTENT_CHANGED = 1 << 0, + CLEAN_STATUS_ATTR_NAMESPACE_CHANGED = 1 << 1, +}; + void clean_status_set_config_digest( struct repository *repo, const struct clean_status_config_digest *digest); void clean_status_attach_config(struct index_state *istate); +int clean_status_capture_attr_snapshot( + struct index_state *istate, + struct attr_source_snapshot **snapshot); +int clean_status_fsmonitor_strong_mismatch(const struct index_state *istate); void clean_status_record_source_identity(struct index_state *istate, const struct stat *st); diff --git a/fsmonitor.c b/fsmonitor.c index 9e90d158402fdd..02408ba801ad2f 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -905,6 +905,15 @@ static void invalidate_all_fsmonitor_strong(struct index_state *istate) fsmonitor_invalidate_cache_entry(istate->cache[i]); } +void fsmonitor_invalidate_semantics(struct index_state *istate) +{ + git_attr_invalidate_all(); + invalidate_all_fsmonitor_strong(istate); + istate->cache_changed |= FSMONITOR_CHANGED; + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/strong-invalidation", 1); +} + void refresh_fsmonitor(struct index_state *istate) { static int warn_once = 0; diff --git a/fsmonitor.h b/fsmonitor.h index e20d280e06a220..e6c617bec77f04 100644 --- a/fsmonitor.h +++ b/fsmonitor.h @@ -51,6 +51,8 @@ static inline int fsmonitor_stat_can_be_valid(const struct stat *st) return !S_ISREG(st->st_mode) || st->st_nlink <= 1; } +void fsmonitor_invalidate_semantics(struct index_state *istate); + /* * Check if refresh_fsmonitor has been called at least once. * refresh_fsmonitor is idempotent. Returns true if fsmonitor is diff --git a/t/t7531-semantic-verify.sh b/t/t7531-semantic-verify.sh index 15d427fdc5eddf..8c8293d8d689da 100755 --- a/t/t7531-semantic-verify.sh +++ b/t/t7531-semantic-verify.sh @@ -140,4 +140,114 @@ test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ test_grep "^tracked raw-clean persist=1" actual ' +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'immutable attribute sources preserve file parsing' ' + attrs="$TRASH_DIRECTORY/attribute-parser-file" && + printf "\357\273\277*.dat text\nignored\0junk\n*.txt text\r\n*.bin text\r" \ + >"$attrs" && + test_create_repo attribute-parser && + git -C attribute-parser config core.attributesFile "$attrs" && + git -C attribute-parser config core.fsmonitor false && + git -C attribute-parser config core.untrackedCache false && + for extension in dat txt bin + do + printf "alpha\r\n" \ + >"attribute-parser/tracked.$extension" || return 1 + done && + git -C attribute-parser add . && + git -C attribute-parser commit -m base && + + GIT_OPTIONAL_LOCKS=0 GIT_TEST_COLD_BULK_STATUS=0 \ + git -C attribute-parser status --porcelain=v2 >actual && + test_must_be_empty actual +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'missing attribute history invalidates cached stats' ' + attrs="$TRASH_DIRECTORY/external-attributes-file" && + printf "*.txt text eol=crlf\n" >"$attrs" && + test_create_repo external-attributes && + git -C external-attributes config core.attributesFile "$attrs" && + git -C external-attributes config core.fsmonitor false && + git -C external-attributes config core.untrackedCache false && + printf "alpha\r\n" >external-attributes/tracked.txt && + git -C external-attributes add tracked.txt && + git -C external-attributes commit -m base && + + printf "*.txt -text\n" >"$attrs" && + GIT_OPTIONAL_LOCKS=0 GIT_TEST_COLD_BULK_STATUS=0 \ + git -C external-attributes status --porcelain=v2 >actual && + test_grep "^1 \.M .* tracked.txt$" actual +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'hook cannot hide an observed external attribute change' ' + attrs="$TRASH_DIRECTORY/hook-attribute-change.rules" && + marker="$TRASH_DIRECTORY/hook-attribute-change.marker" && + printf "*.txt text eol=crlf\n" >"$attrs" && + test_create_repo hook-attribute-change && + ( + cd hook-attribute-change && + git config core.attributesFile "$attrs" && + git config core.untrackedCache false && + printf "alpha\r\n" >tracked.txt && + git add tracked.txt && + git commit -m base && + test_hook --setup fsmonitor-test <<-\EOF && + if test -n "$GIT_TEST_ATTR_FILE" + then + printf "*.txt -text\n" >"$GIT_TEST_ATTR_FILE" + : >"$GIT_TEST_ATTR_MARKER" + fi + printf "token\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git status --porcelain=v2 >/dev/null && + git status --porcelain=v2 >/dev/null && + + GIT_TEST_ATTR_FILE="$attrs" \ + GIT_TEST_ATTR_MARKER="$marker" \ + git status --porcelain=v2 >actual && + test_path_is_file "$marker" && + test_grep "^1 \.M .* tracked.txt$" actual + ) +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'hook missing-history exception preserves reported paths' ' + attrs="$TRASH_DIRECTORY/hook-missing-history.rules" && + printf "*.txt -text\n" >"$attrs" && + test_create_repo hook-missing-history && + ( + cd hook-missing-history && + git config core.attributesFile "$attrs" && + git config core.untrackedCache false && + git config core.trustctime false && + git config core.checkStat minimal && + printf "aaaa\n" >tracked.txt && + git add tracked.txt && + git commit -m base && + test-tool chmtime =-60 tracked.txt && + git update-index --refresh && + mtime=$(test-tool chmtime --get tracked.txt) && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git status --porcelain=v2 >/dev/null && + git status --porcelain=v2 >/dev/null && + test_grep ! FSCF .git/index && + + printf "bbbb\n" >tracked.txt && + test-tool chmtime =$mtime tracked.txt && + GIT_OPTIONAL_LOCKS=0 \ + git status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual + ) +' + test_done diff --git a/t/unit-tests/u-clean-status-history.c b/t/unit-tests/u-clean-status-history.c index 4a39bfa8568811..899e6838d8f02a 100644 --- a/t/unit-tests/u-clean-status-history.c +++ b/t/unit-tests/u-clean-status-history.c @@ -114,7 +114,7 @@ void test_clean_status_history__adopts_only_coherent_proofs(void) state->current_semantic_hash[0] ^= 1; clean_status_prepare_fsmonitor_config(&fixture.istate); - cl_assert(state->strong_mismatch); + cl_assert(clean_status_fsmonitor_strong_mismatch(&fixture.istate)); cl_assert(!state->initial_coherent); fixture_release(&fixture); } diff --git a/wt-status.c b/wt-status.c index 57e2321275dc07..7146f3e42f1760 100644 --- a/wt-status.c +++ b/wt-status.c @@ -3,10 +3,13 @@ #include "git-compat-util.h" #include "advice.h" +#include "attr.h" +#include "attr-fingerprint.h" #include "wt-status.h" #include "object.h" #include "dir.h" #include "commit.h" +#include "clean-status.h" #include "diff.h" #include "environment.h" #include "gettext.h" @@ -811,6 +814,46 @@ static unsigned int wt_status_untracked_dir_flags(const struct wt_status *s) return DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES; } +static int wt_status_begin_attr_snapshot(struct wt_status *s) +{ + int ret; + int hook_provider = + fsm_settings__get_mode(s->repo) == FSMONITOR_MODE_HOOK; + + if (s->attr_source_snapshot) + return 0; + if (s->attr_snapshot_failed) + return -1; + ret = clean_status_capture_attr_snapshot( + s->repo->index, &s->attr_source_snapshot); + if (ret < 0) { + s->attr_snapshot_failed = 1; + untracked_cache_invalidate_all(s->repo->index); + fsmonitor_invalidate_semantics(s->repo->index); + return -1; + } + if (s->attr_source_snapshot) + git_attr_source_snapshot_begin(s->attr_source_snapshot); + if ((ret > 0 && + (!hook_provider || + (ret & CLEAN_STATUS_ATTR_CONTENT_CHANGED))) || + (clean_status_fsmonitor_strong_mismatch(s->repo->index) && + !hook_provider)) { + /* + * Hook providers have no closing query with which to adopt + * missing semantic history, so absence alone must preserve + * their established path-reporting contract. Namespace-only + * churn can arise from an index rewrite and is likewise not + * evidence that the hook missed a semantic change. Changed + * attribute contents are current evidence and invalidate + * cached semantics regardless of provider. + */ + untracked_cache_invalidate_all(s->repo->index); + fsmonitor_invalidate_semantics(s->repo->index); + } + return ret; +} + void wt_status_start_untracked_cache_preload(struct wt_status *s) { struct index_state *istate = s->repo->index; @@ -820,6 +863,7 @@ void wt_status_start_untracked_cache_preload(struct wt_status *s) if (s->untracked_cache_preload) BUG("untracked-cache preload already started"); + wt_status_begin_attr_snapshot(s); if (s->pathspec.nr || s->show_untracked_files == SHOW_NO_UNTRACKED_FILES || s->show_ignored_mode) @@ -1087,6 +1131,7 @@ void wt_status_collect(struct wt_status *s) if (fsm_settings__get_mode(s->repo) > FSMONITOR_MODE_DISABLED) wt_status_finish_untracked_cache_preload(s); + wt_status_begin_attr_snapshot(s); wt_status_close_fsmonitor_token( s, REFRESH_QUIET | REFRESH_UNMERGED, s->show_untracked_files != SHOW_NO_UNTRACKED_FILES && @@ -1130,6 +1175,11 @@ void wt_status_collect_free_buffers(struct wt_status *s) { untracked_cache_preload_release(s->untracked_cache_preload); s->untracked_cache_preload = NULL; + if (s->attr_source_snapshot) + git_attr_source_snapshot_end(s->attr_source_snapshot); + attr_source_snapshot_free(s->attr_source_snapshot); + s->attr_source_snapshot = NULL; + s->attr_snapshot_failed = 0; wt_status_state_free_buffers(&s->state); } diff --git a/wt-status.h b/wt-status.h index 34beac22576fc9..74798dd593aacb 100644 --- a/wt-status.h +++ b/wt-status.h @@ -7,6 +7,7 @@ #include "remote.h" struct repository; +struct attr_source_snapshot; struct worktree; struct untracked_cache_preload; @@ -148,7 +149,9 @@ struct wt_status { struct string_list ignored; uint32_t untracked_in_ms; struct untracked_cache_preload *untracked_cache_preload; + struct attr_source_snapshot *attr_source_snapshot; unsigned untracked_cache_preloaded : 1; + unsigned attr_snapshot_failed : 1; }; size_t wt_status_locate_end(const char *s, size_t len); From ea91fb9491f176d264da33a807f78f27a6ed8b3a Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 01:06:21 -0500 Subject: [PATCH 077/105] fsmonitor: seed missing-history baselines from legacy tokens An fsmonitor token can mark an entry valid even when the index has no coherent history for the configuration and attributes that determine its content. With minimal stat checks, a same-size rewrite can then be reported as clean. Rebuild the attribute manifest for expanded indexes during IPC bootstrap. Compare it with the current in-process or retained on-disk manifest, invalidate only the tracked and untracked scopes whose attribute sources changed, and preserve the last complete manifest when a rebuild fails. A legacy index with no FSCF extension is different from a mismatched proof: it contains no claim about semantic history to disprove. When it also has a valid nontrivial FSMN token with core.trustctime enabled and full core.checkStat, clear FSMN validity and seed a forward baseline through ordinary configured stat checks. This avoids hashing every tracked file solely because the index predates FSCF. The baseline still needs to finish in the bootstrap command. Preserve the freshly-proven FSMN-valid bit on entries replaced by that refresh, so that the accepted token does not defer the same migration work into the next status. Keep strong global invalidation for semantic or attribute mismatches, weak stat settings, a present FSCF without complete manifest history, provider reset or failure, manifest rebuild failure, and fresh indexes without a prior nontrivial FSMN token. Retain ordinary provider handling when reliable file identity is unavailable. The migration exception has ordinary Git stat semantics rather than a content-proof guarantee; same-size changes hidden by the platform's configured stat identity can remain hidden at that boundary. Add coverage for the forward-baseline lane, the weak-stat same-size rewrite, and the refreshed baseline FSMN bits, along with unit coverage for coherent, manifest-only, missing, and present-without-manifest history. Signed-off-by: Taylor Blau --- clean-status-history.c | 85 +++++++++++++++++++ clean-status-internal.h | 1 + clean-status-manifest.c | 104 ++++++++++++++++++++++++ clean-status-manifest.h | 7 ++ clean-status.c | 21 +++++ clean-status.h | 18 +++++ fsmonitor.c | 79 +++++++++++++++++- read-cache.c | 12 ++- t/t7527-builtin-fsmonitor.sh | 63 +++++++++++++++ t/unit-tests/u-clean-status-history.c | 78 ++++++++++++++++++ t/unit-tests/u-clean-status-manifest.c | 108 +++++++++++++++++++++++++ 11 files changed, 572 insertions(+), 4 deletions(-) diff --git a/clean-status-history.c b/clean-status-history.c index 61fc2f02b550c1..56e1ac8079e131 100644 --- a/clean-status-history.c +++ b/clean-status-history.c @@ -111,6 +111,91 @@ void clean_status_prepare_fsmonitor_config(struct index_state *istate) "semantic/initial-mismatch", state->strong_mismatch); } +int clean_status_has_persistent_fsmonitor_semantic_history( + const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + + return state && state->disk_config_valid && + !state->disk_config_invalid && state->disk_semantic_valid && + state->disk_attr_valid && state->manifest.disk_valid && + (state->manifest.disk_flags & FSMONITOR_CLEAN_PROOF_ALL) == + FSMONITOR_CLEAN_PROOF_ALL && + state->disk_config_raw.len; +} + +int clean_status_has_worktree_manifest_history( + const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + uint32_t required = FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | + FSMONITOR_CLEAN_PROOF_FULL_INDEX; + + return state && state->disk_config_valid && + !state->disk_config_invalid && state->manifest.disk_valid && + (state->manifest.disk_flags & required) == required; +} + +int clean_status_fsmonitor_semantic_adoption_needed( + const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + int missing_history; + + if (!state || !state->current_config_valid || !state->config_enforced) + return 0; + if (state->semantic_baseline_pending) + return 0; + missing_history = + !clean_status_has_persistent_fsmonitor_semantic_history(istate) && + !clean_status_has_worktree_manifest_history(istate); + /* + * Keep missing history on the proof path until fsmonitor explicitly + * chooses the narrow forward-baseline lane for a valid legacy token. + */ + return state->strong_mismatch || missing_history; +} + +int clean_status_fsmonitor_semantic_baseline_needed( + const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + const struct repo_config_values *cfg; + + if (!state || !state->current_config_valid || !state->config_enforced || + state->strong_mismatch || state->disk_config_seen || + !istate->fsmonitor_token_valid || + !istate->fsmonitor_last_update || + !*istate->fsmonitor_last_update) + return 0; + /* + * This helper is exercised by isolated index-state unit fixtures, + * which are not the_repository. The config values are already + * initialized with the repository and need no lazy parsing here. + */ + cfg = &istate->repo->config_values_private_; + if (!cfg->trust_ctime || !cfg->check_stat) + return 0; + return !clean_status_has_persistent_fsmonitor_semantic_history(istate) && + !clean_status_has_worktree_manifest_history(istate); +} + +int clean_status_fsmonitor_semantic_baseline_pending( + const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + + return state && state->semantic_baseline_pending; +} + +void clean_status_begin_fsmonitor_semantic_baseline( + struct index_state *istate) +{ + struct clean_status_state *state = clean_status_get_state(istate); + + state->semantic_baseline_pending = 1; +} + static int current_proof_is_writable(const struct index_state *istate) { const struct clean_status_state *state = istate->clean_status; diff --git a/clean-status-internal.h b/clean-status-internal.h index 755d9ea059daec..d4dc54e6616f61 100644 --- a/clean-status-internal.h +++ b/clean-status-internal.h @@ -36,6 +36,7 @@ struct clean_status_state { unsigned disk_attr_valid : 1; unsigned disk_config_seen : 1; unsigned disk_config_invalid : 1; + unsigned semantic_baseline_pending : 1; }; struct clean_status_state *clean_status_get_state(struct index_state *istate); diff --git a/clean-status-manifest.c b/clean-status-manifest.c index 713d8bd4d5e104..b13066a23c2c47 100644 --- a/clean-status-manifest.c +++ b/clean-status-manifest.c @@ -1,8 +1,30 @@ #include "git-compat-util.h" #include "attr-manifest.h" #include "clean-status-manifest.h" +#include "dir.h" #include "fsmonitor-clean-proof.h" +#include "fsmonitor-ll.h" #include "hash-framing.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "trace2.h" +#include "worktree-attr-manifest.h" + +struct invalidate_manifest_data { + struct index_state *istate; + int invalidated; +}; + +static int build_manifest(struct index_state *istate, + struct strbuf *manifest, + unsigned char *manifest_hash, + struct worktree_attr_manifest_stats *stats) +{ + if (istate->sparse_index != INDEX_EXPANDED) + return -1; + return worktree_attr_manifest_build( + istate, manifest, manifest_hash, stats); +} void clean_status_manifest_init(struct clean_status_manifest_state *state) { @@ -48,6 +70,88 @@ void clean_status_manifest_adopt_disk( state->checked = 1; } +static int invalidate_manifest_path(const struct attr_manifest_entry *entry, + void *cb_data) +{ + struct invalidate_manifest_data *data = cb_data; + char *path = xmemdupz(entry->path, entry->path_len); + + untracked_cache_invalidate_trimmed_path(data->istate, path, 0); + data->invalidated += + fsmonitor_invalidate_attributes_path(data->istate, path); + free(path); + return 0; +} + +int clean_status_manifest_refresh(struct index_state *istate, + struct clean_status_manifest_state *state) +{ + struct worktree_attr_manifest_stats stats; + struct invalidate_manifest_data invalidation = { .istate = istate }; + const struct git_hash_algo *algo = istate->repo->hash_algo; + const struct strbuf *baseline = NULL; + struct strbuf next = STRBUF_INIT; + unsigned char next_hash[GIT_MAX_RAWSZ]; + + state->scan_count++; + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-scan-count", state->scan_count); + if (attr_manifest_valid(state->current.buf, state->current.len, algo)) + baseline = &state->current; + else if (state->disk_valid) + baseline = &state->disk; + state->checked = 1; + state->changed = 0; + state->global_fallback = 0; + state->current_valid = 0; + state->current_flags = 0; + if (build_manifest(istate, &next, next_hash, &stats)) { + state->global_fallback = !!baseline; + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-scan-failed", 1); + strbuf_release(&next); + return -1; + } + if (baseline) { + if (attr_manifest_for_each_changed( + baseline->buf, baseline->len, + next.buf, next.len, algo, + invalidate_manifest_path, &invalidation)) { + state->global_fallback = 1; + strbuf_release(&next); + return -1; + } + state->changed = baseline->len != next.len || + memcmp(baseline->buf, next.buf, next.len); + } + strbuf_swap(&state->current, &next); + strbuf_release(&next); + memcpy(state->current_hash, next_hash, algo->rawsz); + state->current_valid = 1; + state->current_flags = FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | + FSMONITOR_CLEAN_PROOF_FULL_INDEX; + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-candidates", stats.candidates); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-threads", stats.threads); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-thread-failures", + stats.thread_failures); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-worktree-sources", + stats.worktree_sources); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-index-sources", stats.index_sources); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-bytes", state->current.len); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-changed", state->changed); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-invalidated", + invalidation.invalidated); + return invalidation.invalidated; +} + void clean_status_manifest_invalidate( struct clean_status_manifest_state *state) { diff --git a/clean-status-manifest.h b/clean-status-manifest.h index e924ba9fade2fe..04a56d56abe65c 100644 --- a/clean-status-manifest.h +++ b/clean-status-manifest.h @@ -4,6 +4,8 @@ #include "hash.h" #include "strbuf.h" +struct index_state; + struct clean_status_manifest_state { struct strbuf disk; struct strbuf current; @@ -11,9 +13,12 @@ struct clean_status_manifest_state { unsigned char current_hash[GIT_MAX_RAWSZ]; uint32_t disk_flags; uint32_t current_flags; + uint32_t scan_count; unsigned disk_valid : 1; unsigned current_valid : 1; unsigned checked : 1; + unsigned changed : 1; + unsigned global_fallback : 1; }; void clean_status_manifest_init(struct clean_status_manifest_state *state); @@ -23,6 +28,8 @@ int clean_status_manifest_load(struct clean_status_manifest_state *state, const struct git_hash_algo *algo); void clean_status_manifest_adopt_disk( struct clean_status_manifest_state *state); +int clean_status_manifest_refresh(struct index_state *istate, + struct clean_status_manifest_state *state); void clean_status_manifest_invalidate( struct clean_status_manifest_state *state); diff --git a/clean-status.c b/clean-status.c index f4d123c0dab190..7a0db98473ef76 100644 --- a/clean-status.c +++ b/clean-status.c @@ -85,6 +85,7 @@ void clean_status_invalidate_current_proof(struct index_state *istate) return; istate->clean_status->config_revalidated = 0; istate->clean_status->initial_coherent = 0; + istate->clean_status->semantic_baseline_pending = 0; } int clean_status_capture_attr_snapshot( @@ -141,6 +142,13 @@ int clean_status_capture_attr_snapshot( return changed; } +int clean_status_fsmonitor_config_mismatch(const struct index_state *istate) +{ + return istate->clean_status && + istate->clean_status->current_config_valid && + istate->clean_status->config_mismatch; +} + int clean_status_fsmonitor_strong_mismatch(const struct index_state *istate) { return istate->clean_status && @@ -148,6 +156,19 @@ int clean_status_fsmonitor_strong_mismatch(const struct index_state *istate) istate->clean_status->strong_mismatch; } +int clean_status_refresh_worktree_manifest(struct index_state *istate) +{ + struct clean_status_state *state = clean_status_get_state(istate); + + return clean_status_manifest_refresh(istate, &state->manifest); +} + +int clean_status_manifest_global_fallback(const struct index_state *istate) +{ + return istate->clean_status && + istate->clean_status->manifest.global_fallback; +} + void clean_status_release(struct index_state *istate) { if (!istate->clean_status) diff --git a/clean-status.h b/clean-status.h index b7ba330760024e..a6aea7b136cd4d 100644 --- a/clean-status.h +++ b/clean-status.h @@ -21,8 +21,26 @@ void clean_status_attach_config(struct index_state *istate); int clean_status_capture_attr_snapshot( struct index_state *istate, struct attr_source_snapshot **snapshot); + +int clean_status_fsmonitor_config_mismatch(const struct index_state *istate); int clean_status_fsmonitor_strong_mismatch(const struct index_state *istate); +int clean_status_has_persistent_fsmonitor_semantic_history( + const struct index_state *istate); +int clean_status_has_worktree_manifest_history( + const struct index_state *istate); +int clean_status_fsmonitor_semantic_adoption_needed( + const struct index_state *istate); +int clean_status_fsmonitor_semantic_baseline_needed( + const struct index_state *istate); +int clean_status_fsmonitor_semantic_baseline_pending( + const struct index_state *istate); +void clean_status_begin_fsmonitor_semantic_baseline( + struct index_state *istate); + +int clean_status_refresh_worktree_manifest(struct index_state *istate); +int clean_status_manifest_global_fallback(const struct index_state *istate); + void clean_status_record_source_identity(struct index_state *istate, const struct stat *st); int clean_status_verify_null_index(const struct index_state *istate, diff --git a/fsmonitor.c b/fsmonitor.c index 02408ba801ad2f..c90bfebc2e4712 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -896,6 +896,22 @@ static void invalidate_all_fsmonitor(struct index_state *istate) istate->cache_changed |= FSMONITOR_CHANGED; } +/* + * A forward baseline still needs one ordinary stat refresh before its + * provider token can certify the index. Clear only process-local + * uptodate state so that refresh_index() performs those stats without + * escalating to content checks. + */ +static void invalidate_all_fsmonitor_for_baseline( + struct index_state *istate) +{ + unsigned int i; + + invalidate_all_fsmonitor(istate); + for (i = 0; i < istate->cache_nr; i++) + istate->cache[i]->ce_flags &= ~CE_UPTODATE; +} + static void invalidate_all_fsmonitor_strong(struct index_state *istate) { unsigned int i; @@ -914,6 +930,45 @@ void fsmonitor_invalidate_semantics(struct index_state *istate) "semantic/strong-invalidation", 1); } +static void invalidate_fsmonitor_for_bootstrap( + struct index_state *istate, enum fsmonitor_mode mode, + int semantic_adoption_needed, int semantic_baseline_needed, + int physical_history_unavailable) +{ + int manifest_refresh_failed; + + if (!fstat_is_reliable() || mode != FSMONITOR_MODE_IPC || + istate->split_index) { + invalidate_all_fsmonitor(istate); + return; + } + + if (physical_history_unavailable) { + if (semantic_adoption_needed) + clean_status_refresh_worktree_manifest(istate); + fsmonitor_invalidate_semantics(istate); + untracked_cache_invalidate_all(istate); + return; + } + + manifest_refresh_failed = + clean_status_refresh_worktree_manifest(istate) < 0; + if (manifest_refresh_failed || + clean_status_manifest_global_fallback(istate) || + (semantic_adoption_needed && !semantic_baseline_needed)) { + fsmonitor_invalidate_semantics(istate); + } else { + if (semantic_baseline_needed) { + clean_status_begin_fsmonitor_semantic_baseline(istate); + invalidate_all_fsmonitor_for_baseline(istate); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/adoption-baseline", 1); + } else { + invalidate_all_fsmonitor(istate); + } + } +} + void refresh_fsmonitor(struct index_state *istate) { static int warn_once = 0; @@ -927,6 +982,8 @@ void refresh_fsmonitor(struct index_state *istate) int is_trivial = 0; int tracked_requires_bootstrap; int untracked_requires_bootstrap; + int semantic_adoption_needed; + int semantic_baseline_needed; struct repository *r = istate->repo; enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); enum fsmonitor_reason reason = fsm_settings__get_reason(r); @@ -943,6 +1000,14 @@ void refresh_fsmonitor(struct index_state *istate) return; istate->fsmonitor_has_run_once = 1; + semantic_adoption_needed = fstat_is_reliable() && + !istate->split_index && + fsm_mode == FSMONITOR_MODE_IPC && + clean_status_fsmonitor_semantic_adoption_needed(istate); + semantic_baseline_needed = fstat_is_reliable() && + !istate->split_index && + fsm_mode == FSMONITOR_MODE_IPC && + clean_status_fsmonitor_semantic_baseline_needed(istate); trace_printf_key(&trace_fsmonitor, "refresh fsmonitor"); @@ -1068,7 +1133,10 @@ void refresh_fsmonitor(struct index_state *istate) trace2_region_enter("fsmonitor", "apply_results", istate->repo); tracked_requires_bootstrap = !query_success || is_trivial || - !istate->fsmonitor_token_valid; + !istate->fsmonitor_token_valid || + (fstat_is_reliable() && !istate->split_index && + fsm_mode == FSMONITOR_MODE_IPC && + clean_status_fsmonitor_config_mismatch(istate)); untracked_requires_bootstrap = !istate->fsmonitor_untracked_valid; if (query_success && !is_trivial) { @@ -1099,7 +1167,10 @@ void refresh_fsmonitor(struct index_state *istate) } if (tracked_requires_bootstrap) - invalidate_all_fsmonitor(istate); + invalidate_fsmonitor_for_bootstrap( + istate, fsm_mode, semantic_adoption_needed, + semantic_baseline_needed, + !istate->fsmonitor_token_valid); /* Now mark the untracked cache for fsmonitor usage */ if (istate->untracked) @@ -1122,7 +1193,9 @@ void refresh_fsmonitor(struct index_state *istate) * we've actually changed entries, so keep track if we * actually changed entries or not. */ - invalidate_all_fsmonitor(istate); + invalidate_fsmonitor_for_bootstrap( + istate, fsm_mode, semantic_adoption_needed, + semantic_baseline_needed, 1); } trace2_region_leave("fsmonitor", "apply_results", istate->repo); diff --git a/read-cache.c b/read-cache.c index 43cbfc19240862..1ca79826e2b513 100644 --- a/read-cache.c +++ b/read-cache.c @@ -1633,7 +1633,17 @@ int refresh_index(struct index_state *istate, unsigned int flags, continue; } - replace_index_entry(istate, i, new_entry); + { + int baseline_valid = + clean_status_fsmonitor_semantic_baseline_pending( + istate) && + (new_entry->ce_flags & CE_FSMONITOR_VALID); + + replace_index_entry(istate, i, new_entry); + if (baseline_valid) + mark_fsmonitor_valid(istate, + istate->cache[i]); + } } trace2_data_intmax("index", NULL, "refresh/sum_lstat", t2_sum_lstat); trace2_data_intmax("index", NULL, "refresh/sum_scan", t2_sum_scan); diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index dec4a493d4eb8a..83d7c436e5ca1f 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -1684,4 +1684,67 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ ) ' +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'missing semantic history seeds a forward baseline' ' + test_when_finished \ + "stop_daemon_delete_repo missing-semantic-baseline" && + test_create_repo missing-semantic-baseline && + ( + cd missing-semantic-baseline && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid tracked && + test_grep ! FSCF .git/index && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual && + test_grep FSCF .git/index && + test_trace2_data fsmonitor semantic/adoption-baseline 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor semantic/strong-invalidation 1 \ + <.git/status.trace && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace + ) +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'missing semantic history with weak stat identity forces content verification' ' + test_when_finished \ + "stop_daemon_delete_repo missing-semantic-history" && + test_create_repo missing-semantic-history && + ( + cd missing-semantic-history && + printf "aaaa\n" >tracked && + git add tracked && + git commit -m base && + git config core.trustctime false && + git config core.checkStat minimal && + test-tool chmtime =-60 tracked && + git update-index --refresh && + mtime=$(test-tool chmtime --get tracked) && + printf "bbbb\n" >tracked && + test-tool chmtime =$mtime tracked && + git config core.fsmonitor true && + git update-index --fsmonitor && + git update-index --fsmonitor-valid tracked && + test_grep FSMN .git/index && + test_grep ! FSCF .git/index && + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_line_count = 1 .git/actual && + test_grep "^1 \.M .* tracked$" .git/actual && + test_trace2_data fsmonitor semantic/strong-invalidation 1 \ + <.git/status.trace + ) +' + test_done diff --git a/t/unit-tests/u-clean-status-history.c b/t/unit-tests/u-clean-status-history.c index 899e6838d8f02a..feb813c185c624 100644 --- a/t/unit-tests/u-clean-status-history.c +++ b/t/unit-tests/u-clean-status-history.c @@ -27,6 +27,7 @@ static void fixture_init(struct history_fixture *fixture, memset(fixture, 0, sizeof(*fixture)); fixture->repo.hash_algo = algo; + repo_config_values_init(&fixture->repo.config_values_private_); index_state_init(&fixture->istate, &fixture->repo); fixture->manifest = (struct strbuf)STRBUF_INIT; fixture->encoded = (struct strbuf)STRBUF_INIT; @@ -54,6 +55,7 @@ static void fixture_release(struct history_fixture *fixture) { clean_status_release(&fixture->istate); free(fixture->istate.fsmonitor_last_update); + repo_config_values_clear(&fixture->repo.config_values_private_); strbuf_release(&fixture->encoded); strbuf_release(&fixture->manifest); } @@ -72,6 +74,7 @@ static struct clean_status_state *install_current( state->current_semantic_valid = 1; state->current_attr_valid = 1; state->config_enforced = 1; + FREE_AND_NULL(fixture->istate.fsmonitor_last_update); fixture->istate.fsmonitor_last_update = xstrdup("builtin:1:2"); fixture->istate.fsmonitor_token_valid = 1; return state; @@ -119,6 +122,81 @@ void test_clean_status_history__adopts_only_coherent_proofs(void) fixture_release(&fixture); } +void test_clean_status_history__distinguishes_available_history(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct history_fixture fixture; + struct clean_status_state *state; + struct strbuf manifest_only = STRBUF_INIT; + + fixture_init(&fixture, algo); + clean_status_read_fsmonitor_config( + &fixture.istate, fixture.encoded.buf, fixture.encoded.len); + cl_assert(clean_status_has_persistent_fsmonitor_semantic_history( + &fixture.istate)); + cl_assert(clean_status_has_worktree_manifest_history(&fixture.istate)); + state = install_current(&fixture); + cl_assert(!clean_status_fsmonitor_semantic_adoption_needed( + &fixture.istate)); + cl_assert(!clean_status_fsmonitor_semantic_baseline_needed( + &fixture.istate)); + state->strong_mismatch = 1; + cl_assert(clean_status_fsmonitor_semantic_adoption_needed( + &fixture.istate)); + cl_assert(!clean_status_fsmonitor_semantic_baseline_needed( + &fixture.istate)); + fixture_release(&fixture); + + fixture_init(&fixture, algo); + cl_assert_equal_i(fsmonitor_clean_proof_copy_without_bindings( + &manifest_only, fixture.encoded.buf, fixture.encoded.len, + algo), 0); + clean_status_read_fsmonitor_config( + &fixture.istate, manifest_only.buf, manifest_only.len); + cl_assert(!clean_status_has_persistent_fsmonitor_semantic_history( + &fixture.istate)); + cl_assert(clean_status_has_worktree_manifest_history(&fixture.istate)); + install_current(&fixture); + cl_assert(!clean_status_fsmonitor_semantic_adoption_needed( + &fixture.istate)); + cl_assert(!clean_status_fsmonitor_semantic_baseline_needed( + &fixture.istate)); + fixture_release(&fixture); + + fixture_init(&fixture, algo); + state = install_current(&fixture); + fixture.istate.fsmonitor_token_valid = 1; + FREE_AND_NULL(fixture.istate.fsmonitor_last_update); + fixture.istate.fsmonitor_last_update = xstrdup("builtin:test:1"); + cl_assert(clean_status_fsmonitor_semantic_adoption_needed( + &fixture.istate)); + cl_assert(clean_status_fsmonitor_semantic_baseline_needed( + &fixture.istate)); + clean_status_begin_fsmonitor_semantic_baseline(&fixture.istate); + cl_assert(!clean_status_fsmonitor_semantic_adoption_needed( + &fixture.istate)); + state->semantic_baseline_pending = 0; + state->disk_config_seen = 1; + cl_assert(clean_status_fsmonitor_semantic_adoption_needed( + &fixture.istate)); + cl_assert(!clean_status_fsmonitor_semantic_baseline_needed( + &fixture.istate)); + state->disk_config_seen = 0; + fixture.repo.config_values_private_.trust_ctime = 0; + cl_assert(clean_status_fsmonitor_semantic_adoption_needed( + &fixture.istate)); + cl_assert(!clean_status_fsmonitor_semantic_baseline_needed( + &fixture.istate)); + fixture.repo.config_values_private_.trust_ctime = 1; + state->strong_mismatch = 1; + cl_assert(clean_status_fsmonitor_semantic_adoption_needed( + &fixture.istate)); + cl_assert(!clean_status_fsmonitor_semantic_baseline_needed( + &fixture.istate)); + fixture_release(&fixture); + strbuf_release(&manifest_only); +} + void test_clean_status_history__preserves_unbound_manifests(void) { const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; diff --git a/t/unit-tests/u-clean-status-manifest.c b/t/unit-tests/u-clean-status-manifest.c index e6d83c564a8a6b..fc17fd8ec0dbb8 100644 --- a/t/unit-tests/u-clean-status-manifest.c +++ b/t/unit-tests/u-clean-status-manifest.c @@ -1,7 +1,12 @@ #include "unit-test.h" #include "attr-manifest.h" #include "clean-status-manifest.h" +#include "dir.h" #include "fsmonitor-clean-proof.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "semantic-verify-internal.h" +#include "wrapper.h" static void make_manifest(struct strbuf *manifest, const struct git_hash_algo *algo) @@ -68,3 +73,106 @@ void test_clean_status_manifest__rejects_invalid_history(void) clean_status_manifest_release(&state); strbuf_release(&manifest); } +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +static char *create_worktree(void) +{ + const char *tmp = getenv("TMPDIR"); + char *path = xstrfmt("%s/status-manifest.XXXXXX", + tmp ? tmp : "/tmp"); + + cl_assert(mkdtemp(path) != NULL); + return path; +} + +static void add_index_path(struct index_state *istate, size_t pos, + const char *path) +{ + size_t len = strlen(path); + struct cache_entry *ce = make_empty_cache_entry(istate, len); + + ce->ce_mode = S_IFREG | 0644; + ce->ce_namelen = len; + memcpy(ce->name, path, len + 1); + istate->cache[pos] = ce; +} +#endif + +void test_clean_status_manifest__invalidates_only_changed_scopes(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *worktree = create_worktree(); + struct repository repo = { .worktree = worktree, .hash_algo = algo }; + struct index_state istate = INDEX_STATE_INIT(&repo); + struct clean_status_manifest_state state; + struct strbuf path = STRBUF_INIT, old = STRBUF_INIT, cleanup = STRBUF_INIT; + + strbuf_addf(&path, "%s/a", worktree); + cl_assert_equal_i(mkdir(path.buf, 0777), 0); + strbuf_reset(&path); + strbuf_addf(&path, "%s/b", worktree); + cl_assert_equal_i(mkdir(path.buf, 0777), 0); + strbuf_reset(&path); + strbuf_addf(&path, "%s/a/.gitattributes", worktree); + write_file(path.buf, "*.txt text\n"); + + CALLOC_ARRAY(istate.cache, 2); + istate.cache_alloc = istate.cache_nr = 2; + add_index_path(&istate, 0, "a/file"); + add_index_path(&istate, 1, "b/file"); + clean_status_manifest_init(&state); + cl_assert_equal_i(clean_status_manifest_refresh(&istate, &state), 0); + strbuf_addbuf(&old, &state.current); + cl_assert_equal_i(clean_status_manifest_load( + &state, old.buf, old.len, FSMONITOR_CLEAN_PROOF_ALL, algo), 0); + + write_file(path.buf, "*.txt -text\n"); + for (size_t i = 0; i < istate.cache_nr; i++) { + istate.cache[i]->ce_flags = CE_FSMONITOR_VALID | CE_UPTODATE; + memset(&istate.cache[i]->ce_stat_data, 1, + sizeof(istate.cache[i]->ce_stat_data)); + } + cl_assert_equal_i(clean_status_manifest_refresh(&istate, &state), 1); + cl_assert(state.changed); + cl_assert(!(istate.cache[0]->ce_flags & CE_FSMONITOR_VALID)); + cl_assert(istate.cache[0]->ce_flags & CE_CONTENT_CHECK_REQUIRED); + cl_assert(istate.cache[1]->ce_flags & CE_FSMONITOR_VALID); + + /* + * Preserve the last complete in-process value when a rebuild fails, + * then return to the on-disk value. The final comparison must use + * the preserved value, not the matching on-disk history. + */ + strbuf_reset(&old); + strbuf_addbuf(&old, &state.current); + clean_status_manifest_invalidate(&state); + istate.cache[0]->ce_flags = create_ce_flags(1); + cl_assert_equal_i(clean_status_manifest_refresh(&istate, &state), -1); + cl_assert(!state.current_valid); + cl_assert(state.global_fallback); + cl_assert_equal_i(strbuf_cmp(&state.current, &old), 0); + + write_file(path.buf, "*.txt text\n"); + for (size_t i = 0; i < istate.cache_nr; i++) { + istate.cache[i]->ce_flags = CE_FSMONITOR_VALID | CE_UPTODATE; + memset(&istate.cache[i]->ce_stat_data, 1, + sizeof(istate.cache[i]->ce_stat_data)); + } + cl_assert_equal_i(clean_status_manifest_refresh(&istate, &state), 1); + cl_assert(state.changed); + cl_assert(!(istate.cache[0]->ce_flags & CE_FSMONITOR_VALID)); + cl_assert(istate.cache[0]->ce_flags & CE_CONTENT_CHECK_REQUIRED); + cl_assert(istate.cache[1]->ce_flags & CE_FSMONITOR_VALID); + + clean_status_manifest_release(&state); + strbuf_release(&old); + strbuf_release(&path); + release_index(&istate); + strbuf_addstr(&cleanup, worktree); + cl_assert_equal_i(remove_dir_recursively(&cleanup, 0), 0); + strbuf_release(&cleanup); + free(worktree); +#endif +} From 25087947d858bc85cce19e1a8eda7a11f55c35be Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Wed, 29 Jul 2026 16:51:39 -0500 Subject: [PATCH 078/105] status: pin index-state snapshots to the named index Checking the index pathname after a scan does not establish that it still names the file whose header and stored trailer agree with the in-memory index. Replacement or a final-component symlink can otherwise bind a semantic proof to the wrong file. Introduce an index-state-based snapshot primitive. Open the repository's named index without following its final symlink and retain the descriptor. Parse the DIRC header and stored trailer for the repository's object format. Require the held descriptor, current pathname, in-memory version, entry count, and stored checksum to identify the same index when pinning and rechecking the snapshot. A skipHash index deliberately stores a null checksum. Permit that value only when pinning an index state whose parsed, durable source identity matches the held descriptor. Recheck the supplied index state's source identity as well as the pathname, closing an A-to-B-to-A replacement window. Keep the path-only snapshot opener checksummed-only because it has no parsed source identity to authenticate a null trailer. Add direct unit coverage for SHA-1 and SHA-256, malformed headers, checksummed and null-checksum indexes, pathname replacement, in-memory identity drift, A-to-B-to-A replacement, and final-component symlinks where supported. The following sparse-manifest change supplies the first production consumer. Signed-off-by: Taylor Blau --- clean-status-index.c | 136 ++++++++++++++ clean-status-index.h | 26 +++ t/unit-tests/u-clean-status-index.c | 274 ++++++++++++++++++++++++++++ 3 files changed, 436 insertions(+) create mode 100644 clean-status-index.h diff --git a/clean-status-index.c b/clean-status-index.c index 4733e53e3a9fcc..ba38d4411b9bbb 100644 --- a/clean-status-index.c +++ b/clean-status-index.c @@ -1,7 +1,143 @@ #include "git-compat-util.h" #include "clean-status.h" +#include "clean-status-index.h" #include "clean-status-internal.h" #include "read-cache-ll.h" +#include "repository.h" +#include "wrapper.h" + +static int snapshot_read( + int fd, const struct stat *st, const struct git_hash_algo *algo, + uint32_t *version, uint32_t *cache_nr, struct object_id *checksum) +{ + unsigned char header[12]; + unsigned char trailer[GIT_MAX_RAWSZ]; + + if (st->st_size < 0 || + (uintmax_t)st->st_size < sizeof(header) + algo->rawsz || + (size_t)pread_in_full(fd, header, sizeof(header), 0) != + sizeof(header) || + memcmp(header, "DIRC", 4) || + (size_t)pread_in_full(fd, trailer, algo->rawsz, + st->st_size - (off_t)algo->rawsz) != + algo->rawsz) + return -1; + *version = get_be32(header + 4); + *cache_nr = get_be32(header + 8); + if (*version < 2 || *version > 4) + return -1; + oidread(checksum, trailer, algo); + return 0; +} + +static int snapshot_matches( + int fd, const struct stat *st, uint32_t expected_version, + uint32_t expected_cache_nr, const struct object_id *expected_checksum, + const struct git_hash_algo *algo) +{ + struct object_id checksum; + uint32_t version, cache_nr; + + return !snapshot_read(fd, st, algo, &version, &cache_nr, &checksum) && + version == expected_version && cache_nr == expected_cache_nr && + oideq(&checksum, expected_checksum); +} + +static int snapshot_open( + struct clean_status_index_snapshot *snapshot, const char *path, + const struct git_hash_algo *algo, int allow_null_checksum) +{ + struct clean_status_identity named; + struct stat fd_st, named_st; + int fd; + + memset(snapshot, 0, sizeof(*snapshot)); + snapshot->fd = -1; + fd = open_nofollow(path, O_RDONLY); + if (fd < 0 || + fstat(fd, &fd_st) || + lstat(path, &named_st) || + clean_status_identity_from_stat(&snapshot->identity, &fd_st) || + clean_status_identity_from_stat(&named, &named_st) || + !clean_status_identity_equal(&snapshot->identity, &named) || + snapshot_read(fd, &fd_st, algo, &snapshot->version, + &snapshot->cache_nr, &snapshot->checksum) || + (!allow_null_checksum && is_null_oid(&snapshot->checksum))) + goto fail; + snapshot->fd = fd; + return 0; + +fail: + if (fd >= 0) + close(fd); + return -1; +} + +static int clean_status_index_snapshot_still_matches_path( + const struct clean_status_index_snapshot *snapshot, const char *path, + const struct git_hash_algo *algo) +{ + struct clean_status_identity fd_identity, named_identity; + struct stat fd_st, named_st; + + return snapshot->fd >= 0 && + !fstat(snapshot->fd, &fd_st) && + !lstat(path, &named_st) && + !clean_status_identity_from_stat(&fd_identity, &fd_st) && + !clean_status_identity_from_stat(&named_identity, &named_st) && + clean_status_identity_equal(&fd_identity, &snapshot->identity) && + clean_status_identity_equal(&named_identity, + &snapshot->identity) && + snapshot_matches(snapshot->fd, &fd_st, snapshot->version, + snapshot->cache_nr, &snapshot->checksum, algo); +} + +static int snapshot_matches_index_state( + const struct clean_status_index_snapshot *snapshot, + const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + + return istate->version == snapshot->version && + istate->cache_nr == snapshot->cache_nr && + oideq(&istate->oid, &snapshot->checksum) && + (!is_null_oid(&snapshot->checksum) || + (clean_status_identity_is_durable() && state && + state->source_identity_valid && + clean_status_identity_equal(&snapshot->identity, + &state->source_identity))); +} + +int clean_status_index_snapshot_pin( + struct clean_status_index_snapshot *snapshot, + struct index_state *istate) +{ + if (snapshot_open(snapshot, istate->repo->index_file, + istate->repo->hash_algo, 1)) + return -1; + if (snapshot_matches_index_state(snapshot, istate)) + return 0; + clean_status_index_snapshot_release(snapshot); + return -1; +} + +int clean_status_index_snapshot_still_matches( + const struct clean_status_index_snapshot *snapshot, + const struct index_state *istate) +{ + return snapshot_matches_index_state(snapshot, istate) && + clean_status_index_snapshot_still_matches_path( + snapshot, istate->repo->index_file, + istate->repo->hash_algo); +} + +void clean_status_index_snapshot_release( + struct clean_status_index_snapshot *snapshot) +{ + if (snapshot->fd >= 0) + close(snapshot->fd); + snapshot->fd = -1; +} void clean_status_record_source_identity(struct index_state *istate, const struct stat *st) diff --git a/clean-status-index.h b/clean-status-index.h new file mode 100644 index 00000000000000..02dea99f71ae96 --- /dev/null +++ b/clean-status-index.h @@ -0,0 +1,26 @@ +#ifndef CLEAN_STATUS_INDEX_H +#define CLEAN_STATUS_INDEX_H + +#include "clean-status-identity.h" +#include "hash.h" + +struct index_state; + +struct clean_status_index_snapshot { + struct clean_status_identity identity; + uint32_t version; + uint32_t cache_nr; + struct object_id checksum; + int fd; +}; + +int clean_status_index_snapshot_pin( + struct clean_status_index_snapshot *snapshot, + struct index_state *istate); +int clean_status_index_snapshot_still_matches( + const struct clean_status_index_snapshot *snapshot, + const struct index_state *istate); +void clean_status_index_snapshot_release( + struct clean_status_index_snapshot *snapshot); + +#endif /* CLEAN_STATUS_INDEX_H */ diff --git a/t/unit-tests/u-clean-status-index.c b/t/unit-tests/u-clean-status-index.c index 5d769692eea894..2f9444ab3140e9 100644 --- a/t/unit-tests/u-clean-status-index.c +++ b/t/unit-tests/u-clean-status-index.c @@ -1,11 +1,285 @@ #include "unit-test.h" #include "clean-status.h" +#include "clean-status-index.h" #include "clean-status-internal.h" #include "dir.h" #include "read-cache-ll.h" +#include "repository.h" #include "strbuf.h" #include "wrapper.h" +struct index_fixture { + char *path; + int fd; + struct stat st; + struct object_id checksum; +}; + +static void fixture_init(struct index_fixture *fixture, + const struct git_hash_algo *algo) +{ + const char *tmp = getenv("TMPDIR"); + unsigned char header[12] = "DIRC"; + unsigned char hash[GIT_MAX_RAWSZ]; + static const char payload[] = "payload"; + + memset(fixture, 0, sizeof(*fixture)); + fixture->path = xstrfmt("%s/index-snapshot.XXXXXX", + tmp ? tmp : "/tmp"); + fixture->fd = mkstemp(fixture->path); + cl_assert(fixture->fd >= 0); + put_be32(header + 4, 4); + put_be32(header + 8, 7); + memset(hash, 1, algo->rawsz); + oidread(&fixture->checksum, hash, algo); + cl_assert_equal_i(write_in_full(fixture->fd, header, sizeof(header)), + sizeof(header)); + cl_assert_equal_i(write_in_full(fixture->fd, payload, sizeof(payload)), + sizeof(payload)); + cl_assert_equal_i(write_in_full(fixture->fd, hash, algo->rawsz), + algo->rawsz); + cl_assert_equal_i(fstat(fixture->fd, &fixture->st), 0); +} + +static void fixture_release(struct index_fixture *fixture) +{ + cl_assert_equal_i(close(fixture->fd), 0); + cl_assert_equal_i(unlink(fixture->path), 0); + free(fixture->path); +} + +static void write_at(int fd, const void *data, size_t len, off_t offset) +{ + cl_assert_equal_i(lseek(fd, offset, SEEK_SET), offset); + cl_assert_equal_i(write_in_full(fd, data, len), len); +} + +static void fixture_clear_checksum(struct index_fixture *fixture, + const struct git_hash_algo *algo) +{ + unsigned char null_hash[GIT_MAX_RAWSZ] = { 0 }; + + write_at(fixture->fd, null_hash, algo->rawsz, + fixture->st.st_size - algo->rawsz); + oidclr(&fixture->checksum, algo); + cl_assert_equal_i(fstat(fixture->fd, &fixture->st), 0); +} + +static void assert_reads_snapshot(const struct git_hash_algo *algo) +{ + struct index_fixture fixture; + struct clean_status_index_snapshot snapshot; + struct repository repo = { 0 }; + struct index_state istate = INDEX_STATE_INIT(&repo); + + fixture_init(&fixture, algo); + repo.hash_algo = algo; + repo.index_file = fixture.path; + istate.version = 4; + istate.cache_nr = 7; + oidcpy(&istate.oid, &fixture.checksum); + cl_assert_equal_i(clean_status_index_snapshot_pin( + &snapshot, &istate), 0); + cl_assert_equal_i(snapshot.version, 4); + cl_assert_equal_i(snapshot.cache_nr, 7); + cl_assert(oideq(&snapshot.checksum, &fixture.checksum)); + cl_assert(clean_status_index_snapshot_still_matches( + &snapshot, &istate)); + clean_status_index_snapshot_release(&snapshot); + fixture_release(&fixture); +} + +void test_clean_status_index__reads_both_object_formats(void) +{ + assert_reads_snapshot(&hash_algos[GIT_HASH_SHA1]); + assert_reads_snapshot(&hash_algos[GIT_HASH_SHA256]); +} + +void test_clean_status_index__rejects_invalid_headers(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct index_fixture fixture; + struct clean_status_index_snapshot snapshot; + struct repository repo = { 0 }; + struct index_state istate = INDEX_STATE_INIT(&repo); + uint32_t value; + + fixture_init(&fixture, algo); + repo.hash_algo = algo; + repo.index_file = fixture.path; + istate.version = 4; + istate.cache_nr = 7; + oidcpy(&istate.oid, &fixture.checksum); + write_at(fixture.fd, "NOPE", 4, 0); + cl_assert_equal_i(clean_status_index_snapshot_pin( + &snapshot, &istate), -1); + write_at(fixture.fd, "DIRC", 4, 0); + put_be32(&value, 1); + write_at(fixture.fd, &value, sizeof(value), 4); + cl_assert_equal_i(clean_status_index_snapshot_pin( + &snapshot, &istate), -1); + fixture_release(&fixture); +} + +static void assert_rejects_null_checksum(const struct git_hash_algo *algo) +{ + struct clean_status_index_snapshot snapshot; + struct index_fixture fixture; + struct repository repo = { 0 }; + struct index_state istate = INDEX_STATE_INIT(&repo); + + fixture_init(&fixture, algo); + repo.hash_algo = algo; + repo.index_file = fixture.path; + istate.version = 4; + istate.cache_nr = 7; + fixture_clear_checksum(&fixture, algo); + oidcpy(&istate.oid, &fixture.checksum); + cl_assert_equal_i(clean_status_index_snapshot_pin( + &snapshot, &istate), -1); + fixture_release(&fixture); +} + +void test_clean_status_index__rejects_null_checksums(void) +{ + assert_rejects_null_checksum(&hash_algos[GIT_HASH_SHA1]); + assert_rejects_null_checksum(&hash_algos[GIT_HASH_SHA256]); +} + +static void assert_pins_null_checksum_source( + const struct git_hash_algo *algo) +{ + struct clean_status_index_snapshot snapshot; + struct index_fixture fixture, replacement; + struct repository repo = { 0 }; + struct index_state istate = INDEX_STATE_INIT(&repo); + struct index_state parsed = INDEX_STATE_INIT(&repo); + char *moved; + + fixture_init(&fixture, algo); + fixture_init(&replacement, algo); + fixture_clear_checksum(&fixture, algo); + fixture_clear_checksum(&replacement, algo); + repo.hash_algo = algo; + repo.index_file = fixture.path; + istate.version = 4; + istate.cache_nr = 7; + oidcpy(&istate.oid, &fixture.checksum); + parsed.version = 4; + parsed.cache_nr = 7; + oidcpy(&parsed.oid, &replacement.checksum); + clean_status_get_state(&istate); + clean_status_record_source_identity(&istate, &fixture.st); + clean_status_get_state(&parsed); + clean_status_record_source_identity(&parsed, &replacement.st); + + if (clean_status_identity_is_durable()) { + cl_assert_equal_i(clean_status_index_snapshot_pin( + &snapshot, &istate), 0); + cl_assert(clean_status_index_snapshot_still_matches( + &snapshot, &istate)); + + /* + * Model an A-to-B-to-A replacement while a second index state + * parses B. The named path and held descriptor are back on A, + * while the parsed state's source identity still binds it to B. + */ + cl_assert(!clean_status_index_snapshot_still_matches( + &snapshot, &parsed)); + clean_status_index_snapshot_release(&snapshot); + } else { + cl_assert_equal_i(clean_status_index_snapshot_pin( + &snapshot, &istate), -1); + clean_status_release(&istate); + clean_status_release(&parsed); + fixture_release(&fixture); + fixture_release(&replacement); + return; + } + + moved = xstrfmt("%s.old", fixture.path); + cl_assert_equal_i(rename(fixture.path, moved), 0); + cl_assert_equal_i(rename(replacement.path, fixture.path), 0); + cl_assert_equal_i(clean_status_index_snapshot_pin( + &snapshot, &istate), -1); + + clean_status_release(&istate); + clean_status_release(&parsed); + cl_assert_equal_i(close(fixture.fd), 0); + cl_assert_equal_i(close(replacement.fd), 0); + cl_assert_equal_i(unlink(fixture.path), 0); + cl_assert_equal_i(unlink(moved), 0); + free(moved); + free(fixture.path); + free(replacement.path); +} + +void test_clean_status_index__pins_null_checksum_source_identity(void) +{ + assert_pins_null_checksum_source(&hash_algos[GIT_HASH_SHA1]); + assert_pins_null_checksum_source(&hash_algos[GIT_HASH_SHA256]); +} + +void test_clean_status_index__pins_named_index_identity(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct clean_status_index_snapshot snapshot; + struct index_fixture fixture; + struct repository repo = { 0 }; + struct index_state istate = INDEX_STATE_INIT(&repo); +#ifndef GIT_WINDOWS_NATIVE + char *moved; + int replacement; +#endif + + fixture_init(&fixture, algo); + repo.hash_algo = algo; + repo.index_file = fixture.path; + istate.version = 4; + istate.cache_nr = 7; + oidcpy(&istate.oid, &fixture.checksum); + + cl_assert_equal_i(clean_status_index_snapshot_pin( + &snapshot, &istate), 0); + cl_assert(clean_status_index_snapshot_still_matches( + &snapshot, &istate)); + + istate.cache_nr++; + cl_assert(!clean_status_index_snapshot_still_matches( + &snapshot, &istate)); + istate.cache_nr--; + +#ifndef GIT_WINDOWS_NATIVE + moved = xstrfmt("%s.old", fixture.path); + cl_assert_equal_i(rename(fixture.path, moved), 0); + replacement = open(fixture.path, O_WRONLY | O_CREAT | O_EXCL, 0600); + cl_assert(replacement >= 0); + cl_assert_equal_i(close(replacement), 0); + cl_assert(!clean_status_index_snapshot_still_matches( + &snapshot, &istate)); + cl_assert_equal_i(unlink(fixture.path), 0); + cl_assert_equal_i(rename(moved, fixture.path), 0); + free(moved); +#endif + clean_status_index_snapshot_release(&snapshot); + +#ifndef GIT_WINDOWS_NATIVE + { + char *symlink_path = xstrfmt("%s.link", fixture.path); + + cl_assert_equal_i(symlink(fixture.path, symlink_path), 0); + repo.index_file = symlink_path; + cl_assert_equal_i(clean_status_index_snapshot_pin( + &snapshot, &istate), -1); + repo.index_file = fixture.path; + cl_assert_equal_i(unlink(symlink_path), 0); + free(symlink_path); + } +#endif + + fixture_release(&fixture); +} + void test_clean_status_index__binds_the_parsed_source(void) { const char *tmp = getenv("TMPDIR"); From cc83455382e174f207c7ee4d13c7f65a5b2fca47 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 01:07:02 -0500 Subject: [PATCH 079/105] fsmonitor: rebuild sparse-index history without expanding the live index A collapsed sparse index cannot enumerate every tracked path needed for a complete attribute manifest. Expanding the live index would discard the sparse representation that status is supposed to preserve. Pin the named index with S09/P02, reread the verified index into a scratch index, and expand only that scratch copy. Build the complete manifest from the expanded scratch index. Check that both the parsed scratch state and original live state still match the held descriptor and stored trailer checksum; discard the manifest if either check fails. Add a sparse-checkout regression that checks the collapsed outside entry before and after status while detecting a same-size tracked rewrite. Extend the existing index unit case with a parsed A-to-B-to-A mismatch. Failed snapshot validation retains ordinary full-invalidation fallback. Signed-off-by: Taylor Blau --- clean-status-manifest.c | 31 +++++++++++++++-- t/t7527-builtin-fsmonitor.sh | 52 +++++++++++++++++++++++++++++ t/unit-tests/u-clean-status-index.c | 14 ++++++++ 3 files changed, 94 insertions(+), 3 deletions(-) diff --git a/clean-status-manifest.c b/clean-status-manifest.c index b13066a23c2c47..b2c8aaec97e71e 100644 --- a/clean-status-manifest.c +++ b/clean-status-manifest.c @@ -1,5 +1,6 @@ #include "git-compat-util.h" #include "attr-manifest.h" +#include "clean-status-index.h" #include "clean-status-manifest.h" #include "dir.h" #include "fsmonitor-clean-proof.h" @@ -7,6 +8,7 @@ #include "hash-framing.h" #include "read-cache-ll.h" #include "repository.h" +#include "sparse-index.h" #include "trace2.h" #include "worktree-attr-manifest.h" @@ -20,10 +22,33 @@ static int build_manifest(struct index_state *istate, unsigned char *manifest_hash, struct worktree_attr_manifest_stats *stats) { - if (istate->sparse_index != INDEX_EXPANDED) + struct clean_status_index_snapshot snapshot; + struct index_state scratch = INDEX_STATE_INIT(istate->repo); + int ret = -1; + + if (istate->sparse_index == INDEX_EXPANDED) + return worktree_attr_manifest_build( + istate, manifest, manifest_hash, stats); + if (clean_status_index_snapshot_pin(&snapshot, istate)) return -1; - return worktree_attr_manifest_build( - istate, manifest, manifest_hash, stats); + scratch.fsmonitor_has_run_once = 1; + if (read_index_from(&scratch, istate->repo->index_file, + istate->repo->gitdir) < 0 || + !clean_status_index_snapshot_still_matches(&snapshot, &scratch)) + goto done; + ensure_full_index(&scratch); + ret = worktree_attr_manifest_build( + &scratch, manifest, manifest_hash, stats); + if (ret || + !clean_status_index_snapshot_still_matches(&snapshot, istate)) { + strbuf_reset(manifest); + ret = -1; + } + +done: + release_index(&scratch); + clean_status_index_snapshot_release(&snapshot); + return ret; } void clean_status_manifest_init(struct clean_status_manifest_state *state) diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index 83d7c436e5ca1f..c92793b0f4c7f7 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -1747,4 +1747,56 @@ test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ ) ' +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'sparse index rebuilds semantic history without expansion' ' + test_when_finished "rm -rf sparse-semantic" && + test_create_repo sparse-semantic && + ( + cd sparse-semantic && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir in outside && + printf "aaaa\n" >in/tracked && + printf "outside\n" >outside/file && + git add . && + git commit -m base && + git sparse-checkout set --cone --sparse-index in && + git ls-files --sparse >.git/sparse.before && + test_grep "^outside/$" .git/sparse.before && + git config core.trustctime false && + git config core.checkStat minimal && + git config core.untrackedCache true && + test-tool chmtime =-60 in/tracked && + git update-index --refresh && + mtime=$(test-tool chmtime --get in/tracked) && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCC \ + GIT_TRACE2_EVENT="$PWD/.git/prime.trace" \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_trace2_data fsm_client query/trivial-response 1 \ + <.git/prime.trace && + test_trace2_data fsmonitor semantic/manifest-scan-count \ + "[1-9]" <.git/prime.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/prime.trace && + test_grep FSMN .git/index && + git -c core.fsmonitor=false ls-files --sparse \ + >.git/sparse.after-prime && + test_grep "^outside/$" .git/sparse.after-prime && + printf "bbbb\n" >in/tracked && + test-tool chmtime =$mtime in/tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=in/tracked \ + GIT_TRACE2_EVENT="$PWD/.git/change.trace" \ + git status --porcelain=v2 >.git/actual && + test_grep "^1 \.M .* in/tracked$" .git/actual && + test_trace2_data fsmonitor apply_count 1 \ + <.git/change.trace && + test_grep FSMN .git/index && + git -c core.fsmonitor=false ls-files --sparse \ + >.git/sparse.after-change && + test_grep "^outside/$" .git/sparse.after-change + ) +' + test_done diff --git a/t/unit-tests/u-clean-status-index.c b/t/unit-tests/u-clean-status-index.c index 2f9444ab3140e9..979f67634d03d3 100644 --- a/t/unit-tests/u-clean-status-index.c +++ b/t/unit-tests/u-clean-status-index.c @@ -227,6 +227,8 @@ void test_clean_status_index__pins_named_index_identity(void) struct index_fixture fixture; struct repository repo = { 0 }; struct index_state istate = INDEX_STATE_INIT(&repo); + struct index_state parsed = INDEX_STATE_INIT(&repo); + unsigned char replacement_hash[GIT_MAX_RAWSZ]; #ifndef GIT_WINDOWS_NATIVE char *moved; int replacement; @@ -244,6 +246,18 @@ void test_clean_status_index__pins_named_index_identity(void) cl_assert(clean_status_index_snapshot_still_matches( &snapshot, &istate)); + /* + * Model an A-to-B-to-A replacement while a consumer parses B. + * Matching the restored named path is insufficient unless the parsed + * state is also bound to the pinned A contents. + */ + memset(replacement_hash, 2, algo->rawsz); + parsed.version = 4; + parsed.cache_nr = 7; + oidread(&parsed.oid, replacement_hash, algo); + cl_assert(!clean_status_index_snapshot_still_matches( + &snapshot, &parsed)); + istate.cache_nr++; cl_assert(!clean_status_index_snapshot_still_matches( &snapshot, &istate)); From ee4b9b54792eb5589738ba427bb3a03453598175 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 13:41:16 -0500 Subject: [PATCH 080/105] status: bind each closing token to its complete proof epoch A clean provider response closes only the filesystem interval after its starting token. It cannot certify a refresh that started before the named index, configuration, attributes, and manifest were captured, or one whose semantic inputs subsequently changed. Capture the proof epoch before each refresh whose provider token may be accepted. Pin the named index, starting token, repository configuration, external attribute fingerprint, and complete full-index manifest. Recheck those inputs after the closing query. Record semantic history only for the accepted token; reject missing or changed inputs and fall back to a complete refresh. Always rebuild the manifest when physical history is unavailable, even if the stored semantic configuration already matches. Without that manifest, a trivial response invalidates the old binding and leaves the closing query with no complete epoch to bind, so each later status repeats the fallback. Register clean-status-epoch.c in Make and Meson alongside its first production consumer in wt-status.c. Add scripted regressions for capture-before-refresh ordering and recovery from unbound physical history. Add a manifest unit case requiring complete full-index history bound to the closed token. Signed-off-by: Taylor Blau --- Makefile | 1 + clean-status-config.c | 26 ++++ clean-status-config.h | 4 + clean-status-epoch.c | 181 +++++++++++++++++++++++++ clean-status.c | 35 +++++ clean-status.h | 15 ++ fsmonitor.c | 3 +- meson.build | 1 + t/t7519-status-fsmonitor.sh | 23 ++++ t/t7527-builtin-fsmonitor.sh | 90 +++++++++++- t/unit-tests/u-clean-status-manifest.c | 47 +++++++ wt-status.c | 82 +++++++++-- 12 files changed, 493 insertions(+), 15 deletions(-) create mode 100644 clean-status-epoch.c diff --git a/Makefile b/Makefile index cde58e8bd4728f..85431baa7dc54d 100644 --- a/Makefile +++ b/Makefile @@ -1127,6 +1127,7 @@ LIB_OBJS += checkout.o LIB_OBJS += chunk-format.o LIB_OBJS += clean-status.o LIB_OBJS += clean-status-config.o +LIB_OBJS += clean-status-epoch.o LIB_OBJS += clean-status-history.o LIB_OBJS += clean-status-identity.o LIB_OBJS += clean-status-index.o diff --git a/clean-status-config.c b/clean-status-config.c index 96037d61ad1104..44481510bcc1c5 100644 --- a/clean-status-config.c +++ b/clean-status-config.c @@ -2,6 +2,7 @@ #include "clean-status-config.h" #include "config.h" #include "hash-framing.h" +#include "repository.h" #include "strbuf.h" void clean_status_config_init(struct clean_status_config_digest *digest, @@ -78,3 +79,28 @@ void clean_status_config_final(struct clean_status_config_digest *digest) git_hash_final(digest->semantic_hash, &digest->semantic_ctx); digest->finalized = 1; } + +static int config_digest_callback(const char *key, const char *value, + const struct config_context *ctx, + void *data) +{ + clean_status_config_add(data, key, value, ctx); + return 0; +} + +int clean_status_config_read_repository( + struct repository *repo, + struct clean_status_config_digest *digest) +{ + struct config_options opts = { 0 }; + + clean_status_config_init(digest, repo->hash_algo); + opts.respect_includes = 1; + opts.commondir = repo->commondir; + opts.git_dir = repo->gitdir; + if (config_with_options(config_digest_callback, digest, NULL, + repo, &opts) < 0) + return -1; + clean_status_config_final(digest); + return 0; +} diff --git a/clean-status-config.h b/clean-status-config.h index dd4f0647420858..1ca173547a1154 100644 --- a/clean-status-config.h +++ b/clean-status-config.h @@ -4,6 +4,7 @@ #include "hash.h" struct config_context; +struct repository; struct clean_status_config_digest { struct git_hash_ctx ctx; @@ -22,5 +23,8 @@ void clean_status_config_add(struct clean_status_config_digest *digest, const char *key, const char *value, const struct config_context *ctx); void clean_status_config_final(struct clean_status_config_digest *digest); +int clean_status_config_read_repository( + struct repository *repo, + struct clean_status_config_digest *digest); #endif /* CLEAN_STATUS_CONFIG_H */ diff --git a/clean-status-epoch.c b/clean-status-epoch.c new file mode 100644 index 00000000000000..af39eacf3f6610 --- /dev/null +++ b/clean-status-epoch.c @@ -0,0 +1,181 @@ +#include "git-compat-util.h" +#include "attr-fingerprint.h" +#include "clean-status.h" +#include "clean-status-index.h" +#include "clean-status-internal.h" +#include "fsmonitor-clean-proof.h" +#include "fsmonitor-ll.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "trace2.h" + +/* + * State captured before a worktree scan. Every recorded input must still + * match after the closing provider query before scan results are accepted. + */ +struct clean_status_proof_epoch { + struct index_state *istate; + struct clean_status_index_snapshot index; + char *scan_start_token; + unsigned char config_hash[GIT_MAX_RAWSZ]; + unsigned char semantic_hash[GIT_MAX_RAWSZ]; + unsigned char attr_hash[GIT_MAX_RAWSZ]; + unsigned char attr_namespace_hash[GIT_MAX_RAWSZ]; + unsigned char manifest_hash[GIT_MAX_RAWSZ]; + uint32_t manifest_flags; + unsigned semantic_explicit : 1; + unsigned attr_sources_present : 1; + unsigned strong_mismatch : 1; + unsigned config_mismatch : 1; +}; + +static int config_matches_epoch( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch) +{ + struct clean_status_state *state = istate->clean_status; + struct clean_status_config_digest digest; + const struct git_hash_algo *algo = istate->repo->hash_algo; + + if (clean_status_config_read_repository(istate->repo, &digest)) + return 0; + return digest.finalized && !digest.unsafe_filter && + digest.semantic_config_explicit == epoch->semantic_explicit && + !memcmp(digest.hash, epoch->config_hash, algo->rawsz) && + !memcmp(digest.semantic_hash, epoch->semantic_hash, algo->rawsz) && + state && state->current_config_valid && + state->current_semantic_valid && + !memcmp(state->current_config_hash, epoch->config_hash, + algo->rawsz) && + !memcmp(state->current_semantic_hash, epoch->semantic_hash, + algo->rawsz); +} + +struct clean_status_proof_epoch *clean_status_capture_proof_epoch( + struct index_state *istate, + const struct attr_source_snapshot *attrs) +{ + struct clean_status_state *state = istate->clean_status; + struct clean_status_proof_epoch *epoch; + struct clean_status_config_digest digest; + struct clean_status_index_snapshot index; + const struct attr_fingerprint *fingerprint = + attr_source_snapshot_fingerprint(attrs); + uint32_t manifest_requirements = + FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | + FSMONITOR_CLEAN_PROOF_FULL_INDEX; + + if (istate->split_index || !state || !state->current_config_valid || + !state->config_enforced || + !state->current_semantic_valid || !state->current_attr_valid || + !state->manifest.current_valid || !state->manifest.checked || + state->manifest.global_fallback || state->unsafe_filter || + (state->manifest.current_flags & manifest_requirements) != + manifest_requirements || + !fsmonitor_pending_token_from_provider(istate) || + !istate->fsmonitor_last_update_pending || !fingerprint || + memcmp(fingerprint->content_hash, state->current_attr_hash, + istate->repo->hash_algo->rawsz) || + memcmp(fingerprint->namespace_hash, + state->current_attr_namespace_hash, + istate->repo->hash_algo->rawsz) || + fingerprint->sources_present != + state->current_attr_sources_present) + return NULL; + if (clean_status_config_read_repository(istate->repo, &digest) || + digest.unsafe_filter || !digest.finalized || + digest.semantic_config_explicit != + state->current_semantic_explicit || + memcmp(digest.hash, state->current_config_hash, + istate->repo->hash_algo->rawsz) || + memcmp(digest.semantic_hash, state->current_semantic_hash, + istate->repo->hash_algo->rawsz) || + clean_status_index_snapshot_pin(&index, istate)) + return NULL; + + CALLOC_ARRAY(epoch, 1); + epoch->istate = istate; + epoch->index = index; + epoch->scan_start_token = xstrdup(istate->fsmonitor_last_update_pending); + memcpy(epoch->config_hash, state->current_config_hash, + istate->repo->hash_algo->rawsz); + memcpy(epoch->semantic_hash, state->current_semantic_hash, + istate->repo->hash_algo->rawsz); + memcpy(epoch->attr_hash, state->current_attr_hash, + istate->repo->hash_algo->rawsz); + memcpy(epoch->attr_namespace_hash, + state->current_attr_namespace_hash, + istate->repo->hash_algo->rawsz); + memcpy(epoch->manifest_hash, state->manifest.current_hash, + istate->repo->hash_algo->rawsz); + epoch->manifest_flags = state->manifest.current_flags; + epoch->semantic_explicit = state->current_semantic_explicit; + epoch->attr_sources_present = state->current_attr_sources_present; + epoch->strong_mismatch = state->strong_mismatch; + epoch->config_mismatch = state->config_mismatch; + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/proof-epoch-captured", 1); + return epoch; +} + +int clean_status_proof_epoch_start_token_matches( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch) +{ + return epoch && epoch->istate == istate && epoch->scan_start_token && + fsmonitor_pending_token_from_provider(istate) && + istate->fsmonitor_last_update_pending && + !strcmp(epoch->scan_start_token, + istate->fsmonitor_last_update_pending); +} + +int clean_status_proof_epoch_matches( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch) +{ + struct clean_status_state *state; + struct attr_fingerprint attrs; + const struct git_hash_algo *algo = istate->repo->hash_algo; + int matched = 0; + + if (!epoch || epoch->istate != istate || + !fsmonitor_pending_token_from_provider(istate) || + !istate->fsmonitor_last_update_pending) + goto done; + state = istate->clean_status; + if (!state || !state->current_config_valid || !state->config_enforced || + !state->current_semantic_valid || !state->current_attr_valid || + !state->manifest.current_valid || !state->manifest.checked || + state->manifest.global_fallback || state->unsafe_filter || + state->manifest.current_flags != epoch->manifest_flags || + state->current_semantic_explicit != epoch->semantic_explicit || + state->current_attr_sources_present != epoch->attr_sources_present || + state->strong_mismatch != epoch->strong_mismatch || + state->config_mismatch != epoch->config_mismatch) + goto done; + if (attr_fingerprint_repository(istate->repo, &attrs) || + memcmp(attrs.content_hash, epoch->attr_hash, algo->rawsz) || + memcmp(attrs.namespace_hash, epoch->attr_namespace_hash, + algo->rawsz) || + attrs.sources_present != epoch->attr_sources_present || + memcmp(state->manifest.current_hash, epoch->manifest_hash, + algo->rawsz) || + !config_matches_epoch(istate, epoch) || + !clean_status_index_snapshot_still_matches(&epoch->index, istate)) + goto done; + matched = 1; +done: + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/proof-epoch-matched", matched); + return matched; +} + +void clean_status_release_proof_epoch( + struct clean_status_proof_epoch *epoch) +{ + if (!epoch) + return; + clean_status_index_snapshot_release(&epoch->index); + free(epoch->scan_start_token); + free(epoch); +} diff --git a/clean-status.c b/clean-status.c index 7a0db98473ef76..76b0bc129256ae 100644 --- a/clean-status.c +++ b/clean-status.c @@ -2,6 +2,7 @@ #include "attr-fingerprint.h" #include "clean-status.h" #include "clean-status-internal.h" +#include "fsmonitor-clean-proof.h" #include "read-cache-ll.h" #include "repository.h" #include "trace2.h" @@ -169,6 +170,40 @@ int clean_status_manifest_global_fallback(const struct index_state *istate) istate->clean_status->manifest.global_fallback; } +void clean_status_mark_fsmonitor_config_valid(struct index_state *istate, + const char *closed_token) +{ + struct clean_status_state *state = istate->clean_status; + + if (!state || !state->current_config_valid) + return; + if (!closed_token || + !state->manifest.current_valid || !state->manifest.checked || + (state->manifest.current_flags & + (FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | + FSMONITOR_CLEAN_PROOF_FULL_INDEX)) != + (FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | + FSMONITOR_CLEAN_PROOF_FULL_INDEX)) { + state->config_revalidated = 0; + FREE_AND_NULL(state->config_revalidated_token); + trace2_data_intmax("fsmonitor", istate->repo, + "config/manifest-unbound", 1); + return; + } + state->config_mismatch = 0; + state->strong_mismatch = 0; + state->semantic_baseline_pending = 0; + state->manifest.current_flags = FSMONITOR_CLEAN_PROOF_ALL; + state->config_revalidated = state->current_semantic_valid && + state->current_attr_valid && state->manifest.current_valid; + state->initial_coherent = state->config_revalidated; + FREE_AND_NULL(state->config_revalidated_token); + if (state->config_revalidated) + state->config_revalidated_token = xstrdup(closed_token); + trace2_data_intmax("fsmonitor", istate->repo, + "config/revalidated", 1); +} + void clean_status_release(struct index_state *istate) { if (!istate->clean_status) diff --git a/clean-status.h b/clean-status.h index a6aea7b136cd4d..eb4c8dd7c69c90 100644 --- a/clean-status.h +++ b/clean-status.h @@ -5,6 +5,7 @@ struct index_state; struct attr_source_snapshot; +struct clean_status_proof_epoch; struct repository; struct stat; struct strbuf; @@ -21,6 +22,17 @@ void clean_status_attach_config(struct index_state *istate); int clean_status_capture_attr_snapshot( struct index_state *istate, struct attr_source_snapshot **snapshot); +struct clean_status_proof_epoch *clean_status_capture_proof_epoch( + struct index_state *istate, + const struct attr_source_snapshot *attrs); +int clean_status_proof_epoch_start_token_matches( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch); +int clean_status_proof_epoch_matches( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch); +void clean_status_release_proof_epoch( + struct clean_status_proof_epoch *epoch); int clean_status_fsmonitor_config_mismatch(const struct index_state *istate); int clean_status_fsmonitor_strong_mismatch(const struct index_state *istate); @@ -41,6 +53,9 @@ void clean_status_begin_fsmonitor_semantic_baseline( int clean_status_refresh_worktree_manifest(struct index_state *istate); int clean_status_manifest_global_fallback(const struct index_state *istate); +void clean_status_mark_fsmonitor_config_valid( + struct index_state *istate, const char *closed_token); + void clean_status_record_source_identity(struct index_state *istate, const struct stat *st); int clean_status_verify_null_index(const struct index_state *istate, diff --git a/fsmonitor.c b/fsmonitor.c index c90bfebc2e4712..06674377f98dc1 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -944,8 +944,7 @@ static void invalidate_fsmonitor_for_bootstrap( } if (physical_history_unavailable) { - if (semantic_adoption_needed) - clean_status_refresh_worktree_manifest(istate); + clean_status_refresh_worktree_manifest(istate); fsmonitor_invalidate_semantics(istate); untracked_cache_invalidate_all(istate); return; diff --git a/meson.build b/meson.build index 59b7097fca049d..754cd00c979934 100644 --- a/meson.build +++ b/meson.build @@ -335,6 +335,7 @@ libgit_sources = [ 'chunk-format.c', 'clean-status.c', 'clean-status-config.c', + 'clean-status-epoch.c', 'clean-status-history.c', 'clean-status-identity.c', 'clean-status-index.c', diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 90228af9d007d2..c4cbd1adfb382b 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -69,6 +69,29 @@ test_expect_success 'FSUC parser fails closed' ' test-tool read-cache --test-fsuc-parser ' +test_expect_success !SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'unsupported identity preserves an ordinary provider token' ' + test_when_finished "rm -rf unsupported-provider-token" && + test_create_repo unsupported-provider-token && + ( + cd unsupported-provider-token && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual && + test-tool dump-fsmonitor >.git/fsmonitor && + test_grep "^fsmonitor last update builtin:test:" \ + .git/fsmonitor && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor semantic/strong-invalidation 1 \ + <.git/status.trace + ) +' + test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ 'FSCF survives index I/O and generic rewrites' ' test_when_finished "rm -rf fscf-round-trip" && diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index c92793b0f4c7f7..bae13a6bd1cd1d 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -1645,9 +1645,6 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ git status --porcelain=v2 >.git/prime && test_must_be_empty .git/prime && test_grep FSUC .git/index && - GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ - test-tool read-cache --test-fscf-seed && - test_grep FSUC .git/index && test_grep FSCF .git/index && touch x && @@ -1747,6 +1744,93 @@ test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ ) ' +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'token closure refresh starts inside its proof epoch' ' + test_when_finished "rm -rf proof-epoch-refresh" && + test_create_repo proof-epoch-refresh && + ( + cd proof-epoch-refresh && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + # Leave the next refresh with untracked history to bootstrap. + git update-index --no-untracked-cache 2>.git/no-uc.err && + test_grep FSCF .git/index && + test_grep ! FSUC .git/index && + + test_env GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + test_must_fail git commit --dry-run --porcelain \ + >.git/actual && + test_must_be_empty .git/actual && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + captured=$(test_grep -n \ + "\"key\":\"semantic/proof-epoch-captured\"" \ + .git/status.trace | sed -n "1s/:.*//p") && + refreshed=$(test_grep -n \ + "\"category\":\"index\",\"label\":\"refresh\"" \ + .git/status.trace | sed -n "\$s/:.*//p") && + test -n "$captured" && + test -n "$refreshed" && + test "$captured" -lt "$refreshed" + ) +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'trivial query closes unbound semantic history' ' + test_when_finished "rm -rf unbound-trivial" && + test_create_repo unbound-trivial && + ( + cd unbound-trivial && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + test-tool read-cache --test-fscf-round-trip && + test_grep FSMN .git/index && + test_grep FSCF .git/index && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TC \ + GIT_TRACE2_EVENT="$PWD/.git/recovery.trace" \ + git status --porcelain=v2 --untracked-files=no \ + >.git/recovery.out && + test_must_be_empty .git/recovery.out && + test_trace2_data fsm_client query/trivial-response 1 \ + <.git/recovery.trace && + test_trace2_data fsmonitor semantic/manifest-scan-count \ + "[1-9]" <.git/recovery.trace && + test_trace2_data fsmonitor semantic/proof-epoch-captured 1 \ + <.git/recovery.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/recovery.trace && + ! test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/recovery.trace && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + GIT_TRACE2_EVENT="$PWD/.git/warm.trace" \ + git status --porcelain=v2 --untracked-files=no \ + >.git/warm.out && + test_must_be_empty .git/warm.out && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/warm.trace && + ! test_trace2_data fsmonitor semantic/manifest-scan-count \ + "[1-9]" <.git/warm.trace && + ! test_trace2_data fsm_client query/trivial-response 1 \ + <.git/warm.trace && + ! test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/warm.trace + ) +' + test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ 'sparse index rebuilds semantic history without expansion' ' test_when_finished "rm -rf sparse-semantic" && diff --git a/t/unit-tests/u-clean-status-manifest.c b/t/unit-tests/u-clean-status-manifest.c index fc17fd8ec0dbb8..41c565330e5cde 100644 --- a/t/unit-tests/u-clean-status-manifest.c +++ b/t/unit-tests/u-clean-status-manifest.c @@ -1,5 +1,7 @@ #include "unit-test.h" #include "attr-manifest.h" +#include "clean-status.h" +#include "clean-status-internal.h" #include "clean-status-manifest.h" #include "dir.h" #include "fsmonitor-clean-proof.h" @@ -73,6 +75,51 @@ void test_clean_status_manifest__rejects_invalid_history(void) clean_status_manifest_release(&state); strbuf_release(&manifest); } + +void test_clean_status_manifest__requires_complete_full_index(void) +{ + const char *current_token = "builtin:1:2"; + const char *closed_token = "builtin:1:3"; + struct repository repo = { .hash_algo = &hash_algos[GIT_HASH_SHA1] }; + struct index_state istate = INDEX_STATE_INIT(&repo); + struct clean_status_state *state = clean_status_get_state(&istate); + + istate.fsmonitor_last_update = xstrdup(current_token); + istate.fsmonitor_token_valid = 1; + state->current_config_valid = 1; + state->current_semantic_valid = 1; + state->current_attr_valid = 1; + state->config_enforced = 1; + state->manifest.current_valid = 1; + state->manifest.checked = 1; + state->manifest.current_flags = + FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE; + + clean_status_mark_fsmonitor_config_valid(&istate, closed_token); + cl_assert(!state->config_revalidated); + cl_assert(!state->config_revalidated_token); + cl_assert(!clean_status_should_write_fsmonitor_config(&istate)); + cl_assert_equal_i(state->manifest.current_flags, + FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE); + + state->manifest.current_flags |= FSMONITOR_CLEAN_PROOF_FULL_INDEX; + clean_status_mark_fsmonitor_config_valid(&istate, closed_token); + cl_assert(state->config_revalidated); + cl_assert_equal_s(state->config_revalidated_token, closed_token); + cl_assert(!clean_status_revalidated_token_matches(&istate)); + cl_assert(!clean_status_should_write_fsmonitor_config(&istate)); + + FREE_AND_NULL(istate.fsmonitor_last_update); + istate.fsmonitor_last_update = xstrdup(closed_token); + cl_assert(clean_status_revalidated_token_matches(&istate)); + cl_assert(clean_status_should_write_fsmonitor_config(&istate)); + cl_assert_equal_i(state->manifest.current_flags, + FSMONITOR_CLEAN_PROOF_ALL); + + clean_status_release(&istate); + FREE_AND_NULL(istate.fsmonitor_last_update); +} + #if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN static char *create_worktree(void) { diff --git a/wt-status.c b/wt-status.c index 7146f3e42f1760..b95ecb250f4c5e 100644 --- a/wt-status.c +++ b/wt-status.c @@ -1015,45 +1015,102 @@ static void wt_status_publish_staged_untracked( closure->staged_untracked_ready = 0; } +static int fsmonitor_token_requires_rescan(enum fsmonitor_token_result result) +{ + return result == FSMONITOR_TOKEN_CHANGED || + result == FSMONITOR_TOKEN_TRIVIAL; +} + +static void wt_status_refresh_for_token( + struct wt_status *s, unsigned int refresh_flags, + struct clean_status_proof_epoch **epoch, int *refresh_result) +{ + struct index_state *istate = s->repo->index; + + clean_status_release_proof_epoch(*epoch); + *epoch = clean_status_capture_proof_epoch( + istate, s->attr_source_snapshot); + if (*epoch) + *refresh_result |= refresh_index( + istate, refresh_flags, &s->pathspec, NULL, NULL); +} + static int wt_status_close_ordinary_fsmonitor_token( struct wt_status_token_closure *closure, int refreshed_before_closure) { struct wt_status *s = closure->status; struct index_state *istate = s->repo->index; + struct clean_status_proof_epoch *scan_epoch = NULL; + int reliable_stat = fstat_is_reliable(); - if (!refreshed_before_closure) - closure->refresh_result = refresh_index( + /* + * A pending token must close a refresh begun after its epoch was + * captured. A refresh performed before entering token closure cannot + * be validated by capturing its inputs afterward. + */ + if (reliable_stat) { + wt_status_refresh_for_token( + s, closure->refresh_flags, &scan_epoch, + &closure->refresh_result); + if (!scan_epoch) + return 0; + } else if (!refreshed_before_closure) { + closure->refresh_result |= refresh_index( istate, closure->refresh_flags, &s->pathspec, NULL, NULL); + } if (!closure->untracked_ready && closure->can_prime) closure->untracked_ready = wt_status_stage_untracked(closure); while (closure->queries < FSMONITOR_TOKEN_MAX_QUERIES) { - enum fsmonitor_token_result result = - fsmonitor_query_pending_token( - istate, closure->untracked_ready); + enum fsmonitor_token_result result; + if (reliable_stat && + !clean_status_proof_epoch_start_token_matches( + istate, scan_epoch)) + break; closure->queries++; + result = fsmonitor_query_pending_token( + istate, closure->untracked_ready); if (result == FSMONITOR_TOKEN_CLEAN) { + if (reliable_stat && + !clean_status_proof_epoch_matches( + istate, scan_epoch)) + break; if (closure->untracked_ready) { + if (reliable_stat) + clean_status_mark_fsmonitor_config_valid( + istate, + istate->fsmonitor_last_update_pending); + clean_status_release_proof_epoch(scan_epoch); fsmonitor_accept_pending_token(istate); return 1; } break; } - if (result == FSMONITOR_TOKEN_ERROR || - result == FSMONITOR_TOKEN_NOT_PENDING) + clean_status_release_proof_epoch(scan_epoch); + scan_epoch = NULL; + if (!fsmonitor_token_requires_rescan(result)) break; /* Rescan invalidations returned by the closure query. */ - closure->refresh_result |= refresh_index( - istate, closure->refresh_flags, &s->pathspec, - NULL, NULL); + if (reliable_stat) { + wt_status_refresh_for_token( + s, closure->refresh_flags, &scan_epoch, + &closure->refresh_result); + if (!scan_epoch) + break; + } else { + closure->refresh_result |= refresh_index( + istate, closure->refresh_flags, + &s->pathspec, NULL, NULL); + } if (closure->can_prime) closure->untracked_ready = wt_status_stage_untracked(closure); } + clean_status_release_proof_epoch(scan_epoch); return 0; } @@ -1095,6 +1152,11 @@ static int wt_status_close_fsmonitor_token( /* Keep the last valid token and fall back to complete scans. */ wt_status_discard_staged_untracked(&closure); fsmonitor_reject_pending_token(istate); + if (fstat_is_reliable()) { + if (closure.can_prime) + untracked_cache_invalidate_all(istate); + fsmonitor_invalidate_semantics(istate); + } closure.refresh_result |= refresh_index( istate, refresh_flags, &s->pathspec, NULL, NULL); accepted: From c4dfd30288ce8ea5c9c049d81fd44d9c736fd19e Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 13:42:44 -0500 Subject: [PATCH 081/105] fsmonitor: reopen semantic proofs after attribute events A provider event can change a tracked .gitattributes file after status captures its conversion inputs. Accepting the resulting token against the previous manifest can incorrectly reuse tracked validity when a complete status would report a content change. Record when provider invalidation expires the current manifest and semantic proof. Before retrying token closure, rebuild that manifest and recapture external attribute sources when their content changes. Keep the response token pending until the new scan and current attribute epoch are both closed. Preserve ordinary provider handling when file identity is unreliable. Preserve reusable manifest history across ordinary index rewrites without retaining expired bindings. Extend the history and manifest unit cases and the index round-trip helper. Add a scripted regression for tracked attribute changes. Manifest or snapshot failure still forces a complete scan. Signed-off-by: Taylor Blau --- attr-fingerprint.c | 15 +++++ attr-fingerprint.h | 3 + clean-status-manifest.c | 4 ++ clean-status-manifest.h | 1 + clean-status.c | 20 ++++++- clean-status.h | 4 +- fsmonitor.c | 44 ++++++++++++-- t/helper/test-read-cache.c | 12 ++++ t/t7527-builtin-fsmonitor.sh | 39 +++++++++++++ t/unit-tests/u-clean-status-history.c | 22 +++++++ t/unit-tests/u-clean-status-manifest.c | 4 ++ t/unit-tests/u-fsmonitor-attributes.c | 27 +++++++++ wt-status.c | 81 ++++++++++++++++++++++---- 13 files changed, 258 insertions(+), 18 deletions(-) diff --git a/attr-fingerprint.c b/attr-fingerprint.c index d7fdc1870dd2c3..d0152d6fe23963 100644 --- a/attr-fingerprint.c +++ b/attr-fingerprint.c @@ -213,6 +213,21 @@ int attr_source_snapshot_repository(struct repository *repo, return 0; } +int attr_source_snapshot_matches_repository( + struct repository *repo, + const struct attr_source_snapshot *snapshot) +{ + struct attr_fingerprint current; + + return snapshot && + !attr_fingerprint_repository(repo, ¤t) && + current.sources_present == + snapshot->fingerprint.sources_present && + !memcmp(current.content_hash, + snapshot->fingerprint.content_hash, + repo->hash_algo->rawsz); +} + const struct attr_fingerprint *attr_source_snapshot_fingerprint( const struct attr_source_snapshot *snapshot) { diff --git a/attr-fingerprint.h b/attr-fingerprint.h index 6d15646fcd1975..518a5b31b4e485 100644 --- a/attr-fingerprint.h +++ b/attr-fingerprint.h @@ -32,6 +32,9 @@ int attr_fingerprint_repository(struct repository *repo, struct attr_fingerprint *result); int attr_source_snapshot_repository(struct repository *repo, struct attr_source_snapshot **result); +int attr_source_snapshot_matches_repository( + struct repository *repo, + const struct attr_source_snapshot *snapshot); const struct attr_fingerprint *attr_source_snapshot_fingerprint( const struct attr_source_snapshot *snapshot); int attr_source_snapshot_read( diff --git a/clean-status-manifest.c b/clean-status-manifest.c index b2c8aaec97e71e..2750ddeb7280a9 100644 --- a/clean-status-manifest.c +++ b/clean-status-manifest.c @@ -93,6 +93,7 @@ void clean_status_manifest_adopt_disk( state->current_flags = state->disk_flags; state->current_valid = 1; state->checked = 1; + state->current_invalidated = 0; } static int invalidate_manifest_path(const struct attr_manifest_entry *entry, @@ -155,6 +156,7 @@ int clean_status_manifest_refresh(struct index_state *istate, state->current_valid = 1; state->current_flags = FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | FSMONITOR_CLEAN_PROOF_FULL_INDEX; + state->current_invalidated = 0; trace2_data_intmax("fsmonitor", istate->repo, "semantic/manifest-candidates", stats.candidates); trace2_data_intmax("fsmonitor", istate->repo, @@ -180,6 +182,8 @@ int clean_status_manifest_refresh(struct index_state *istate, void clean_status_manifest_invalidate( struct clean_status_manifest_state *state) { + if (state->current_valid) + state->current_invalidated = 1; state->current_valid = 0; state->current_flags = 0; } diff --git a/clean-status-manifest.h b/clean-status-manifest.h index 04a56d56abe65c..394fe25a888c0d 100644 --- a/clean-status-manifest.h +++ b/clean-status-manifest.h @@ -19,6 +19,7 @@ struct clean_status_manifest_state { unsigned checked : 1; unsigned changed : 1; unsigned global_fallback : 1; + unsigned current_invalidated : 1; }; void clean_status_manifest_init(struct clean_status_manifest_state *state); diff --git a/clean-status.c b/clean-status.c index 76b0bc129256ae..2320bca5098cec 100644 --- a/clean-status.c +++ b/clean-status.c @@ -147,7 +147,8 @@ int clean_status_fsmonitor_config_mismatch(const struct index_state *istate) { return istate->clean_status && istate->clean_status->current_config_valid && - istate->clean_status->config_mismatch; + (istate->clean_status->config_mismatch || + !istate->clean_status->config_revalidated); } int clean_status_fsmonitor_strong_mismatch(const struct index_state *istate) @@ -170,6 +171,23 @@ int clean_status_manifest_global_fallback(const struct index_state *istate) istate->clean_status->manifest.global_fallback; } +int clean_status_worktree_manifest_needs_refresh( + const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + + return state && state->config_enforced && + state->manifest.current_invalidated; +} + +void clean_status_invalidate_current_manifest(struct index_state *istate) +{ + if (!istate->clean_status) + return; + clean_status_manifest_invalidate(&istate->clean_status->manifest); + clean_status_invalidate_current_proof(istate); +} + void clean_status_mark_fsmonitor_config_valid(struct index_state *istate, const char *closed_token) { diff --git a/clean-status.h b/clean-status.h index eb4c8dd7c69c90..fd5212364a5e0c 100644 --- a/clean-status.h +++ b/clean-status.h @@ -52,7 +52,9 @@ void clean_status_begin_fsmonitor_semantic_baseline( int clean_status_refresh_worktree_manifest(struct index_state *istate); int clean_status_manifest_global_fallback(const struct index_state *istate); - +int clean_status_worktree_manifest_needs_refresh( + const struct index_state *istate); +void clean_status_invalidate_current_manifest(struct index_state *istate); void clean_status_mark_fsmonitor_config_valid( struct index_state *istate, const char *closed_token); diff --git a/fsmonitor.c b/fsmonitor.c index 06674377f98dc1..0fd64efc2d8342 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -647,7 +647,7 @@ static void fsmonitor_refresh_callback(struct index_state *istate, char *name) if (!strcmp(name, FSMONITOR_PATH_GLOBAL_INVALIDATE)) { unsigned int i; - clean_status_invalidate_current_proof(istate); + clean_status_invalidate_current_manifest(istate); git_attr_invalidate_all(); untracked_cache_invalidate_all(istate); for (i = 0; i < istate->cache_nr; i++) @@ -684,7 +684,7 @@ static void fsmonitor_refresh_callback(struct index_state *istate, char *name) } } if (attributes_may_have_changed) - clean_status_invalidate_current_proof(istate); + clean_status_invalidate_current_manifest(istate); if (nr_in_cone) trace_printf_key(&trace_fsmonitor, @@ -915,14 +915,22 @@ static void invalidate_all_fsmonitor_for_baseline( static void invalidate_all_fsmonitor_strong(struct index_state *istate) { unsigned int i; + int provider_disabled = + fsm_settings__get_mode(istate->repo) == FSMONITOR_MODE_DISABLED; invalidate_all_fsmonitor(istate); - for (i = 0; i < istate->cache_nr; i++) - fsmonitor_invalidate_cache_entry(istate->cache[i]); + for (i = 0; i < istate->cache_nr; i++) { + struct cache_entry *ce = istate->cache[i]; + + if (provider_disabled && ce_skip_worktree(ce)) + continue; + fsmonitor_invalidate_cache_entry(ce); + } } void fsmonitor_invalidate_semantics(struct index_state *istate) { + clean_status_invalidate_current_proof(istate); git_attr_invalidate_all(); invalidate_all_fsmonitor_strong(istate); istate->cache_changed |= FSMONITOR_CHANGED; @@ -1165,11 +1173,37 @@ void refresh_fsmonitor(struct index_state *istate) } } - if (tracked_requires_bootstrap) + /* + * Applying a provider event may expire semantic history after + * the initial bootstrap decision. Keep the new token pending + * until status has rescanned against rebuilt inputs. + */ + if (fstat_is_reliable() && !istate->split_index && + fsm_mode == FSMONITOR_MODE_IPC && + clean_status_fsmonitor_config_mismatch(istate)) + tracked_requires_bootstrap = 1; + + if (tracked_requires_bootstrap) { + /* + * Provider paths can invalidate the manifest or + * semantic inputs after our pre-query snapshot. + * Recheck before choosing the narrow baseline lane. + */ + semantic_adoption_needed = fstat_is_reliable() && + !istate->split_index && + fsm_mode == FSMONITOR_MODE_IPC && + clean_status_fsmonitor_semantic_adoption_needed( + istate); + semantic_baseline_needed = fstat_is_reliable() && + !istate->split_index && + fsm_mode == FSMONITOR_MODE_IPC && + clean_status_fsmonitor_semantic_baseline_needed( + istate); invalidate_fsmonitor_for_bootstrap( istate, fsm_mode, semantic_adoption_needed, semantic_baseline_needed, !istate->fsmonitor_token_valid); + } /* Now mark the untracked cache for fsmonitor usage */ if (istate->untracked) diff --git a/t/helper/test-read-cache.c b/t/helper/test-read-cache.c index 8c2e97056b7e01..f01b8d9f19a824 100644 --- a/t/helper/test-read-cache.c +++ b/t/helper/test-read-cache.c @@ -345,6 +345,18 @@ static int test_fscf_history(int seed_existing_token) return error("unable to reread test index"); if (!test_fscf_history_is_coherent(the_repository->index)) return error("FSCF did not survive an index round trip"); + + clean_status_invalidate_current_manifest(the_repository->index); + if (write_test_index()) + goto done; + discard_index(the_repository->index); + if (repo_read_index(the_repository) < 0) + return error("unable to reread preserved test index"); + if (clean_status_has_persistent_fsmonitor_semantic_history( + the_repository->index)) + return error("generic rewrite retained FSCF epoch bindings"); + if (!clean_status_has_worktree_manifest_history(the_repository->index)) + return error("generic rewrite discarded FSCF manifest history"); ret = 0; done: diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index bae13a6bd1cd1d..e0077f40f00006 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -1883,4 +1883,43 @@ test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ ) ' +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'tracked attribute events reopen semantic history' ' + test_when_finished "rm -rf tracked-attr-change" && + test_create_repo tracked-attr-change && + ( + cd tracked-attr-change && + sane_unset GIT_TEST_SPLIT_INDEX && + printf "*.txt text eol=crlf\n" >.gitattributes && + printf "alpha\r\n" >tracked.txt && + git add .gitattributes tracked.txt && + git commit -m base && + git config core.fsmonitor true && + git config core.untrackedCache true && + git config core.trustctime false && + git config core.checkStat minimal && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime.actual && + test_must_be_empty .git/prime.actual && + test_grep FSCF .git/index && + test-tool chmtime =-60 tracked.txt && + + printf "*.txt -text\n" >.gitattributes && + test-tool chmtime +1 tracked.txt && + GIT_OPTIONAL_LOCKS=0 git -c core.fsmonitor=false \ + -c core.untrackedCache=false status --porcelain=v2 \ + >.git/expect && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=.gitattributes \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_cmp .git/expect .git/actual && + test_grep "^1 \.M .* tracked.txt$" .git/actual && + test_trace2_data fsmonitor semantic/manifest-scan-count \ + "[1-9]" <.git/status.trace + ) +' + test_done diff --git a/t/unit-tests/u-clean-status-history.c b/t/unit-tests/u-clean-status-history.c index feb813c185c624..3d196c73c8236e 100644 --- a/t/unit-tests/u-clean-status-history.c +++ b/t/unit-tests/u-clean-status-history.c @@ -277,6 +277,28 @@ void test_clean_status_history__advances_only_current_proofs(void) fixture_release(&fixture); } +void test_clean_status_history__expires_invalidated_proofs(void) +{ + struct history_fixture fixture; + struct clean_status_state *state; + + fixture_init(&fixture, &hash_algos[GIT_HASH_SHA1]); + clean_status_read_fsmonitor_config( + &fixture.istate, fixture.encoded.buf, fixture.encoded.len); + state = install_current(&fixture); + clean_status_prepare_fsmonitor_config(&fixture.istate); + cl_assert(state->manifest.current_valid); + cl_assert(state->config_revalidated); + cl_assert(state->initial_coherent); + + clean_status_invalidate_current_manifest(&fixture.istate); + cl_assert(!state->manifest.current_valid); + cl_assert(!state->config_revalidated); + cl_assert(!state->initial_coherent); + cl_assert(clean_status_fsmonitor_config_mismatch(&fixture.istate)); + fixture_release(&fixture); +} + void test_clean_status_history__copies_validated_history(void) { struct history_fixture fixture; diff --git a/t/unit-tests/u-clean-status-manifest.c b/t/unit-tests/u-clean-status-manifest.c index 41c565330e5cde..971b4bea973c0c 100644 --- a/t/unit-tests/u-clean-status-manifest.c +++ b/t/unit-tests/u-clean-status-manifest.c @@ -42,6 +42,7 @@ void test_clean_status_manifest__loads_and_adopts_valid_history(void) cl_assert(!memcmp(state.current.buf, manifest.buf, manifest.len)); clean_status_manifest_invalidate(&state); cl_assert(!state.current_valid); + cl_assert(state.current_invalidated); cl_assert_equal_i(state.current_flags, 0); clean_status_manifest_release(&state); strbuf_release(&manifest); @@ -183,6 +184,7 @@ void test_clean_status_manifest__invalidates_only_changed_scopes(void) } cl_assert_equal_i(clean_status_manifest_refresh(&istate, &state), 1); cl_assert(state.changed); + cl_assert(!state.current_invalidated); cl_assert(!(istate.cache[0]->ce_flags & CE_FSMONITOR_VALID)); cl_assert(istate.cache[0]->ce_flags & CE_CONTENT_CHECK_REQUIRED); cl_assert(istate.cache[1]->ce_flags & CE_FSMONITOR_VALID); @@ -195,6 +197,7 @@ void test_clean_status_manifest__invalidates_only_changed_scopes(void) strbuf_reset(&old); strbuf_addbuf(&old, &state.current); clean_status_manifest_invalidate(&state); + cl_assert(state.current_invalidated); istate.cache[0]->ce_flags = create_ce_flags(1); cl_assert_equal_i(clean_status_manifest_refresh(&istate, &state), -1); cl_assert(!state.current_valid); @@ -209,6 +212,7 @@ void test_clean_status_manifest__invalidates_only_changed_scopes(void) } cl_assert_equal_i(clean_status_manifest_refresh(&istate, &state), 1); cl_assert(state.changed); + cl_assert(!state.current_invalidated); cl_assert(!(istate.cache[0]->ce_flags & CE_FSMONITOR_VALID)); cl_assert(istate.cache[0]->ce_flags & CE_CONTENT_CHECK_REQUIRED); cl_assert(istate.cache[1]->ce_flags & CE_FSMONITOR_VALID); diff --git a/t/unit-tests/u-fsmonitor-attributes.c b/t/unit-tests/u-fsmonitor-attributes.c index 5a2b7a25f137b3..3eedefca7aad0b 100644 --- a/t/unit-tests/u-fsmonitor-attributes.c +++ b/t/unit-tests/u-fsmonitor-attributes.c @@ -1,5 +1,7 @@ #include "unit-test.h" +#include "fsmonitor.h" #include "fsmonitor-ll.h" +#include "fsmonitor-settings.h" #include "read-cache-ll.h" #include "repository.h" @@ -70,3 +72,28 @@ void test_fsmonitor_attributes__root_source_invalidates_every_entry(void) } release_index(&istate); } + +void test_fsmonitor_attributes__disabled_provider_preserves_skipped_stat(void) +{ + struct repository repo = { .hash_algo = &hash_algos[GIT_HASH_SHA1] }; + struct index_state istate = INDEX_STATE_INIT(&repo); + + fsm_settings__set_disabled(&repo); + CALLOC_ARRAY(istate.cache, 2); + istate.cache_alloc = istate.cache_nr = 2; + add_entry(&istate, 0, "skipped"); + add_entry(&istate, 1, "tracked"); + istate.cache[0]->ce_flags |= CE_SKIP_WORKTREE; + + fsmonitor_invalidate_semantics(&istate); + + cl_assert(!(istate.cache[0]->ce_flags & CE_FSMONITOR_VALID)); + cl_assert(!(istate.cache[0]->ce_flags & CE_CONTENT_CHECK_REQUIRED)); + cl_assert(!stat_data_is_zero(istate.cache[0])); + cl_assert(!(istate.cache[1]->ce_flags & CE_FSMONITOR_VALID)); + cl_assert(istate.cache[1]->ce_flags & CE_CONTENT_CHECK_REQUIRED); + cl_assert(stat_data_is_zero(istate.cache[1])); + + release_index(&istate); + FREE_AND_NULL(repo.settings.fsmonitor); +} diff --git a/wt-status.c b/wt-status.c index b95ecb250f4c5e..dbb21d31a15999 100644 --- a/wt-status.c +++ b/wt-status.c @@ -820,10 +820,10 @@ static int wt_status_begin_attr_snapshot(struct wt_status *s) int hook_provider = fsm_settings__get_mode(s->repo) == FSMONITOR_MODE_HOOK; - if (s->attr_source_snapshot) - return 0; if (s->attr_snapshot_failed) return -1; + if (s->attr_source_snapshot) + return 0; ret = clean_status_capture_attr_snapshot( s->repo->index, &s->attr_source_snapshot); if (ret < 0) { @@ -1021,6 +1021,35 @@ static int fsmonitor_token_requires_rescan(enum fsmonitor_token_result result) result == FSMONITOR_TOKEN_TRIVIAL; } +static void wt_status_release_attr_snapshot(struct wt_status *s); + +static int wt_status_attr_snapshot_matches(struct wt_status *s) +{ + if (s->attr_snapshot_failed) + return 0; + return !s->attr_source_snapshot || + attr_source_snapshot_matches_repository( + s->repo, s->attr_source_snapshot); +} + +static int wt_status_refresh_invalidated_manifest(struct wt_status *s) +{ + if (!clean_status_worktree_manifest_needs_refresh(s->repo->index)) + return 0; + return clean_status_refresh_worktree_manifest(s->repo->index) < 0 ? + -1 : 0; +} + +static void wt_status_reset_attr_snapshot_if_changed(struct wt_status *s) +{ + if (wt_status_attr_snapshot_matches(s)) + return; + wt_status_release_attr_snapshot(s); + wt_status_begin_attr_snapshot(s); + trace2_data_intmax("status", s->repo, + "semantic/attribute-epoch-rejected", 1); +} + static void wt_status_refresh_for_token( struct wt_status *s, unsigned int refresh_flags, struct clean_status_proof_epoch **epoch, int *refresh_result) @@ -1030,9 +1059,10 @@ static void wt_status_refresh_for_token( clean_status_release_proof_epoch(*epoch); *epoch = clean_status_capture_proof_epoch( istate, s->attr_source_snapshot); - if (*epoch) + if (*epoch) { *refresh_result |= refresh_index( istate, refresh_flags, &s->pathspec, NULL, NULL); + } } static int wt_status_close_ordinary_fsmonitor_token( @@ -1076,8 +1106,10 @@ static int wt_status_close_ordinary_fsmonitor_token( if (result == FSMONITOR_TOKEN_CLEAN) { if (reliable_stat && !clean_status_proof_epoch_matches( - istate, scan_epoch)) + istate, scan_epoch)) { + wt_status_reset_attr_snapshot_if_changed(s); break; + } if (closure->untracked_ready) { if (reliable_stat) clean_status_mark_fsmonitor_config_valid( @@ -1095,6 +1127,9 @@ static int wt_status_close_ordinary_fsmonitor_token( break; /* Rescan invalidations returned by the closure query. */ + wt_status_reset_attr_snapshot_if_changed(s); + if (wt_status_refresh_invalidated_manifest(s)) + break; if (reliable_stat) { wt_status_refresh_for_token( s, closure->refresh_flags, &scan_epoch, @@ -1129,10 +1164,24 @@ static int wt_status_close_fsmonitor_token( refresh_fsmonitor(istate); if (!fsmonitor_has_pending_token(istate) || s->pathspec.nr || fsm_settings__get_mode(s->repo) != FSMONITOR_MODE_IPC) { - if (!refreshed_before_closure) - closure.refresh_result = refresh_index( + int attr_inputs_match = + wt_status_attr_snapshot_matches(s) && + !clean_status_worktree_manifest_needs_refresh(istate); + + if (!refreshed_before_closure && attr_inputs_match) + return refresh_index( istate, refresh_flags, &s->pathspec, NULL, NULL); + if (refreshed_before_closure && attr_inputs_match) + return closure.refresh_result; + + wt_status_reset_attr_snapshot_if_changed(s); + if (fsmonitor_has_pending_token(istate)) + fsmonitor_reject_pending_token(istate); + untracked_cache_invalidate_all(istate); + fsmonitor_invalidate_semantics(istate); + closure.refresh_result |= refresh_index( + istate, refresh_flags, &s->pathspec, NULL, NULL); return closure.refresh_result; } @@ -1145,11 +1194,15 @@ static int wt_status_close_fsmonitor_token( !closure.untracked_ready) BUG("cannot close required untracked scan"); trace2_region_enter("status", "fsmonitor_token_closure", s->repo); + wt_status_reset_attr_snapshot_if_changed(s); + if (wt_status_refresh_invalidated_manifest(s)) + goto fallback; if (wt_status_close_ordinary_fsmonitor_token( &closure, refreshed_before_closure)) goto accepted; /* Keep the last valid token and fall back to complete scans. */ +fallback: wt_status_discard_staged_untracked(&closure); fsmonitor_reject_pending_token(istate); if (fstat_is_reliable()) { @@ -1170,10 +1223,20 @@ int wt_status_refresh_index(struct wt_status *s, unsigned int refresh_flags, int require_untracked) { + wt_status_begin_attr_snapshot(s); return wt_status_close_fsmonitor_token( s, refresh_flags, require_untracked, 0); } +static void wt_status_release_attr_snapshot(struct wt_status *s) +{ + if (s->attr_source_snapshot) + git_attr_source_snapshot_end(s->attr_source_snapshot); + attr_source_snapshot_free(s->attr_source_snapshot); + s->attr_source_snapshot = NULL; + s->attr_snapshot_failed = 0; +} + static int has_unmerged(struct wt_status *s) { int i; @@ -1237,11 +1300,7 @@ void wt_status_collect_free_buffers(struct wt_status *s) { untracked_cache_preload_release(s->untracked_cache_preload); s->untracked_cache_preload = NULL; - if (s->attr_source_snapshot) - git_attr_source_snapshot_end(s->attr_source_snapshot); - attr_source_snapshot_free(s->attr_source_snapshot); - s->attr_source_snapshot = NULL; - s->attr_snapshot_failed = 0; + wt_status_release_attr_snapshot(s); wt_status_state_free_buffers(&s->state); } From 95ef8d0f58bf715dfdac06a7d7ef931d5fe4c5be Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 13:44:44 -0500 Subject: [PATCH 082/105] status: adopt tracked history only after closing its proof An IPC provider cannot safely adopt missing semantic history merely because tracked entries are marked fsmonitor-valid. Minimal stat checks can conceal a content rewrite, and a clean token cannot retroactively certify workers started under different attributes. Capture the complete proof epoch before preparing semantic workers. Prime each worker's attribute frames and verify the starting token and complete epoch before hashing. After a clean closing query, apply the proof only if the pinned index, configuration, attribute content, manifest, worktree identity, and token remain consistent. Permit attribute-namespace bookkeeping to change only after its source bytes and initial namespace were verified. Accept tracked validity independently of untracked validity. Keep a query pending when the untracked scan has not run. Leave collapsed sparse indexes, pathspecs, ignored-file requests, unreliable file identity, non-IPC providers, and failed proofs on ordinary closure or complete refresh. Add scripted regressions for adopting missing tracked history without hiding a same-size rewrite and for preserving a collapsed sparse index on the ordinary closure path. Signed-off-by: Taylor Blau --- clean-status-epoch.c | 37 ++++++++- clean-status.h | 6 ++ fsmonitor-ll.h | 3 +- fsmonitor.c | 22 ++++-- semantic-verify-internal.h | 4 + semantic-verify-worker.c | 6 +- semantic-verify.c | 58 +++++++++++++- semantic-verify.h | 6 ++ t/t7527-builtin-fsmonitor.sh | 85 ++++++++++++++++++++ wt-status.c | 147 +++++++++++++++++++++++++++++++++-- 10 files changed, 354 insertions(+), 20 deletions(-) diff --git a/clean-status-epoch.c b/clean-status-epoch.c index af39eacf3f6610..1257a455bf94c1 100644 --- a/clean-status-epoch.c +++ b/clean-status-epoch.c @@ -129,9 +129,10 @@ int clean_status_proof_epoch_start_token_matches( istate->fsmonitor_last_update_pending); } -int clean_status_proof_epoch_matches( +static int proof_epoch_matches( struct index_state *istate, - const struct clean_status_proof_epoch *epoch) + const struct clean_status_proof_epoch *epoch, + int check_attr_namespace) { struct clean_status_state *state; struct attr_fingerprint attrs; @@ -155,8 +156,9 @@ int clean_status_proof_epoch_matches( goto done; if (attr_fingerprint_repository(istate->repo, &attrs) || memcmp(attrs.content_hash, epoch->attr_hash, algo->rawsz) || - memcmp(attrs.namespace_hash, epoch->attr_namespace_hash, - algo->rawsz) || + (check_attr_namespace && + memcmp(attrs.namespace_hash, epoch->attr_namespace_hash, + algo->rawsz)) || attrs.sources_present != epoch->attr_sources_present || memcmp(state->manifest.current_hash, epoch->manifest_hash, algo->rawsz) || @@ -170,6 +172,33 @@ int clean_status_proof_epoch_matches( return matched; } +int clean_status_proof_epoch_prime_matches( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch) +{ + int matched = + clean_status_proof_epoch_start_token_matches(istate, epoch) && + proof_epoch_matches(istate, epoch, 1); + + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/proof-epoch-primed", matched); + return matched; +} + +int clean_status_proof_epoch_matches( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch) +{ + return proof_epoch_matches(istate, epoch, 1); +} + +int clean_status_proof_epoch_content_matches( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch) +{ + return proof_epoch_matches(istate, epoch, 0); +} + void clean_status_release_proof_epoch( struct clean_status_proof_epoch *epoch) { diff --git a/clean-status.h b/clean-status.h index fd5212364a5e0c..488810ae6a7d11 100644 --- a/clean-status.h +++ b/clean-status.h @@ -28,9 +28,15 @@ struct clean_status_proof_epoch *clean_status_capture_proof_epoch( int clean_status_proof_epoch_start_token_matches( struct index_state *istate, const struct clean_status_proof_epoch *epoch); +int clean_status_proof_epoch_prime_matches( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch); int clean_status_proof_epoch_matches( struct index_state *istate, const struct clean_status_proof_epoch *epoch); +int clean_status_proof_epoch_content_matches( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch); void clean_status_release_proof_epoch( struct clean_status_proof_epoch *epoch); diff --git a/fsmonitor-ll.h b/fsmonitor-ll.h index 7e7564e5e2c493..01eb5d8e9856cf 100644 --- a/fsmonitor-ll.h +++ b/fsmonitor-ll.h @@ -69,7 +69,8 @@ int fsmonitor_has_pending_token(const struct index_state *istate); int fsmonitor_pending_token_from_provider(const struct index_state *istate); enum fsmonitor_token_result fsmonitor_query_pending_token( struct index_state *istate, int untracked_ready); -void fsmonitor_accept_pending_token(struct index_state *istate); +void fsmonitor_accept_pending_token(struct index_state *istate, + int untracked_ready); void fsmonitor_reject_pending_token(struct index_state *istate); void fsmonitor_mark_untracked_cache_valid(struct index_state *istate); diff --git a/fsmonitor.c b/fsmonitor.c index 0fd64efc2d8342..e7100bceba9c5f 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -1345,7 +1345,8 @@ enum fsmonitor_token_result fsmonitor_query_pending_token( return ret; } -void fsmonitor_accept_pending_token(struct index_state *istate) +void fsmonitor_accept_pending_token(struct index_state *istate, + int untracked_ready) { if (!fsmonitor_pending_token_from_provider(istate)) return; @@ -1354,13 +1355,24 @@ void fsmonitor_accept_pending_token(struct index_state *istate) istate->fsmonitor_last_update_pending = NULL; istate->fsmonitor_pending_token_from_provider = 0; istate->fsmonitor_token_valid = 1; - istate->fsmonitor_untracked_valid = 1; + istate->fsmonitor_untracked_valid = !!untracked_ready; if (istate->untracked) - istate->untracked->use_fsmonitor = 1; + istate->untracked->use_fsmonitor = !!untracked_ready; istate->cache_changed |= FSMONITOR_CHANGED; FREE_AND_NULL(istate->fsmonitor_untracked_token); - istate->fsmonitor_untracked_token = - xstrdup(istate->fsmonitor_last_update); + if (untracked_ready) + istate->fsmonitor_untracked_token = + xstrdup(istate->fsmonitor_last_update); + else { + /* + * Keep a query anchored at the accepted tracked token. A + * later in-process status may need to close work done after + * this point before validating its untracked cache. + */ + istate->fsmonitor_last_update_pending = + xstrdup(istate->fsmonitor_last_update); + istate->fsmonitor_pending_token_from_provider = 1; + } trace2_data_intmax("fsmonitor", istate->repo, "token_closure/accepted", 1); } diff --git a/semantic-verify-internal.h b/semantic-verify-internal.h index 44d7fa49bee013..de1003226b1ff2 100644 --- a/semantic-verify-internal.h +++ b/semantic-verify-internal.h @@ -30,6 +30,7 @@ struct attr_check; struct repository; +struct clean_status_proof_epoch; struct cache_entry; struct git_hash_algo; struct index_state; @@ -104,6 +105,7 @@ struct semantic_verify_worker { struct index_state *istate; struct semantic_verify_root *root; struct semantic_verify_result *results; + struct attr_check *check; size_t start; size_t end; struct semantic_verify_stat_update *updates; @@ -126,6 +128,7 @@ void semantic_verify_worker_run(struct semantic_verify_worker *worker); struct semantic_verify_proof { struct index_state *istate; struct semantic_verify_root *root; + struct clean_status_proof_epoch *epoch; struct semantic_verify_result *results; struct semantic_verify_entry_identity *entry_identities; struct semantic_verify_stat_update *stat_updates; @@ -141,6 +144,7 @@ struct semantic_verify_proof { size_t errors; size_t hardlinks; unsigned int namespace_unstable; + unsigned int epoch_required; }; #endif /* SEMANTIC_VERIFY_INTERNAL_H */ diff --git a/semantic-verify-worker.c b/semantic-verify-worker.c index 7bbfabcd7a4f5e..81c33ee59fabe2 100644 --- a/semantic-verify-worker.c +++ b/semantic-verify-worker.c @@ -56,10 +56,14 @@ void semantic_verify_worker_run(struct semantic_verify_worker *worker) { struct semantic_verify_path *path = semantic_verify_path_new(worker->root); - struct attr_check *check = convert_attrs_check_alloc(); + struct attr_check *check = worker->check; void *buffer = xmalloc(SEMANTIC_VERIFY_HASH_BUFFER_SIZE); size_t unstable_from = SIZE_MAX; + worker->check = NULL; + if (!check) + check = convert_attrs_check_alloc(); + for (size_t i = worker->start; i < worker->end; i++) { struct cache_entry *ce = worker->istate->cache[i]; struct semantic_verify_result *result = &worker->results[i]; diff --git a/semantic-verify.c b/semantic-verify.c index 0604ae6cde7d4a..347a8636d4ea3b 100644 --- a/semantic-verify.c +++ b/semantic-verify.c @@ -1,6 +1,8 @@ #define USE_THE_REPOSITORY_VARIABLE #include "git-compat-util.h" +#include "attr.h" +#include "clean-status.h" #include "convert.h" #include "fsmonitor.h" #include "object.h" @@ -88,6 +90,7 @@ int semantic_verify_prepare(struct index_state *istate, (uintmax_t)sizeof(struct semantic_verify_result)); CALLOC_ARRAY(proof, 1); proof->istate = istate; + proof->epoch_required = options && options->require_proof_epoch; proof->cache_nr = istate->cache_nr; CALLOC_ARRAY(proof->results, proof->cache_nr); CALLOC_ARRAY(proof->entry_identities, proof->cache_nr); @@ -105,7 +108,7 @@ int semantic_verify_prepare(struct index_state *istate, identity->flags = ce->ce_flags & SEMANTIC_VERIFY_ENTRY_FLAGS; } *proof_out = proof; - if (!proof->cache_nr) + if (!proof->cache_nr && !proof->epoch_required) return 0; if (istate->sparse_index != INDEX_EXPANDED) { for (size_t i = 0; i < proof->cache_nr; i++) { @@ -125,11 +128,48 @@ int semantic_verify_prepare(struct index_state *istate, proof->errors = proof->cache_nr; return -1; } + if (proof->epoch_required) { + proof->epoch = clean_status_capture_proof_epoch( + istate, options->attr_snapshot); + if (!proof->epoch) { + for (size_t i = 0; i < proof->cache_nr; i++) { + proof->results[i].kind = SEMANTIC_VERIFY_ERROR; + proof->results[i].error = EAGAIN; + } + proof->errors = proof->cache_nr; + return -1; + } + } + if (!proof->cache_nr) + return 0; /* Initialize conversion config and default attribute state serially. */ convert_attrs_prepare(istate); nr_threads = select_thread_count(proof->cache_nr, options); CALLOC_ARRAY(workers, nr_threads); + if (proof->epoch_required) { + /* + * Load each worker's system, global, root, and info + * attribute frames before closing the proof epoch. + */ + for (unsigned int i = 0; i < nr_threads; i++) { + workers[i].check = convert_attrs_check_alloc(); + git_check_attr(istate, "", workers[i].check); + } + if (!clean_status_proof_epoch_prime_matches( + istate, proof->epoch)) { + for (unsigned int i = 0; i < nr_threads; i++) + attr_check_free(workers[i].check); + free(workers); + for (size_t i = 0; i < proof->cache_nr; i++) { + proof->results[i].kind = SEMANTIC_VERIFY_ERROR; + proof->results[i].error = EAGAIN; + } + proof->errors = proof->cache_nr; + git_attr_invalidate_all(); + return -1; + } + } trace2_region_enter("semantic_verify", "prepare", istate->repo); trace2_data_intmax("semantic_verify", istate->repo, "threads", nr_threads); @@ -200,6 +240,16 @@ int semantic_verify_root_is_stable(const struct semantic_verify_proof *proof) return proof && semantic_verify_root_stable(proof->root); } +int semantic_verify_start_token_is_current( + struct index_state *istate, + const struct semantic_verify_proof *proof) +{ + return proof && proof->istate == istate && + (!proof->epoch_required || + clean_status_proof_epoch_start_token_matches( + istate, proof->epoch)); +} + void semantic_verify_get_stats(const struct semantic_verify_proof *proof, struct semantic_verify_stats *stats) { @@ -238,7 +288,10 @@ int semantic_verify_apply_after_closure( if (!istate || !proof || proof->istate != istate || proof->cache_nr != istate->cache_nr || proof->namespace_unstable || - !semantic_verify_root_is_stable(proof)) + !semantic_verify_root_is_stable(proof) || + (proof->epoch_required && + !clean_status_proof_epoch_content_matches( + istate, proof->epoch))) return -1; for (size_t i = 0; i < proof->cache_nr; i++) { @@ -333,6 +386,7 @@ void semantic_verify_proof_clear(struct semantic_verify_proof *proof) { if (!proof) return; + clean_status_release_proof_epoch(proof->epoch); semantic_verify_root_clear(proof->root); for (size_t i = 0; i < proof->cache_nr; i++) free(proof->entry_identities[i].name); diff --git a/semantic-verify.h b/semantic-verify.h index 3c67de0580b6dd..52b4fa5bc647d8 100644 --- a/semantic-verify.h +++ b/semantic-verify.h @@ -2,10 +2,13 @@ #define SEMANTIC_VERIFY_H struct index_state; +struct attr_source_snapshot; struct semantic_verify_proof; struct semantic_verify_options { unsigned int nr_threads; + const struct attr_source_snapshot *attr_snapshot; + unsigned int require_proof_epoch : 1; }; #define SEMANTIC_VERIFY_OPTIONS_INIT { 0 } @@ -58,6 +61,9 @@ int semantic_verify_apply_after_closure( const struct semantic_verify_proof *proof); int semantic_verify_root_is_stable( const struct semantic_verify_proof *proof); +int semantic_verify_start_token_is_current( + struct index_state *istate, + const struct semantic_verify_proof *proof); void semantic_verify_proof_clear(struct semantic_verify_proof *proof); /* Introspection used by the semantic verifier test helper. */ diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index e0077f40f00006..efd75c116391b4 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -1922,4 +1922,89 @@ test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ ) ' +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'tracked-only status adopts missing semantic history' ' + test_when_finished "rm -rf tracked-semantic-adoption" && + test_create_repo tracked-semantic-adoption && + ( + cd tracked-semantic-adoption && + sane_unset GIT_TEST_SPLIT_INDEX && + printf "aaaa\n" >tracked && + git add tracked && + git commit -m base && + git config core.trustctime false && + git config core.checkStat minimal && + git config core.untrackedCache true && + test-tool chmtime -60 tracked && + git update-index --refresh && + mtime=$(test-tool chmtime --get tracked) && + printf "bbbb\n" >tracked && + test-tool chmtime =$mtime tracked && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid tracked && + test_grep ! FSCF .git/index && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 --untracked-files=no \ + >.git/actual && + test_grep "^1 \.M .* tracked$" .git/actual && + test_trace2_data status fsmonitor_token/semantic-closed 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep FSCF .git/index + ) +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'collapsed sparse index uses ordinary token closure' ' + test_when_finished "rm -rf sparse-tracked-only" && + test_create_repo sparse-tracked-only && + ( + cd sparse-tracked-only && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir in outside && + printf "aaaa\n" >in/tracked && + printf "outside\n" >outside/file && + git add . && + git commit -m base && + git sparse-checkout set --cone --sparse-index in && + git config core.trustctime false && + git config core.checkStat minimal && + git config core.untrackedCache true && + test-tool chmtime =-60 in/tracked && + git update-index --refresh && + mtime=$(test-tool chmtime --get in/tracked) && + printf "bbbb\n" >in/tracked && + test-tool chmtime =$mtime in/tracked && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid in/tracked && + test_grep ! FSCF .git/index && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DDC \ + GIT_TEST_FSMONITOR_QUERY_PATH=in/tracked \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 --untracked-files=no \ + >.git/actual && + test_grep "^1 \.M .* in/tracked$" .git/actual && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/apply_count 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep FSCF .git/index && + git -c core.fsmonitor=false ls-files --sparse \ + >.git/sparse.after && + test_grep "^outside/$" .git/sparse.after + ) +' + test_done diff --git a/wt-status.c b/wt-status.c index dbb21d31a15999..ef6544b83896f3 100644 --- a/wt-status.c +++ b/wt-status.c @@ -29,6 +29,7 @@ #include "column.h" #include "read-cache.h" #include "setup.h" +#include "semantic-verify.h" #include "strbuf.h" #include "trace.h" #include "trace2.h" @@ -956,6 +957,39 @@ static int wt_status_collect_untracked_1( return used_untracked_cache; } +static struct semantic_verify_proof *wt_status_prepare_semantic_verify( + struct wt_status *s, int require_untracked) +{ + struct index_state *istate = s->repo->index; + struct semantic_verify_options options = SEMANTIC_VERIFY_OPTIONS_INIT; + struct semantic_verify_proof *proof = NULL; + int ret; + + if (!fstat_is_reliable() || istate->split_index || + require_untracked || + s->show_untracked_files != SHOW_NO_UNTRACKED_FILES || + s->show_ignored_mode || s->pathspec.nr || + fsm_settings__get_mode(s->repo) != FSMONITOR_MODE_IPC || + istate->sparse_index != INDEX_EXPANDED || + !fsmonitor_has_pending_token(istate) || + !fsmonitor_pending_token_from_provider(istate) || + !clean_status_fsmonitor_semantic_adoption_needed(istate)) + return NULL; + + options.require_proof_epoch = 1; + options.attr_snapshot = s->attr_source_snapshot; + trace2_region_enter("status", "semantic_verify", s->repo); + ret = semantic_verify_prepare(istate, &options, &proof); + trace2_data_intmax("status", s->repo, + "semantic_verify/prepared", !ret); + trace2_region_leave("status", "semantic_verify", s->repo); + if (ret) { + semantic_verify_proof_clear(proof); + return NULL; + } + return proof; +} + static int wt_status_collect_untracked(struct wt_status *s) { if (s->untracked_from_token_closure && !s->show_ignored_mode) @@ -969,6 +1003,7 @@ static int wt_status_collect_untracked(struct wt_status *s) struct wt_status_token_closure { struct wt_status *status; unsigned int refresh_flags; + int require_untracked; int can_prime; int untracked_ready; struct string_list staged_untracked; @@ -1050,6 +1085,19 @@ static void wt_status_reset_attr_snapshot_if_changed(struct wt_status *s) "semantic/attribute-epoch-rejected", 1); } +static void wt_status_discard_semantic_verify( + struct wt_status *s, struct semantic_verify_proof **proof, + const char *reason) +{ + if (!*proof) + return; + trace2_data_string("status", s->repo, "semantic_verify/discard", + reason); + semantic_verify_proof_clear(*proof); + *proof = NULL; + git_attr_invalidate_all(); +} + static void wt_status_refresh_for_token( struct wt_status *s, unsigned int refresh_flags, struct clean_status_proof_epoch **epoch, int *refresh_result) @@ -1110,13 +1158,15 @@ static int wt_status_close_ordinary_fsmonitor_token( wt_status_reset_attr_snapshot_if_changed(s); break; } - if (closure->untracked_ready) { + if (closure->untracked_ready || + !closure->require_untracked) { if (reliable_stat) clean_status_mark_fsmonitor_config_valid( istate, istate->fsmonitor_last_update_pending); clean_status_release_proof_epoch(scan_epoch); - fsmonitor_accept_pending_token(istate); + fsmonitor_accept_pending_token( + istate, closure->untracked_ready); return 1; } break; @@ -1149,17 +1199,80 @@ static int wt_status_close_ordinary_fsmonitor_token( return 0; } +enum wt_status_token_closure_result { + WT_STATUS_TOKEN_CLOSURE_FALLBACK = -1, + WT_STATUS_TOKEN_CLOSURE_RETRY, + WT_STATUS_TOKEN_CLOSURE_ACCEPTED, +}; + +static enum wt_status_token_closure_result +wt_status_close_semantic_fsmonitor_token( + struct wt_status_token_closure *closure, + struct semantic_verify_proof **proof) +{ + struct wt_status *s = closure->status; + struct index_state *istate = s->repo->index; + enum fsmonitor_token_result result; + int applied; + + if (!semantic_verify_start_token_is_current(istate, *proof)) { + wt_status_discard_semantic_verify( + s, proof, "start-token-drift"); + return WT_STATUS_TOKEN_CLOSURE_FALLBACK; + } + + closure->queries++; + result = fsmonitor_query_pending_token( + istate, closure->untracked_ready); + if (result != FSMONITOR_TOKEN_CLEAN) { + wt_status_discard_semantic_verify( + s, proof, "token-reset"); + if (fsmonitor_token_requires_rescan(result)) + return WT_STATUS_TOKEN_CLOSURE_RETRY; + return WT_STATUS_TOKEN_CLOSURE_FALLBACK; + } + + applied = semantic_verify_apply_after_closure(istate, *proof); + if (applied < 0) { + wt_status_reset_attr_snapshot_if_changed(s); + wt_status_discard_semantic_verify( + s, proof, "closure-drift"); + return WT_STATUS_TOKEN_CLOSURE_FALLBACK; + } + closure->refresh_result |= refresh_index( + istate, closure->refresh_flags, &s->pathspec, NULL, NULL); + trace2_data_intmax("status", s->repo, + "fsmonitor_token/semantic-closed", 1); + if (!wt_status_attr_snapshot_matches(s) || + clean_status_worktree_manifest_needs_refresh(istate)) { + wt_status_reset_attr_snapshot_if_changed(s); + wt_status_discard_semantic_verify( + s, proof, "attribute-drift"); + return WT_STATUS_TOKEN_CLOSURE_FALLBACK; + } + + clean_status_mark_fsmonitor_config_valid( + istate, istate->fsmonitor_last_update_pending); + semantic_verify_proof_clear(*proof); + *proof = NULL; + fsmonitor_accept_pending_token(istate, closure->untracked_ready); + return WT_STATUS_TOKEN_CLOSURE_ACCEPTED; +} + static int wt_status_close_fsmonitor_token( - struct wt_status *s, unsigned int refresh_flags, - int require_untracked, int refreshed_before_closure) + struct wt_status *s, struct semantic_verify_proof *proof, + unsigned int refresh_flags, int require_untracked, + int refreshed_before_closure) { struct index_state *istate = s->repo->index; struct wt_status_token_closure closure = { .status = s, .refresh_flags = refresh_flags, + .require_untracked = require_untracked, .staged_untracked = STRING_LIST_INIT_DUP, .staged_ignored = STRING_LIST_INIT_DUP, }; + enum wt_status_token_closure_result result; refresh_fsmonitor(istate); if (!fsmonitor_has_pending_token(istate) || s->pathspec.nr || @@ -1168,6 +1281,8 @@ static int wt_status_close_fsmonitor_token( wt_status_attr_snapshot_matches(s) && !clean_status_worktree_manifest_needs_refresh(istate); + wt_status_discard_semantic_verify( + s, &proof, "provider-unavailable"); if (!refreshed_before_closure && attr_inputs_match) return refresh_index( istate, refresh_flags, &s->pathspec, @@ -1197,12 +1312,26 @@ static int wt_status_close_fsmonitor_token( wt_status_reset_attr_snapshot_if_changed(s); if (wt_status_refresh_invalidated_manifest(s)) goto fallback; + + if (proof) { + result = wt_status_close_semantic_fsmonitor_token( + &closure, &proof); + if (result == WT_STATUS_TOKEN_CLOSURE_ACCEPTED) + goto accepted; + if (result == WT_STATUS_TOKEN_CLOSURE_FALLBACK) + goto fallback; + wt_status_reset_attr_snapshot_if_changed(s); + if (wt_status_refresh_invalidated_manifest(s)) + goto fallback; + } + if (wt_status_close_ordinary_fsmonitor_token( &closure, refreshed_before_closure)) goto accepted; /* Keep the last valid token and fall back to complete scans. */ fallback: + wt_status_discard_semantic_verify(s, &proof, "fallback"); wt_status_discard_staged_untracked(&closure); fsmonitor_reject_pending_token(istate); if (fstat_is_reliable()) { @@ -1223,9 +1352,13 @@ int wt_status_refresh_index(struct wt_status *s, unsigned int refresh_flags, int require_untracked) { + struct semantic_verify_proof *proof; + wt_status_begin_attr_snapshot(s); + refresh_fsmonitor(s->repo->index); + proof = wt_status_prepare_semantic_verify(s, require_untracked); return wt_status_close_fsmonitor_token( - s, refresh_flags, require_untracked, 0); + s, proof, refresh_flags, require_untracked, 0); } static void wt_status_release_attr_snapshot(struct wt_status *s) @@ -1258,7 +1391,7 @@ void wt_status_collect(struct wt_status *s) wt_status_finish_untracked_cache_preload(s); wt_status_begin_attr_snapshot(s); wt_status_close_fsmonitor_token( - s, REFRESH_QUIET | REFRESH_UNMERGED, + s, NULL, REFRESH_QUIET | REFRESH_UNMERGED, s->show_untracked_files != SHOW_NO_UNTRACKED_FILES && !s->show_ignored_mode, 1); @@ -1286,7 +1419,7 @@ void wt_status_collect(struct wt_status *s) (used_untracked_cache || !s->repo->index->untracked || !s->repo->index->untracked->root)) { if (fsmonitor_pending_token_from_provider(s->repo->index)) - fsmonitor_accept_pending_token(s->repo->index); + fsmonitor_accept_pending_token(s->repo->index, 1); else fsmonitor_reject_pending_token(s->repo->index); } From 62676196b61db5d30c4d92d25485224d5de01ef1 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Thu, 23 Jul 2026 23:46:57 -0500 Subject: [PATCH 083/105] wt-status: close tracked proofs before scanning untracked paths An untracked-cache preload can inspect cached excludes and directory state before tracked semantic adoption restores verified stat data. One provider response also cannot certify an untracked traversal performed after the tracked scan that response closes. Defer provider-backed untracked validation until the tracked proof has been applied and its first query has closed. Prime the untracked cache afterward, issue a second closing query, and recheck the full tracked proof before accepting either result. If the later query reports a change, invalidate both results, reprime during ordinary closure, and retry within the existing query bound. Factor the existing proof-current checks into the predicate used by proof application and deferred closure. Preserve automatic untracked preload when no provider is enabled or file identity is unreliable. Fall back to a complete scan if untracked validation or token closure fails. Add prerequisite-guarded scripted cases for successful deferred scans, failed untracked closure, and a change reported by the second closing query. Signed-off-by: Taylor Blau --- dir.c | 3 +- semantic-verify.c | 21 +++++--- semantic-verify.h | 3 ++ t/t7519-status-fsmonitor.sh | 75 ++++++++++++++++++++++++++--- t/t7527-builtin-fsmonitor.sh | 92 ++++++++++++++++++++++++++++++++++++ wt-status.c | 77 ++++++++++++++++++++++++++---- 6 files changed, 246 insertions(+), 25 deletions(-) diff --git a/dir.c b/dir.c index b2a4e4b2e5adc4..7e8b55638bdc8d 100644 --- a/dir.c +++ b/dir.c @@ -312,7 +312,8 @@ static void preload_fsmonitor_excludes_from_index( goto next; ce = preload->istate->cache[pos]; if (!S_ISREG(ce->ce_mode) || - !(ce->ce_flags & CE_FSMONITOR_VALID) || + (!(ce->ce_flags & CE_FSMONITOR_VALID) && + fstat_is_reliable()) || ce_skip_worktree(ce) || (ce->ce_flags & CE_REMOVE) || ce_intent_to_add(ce)) diff --git a/semantic-verify.c b/semantic-verify.c index 347a8636d4ea3b..7852dcf2ad630c 100644 --- a/semantic-verify.c +++ b/semantic-verify.c @@ -250,6 +250,19 @@ int semantic_verify_start_token_is_current( istate, proof->epoch)); } +int semantic_verify_proof_is_current( + struct index_state *istate, + const struct semantic_verify_proof *proof) +{ + return istate && proof && proof->istate == istate && + proof->cache_nr == istate->cache_nr && + !proof->namespace_unstable && + semantic_verify_root_is_stable(proof) && + (!proof->epoch_required || + clean_status_proof_epoch_content_matches( + istate, proof->epoch)); +} + void semantic_verify_get_stats(const struct semantic_verify_proof *proof, struct semantic_verify_stats *stats) { @@ -285,13 +298,7 @@ int semantic_verify_apply_after_closure( int poisoned = 0; size_t validated_updates = 0; - if (!istate || !proof || proof->istate != istate || - proof->cache_nr != istate->cache_nr || - proof->namespace_unstable || - !semantic_verify_root_is_stable(proof) || - (proof->epoch_required && - !clean_status_proof_epoch_content_matches( - istate, proof->epoch))) + if (!semantic_verify_proof_is_current(istate, proof)) return -1; for (size_t i = 0; i < proof->cache_nr; i++) { diff --git a/semantic-verify.h b/semantic-verify.h index 52b4fa5bc647d8..21d4acb6295158 100644 --- a/semantic-verify.h +++ b/semantic-verify.h @@ -64,6 +64,9 @@ int semantic_verify_root_is_stable( int semantic_verify_start_token_is_current( struct index_state *istate, const struct semantic_verify_proof *proof); +int semantic_verify_proof_is_current( + struct index_state *istate, + const struct semantic_verify_proof *proof); void semantic_verify_proof_clear(struct semantic_verify_proof *proof); /* Introspection used by the semantic verifier test helper. */ diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index c4cbd1adfb382b..002d5f1a83c1fc 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -594,7 +594,8 @@ prepare_builtin_closure_repo () { ) } -test_expect_success UNTRACKED_CACHE 'builtin clean closure publishes its proof' ' +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'builtin clean closure publishes its proof' ' test_when_finished "rm -rf builtin-closure-clean" && prepare_builtin_closure_repo builtin-closure-clean untracked && ( @@ -616,7 +617,8 @@ test_expect_success UNTRACKED_CACHE 'builtin clean closure publishes its proof' ) ' -test_expect_success 'builtin changed closure rescans before acceptance' ' +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'builtin changed closure rescans before acceptance' ' test_when_finished "rm -rf builtin-closure-changed" && prepare_builtin_closure_repo builtin-closure-changed && ( @@ -638,7 +640,8 @@ test_expect_success 'builtin changed closure rescans before acceptance' ' ) ' -test_expect_success 'builtin initial trivial response anchors a closure' ' +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'builtin initial trivial response anchors a closure' ' test_when_finished "rm -rf builtin-initial-trivial" && prepare_builtin_closure_repo builtin-initial-trivial && ( @@ -661,7 +664,8 @@ test_expect_success 'builtin initial trivial response anchors a closure' ' ) ' -test_expect_success 'builtin trivial closure can rescan and accept' ' +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'builtin trivial closure can rescan and accept' ' test_when_finished "rm -rf builtin-closure-trivial" && prepare_builtin_closure_repo builtin-closure-trivial && ( @@ -681,7 +685,8 @@ test_expect_success 'builtin trivial closure can rescan and accept' ' ) ' -test_expect_success 'builtin closure rejects three intervening changes' ' +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'builtin closure rejects three intervening changes' ' test_when_finished "rm -rf builtin-closure-exhausted" && prepare_builtin_closure_repo builtin-closure-exhausted && ( @@ -704,14 +709,15 @@ test_expect_success 'builtin closure rejects three intervening changes' ' ) ' -test_expect_success 'builtin closure query errors fall back completely' ' +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'builtin closure query errors fall back completely' ' test_when_finished "rm -rf builtin-closure-error" && prepare_builtin_closure_repo builtin-closure-error untracked && ( cd builtin-closure-error && sane_unset GIT_TEST_SPLIT_INDEX && test_write_lines visible >visible && - GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CE \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCE \ GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ git status --porcelain=v2 >.git/actual && test_grep "^? visible$" .git/actual && @@ -1419,4 +1425,59 @@ test_expect_success HARDLINKS,!MINGW,!CYGWIN \ ) ' +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'second closing-query change reprimes untracked cache' ' + test_when_finished "rm -rf second-query-changed" && + test_create_repo second-query-changed && + ( + cd second-query-changed && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir cached && + test_write_lines "*.ignored" >.gitignore && + test_write_lines "*.ignored" >cached/.gitignore && + printf "aaaa\n" >cached/tracked && + test_write_lines ignored >cached/junk.ignored && + git add .gitignore cached/.gitignore cached/tracked && + git commit -m base && + git config core.trustctime false && + git config core.checkStat minimal && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain=v2 \ + >.git/prime && + test_must_be_empty .git/prime && + test-tool chmtime =-60 cached/tracked && + git update-index --refresh && + mtime=$(test-tool chmtime --get cached/tracked) && + printf "bbbb\n" >cached/tracked && + test-tool chmtime =$mtime cached/tracked && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid cached/tracked && + test_grep ! FSCF .git/index && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCDC \ + GIT_TEST_FSMONITOR_QUERY_PATH=cached/tracked \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_line_count = 1 .git/actual && + test_grep "^1 \.M .* cached/tracked$" .git/actual && + test_trace2_data status fsmonitor_token/semantic-closed 1 \ + <.git/status.trace && + test_trace2_data status \ + fsmonitor_token/untracked-after-semantic 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/apply_count 1 \ + <.git/status.trace && + test_trace2_data status \ + fsmonitor_token/untracked-after-retry 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep FSCF .git/index && + test_grep FSUC .git/index + ) +' + test_done diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index efd75c116391b4..ec527140f59680 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -1883,6 +1883,98 @@ test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ ) ' +prepare_semantic_untracked_repo () { + r=$1 && + test_create_repo "$r" && + ( + cd "$r" && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir cached && + test_write_lines "*.ignored" >.gitignore && + test_write_lines "*.ignored" >cached/.gitignore && + printf "aaaa\n" >cached/tracked && + git add .gitignore cached/.gitignore cached/tracked && + git commit -m base && + git config core.trustctime false && + git config core.checkStat minimal && + git config core.untrackedCache true && + test_write_lines ignored >cached/junk.ignored && + git status --porcelain=v2 >.git/prime.actual && + test_must_be_empty .git/prime.actual && + test_grep UNTR .git/index && + test-tool chmtime =-60 cached/tracked && + git update-index --refresh && + mtime=$(test-tool chmtime --get cached/tracked) && + printf "bbbb\n" >cached/tracked && + test-tool chmtime =$mtime cached/tracked && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid cached/tracked && + test_grep FSMN .git/index && + test_grep ! FSCF .git/index + ) +} + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'semantic adoption closes the untracked scan' ' + test_when_finished "rm -rf semantic-untracked" && + prepare_semantic_untracked_repo semantic-untracked && + ( + cd semantic-untracked && + sane_unset GIT_TEST_SPLIT_INDEX && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_line_count = 1 .git/actual && + test_grep "^1 \.M .* cached/tracked$" .git/actual && + test_trace2_data status fsmonitor_token/untracked-deferred 1 \ + <.git/status.trace && + test_trace2_data status fsmonitor_token/semantic-closed 1 \ + <.git/status.trace && + test_trace2_data status \ + fsmonitor_token/untracked-after-semantic 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/apply_count \ + "[0-9][0-9]*" <.git/status.trace >.git/apply-count && + test_line_count = 2 .git/apply-count && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep FSCF .git/index && + test_grep FSUC .git/index + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'failed untracked closure discards semantic adoption' ' + test_when_finished "rm -rf failed-semantic-untracked" && + prepare_semantic_untracked_repo failed-semantic-untracked && + ( + cd failed-semantic-untracked && + sane_unset GIT_TEST_SPLIT_INDEX && + GIT_DISABLE_UNTRACKED_CACHE=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_line_count = 1 .git/actual && + test_grep "^1 \.M .* cached/tracked$" .git/actual && + test_trace2_data status fsmonitor_token/semantic-closed 1 \ + <.git/status.trace && + test_trace2_data status \ + fsmonitor_token/untracked-after-semantic 0 \ + <.git/status.trace && + test_trace2_data fsmonitor semantic/strong-invalidation 1 \ + <.git/status.trace >.git/strong-invalidations && + test_line_count = 2 .git/strong-invalidations && + test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep ! FSCF .git/index + ) +' + test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ 'tracked attribute events reopen semantic history' ' test_when_finished "rm -rf tracked-attr-change" && diff --git a/wt-status.c b/wt-status.c index ef6544b83896f3..2f23879edf3355 100644 --- a/wt-status.c +++ b/wt-status.c @@ -865,18 +865,36 @@ void wt_status_start_untracked_cache_preload(struct wt_status *s) if (s->untracked_cache_preload) BUG("untracked-cache preload already started"); wt_status_begin_attr_snapshot(s); + /* Record the provider token before either filesystem traversal. */ + refresh_fsmonitor(istate); if (s->pathspec.nr || s->show_untracked_files == SHOW_NO_UNTRACKED_FILES || s->show_ignored_mode) return; + dir_flags = wt_status_untracked_dir_flags(s); - if (has_fsmonitor) { + if (has_fsmonitor && + (!fsmonitor_has_pending_token(istate) || + !fstat_is_reliable())) { s->untracked_cache_preload = untracked_cache_preload_start_fsmonitor_excludes( istate, dir_flags); return; } + /* Restore verified stats before cached excludes inspect them. */ + if (fstat_is_reliable() && !istate->split_index && + fsm_settings__get_mode(s->repo) == FSMONITOR_MODE_IPC && + fsmonitor_pending_token_from_provider(istate) && + clean_status_fsmonitor_semantic_adoption_needed(istate) && + istate->untracked && istate->untracked->root) { + trace2_data_intmax("status", s->repo, + "fsmonitor_token/untracked-deferred", 1); + return; + } + if (has_fsmonitor) + return; + s->untracked_cache_preload = untracked_cache_preload_start_ordinary(istate, dir_flags); } @@ -958,7 +976,7 @@ static int wt_status_collect_untracked_1( } static struct semantic_verify_proof *wt_status_prepare_semantic_verify( - struct wt_status *s, int require_untracked) + struct wt_status *s) { struct index_state *istate = s->repo->index; struct semantic_verify_options options = SEMANTIC_VERIFY_OPTIONS_INIT; @@ -966,8 +984,6 @@ static struct semantic_verify_proof *wt_status_prepare_semantic_verify( int ret; if (!fstat_is_reliable() || istate->split_index || - require_untracked || - s->show_untracked_files != SHOW_NO_UNTRACKED_FILES || s->show_ignored_mode || s->pathspec.nr || fsm_settings__get_mode(s->repo) != FSMONITOR_MODE_IPC || istate->sparse_index != INDEX_EXPANDED || @@ -1138,8 +1154,14 @@ static int wt_status_close_ordinary_fsmonitor_token( istate, closure->refresh_flags, &s->pathspec, NULL, NULL); } - if (!closure->untracked_ready && closure->can_prime) + if (!closure->untracked_ready && closure->can_prime) { closure->untracked_ready = wt_status_stage_untracked(closure); + if (closure->queries) + trace2_data_intmax( + "status", s->repo, + "fsmonitor_token/untracked-after-retry", + closure->untracked_ready); + } while (closure->queries < FSMONITOR_TOKEN_MAX_QUERIES) { enum fsmonitor_token_result result; @@ -1213,6 +1235,8 @@ wt_status_close_semantic_fsmonitor_token( struct wt_status *s = closure->status; struct index_state *istate = s->repo->index; enum fsmonitor_token_result result; + int defer_untracked = + closure->can_prime && !closure->untracked_ready; int applied; if (!semantic_verify_start_token_is_current(istate, *proof)) { @@ -1221,9 +1245,10 @@ wt_status_close_semantic_fsmonitor_token( return WT_STATUS_TOKEN_CLOSURE_FALLBACK; } + /* The first query closes the tracked scan and its semantic proof. */ closure->queries++; result = fsmonitor_query_pending_token( - istate, closure->untracked_ready); + istate, defer_untracked ? 0 : closure->untracked_ready); if (result != FSMONITOR_TOKEN_CLEAN) { wt_status_discard_semantic_verify( s, proof, "token-reset"); @@ -1243,14 +1268,45 @@ wt_status_close_semantic_fsmonitor_token( istate, closure->refresh_flags, &s->pathspec, NULL, NULL); trace2_data_intmax("status", s->repo, "fsmonitor_token/semantic-closed", 1); - if (!wt_status_attr_snapshot_matches(s) || - clean_status_worktree_manifest_needs_refresh(istate)) { + if (!semantic_verify_proof_is_current(istate, *proof)) { wt_status_reset_attr_snapshot_if_changed(s); wt_status_discard_semantic_verify( - s, proof, "attribute-drift"); + s, proof, "closure-drift"); return WT_STATUS_TOKEN_CLOSURE_FALLBACK; } + if (defer_untracked) { + closure->untracked_ready = wt_status_stage_untracked(closure); + trace2_data_intmax( + "status", s->repo, + "fsmonitor_token/untracked-after-semantic", + closure->untracked_ready); + if (!closure->untracked_ready || + closure->queries >= FSMONITOR_TOKEN_MAX_QUERIES) + return WT_STATUS_TOKEN_CLOSURE_FALLBACK; + + /* A second query closes the subsequent untracked scan. */ + closure->queries++; + result = fsmonitor_query_pending_token( + istate, closure->untracked_ready); + if (result != FSMONITOR_TOKEN_CLEAN) { + untracked_cache_invalidate_all(istate); + fsmonitor_invalidate_semantics(istate); + closure->untracked_ready = 0; + wt_status_discard_semantic_verify( + s, proof, "token-reset"); + if (fsmonitor_token_requires_rescan(result)) + return WT_STATUS_TOKEN_CLOSURE_RETRY; + return WT_STATUS_TOKEN_CLOSURE_FALLBACK; + } + if (!semantic_verify_proof_is_current(istate, *proof)) { + wt_status_reset_attr_snapshot_if_changed(s); + wt_status_discard_semantic_verify( + s, proof, "closure-drift"); + return WT_STATUS_TOKEN_CLOSURE_FALLBACK; + } + } + clean_status_mark_fsmonitor_config_valid( istate, istate->fsmonitor_last_update_pending); semantic_verify_proof_clear(*proof); @@ -1301,6 +1357,7 @@ static int wt_status_close_fsmonitor_token( } closure.can_prime = require_untracked && + istate->untracked && istate->untracked->root && s->show_untracked_files != SHOW_NO_UNTRACKED_FILES && !s->show_ignored_mode; closure.untracked_ready = !istate->untracked || @@ -1356,7 +1413,7 @@ int wt_status_refresh_index(struct wt_status *s, wt_status_begin_attr_snapshot(s); refresh_fsmonitor(s->repo->index); - proof = wt_status_prepare_semantic_verify(s, require_untracked); + proof = wt_status_prepare_semantic_verify(s); return wt_status_close_fsmonitor_token( s, proof, refresh_flags, require_untracked, 0); } From d35bd74fc68db8a34bc0cd25eb6868cf4b0fdcb3 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 13:46:20 -0500 Subject: [PATCH 084/105] commit: close fsmonitor tokens across pre-commit hooks An as-is commit refreshes its index before running the pre-commit hook. If the hook rewrites a tracked path without changing its size or mtime, the later in-process status must not certify the earlier refresh as though it covered the hook. For a nonsplit index using an IPC provider, perform the initial refresh through status token closure. After an invoked hook, release the saved attribute snapshot and reopen the last accepted provider token before status runs again. Reject unavailable token state and invalidate the manifest, tracked semantics, and untracked cache before falling back to a complete refresh. Pin the post-hook named index before persisting strong invalidation. Write refreshed state only while its held descriptor, pathname, stored trailer checksum, and in-memory index still match. Preserve a hook-replaced index and the existing reread. Split indexes, platforms without reliable file identity, and non-IPC providers retain their original initial refresh. Add prerequisite-guarded scripted cases for successful post-hook closure without an untracked cache, failed closure with complete worktree refresh, and a hook that updates the index itself. Signed-off-by: Taylor Blau --- builtin/commit.c | 61 +++++++++++++++++++++--- fsmonitor-ll.h | 2 + fsmonitor.c | 15 ++++++ t/t7527-builtin-fsmonitor.sh | 92 +++++++++++++++++++++++++++++++++++- wt-status.c | 15 ++++++ wt-status.h | 1 + 6 files changed, 178 insertions(+), 8 deletions(-) diff --git a/builtin/commit.c b/builtin/commit.c index 29f339f89a2254..685959875418c2 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -14,12 +14,14 @@ #include "lockfile.h" #include "cache-tree.h" #include "clean-status.h" +#include "clean-status-index.h" #include "color.h" #include "dir.h" #include "editor.h" #include "environment.h" #include "diff.h" #include "commit.h" +#include "fsmonitor-settings.h" #include "add-interactive.h" #include "gettext.h" #include "revision.h" @@ -373,7 +375,8 @@ static void refresh_cache_or_die(int refresh_flags) } static const char *prepare_index(const char **argv, const char *prefix, - const struct commit *current_head, int is_status) + const struct commit *current_head, int is_status, + struct wt_status *s) { struct string_list partial = STRING_LIST_INIT_DUP; struct pathspec pathspec; @@ -506,7 +509,14 @@ static const char *prepare_index(const char **argv, const char *prefix, if (!only && !pathspec.nr) { repo_hold_locked_index(the_repository, &index_lock, LOCK_DIE_ON_ERROR); - refresh_cache_or_die(refresh_flags); + if (!fstat_is_reliable() || + the_repository->index->split_index || + fsm_settings__get_mode(the_repository) != + FSMONITOR_MODE_IPC) + refresh_cache_or_die(refresh_flags); + else if (wt_status_refresh_index( + s, refresh_flags | REFRESH_IN_PORCELAIN, 0)) + die_resolve_conflict("commit"); if (the_repository->index->cache_changed || !cache_tree_fully_valid(the_repository->index->cache_tree)) cache_tree_update(the_repository->index, WRITE_TREE_SILENT); @@ -797,13 +807,29 @@ static int prepare_to_commit(const char *index_file, const char *prefix, int clean_message_contents = (cleanup_mode != COMMIT_MSG_CLEANUP_NONE); int old_display_comment_prefix; int invoked_hook; + int hook_index_matches = 0; + struct clean_status_index_snapshot hook_index = { .fd = -1 }; /* This checks and barfs if author is badly specified */ determine_author_info(author_ident); - if (!no_verify && run_commit_hook(use_editor, index_file, &invoked_hook, - "pre-commit", NULL)) - return 0; + if (!no_verify) { + int hook_failed = run_commit_hook( + use_editor, index_file, &invoked_hook, + "pre-commit", NULL); + + if (invoked_hook && fstat_is_reliable()) + wt_status_invalidate_refresh(s); + if (invoked_hook && fstat_is_reliable() && + commit_style == COMMIT_AS_IS) + hook_index_matches = + !clean_status_index_snapshot_pin( + &hook_index, the_repository->index); + if (hook_failed) { + clean_status_index_snapshot_release(&hook_index); + return 0; + } + } if (squash_message) { /* @@ -1119,10 +1145,29 @@ static int prepare_to_commit(const char *index_file, const char *prefix, else fputs(_(empty_rebase_pick_advice), stderr); } + clean_status_index_snapshot_release(&hook_index); return 0; } if (!no_verify && invoked_hook) { + struct lock_file refresh_lock = LOCK_INIT; + + /* + * Preserve any strong invalidation recorded while status + * closed the post-hook token. The pinned source prevents this + * write from replacing an index updated by the hook. + */ + if (hook_index_matches && + repo_hold_locked_index(the_repository, &refresh_lock, 0) >= 0) { + if (clean_status_index_snapshot_still_matches( + &hook_index, the_repository->index)) + repo_update_index_if_able( + the_repository, &refresh_lock); + else + rollback_lock_file(&refresh_lock); + } + clean_status_index_snapshot_release(&hook_index); + /* * Re-read the index as the pre-commit-commit hook was invoked * and could have updated it. We must do this before we invoke @@ -1455,7 +1500,7 @@ static int dry_run_commit(const char **argv, const char *prefix, int committable; const char *index_file; - index_file = prepare_index(argv, prefix, current_head, 1); + index_file = prepare_index(argv, prefix, current_head, 1, s); committable = run_status(stdout, index_file, prefix, 0, s); rollback_index_files(); @@ -1871,7 +1916,7 @@ int cmd_commit(int argc, if (dry_run) return dry_run_commit(argv, prefix, current_head, &s); - index_file = prepare_index(argv, prefix, current_head, 0); + index_file = prepare_index(argv, prefix, current_head, 0, &s); /* Set up everything for writing the commit object. This includes running hooks, writing the trees, and interacting with the user. */ @@ -1881,6 +1926,7 @@ int cmd_commit(int argc, rollback_index_files(); goto cleanup; } + wt_status_collect_free_buffers(&s); /* Determine parents */ reflog_msg = getenv("GIT_REFLOG_ACTION"); @@ -2019,6 +2065,7 @@ int cmd_commit(int argc, NULL, NULL, NULL, NULL); cleanup: + wt_status_collect_free_buffers(&s); free_commit_extra_headers(extra); commit_list_free(parents); strbuf_release(&author_ident); diff --git a/fsmonitor-ll.h b/fsmonitor-ll.h index 01eb5d8e9856cf..7268bc18802b76 100644 --- a/fsmonitor-ll.h +++ b/fsmonitor-ll.h @@ -67,6 +67,8 @@ int fsmonitor_invalidate_attributes_path(struct index_state *istate, /* Close a provider token which was obtained before a required scan. */ int fsmonitor_has_pending_token(const struct index_state *istate); int fsmonitor_pending_token_from_provider(const struct index_state *istate); +/* Reopen the last accepted IPC token after an in-process operation. */ +int fsmonitor_reopen_token(struct index_state *istate); enum fsmonitor_token_result fsmonitor_query_pending_token( struct index_state *istate, int untracked_ready); void fsmonitor_accept_pending_token(struct index_state *istate, diff --git a/fsmonitor.c b/fsmonitor.c index e7100bceba9c5f..91ee5ccc33ae6f 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -1302,6 +1302,21 @@ int fsmonitor_pending_token_from_provider(const struct index_state *istate) istate->fsmonitor_pending_token_from_provider; } +int fsmonitor_reopen_token(struct index_state *istate) +{ + if (!fstat_is_reliable() || istate->split_index || + fsm_settings__get_mode(istate->repo) != FSMONITOR_MODE_IPC) + return 0; + if (istate->fsmonitor_last_update_pending) + return istate->fsmonitor_pending_token_from_provider; + if (!istate->fsmonitor_token_valid || !istate->fsmonitor_last_update) + return 0; + istate->fsmonitor_last_update_pending = + xstrdup(istate->fsmonitor_last_update); + istate->fsmonitor_pending_token_from_provider = 1; + return 1; +} + enum fsmonitor_token_result fsmonitor_query_pending_token( struct index_state *istate, int untracked_ready) { diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index ec527140f59680..0ff924e2828c9b 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -1893,7 +1893,9 @@ prepare_semantic_untracked_repo () { test_write_lines "*.ignored" >.gitignore && test_write_lines "*.ignored" >cached/.gitignore && printf "aaaa\n" >cached/tracked && - git add .gitignore cached/.gitignore cached/tracked && + printf "cccc\n" >cached/hook-tracked && + git add .gitignore cached/.gitignore cached/hook-tracked \ + cached/tracked && git commit -m base && git config core.trustctime false && git config core.checkStat minimal && @@ -1975,6 +1977,94 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ ) ' +test_expect_success UNTRACKED_CACHE,!MINGW,!CYGWIN \ + 'commit closes hook changes without an untracked cache' ' + test_when_finished "rm -rf commit-hook-closure" && + prepare_semantic_untracked_repo commit-hook-closure && + ( + cd commit-hook-closure && + sane_unset GIT_TEST_SPLIT_INDEX && + git config core.untrackedCache false && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --no-untracked-cache && + test_grep ! UNTR .git/index && + write_script .git/hooks/pre-commit <<-\EOF && + mtime=$(test-tool chmtime --get cached/hook-tracked) && + printf "dddd\n" >cached/hook-tracked && + test-tool chmtime =$mtime cached/hook-tracked + EOF + GIT_EDITOR=: \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCDC \ + GIT_TEST_FSMONITOR_QUERY_PATH=cached/hook-tracked \ + GIT_TRACE2_EVENT="$PWD/.git/commit.trace" \ + git commit --allow-empty --edit -m adoption && + test_trace2_data status fsmonitor_token/semantic-closed 1 \ + <.git/commit.trace && + test_trace2_data fsmonitor token_closure/apply_count "[1-9]" \ + <.git/commit.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/commit.trace >.git/accepted && + test_line_count = 2 .git/accepted && + test_grep FSCF .git/index && + test_grep ! FSUC .git/index && + GIT_OPTIONAL_LOCKS=0 git -c core.fsmonitor=false \ + -c core.untrackedCache=false status --porcelain=v2 \ + >.git/status.actual && + test_grep "^1 \.M .* cached/hook-tracked$" \ + .git/status.actual && + test_grep ! UNTR .git/index + ) +' + +test_expect_success UNTRACKED_CACHE,!MINGW,!CYGWIN \ + 'failed hook closure refreshes the worktree' ' + test_when_finished "rm -rf commit-hook-fallback" && + prepare_semantic_untracked_repo commit-hook-fallback && + ( + cd commit-hook-fallback && + sane_unset GIT_TEST_SPLIT_INDEX && + write_script .git/hooks/pre-commit <<-\EOF && + mtime=$(test-tool chmtime --get cached/hook-tracked) && + printf "dddd\n" >cached/hook-tracked && + test-tool chmtime =$mtime cached/hook-tracked + EOF + GIT_EDITOR=: \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCE \ + GIT_TRACE2_EVENT="$PWD/.git/commit.trace" \ + git commit --allow-empty --edit -m adoption && + test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/commit.trace && + test_trace2_data status count/changed 2 <.git/commit.trace && + test_grep "cached/hook-tracked$" .git/COMMIT_EDITMSG && + GIT_OPTIONAL_LOCKS=0 git -c core.fsmonitor=false \ + -c core.untrackedCache=false status --porcelain=v2 \ + >.git/status.actual && + test_grep "cached/hook-tracked$" .git/status.actual + ) +' + +test_expect_success UNTRACKED_CACHE,!MINGW,!CYGWIN \ + 'post-hook refresh preserves hook index updates' ' + test_when_finished "rm -rf commit-hook-index" && + prepare_semantic_untracked_repo commit-hook-index && + ( + cd commit-hook-index && + sane_unset GIT_TEST_SPLIT_INDEX && + write_script .git/hooks/pre-commit <<-\EOF && + printf "hook update\n" >cached/hook-tracked && + oid=$(git hash-object -w cached/hook-tracked) && + git update-index --cacheinfo \ + 100644,$oid,cached/hook-tracked + EOF + GIT_EDITOR=: \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git commit --allow-empty --edit -m adoption && + printf "hook update\n" >expect && + git show HEAD:cached/hook-tracked >actual && + test_cmp expect actual + ) +' + test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ 'tracked attribute events reopen semantic history' ' test_when_finished "rm -rf tracked-attr-change" && diff --git a/wt-status.c b/wt-status.c index 2f23879edf3355..03904a69bc085f 100644 --- a/wt-status.c +++ b/wt-status.c @@ -1427,6 +1427,21 @@ static void wt_status_release_attr_snapshot(struct wt_status *s) s->attr_snapshot_failed = 0; } +void wt_status_invalidate_refresh(struct wt_status *s) +{ + struct index_state *istate = s->repo->index; + + wt_status_release_attr_snapshot(s); + if (!s->pathspec.nr && !istate->split_index && + fsmonitor_reopen_token(istate)) + return; + if (fsmonitor_has_pending_token(istate)) + fsmonitor_reject_pending_token(istate); + clean_status_invalidate_current_manifest(istate); + untracked_cache_invalidate_all(istate); + fsmonitor_invalidate_semantics(istate); +} + static int has_unmerged(struct wt_status *s) { int i; diff --git a/wt-status.h b/wt-status.h index 74798dd593aacb..0f7104b4c6ac5f 100644 --- a/wt-status.h +++ b/wt-status.h @@ -168,6 +168,7 @@ void wt_status_start_untracked_cache_preload(struct wt_status *s); int wt_status_refresh_index(struct wt_status *s, unsigned int refresh_flags, int require_untracked); +void wt_status_invalidate_refresh(struct wt_status *s); /* * Collect all changes between the two trees. Changes will be displayed as if From fcc2ebd371c37a58583b4bc2bc65b254ed94d983 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 14:03:03 -0500 Subject: [PATCH 085/105] exclude: prove the contents of observed ignore sources A bulk untracked scan cannot reuse its result merely because an ignore file has familiar stat data. A file or its parent may be replaced while the scan runs, an absent source may appear, and repeated reads of the same source may observe different patterns. Record each source beneath its nearest available anchored parent, along with its path, symlink policy, presence, size, and blob identity. Check descriptor and parent identities while capturing an observation, then resolve the current parent again and compare the actual source bytes at validation. Coalesce equivalent observations and invalidate the proof immediately when observations conflict. Validation uses nonblocking opens, so replacing a source with a FIFO cannot hang. Equal contents remain acceptable even if the source or its parent has a different identity. This also preserves an empty /dev/null and an equivalent empty FIFO; changed or missing contents, unavailable anchored primitives, and failed parent callbacks invalidate the proof. Register the implementation and focused unit suite in both Make and Meson. The tests cover source and parent replacement, stable absence, repeated and conflicting observations, missing buffers, no-follow policy, /dev/null, FIFO replacement, and parent-opener failure. Signed-off-by: Taylor Blau --- Makefile | 2 + exclude-source-proof.c | 422 ++++++++++++++++++++++++++ exclude-source-proof.h | 37 +++ meson.build | 1 + t/meson.build | 1 + t/unit-tests/u-exclude-source-proof.c | 404 ++++++++++++++++++++++++ 6 files changed, 867 insertions(+) create mode 100644 exclude-source-proof.c create mode 100644 exclude-source-proof.h create mode 100644 t/unit-tests/u-exclude-source-proof.c diff --git a/Makefile b/Makefile index a261dfabb049b1..1c3d6e1520a9c1 100644 --- a/Makefile +++ b/Makefile @@ -1180,6 +1180,7 @@ LIB_OBJS += ewah/bitmap.o LIB_OBJS += ewah/ewah_bitmap.o LIB_OBJS += ewah/ewah_io.o LIB_OBJS += ewah/ewah_rlw.o +LIB_OBJS += exclude-source-proof.o LIB_OBJS += exec-cmd.o LIB_OBJS += fetch-negotiator.o LIB_OBJS += fetch-object-info.o @@ -1576,6 +1577,7 @@ CLAR_TEST_SUITES += u-clean-status-manifest CLAR_TEST_SUITES += u-ctype CLAR_TEST_SUITES += u-dir CLAR_TEST_SUITES += u-example-decorate +CLAR_TEST_SUITES += u-exclude-source-proof CLAR_TEST_SUITES += u-fsmonitor-attributes CLAR_TEST_SUITES += u-fsmonitor-clean-proof CLAR_TEST_SUITES += u-fsmonitor-response diff --git a/exclude-source-proof.c b/exclude-source-proof.c new file mode 100644 index 00000000000000..ec5194a1fa355e --- /dev/null +++ b/exclude-source-proof.c @@ -0,0 +1,422 @@ +#include "git-compat-util.h" +#include "exclude-source-proof.h" +#include "object-file.h" +#include "path-namespace.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "strmap.h" +#include "trace2.h" + +/* + * Each entry describes one path/policy observation. Filesystem identities + * are used only to make capture and validation coherent; the durable + * observation is the source's existence and bytes. + */ +struct exclude_source_proof_entry { + char *path; + size_t size; + struct object_id oid; + unsigned exists : 1; + unsigned nofollow : 1; +}; + +struct exclude_source_proof { + struct index_state *istate; + void *open_data; + exclude_source_open_parent_fn open_parent; + struct exclude_source_proof_entry *entries; + struct strintmap entries_by_path[2]; + size_t nr; + size_t alloc; + unsigned invalid : 1; +}; + +struct exclude_source_capture { + struct exclude_source_proof *proof; + char *path; + char *parent; + char *relative; + int parent_fd; + struct stat parent_stat; + unsigned nofollow : 1; +}; + +static char *source_parent(const char *path) +{ + const char *slash = strrchr(path, '/'); + + if (!slash) + return xstrdup("."); + if (slash == path) + return xstrdup("/"); + return xmemdupz(path, slash - path); +} + +static char *source_relative(const char *path, const char *parent) +{ + const char *relative; + size_t len; + + if (!strcmp(parent, ".")) + return xstrdup(path); + if (!strcmp(parent, "/")) { + relative = path + 1; + } else { + len = strlen(parent); + if (strncmp(path, parent, len) || path[len] != '/') + BUG("exclude source is not below its parent"); + relative = path + len + 1; + } + return xstrdup(*relative ? relative : "."); +} + +static int parent_up(char *parent) +{ + char *slash; + + if (!strcmp(parent, ".") || !strcmp(parent, "/")) + return 0; + slash = strrchr(parent, '/'); + if (!slash) { + parent[0] = '.'; + parent[1] = '\0'; + } else if (slash == parent) { + parent[1] = '\0'; + } else { + *slash = '\0'; + } + return 1; +} + +static int open_source_at(int parent_fd, const char *relative, int nofollow, + int nonblocking) +{ +#if EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN + int flags = O_RDONLY | O_CLOEXEC; + + if (nofollow) + flags |= O_NOFOLLOW; + if (nonblocking) + flags |= O_NONBLOCK; + return openat(parent_fd, relative, flags); +#else + (void)parent_fd; + (void)relative; + (void)nofollow; + (void)nonblocking; + errno = ENOSYS; + return -1; +#endif +} + +static int parent_identity_stable( + struct exclude_source_proof *proof, const char *parent, + int held_fd, const struct stat *expected) +{ + struct stat held, reopened; + int fd = proof->open_parent(proof->open_data, parent); + int stable = !fstat(held_fd, &held) && + fd >= 0 && !fstat(fd, &reopened) && + path_namespace_stat_equal(expected, &held) && + path_namespace_stat_equal(expected, &reopened); + + if (fd >= 0) + close(fd); + return stable; +} + +static int parent_stable(struct exclude_source_capture *capture) +{ + return parent_identity_stable( + capture->proof, capture->parent, capture->parent_fd, + &capture->parent_stat); +} + +static void capture_free(struct exclude_source_capture *capture) +{ + if (!capture) + return; + if (capture->parent_fd >= 0) + close(capture->parent_fd); + free(capture->path); + free(capture->parent); + free(capture->relative); + free(capture); +} + +static struct exclude_source_capture *capture_begin( + struct exclude_source_proof *proof, const char *path, + int nofollow, int invalidate) +{ + struct exclude_source_capture *capture; + + if (!proof || proof->invalid) + return NULL; + if (!path) { + if (invalidate) + proof->invalid = 1; + return NULL; + } + CALLOC_ARRAY(capture, 1); + capture->proof = proof; + capture->nofollow = nofollow; + capture->parent_fd = -1; + capture->path = xstrdup(path); + capture->parent = source_parent(path); + for (;;) { + capture->parent_fd = proof->open_parent(proof->open_data, + capture->parent); + if (capture->parent_fd >= 0) + break; + if (!is_missing_file_error(errno) || + !parent_up(capture->parent)) + break; + } + if (capture->parent_fd < 0 || + fstat(capture->parent_fd, &capture->parent_stat) || + !S_ISDIR(capture->parent_stat.st_mode)) { + if (invalidate) + proof->invalid = 1; + capture_free(capture); + return NULL; + } + capture->relative = source_relative(path, capture->parent); + return capture; +} + +struct exclude_source_proof *exclude_source_proof_create( + struct index_state *istate, void *open_data, + exclude_source_open_parent_fn open_parent) +{ + struct exclude_source_proof *proof; + + CALLOC_ARRAY(proof, 1); + proof->istate = istate; + proof->open_data = open_data; + proof->open_parent = open_parent; + strintmap_init_with_options(&proof->entries_by_path[0], -1, + NULL, 0); + strintmap_init_with_options(&proof->entries_by_path[1], -1, + NULL, 0); + if (!EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN || + !istate || !istate->repo || !istate->repo->hash_algo || + !open_parent) + proof->invalid = 1; + return proof; +} + +struct exclude_source_capture *exclude_source_capture_begin( + struct exclude_source_proof *proof, const char *path, + int nofollow) +{ + return capture_begin(proof, path, nofollow, 1); +} + +int exclude_source_capture_open(struct exclude_source_capture *capture) +{ + if (!capture) { + errno = EINVAL; + return -1; + } + return open_source_at(capture->parent_fd, capture->relative, + capture->nofollow, 0); +} + +int exclude_source_capture_absent(struct exclude_source_capture *capture) +{ +#if EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN + struct stat st; + + if (!capture) + return 0; + if (!fstatat(capture->parent_fd, capture->relative, &st, + AT_SYMLINK_NOFOLLOW)) + return 0; + return is_missing_file_error(errno); +#else + (void)capture; + return 0; +#endif +} + +static int source_matches(struct exclude_source_capture *capture, + const struct stat *expected) +{ + struct stat st; + int fd = open_source_at(capture->parent_fd, capture->relative, + capture->nofollow, 1); + int ret = fd >= 0 && !fstat(fd, &st) && + path_namespace_stat_equal(expected, &st); + + if (fd >= 0) + close(fd); + return ret; +} + +static int same_observation( + const struct exclude_source_proof_entry *entry, + int exists, size_t size, const struct object_id *oid) +{ + return entry->exists == exists && + (!exists || + (entry->size == size && oideq(&entry->oid, oid))); +} + +static void record_observation( + struct exclude_source_capture *capture, int exists, + size_t size, const struct object_id *oid) +{ + struct exclude_source_proof *proof = capture->proof; + struct strintmap *map = + &proof->entries_by_path[!!capture->nofollow]; + struct exclude_source_proof_entry *entry; + int index = strintmap_get(map, capture->path); + + if (index >= 0) { + if (!same_observation(&proof->entries[index], + exists, size, oid)) + proof->invalid = 1; + return; + } + + ALLOC_GROW(proof->entries, proof->nr + 1, proof->alloc); + entry = &proof->entries[proof->nr]; + memset(entry, 0, sizeof(*entry)); + entry->path = xstrdup(capture->path); + entry->nofollow = capture->nofollow; + entry->exists = exists; + if (exists) { + entry->size = size; + oidcpy(&entry->oid, oid); + } + strintmap_set(map, entry->path, proof->nr); + proof->nr++; +} + +void exclude_source_capture_record( + struct exclude_source_capture *capture, + int source_fd, + const struct stat *source_stat, + const void *buf, size_t size) +{ + struct exclude_source_proof *proof; + struct object_id oid; + struct stat final; + + if (!capture) + return; + proof = capture->proof; + if (proof->invalid) + return; + + if (!source_stat) { + if (!exclude_source_capture_absent(capture) || + !parent_stable(capture) || + !exclude_source_capture_absent(capture)) { + proof->invalid = 1; + return; + } + record_observation(capture, 0, 0, NULL); + return; + } + + if (source_fd < 0 || source_stat->st_size < 0 || + (!buf && size) || + xsize_t(source_stat->st_size) != size || + fstat(source_fd, &final) || + !path_namespace_stat_equal(source_stat, &final) || + !source_matches(capture, &final) || + !parent_stable(capture)) { + proof->invalid = 1; + return; + } + hash_object_file(proof->istate->repo->hash_algo, buf, size, + OBJ_BLOB, &oid); + record_observation(capture, 1, size, &oid); +} + +void exclude_source_capture_release(struct exclude_source_capture *capture) +{ + capture_free(capture); +} + +static int proof_entry_matches( + struct exclude_source_proof *proof, + const struct exclude_source_proof_entry *entry) +{ + struct exclude_source_capture *capture = + capture_begin(proof, entry->path, entry->nofollow, 0); + struct object_id oid; + struct stat before, after, final; + char *buf = NULL; + size_t size; + int fd = -1; + int ret = 0; + + if (!capture) + goto done; + if (!entry->exists) { + ret = exclude_source_capture_absent(capture) && + parent_stable(capture) && + exclude_source_capture_absent(capture); + goto done; + } + + fd = open_source_at(capture->parent_fd, capture->relative, + entry->nofollow, 1); + if (fd < 0 || fstat(fd, &before) || before.st_size < 0 || + xsize_t(before.st_size) != entry->size) + goto done; + size = entry->size; + buf = xmalloc(size ? size : 1); + if ((size_t)read_in_full(fd, buf, size) != size || + fstat(fd, &after) || + !path_namespace_stat_equal(&before, &after) || + !source_matches(capture, &after)) + goto done; + hash_object_file(proof->istate->repo->hash_algo, buf, size, + OBJ_BLOB, &oid); + if (!oideq(&oid, &entry->oid) || + !parent_stable(capture) || + fstat(fd, &final) || + !path_namespace_stat_equal(&after, &final) || + !source_matches(capture, &final)) + goto done; + ret = 1; +done: + free(buf); + if (fd >= 0) + close(fd); + capture_free(capture); + return ret; +} + +int exclude_source_proof_validate(struct exclude_source_proof *proof) +{ + int valid; + + if (!proof) + return 0; + valid = !proof->invalid; + for (size_t i = 0; valid && i < proof->nr; i++) + valid = proof_entry_matches(proof, &proof->entries[i]); + if (proof->istate && proof->istate->repo) { + trace2_data_intmax("exclude", proof->istate->repo, + "proof_entries", proof->nr); + trace2_data_intmax("exclude", proof->istate->repo, + "proof_valid", valid); + } + return valid; +} + +void exclude_source_proof_release(struct exclude_source_proof *proof) +{ + if (!proof) + return; + strintmap_clear(&proof->entries_by_path[0]); + strintmap_clear(&proof->entries_by_path[1]); + for (size_t i = 0; i < proof->nr; i++) + free(proof->entries[i].path); + free(proof->entries); + free(proof); +} diff --git a/exclude-source-proof.h b/exclude-source-proof.h new file mode 100644 index 00000000000000..1949afd3929a1c --- /dev/null +++ b/exclude-source-proof.h @@ -0,0 +1,37 @@ +#ifndef EXCLUDE_SOURCE_PROOF_H +#define EXCLUDE_SOURCE_PROOF_H + +#if (defined(__APPLE__) || defined(__linux__)) && \ + defined(O_CLOEXEC) && defined(O_NONBLOCK) && \ + defined(O_NOFOLLOW) && defined(O_DIRECTORY) && \ + defined(AT_SYMLINK_NOFOLLOW) +#define EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN 1 +#else +#define EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN 0 +#endif + +struct exclude_source_capture; +struct exclude_source_proof; +struct index_state; +struct stat; + +typedef int (*exclude_source_open_parent_fn)(void *data, const char *path); + +struct exclude_source_proof *exclude_source_proof_create( + struct index_state *istate, void *open_data, + exclude_source_open_parent_fn open_parent); +struct exclude_source_capture *exclude_source_capture_begin( + struct exclude_source_proof *proof, const char *path, + int nofollow); +int exclude_source_capture_open(struct exclude_source_capture *capture); +int exclude_source_capture_absent(struct exclude_source_capture *capture); +void exclude_source_capture_record( + struct exclude_source_capture *capture, + int source_fd, + const struct stat *source_stat, + const void *buf, size_t size); +void exclude_source_capture_release(struct exclude_source_capture *capture); +int exclude_source_proof_validate(struct exclude_source_proof *proof); +void exclude_source_proof_release(struct exclude_source_proof *proof); + +#endif /* EXCLUDE_SOURCE_PROOF_H */ diff --git a/meson.build b/meson.build index 1ac2402eec2a39..63994b45cdf02f 100644 --- a/meson.build +++ b/meson.build @@ -377,6 +377,7 @@ libgit_sources = [ 'editor.c', 'entry.c', 'environment.c', + 'exclude-source-proof.c', 'ewah/bitmap.c', 'ewah/ewah_bitmap.c', 'ewah/ewah_io.c', diff --git a/t/meson.build b/t/meson.build index d713115cc64b93..ea4a0d2c11380c 100644 --- a/t/meson.build +++ b/t/meson.build @@ -9,6 +9,7 @@ clar_test_suites = [ 'unit-tests/u-ctype.c', 'unit-tests/u-dir.c', 'unit-tests/u-example-decorate.c', + 'unit-tests/u-exclude-source-proof.c', 'unit-tests/u-fsmonitor-attributes.c', 'unit-tests/u-fsmonitor-clean-proof.c', 'unit-tests/u-fsmonitor-response.c', diff --git a/t/unit-tests/u-exclude-source-proof.c b/t/unit-tests/u-exclude-source-proof.c new file mode 100644 index 00000000000000..26d5f16e6f2e9c --- /dev/null +++ b/t/unit-tests/u-exclude-source-proof.c @@ -0,0 +1,404 @@ +#include "unit-test.h" + +#include "dir.h" +#include "exclude-source-proof.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "wrapper.h" + +#if EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN + +static struct repository repo = { + .hash_algo = &hash_algos[GIT_HASH_SHA1], +}; +static struct index_state istate = { + .repo = &repo, +}; +static char *trash; +static int fail_open_parent; + +static int open_parent(void *data UNUSED, const char *path) +{ + if (fail_open_parent) { + errno = EACCES; + return -1; + } + return open(path, O_RDONLY | O_DIRECTORY | O_CLOEXEC); +} + +static struct exclude_source_proof *new_proof(void) +{ + return exclude_source_proof_create( + &istate, NULL, open_parent); +} + +static char *make_path(const char *name) +{ + struct strbuf path = STRBUF_INIT; + + strbuf_addf(&path, "%s/%s", trash, name); + return strbuf_detach(&path, NULL); +} + +static void record_file(struct exclude_source_proof *proof, const char *path) +{ + struct exclude_source_capture *capture = + exclude_source_capture_begin(proof, path, 0); + struct stat before, after; + char *buf; + size_t size; + ssize_t read_size; + int fd; + + cl_assert(capture != NULL); + fd = exclude_source_capture_open(capture); + cl_must_pass(fd); + cl_must_pass(fstat(fd, &before)); + cl_assert(before.st_size >= 0); + size = xsize_t(before.st_size); + buf = xmalloc(size ? size : 1); + read_size = read_in_full(fd, buf, size); + cl_assert(read_size >= 0 && (size_t)read_size == size); + cl_must_pass(fstat(fd, &after)); + exclude_source_capture_record(capture, fd, &after, buf, size); + exclude_source_capture_release(capture); + free(buf); + cl_must_pass(close(fd)); +} + +static void record_absence(struct exclude_source_proof *proof, + const char *path) +{ + struct exclude_source_capture *capture = + exclude_source_capture_begin(proof, path, 0); + + cl_assert(capture != NULL); + cl_assert(exclude_source_capture_absent(capture)); + exclude_source_capture_record(capture, -1, NULL, NULL, 0); + exclude_source_capture_release(capture); +} + +void test_exclude_source_proof__initialize(void) +{ + char template[] = "/tmp/exclude-source-proof-XXXXXX"; + + fail_open_parent = 0; + cl_assert(mkdtemp(template) != NULL); + trash = xstrdup(template); +} + +void test_exclude_source_proof__cleanup(void) +{ + struct strbuf path = STRBUF_INIT; + + strbuf_addstr(&path, trash); + cl_must_pass(remove_dir_recursively( + &path, REMOVE_DIR_PURGE_ORIGINAL_CWD)); + strbuf_release(&path); + FREE_AND_NULL(trash); +} + +void test_exclude_source_proof__accepts_same_content_replacement(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(proof, source); + cl_assert(exclude_source_proof_validate(proof)); + cl_must_pass(unlink(source)); + write_file_buf(source, "content", 7); + cl_assert(exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__rejects_different_content_replacement(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(proof, source); + cl_assert(exclude_source_proof_validate(proof)); + cl_must_pass(unlink(source)); + cl_assert(!exclude_source_proof_validate(proof)); + write_file_buf(source, "changed", 7); + cl_assert(!exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__accepts_repeated_observation(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(proof, source); + record_file(proof, source); + cl_assert(exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__rejects_missing_source_buffer(void) +{ + struct exclude_source_proof *proof = new_proof(); + struct exclude_source_capture *capture; + struct stat st; + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + int fd; + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + capture = exclude_source_capture_begin(proof, source, 0); + cl_assert(capture != NULL); + fd = exclude_source_capture_open(capture); + cl_must_pass(fd); + cl_must_pass(fstat(fd, &st)); + exclude_source_capture_record(capture, fd, &st, NULL, 7); + cl_assert(!exclude_source_proof_validate(proof)); + + cl_must_pass(close(fd)); + exclude_source_capture_release(capture); + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__rejects_conflicting_observations(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(proof, source); + write_file_buf(source, "changed", 7); + record_file(proof, source); + cl_assert(!exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__rejects_open_failure(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(proof, source); + fail_open_parent = 1; + cl_assert(!exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__fails_closed_without_parent_opener(void) +{ + struct exclude_source_proof *proof = + exclude_source_proof_create(&istate, NULL, NULL); + + cl_assert(!exclude_source_capture_begin(proof, "/dev/null", 0)); + cl_assert(!exclude_source_proof_validate(proof)); + exclude_source_proof_release(proof); +} + +void test_exclude_source_proof__honors_nofollow(void) +{ + struct exclude_source_proof *proof = new_proof(); + struct exclude_source_capture *capture; + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + char *target = make_path("parent/target"); + int fd; + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(target, "content", 7); + cl_must_pass(symlink("target", source)); + + capture = exclude_source_capture_begin(proof, source, 0); + cl_assert(capture != NULL); + fd = exclude_source_capture_open(capture); + cl_must_pass(fd); + cl_must_pass(close(fd)); + exclude_source_capture_release(capture); + + capture = exclude_source_capture_begin(proof, source, 1); + cl_assert(capture != NULL); + fd = exclude_source_capture_open(capture); + cl_assert(fd < 0 && errno == ELOOP); + exclude_source_capture_release(capture); + + exclude_source_proof_release(proof); + free(target); + free(source); + free(parent); +} + +void test_exclude_source_proof__opens_directory_sources(void) +{ + struct exclude_source_proof *proof = new_proof(); + struct exclude_source_capture *capture; + struct stat st; + char *parent = make_path("parent"); + char *source = make_path("parent/source/"); + int fd; + + cl_must_pass(mkdir(parent, 0700)); + cl_must_pass(mkdir(source, 0700)); + capture = exclude_source_capture_begin(proof, source, 0); + cl_assert(capture != NULL); + fd = exclude_source_capture_open(capture); + cl_must_pass(fd); + cl_must_pass(fstat(fd, &st)); + cl_assert(S_ISDIR(st.st_mode)); + cl_must_pass(close(fd)); + exclude_source_capture_release(capture); + + capture = exclude_source_capture_begin(proof, "/", 0); + cl_assert(capture != NULL); + fd = exclude_source_capture_open(capture); + cl_must_pass(fd); + cl_must_pass(fstat(fd, &st)); + cl_assert(S_ISDIR(st.st_mode)); + cl_must_pass(close(fd)); + exclude_source_capture_release(capture); + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__accepts_same_content_parent_replacement(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *old_parent = make_path("old-parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(proof, source); + cl_must_pass(rename(parent, old_parent)); + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + cl_assert(exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(old_parent); + free(parent); +} + +void test_exclude_source_proof__reresolves_absent_source_parent(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *missing = make_path("parent/missing"); + char *source = make_path("parent/missing/source"); + + cl_must_pass(mkdir(parent, 0700)); + cl_must_pass(mkdir(missing, 0700)); + record_absence(proof, source); + cl_must_pass(rmdir(missing)); + cl_assert(exclude_source_proof_validate(proof)); + cl_must_pass(mkdir(missing, 0700)); + cl_assert(exclude_source_proof_validate(proof)); + write_file_buf(source, "content", 7); + cl_assert(!exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(missing); + free(parent); +} + +void test_exclude_source_proof__accepts_dev_null(void) +{ + struct exclude_source_proof *proof = new_proof(); + + record_file(proof, "/dev/null"); + cl_assert(exclude_source_proof_validate(proof)); + exclude_source_proof_release(proof); +} + +void test_exclude_source_proof__accepts_empty_fifo_replacement(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "", 0); + record_file(proof, source); + cl_must_pass(unlink(source)); + cl_must_pass(mkfifo(source, 0600)); + cl_assert(exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__rejects_nonempty_fifo_replacement(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(proof, source); + cl_must_pass(unlink(source)); + cl_must_pass(mkfifo(source, 0600)); + cl_assert(!exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +#else + +#define EMPTY_TEST(name) void name(void) {} +#define SKIP_TEST(name) void name(void) { cl_skip(); } + +EMPTY_TEST(test_exclude_source_proof__initialize) +EMPTY_TEST(test_exclude_source_proof__cleanup) +SKIP_TEST(test_exclude_source_proof__accepts_same_content_replacement) +SKIP_TEST(test_exclude_source_proof__rejects_different_content_replacement) +SKIP_TEST(test_exclude_source_proof__accepts_repeated_observation) +SKIP_TEST(test_exclude_source_proof__rejects_missing_source_buffer) +SKIP_TEST(test_exclude_source_proof__rejects_conflicting_observations) +SKIP_TEST(test_exclude_source_proof__rejects_open_failure) +SKIP_TEST(test_exclude_source_proof__fails_closed_without_parent_opener) +SKIP_TEST(test_exclude_source_proof__honors_nofollow) +SKIP_TEST(test_exclude_source_proof__opens_directory_sources) +SKIP_TEST(test_exclude_source_proof__accepts_same_content_parent_replacement) +SKIP_TEST(test_exclude_source_proof__reresolves_absent_source_parent) +SKIP_TEST(test_exclude_source_proof__accepts_dev_null) +SKIP_TEST(test_exclude_source_proof__accepts_empty_fifo_replacement) +SKIP_TEST(test_exclude_source_proof__rejects_nonempty_fifo_replacement) + +#endif From b18020cf6c9ac5baee3080aa0db4b2f7023848fa Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 14:03:31 -0500 Subject: [PATCH 086/105] dir: capture ignore sources beneath anchored parents The ordinary exclude reader opens configured, repository, and per-directory ignore files by pathname. That is sufficient for a one-off walk, but a concurrent replacement or newly created ignore file makes a retained bulk result unsafe. Attach the optional source proof from S13/P01 to dir_struct and capture the exact bytes or stable absence observed by add_patterns(). Preserve symlink-following for standard excludes and the existing no-follow policy for per-directory .gitignore files. Visit configured and repository sources even when absent so their later appearance invalidates the proof. Mark failed, oversized, short, and index-backed reads unprovable rather than treating their results as stable filesystem observations. Existing callers without a proof retain their original opens, error handling, pattern parsing, and oversized-source guard. Signed-off-by: Taylor Blau --- dir.c | 72 +++++++++++++++++++++++++++++++++--------- dir.h | 7 ++++ exclude-source-proof.c | 6 ++++ exclude-source-proof.h | 1 + 4 files changed, 71 insertions(+), 15 deletions(-) diff --git a/dir.c b/dir.c index 8b4d4cab260ed3..4e2e474e083871 100644 --- a/dir.c +++ b/dir.c @@ -15,6 +15,7 @@ #include "convert.h" #include "dir.h" #include "environment.h" +#include "exclude-source-proof.h" #include "gettext.h" #include "name-hash.h" #include "object-file.h" @@ -2017,40 +2018,63 @@ static void invalidate_directory(struct untracked_cache *uc, */ static int add_patterns(const char *fname, const char *base, int baselen, struct pattern_list *pl, struct index_state *istate, - unsigned flags, struct oid_stat *oid_stat) + unsigned flags, struct oid_stat *oid_stat, + struct exclude_source_proof *source_proof) { + struct exclude_source_capture *capture = + exclude_source_capture_begin(source_proof, fname, + !!(flags & PATTERN_NOFOLLOW)); struct stat st; int r; int fd; size_t size = 0; char *buf; - if (flags & PATTERN_NOFOLLOW) + if (capture) + fd = exclude_source_capture_open(capture); + else if (flags & PATTERN_NOFOLLOW) fd = open_nofollow(fname, O_RDONLY); else fd = open(fname, O_RDONLY); if (fd < 0 || fstat(fd, &st) < 0) { - if (fd < 0) + if (fd < 0) { warn_on_fopen_errors(fname); - else + if (capture && exclude_source_capture_absent(capture)) + exclude_source_capture_record(capture, -1, NULL, + NULL, 0); + else + exclude_source_capture_error(capture); + } else { + exclude_source_capture_error(capture); close(fd); - if (!istate) + } + if (!istate) { + exclude_source_capture_release(capture); return -1; + } r = read_skip_worktree_file_from_index(istate, fname, &size, &buf, oid_stat); + if (r == 1) + exclude_source_capture_error(capture); + exclude_source_capture_release(capture); + capture = NULL; if (r != 1) return r; } else { size = xsize_t(st.st_size); if (size > PATTERN_MAX_FILE_SIZE) { + exclude_source_capture_error(capture); + exclude_source_capture_release(capture); warning("ignoring excessively large pattern file: %s", fname); close(fd); return -1; } if (size == 0) { + exclude_source_capture_record(capture, fd, &st, NULL, 0); + exclude_source_capture_release(capture); if (oid_stat) { fill_stat_data(&oid_stat->stat, &st); oidcpy(&oid_stat->oid, the_hash_algo->empty_blob); @@ -2061,10 +2085,15 @@ static int add_patterns(const char *fname, const char *base, int baselen, } buf = xmallocz(size); if (read_in_full(fd, buf, size) != size) { + exclude_source_capture_error(capture); + exclude_source_capture_release(capture); free(buf); close(fd); return -1; } + exclude_source_capture_record(capture, fd, &st, buf, size); + exclude_source_capture_release(capture); + capture = NULL; buf[size++] = '\n'; close(fd); if (oid_stat) { @@ -2137,7 +2166,8 @@ int add_patterns_from_file_to_list(const char *fname, const char *base, struct index_state *istate, unsigned flags) { - return add_patterns(fname, base, baselen, pl, istate, flags, NULL); + return add_patterns(fname, base, baselen, pl, istate, flags, NULL, + NULL); } int add_patterns_from_blob_to_list( @@ -2182,10 +2212,11 @@ struct pattern_list *add_pattern_list(struct dir_struct *dir, /* * Used to set up core.excludesfile and .git/info/exclude lists. */ -static void add_patterns_from_file_1(struct dir_struct *dir, const char *fname, - struct oid_stat *oid_stat) +static int add_patterns_from_file_1(struct dir_struct *dir, const char *fname, + struct oid_stat *oid_stat, int gentle) { struct pattern_list *pl; + int ret; /* * catch setup_standard_excludes() that's called before * dir->untracked is assigned. That function behaves @@ -2194,14 +2225,17 @@ static void add_patterns_from_file_1(struct dir_struct *dir, const char *fname, if (!dir->untracked) dir->internal.unmanaged_exclude_files++; pl = add_pattern_list(dir, EXC_FILE, fname); - if (add_patterns(fname, "", 0, pl, NULL, 0, oid_stat) < 0) + ret = add_patterns(fname, "", 0, pl, NULL, 0, oid_stat, + dir->internal.exclude_source_proof); + if (ret < 0 && !gentle) die(_("cannot use %s as an exclude file"), fname); + return ret; } void add_patterns_from_file(struct dir_struct *dir, const char *fname) { dir->internal.unmanaged_exclude_files++; /* see validate_untracked_cache() */ - add_patterns_from_file_1(dir, fname, NULL); + add_patterns_from_file_1(dir, fname, NULL, 0); } int match_basename(const char *basename, int basenamelen, @@ -2651,7 +2685,8 @@ static void prep_exclude(struct dir_struct *dir, pl->src = strbuf_detach(&sb, NULL); if (add_patterns(pl->src, pl->src, stk->baselen, pl, istate, PATTERN_NOFOLLOW, - untracked ? &oid_stat : NULL) < 0 && + untracked ? &oid_stat : NULL, + dir->internal.exclude_source_proof) < 0 && untracked && is_null_oid(&oid_stat.oid)) { struct stat st; @@ -4453,16 +4488,23 @@ void setup_standard_excludes(struct dir_struct *dir) dir->exclude_per_dir = ".gitignore"; /* core.excludesfile defaulting to $XDG_CONFIG_HOME/git/ignore */ - if (excludes_file && !access_or_warn(excludes_file, R_OK, 0)) + if (excludes_file && + (dir->internal.exclude_source_proof || + !access_or_warn(excludes_file, R_OK, 0))) add_patterns_from_file_1(dir, excludes_file, - dir->untracked ? &dir->internal.ss_excludes_file : NULL); + dir->untracked ? + &dir->internal.ss_excludes_file : NULL, + !!dir->internal.exclude_source_proof); /* per repository user preference */ if (startup_info->have_repository) { const char *path = git_path_info_exclude(); - if (!access_or_warn(path, R_OK, 0)) + if (dir->internal.exclude_source_proof || + !access_or_warn(path, R_OK, 0)) add_patterns_from_file_1(dir, path, - dir->untracked ? &dir->internal.ss_info_exclude : NULL); + dir->untracked ? + &dir->internal.ss_info_exclude : NULL, + !!dir->internal.exclude_source_proof); } } diff --git a/dir.h b/dir.h index f6df0b54d271e9..23eed870a0e235 100644 --- a/dir.h +++ b/dir.h @@ -7,6 +7,7 @@ #include "statinfo.h" #include "strbuf.h" +struct exclude_source_proof; struct repository; /** @@ -364,6 +365,12 @@ struct dir_struct { unsigned visited_paths; unsigned visited_directories; unsigned untracked_cache_preloaded : 1; + + /* + * Optional borrowed proof that covers every exclusion source + * consulted by this traversal. + */ + struct exclude_source_proof *exclude_source_proof; } internal; }; diff --git a/exclude-source-proof.c b/exclude-source-proof.c index ec5194a1fa355e..022aa5a7ba1d5a 100644 --- a/exclude-source-proof.c +++ b/exclude-source-proof.c @@ -335,6 +335,12 @@ void exclude_source_capture_record( record_observation(capture, 1, size, &oid); } +void exclude_source_capture_error(struct exclude_source_capture *capture) +{ + if (capture) + capture->proof->invalid = 1; +} + void exclude_source_capture_release(struct exclude_source_capture *capture) { capture_free(capture); diff --git a/exclude-source-proof.h b/exclude-source-proof.h index 1949afd3929a1c..e2932f535fb7a4 100644 --- a/exclude-source-proof.h +++ b/exclude-source-proof.h @@ -30,6 +30,7 @@ void exclude_source_capture_record( int source_fd, const struct stat *source_stat, const void *buf, size_t size); +void exclude_source_capture_error(struct exclude_source_capture *capture); void exclude_source_capture_release(struct exclude_source_capture *capture); int exclude_source_proof_validate(struct exclude_source_proof *proof); void exclude_source_proof_release(struct exclude_source_proof *proof); From d0dde663842a379fc7cf54f51f752038ef7ffb0f Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 13:14:28 -0500 Subject: [PATCH 087/105] preload-index: stage proven untracked bulk results A tracked-file bulk preload can already walk directories that ordinary status later scans for untracked files. Sharing those observations requires a complete, independently validated result; a partial list must never suppress the conventional untracked traversal. Add an explicit backend capability and optional borrowed destination for visible paths. Serialize the existing ignore matcher across scan workers, collapse an untracked directory after its first visible descendant, and sort the provisional results. Publish them only after the directory scan and the anchored ignore-source proof both complete. Reject duplicate paths and discard incomplete or conflicting untracked results without discarding independently valid tracked observations. Report completeness, visible-path count, and fallback reason through Trace2, and release all temporary path and proof state. No existing backend advertises the new capability and no status caller requests it at this boundary. Ordinary tracked and untracked behavior therefore remains unchanged. Signed-off-by: Taylor Blau --- preload-index-bulk-thread.c | 2 + preload-index-bulk.c | 197 ++++++++++++++++++++++++++++++++++++ preload-index-bulk.h | 33 ++++++ preload-index.c | 17 ++++ read-cache-ll.h | 3 + 5 files changed, 252 insertions(+) diff --git a/preload-index-bulk-thread.c b/preload-index-bulk-thread.c index b1a8d430d8e21e..61702a5a5ba07a 100644 --- a/preload-index-bulk-thread.c +++ b/preload-index-bulk-thread.c @@ -244,6 +244,8 @@ int preload_bulk_run_scan(struct preload_bulk_scan *scan, result->malformed += worker->malformed; } result->threads = started_threads; + result->untracked_complete = + scan->collect_untracked && !scan->queue.untracked_invalid; failed = scan->queue.failed || result->malformed || result->changed_dirs; diff --git a/preload-index-bulk.c b/preload-index-bulk.c index 41b2398d3913f3..cd53761358af4f 100644 --- a/preload-index-bulk.c +++ b/preload-index-bulk.c @@ -1,8 +1,23 @@ #include "git-compat-util.h" +#include "abspath.h" +#include "dir.h" +#include "exclude-source-proof.h" #include "name-hash.h" #include "parse.h" #include "preload-index-bulk.h" #include "read-cache-ll.h" +#include "trace2.h" + +struct preload_bulk_untracked_root { + struct preload_bulk_untracked_root *next; + /* + * Normal-mode status reports an untracked directory after finding + * one visible descendant. Share that decision among workers below + * the directory. + */ + unsigned visible : 1; + char path[FLEX_ARRAY]; +}; static int backend_available(const struct preload_bulk_backend *backend) { @@ -11,6 +26,15 @@ static int backend_available(const struct preload_bulk_backend *backend) backend->scan_directory; } +static int open_exclude_parent(void *data, const char *path) +{ + struct preload_bulk_scan *scan = data; + + if (is_absolute_path(path)) + return open(path, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + return scan->backend->open_proof_parent(scan, path); +} + int preload_bulk_available(void) { return backend_available(preload_bulk_platform_backend()); @@ -35,9 +59,120 @@ int preload_bulk_test_barrier(struct preload_bulk_scan *scan, return result; } +int preload_bulk_path_is_excluded(struct preload_bulk_worker *worker, + const char *path, int dtype) +{ + struct preload_bulk_scan *scan = worker->scan; + int result; + + if (!scan->exclude_dir) + BUG("bulk preload has no exclude state"); + pthread_mutex_lock(&scan->exclude_mutex); + result = is_excluded(scan->exclude_dir, scan->istate, path, &dtype); + pthread_mutex_unlock(&scan->exclude_mutex); + return result; +} + +void preload_bulk_invalidate_untracked( + struct preload_bulk_worker *worker) +{ + struct preload_bulk_queue *queue = &worker->scan->queue; + + pthread_mutex_lock(&queue->mutex); + queue->untracked_invalid = 1; + pthread_mutex_unlock(&queue->mutex); +} + +int preload_bulk_untracked_is_invalid( + struct preload_bulk_worker *worker) +{ + struct preload_bulk_queue *queue = &worker->scan->queue; + int invalid; + + pthread_mutex_lock(&queue->mutex); + invalid = queue->untracked_invalid; + pthread_mutex_unlock(&queue->mutex); + return invalid; +} + +struct preload_bulk_untracked_root *preload_bulk_untracked_root_new( + struct preload_bulk_worker *worker, const char *path, + size_t path_len) +{ + struct preload_bulk_scan *scan = worker->scan; + struct preload_bulk_untracked_root *root; + + FLEX_ALLOC_MEM(root, path, path, path_len + 1); + root->path[path_len] = '/'; + root->path[path_len + 1] = '\0'; + + pthread_mutex_lock(&scan->queue.mutex); + root->next = scan->untracked_roots; + scan->untracked_roots = root; + pthread_mutex_unlock(&scan->queue.mutex); + return root; +} + +int preload_bulk_untracked_root_is_visible( + struct preload_bulk_worker *worker MAYBE_UNUSED, + const struct preload_bulk_untracked_root *root) +{ + int visible; + + if (!root) + return 0; + pthread_mutex_lock(&worker->scan->queue.mutex); + visible = root->visible; + pthread_mutex_unlock(&worker->scan->queue.mutex); + return visible; +} + +void preload_bulk_record_untracked( + struct preload_bulk_worker *worker, + struct preload_bulk_untracked_root *root, + const char *path) +{ + struct preload_bulk_scan *scan = worker->scan; + struct preload_bulk_queue *queue = &scan->queue; + int record = 1; + + pthread_mutex_lock(&queue->mutex); + if (queue->untracked_invalid) + record = 0; + else if (root) { + if (root->visible) + record = 0; + else + root->visible = 1; + } + if (record) + string_list_append(&scan->untracked, + root ? root->path : path); + pthread_mutex_unlock(&queue->mutex); +} + +static int collect_untracked_paths(struct preload_bulk_scan *scan, + struct preload_bulk_result *result) +{ + /* + * Do not publish provisional output until all closing validations + * have succeeded. + */ + string_list_sort(&scan->untracked); + for (size_t i = 1; i < scan->untracked.nr; i++) + if (!strcmp(scan->untracked.items[i - 1].string, + scan->untracked.items[i].string)) + return -1; + result->untracked = scan->untracked; + scan->untracked = (struct string_list)STRING_LIST_INIT_DUP; + return 0; +} + int preload_bulk_collect(struct index_state *istate, int threads, struct preload_bulk_result *result) { + struct dir_struct exclude_dir = DIR_INIT; + struct exclude_source_proof *exclude_proof = NULL; const struct preload_bulk_backend *backend = preload_bulk_platform_backend(); struct preload_bulk_scan scan = { @@ -46,13 +181,16 @@ int preload_bulk_collect(struct index_state *istate, int threads, .backend = backend, .root_fd = -1, .threads = threads, + .untracked = STRING_LIST_INIT_DUP, }; struct preload_bulk_run_result run_result = { 0 }; const char *start_error, *finish_error = NULL; + const char *untracked_reason = NULL; int scan_error = -1; int clean; memset(result, 0, sizeof(*result)); + result->untracked.strdup_strings = 1; result->outcome = "start-fallback"; result->reason = "backend-unavailable"; if (!backend_available(backend)) @@ -69,6 +207,20 @@ int preload_bulk_collect(struct index_state *istate, int threads, scan.case_insensitive = prepare_index_casefolding(istate); scan.can_skip_unseen_preload = 1; } + scan.collect_untracked = + !!istate->preload_untracked && + backend->collects_untracked && + backend->open_proof_parent; + if (istate->preload_untracked && !scan.collect_untracked) + untracked_reason = "backend-unsupported"; + if (scan.collect_untracked) { + scan.exclude_dir = &exclude_dir; +#if HAVE_THREADS + if (pthread_mutex_init(&scan.exclude_mutex, NULL)) { + return -1; + } +#endif + } if (git_env_bool("GIT_TEST_PRELOAD_INDEX_BULK", 0)) { scan.test_barrier_path = getenv( @@ -82,16 +234,44 @@ int preload_bulk_collect(struct index_state *istate, int threads, CALLOC_ARRAY(scan.tracked_state, istate->cache_nr); start_error = backend->start(&scan); if (!start_error) { + if (scan.collect_untracked) { + exclude_proof = exclude_source_proof_create( + istate, &scan, open_exclude_parent); + exclude_dir.internal.exclude_source_proof = + exclude_proof; + setup_standard_excludes(&exclude_dir); + } scan_error = preload_bulk_run_scan(&scan, &run_result); if (!scan_error) scan_error = preload_bulk_test_barrier(&scan, ""); finish_error = backend->finish(&scan); + if (!scan_error && !finish_error && + run_result.untracked_complete) { + int exclude_proof_valid; + + trace2_region_enter( + "index", "preload/bulk_excludes", istate->repo); + exclude_proof_valid = + exclude_source_proof_validate(exclude_proof); + if (!exclude_proof_valid) { + run_result.untracked_complete = 0; + untracked_reason = "exclude-race"; + } + trace2_region_leave( + "index", "preload/bulk_excludes", istate->repo); + } } clean = !start_error && !scan_error && !finish_error && !run_result.changed_dirs && !run_result.malformed; + if (clean && run_result.untracked_complete && + collect_untracked_paths(&scan, result)) { + run_result.untracked_complete = 0; + untracked_reason = "duplicate-path"; + } result->run = run_result; + result->untracked_reason = untracked_reason; if (start_error) { result->outcome = "start-fallback"; result->reason = start_error; @@ -116,10 +296,26 @@ int preload_bulk_collect(struct index_state *istate, int threads, result->nr = istate->cache_nr; result->can_skip_unseen_preload = scan.can_skip_unseen_preload; + result->untracked_complete = run_result.untracked_complete; scan.tracked_state = NULL; } backend->release(&scan); + while (scan.untracked_roots) { + struct preload_bulk_untracked_root *next = + scan.untracked_roots->next; + + free(scan.untracked_roots); + scan.untracked_roots = next; + } + string_list_clear(&scan.untracked, 0); + if (scan.exclude_dir) { +#if HAVE_THREADS + pthread_mutex_destroy(&scan.exclude_mutex); +#endif + dir_clear(&exclude_dir); + exclude_source_proof_release(exclude_proof); + } free(scan.tracked_state); return clean ? 0 : -1; } @@ -127,5 +323,6 @@ int preload_bulk_collect(struct index_state *istate, int threads, void preload_bulk_result_release(struct preload_bulk_result *result) { FREE_AND_NULL(result->tracked_state); + string_list_clear(&result->untracked, 0); memset(result, 0, sizeof(*result)); } diff --git a/preload-index-bulk.h b/preload-index-bulk.h index fff436d23f8d4b..317a9cb244275d 100644 --- a/preload-index-bulk.h +++ b/preload-index-bulk.h @@ -4,8 +4,12 @@ #include "git-compat-util.h" #include "preload-index.h" #include "strbuf.h" +#include "string-list.h" #include "thread-utils.h" +struct dir_struct; +struct preload_bulk_untracked_root; + struct preload_bulk_dir_identity { struct stat stat; unsigned complete : 1; @@ -34,6 +38,7 @@ struct preload_bulk_queue { size_t open_fds; size_t open_fd_limit; int failed; + unsigned untracked_invalid : 1; }; struct preload_bulk_scan; @@ -52,9 +57,12 @@ struct preload_bulk_worker { }; struct preload_bulk_backend { + unsigned collects_untracked : 1; const char *(*start)(struct preload_bulk_scan *scan); const char *(*finish)(struct preload_bulk_scan *scan); void (*release)(struct preload_bulk_scan *scan); + int (*open_proof_parent)(struct preload_bulk_scan *scan, + const char *path); int (*open_dir_at)(struct preload_bulk_worker *worker, int parent_fd, const char *name); /* @@ -76,8 +84,13 @@ struct preload_bulk_scan { struct preload_bulk_queue queue; struct preload_bulk_worker *workers; unsigned char *tracked_state; + struct dir_struct *exclude_dir; + pthread_mutex_t exclude_mutex; + struct preload_bulk_untracked_root *untracked_roots; + struct string_list untracked; int root_fd; int threads; + unsigned collect_untracked : 1; unsigned case_insensitive : 1; unsigned can_skip_unseen_preload : 1; }; @@ -89,6 +102,7 @@ struct preload_bulk_run_result { uint64_t changed_dirs; uint64_t malformed; int threads; + unsigned untracked_complete : 1; }; struct preload_bulk_result { @@ -96,8 +110,11 @@ struct preload_bulk_result { size_t nr; const char *outcome; const char *reason; + const char *untracked_reason; struct preload_bulk_run_result run; unsigned can_skip_unseen_preload : 1; + struct string_list untracked; + unsigned untracked_complete : 1; }; void preload_bulk_schedule_directory( @@ -120,6 +137,22 @@ void preload_bulk_record_tracked_descendants_fallback( int preload_bulk_record_tracked_alias_fallback( struct preload_bulk_worker *worker, const char *path, size_t path_len); +int preload_bulk_path_is_excluded(struct preload_bulk_worker *worker, + const char *path, int dtype); +void preload_bulk_invalidate_untracked( + struct preload_bulk_worker *worker); +int preload_bulk_untracked_is_invalid( + struct preload_bulk_worker *worker); +struct preload_bulk_untracked_root *preload_bulk_untracked_root_new( + struct preload_bulk_worker *worker, const char *path, + size_t path_len); +int preload_bulk_untracked_root_is_visible( + struct preload_bulk_worker *worker, + const struct preload_bulk_untracked_root *root); +void preload_bulk_record_untracked( + struct preload_bulk_worker *worker, + struct preload_bulk_untracked_root *root, + const char *path); int preload_bulk_run_scan(struct preload_bulk_scan *scan, struct preload_bulk_run_result *result); const struct preload_bulk_backend *preload_bulk_platform_backend(void); diff --git a/preload-index.c b/preload-index.c index 72d37ae93e67d0..fbe83f8e7a1a89 100644 --- a/preload-index.c +++ b/preload-index.c @@ -251,6 +251,10 @@ static void preload_bulk_trace_result( if (result->reason) trace2_data_string("index", index->repo, "preload/bulk_reason", result->reason); + if (result->untracked_reason) + trace2_data_string("index", index->repo, + "preload/bulk_untracked_reason", + result->untracked_reason); trace2_data_intmax("index", index->repo, "preload/bulk_applied", applied); trace2_data_intmax("index", index->repo, "preload/bulk_dirs", @@ -271,6 +275,12 @@ static void preload_bulk_trace_result( "preload/bulk_content_check", content_check); trace2_data_intmax("index", index->repo, "preload/bulk_fallback", fallback); + trace2_data_intmax("index", index->repo, + "preload/bulk_untracked_complete", + result->untracked_complete); + trace2_data_intmax("index", index->repo, + "preload/bulk_untracked_count", + result->untracked.nr); } static unsigned char *preload_bulk_try(struct index_state *index) @@ -315,6 +325,11 @@ static unsigned char *preload_bulk_try(struct index_state *index) tracked_state = result.tracked_state; result.tracked_state = NULL; } + if (result.untracked_complete && index->preload_untracked) { + *index->preload_untracked = result.untracked; + result.untracked = + (struct string_list)STRING_LIST_INIT_DUP; + } trace2_region_leave("index", "preload/bulk", index->repo); preload_bulk_result_release(&result); return tracked_state; @@ -353,6 +368,8 @@ void preload_index(struct index_state *index, int core_preload_index = 1; preload_index_bulk_result_clear(index); + if (index->preload_untracked) + string_list_clear(index->preload_untracked, 0); repo_config_get_bool(index->repo, "core.preloadindex", &core_preload_index); if (!core_preload_index) diff --git a/read-cache-ll.h b/read-cache-ll.h index 4733b6800fd145..2318999f2e3899 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -144,6 +144,7 @@ static inline unsigned create_ce_flags(unsigned stage) struct split_index; struct clean_status_state; struct untracked_cache; +struct string_list; struct progress; struct pattern_list; @@ -197,6 +198,8 @@ struct index_state { struct untracked_cache *untracked; unsigned char *preload_bulk_tracked_state; size_t preload_bulk_tracked_nr; + /* Borrowed for the duration of preload_index(). */ + struct string_list *preload_untracked; char *fsmonitor_last_update; char *fsmonitor_last_update_pending; char *fsmonitor_untracked_token; From d4042c13e946600df673999eaa60c595fc0b0d7b Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 13:26:44 -0500 Subject: [PATCH 088/105] preload-index: collect visible paths during the APFS walk The APFS tracked preload encounters untracked entries but ordinarily discards them. Repeating the entire directory walk to rediscover those entries costs work even when the existing scan can establish their visibility. Teach the APFS backend to advertise visible-path collection and supply its root-anchored exclude-parent opener. Classify regular files and symlinks with the normal exclusion machinery, and follow untracked directories only until their first visible descendant establishes the single directory entry that normal-mode status reports. Keep the top-level Git directory and tracked gitlinks out of the untracked result. Case aliases, embedded repositories, and foreign mounts invalidate provisional untracked observations while retaining separately valid tracked results. A replaced directory increments changed_dirs, so scan-wide validation discards the entire bulk result. Carry the collapsed-directory root through queued workers. Ordinary status remains unchanged until a caller explicitly requests the new backend capability. Signed-off-by: Taylor Blau --- compat/preload-index/bulk-darwin.c | 106 ++++++++++++++++++++++++++--- preload-index-bulk-index.c | 7 ++ preload-index-bulk-thread.c | 4 ++ preload-index-bulk.h | 4 ++ 4 files changed, 110 insertions(+), 11 deletions(-) diff --git a/compat/preload-index/bulk-darwin.c b/compat/preload-index/bulk-darwin.c index 882cc64a41f64b..7e65c5a24c78eb 100644 --- a/compat/preload-index/bulk-darwin.c +++ b/compat/preload-index/bulk-darwin.c @@ -6,6 +6,7 @@ #include "compat/precompose_utf8.h" #include "compat/preload-index/bulk-darwin.h" +#include "dir.h" #include "path-namespace.h" #include "preload-index-bulk.h" @@ -329,6 +330,10 @@ static int enumerate_directory(struct preload_bulk_worker *worker, size_t remaining; int pos; + if (scan->collect_untracked && + preload_bulk_untracked_root_is_visible( + worker, task->untracked_root)) + return 0; remaining = buf + PRELOAD_INDEX_BULK_BUFFER_SIZE - record; if (decode_entry(record, remaining, &entry)) goto malformed; @@ -344,6 +349,17 @@ static int enumerate_directory(struct preload_bulk_worker *worker, strbuf_addstr(&worker->path, path_name); if (path_name != entry.name) free((char *)path_name); + if (scan->collect_untracked) { + if (!strcmp(task->path, ".") && + !fspathcmp(worker->path.buf, ".git")) + goto next_record; + if (strcmp(task->path, ".") && + !fspathcmp(entry.name, ".git")) { + preload_bulk_invalidate_untracked( + worker); + goto next_record; + } + } pos = preload_bulk_index_position(scan, worker->path.buf, worker->path.len); @@ -358,20 +374,51 @@ static int enumerate_directory(struct preload_bulk_worker *worker, .st_ctimespec = entry.ctime, }, }; + struct preload_bulk_untracked_root *untracked_root = + task->untracked_root; + int has_tracked_descendants; if (pos >= 0) { + if (scan->collect_untracked && + preload_bulk_index_entry_is_gitlink( + scan, pos)) + goto next_record; preload_bulk_record_tracked_fallback( worker, pos); goto next_record; } - if (!preload_bulk_index_pos_has_tracked_descendants( - scan, worker->path.buf, - worker->path.len, pos)) { - preload_bulk_record_tracked_alias_fallback( - worker, worker->path.buf, - worker->path.len); + has_tracked_descendants = + preload_bulk_index_pos_has_tracked_descendants( + scan, worker->path.buf, + worker->path.len, pos); + if (!has_tracked_descendants && + preload_bulk_record_tracked_alias_fallback( + worker, worker->path.buf, + worker->path.len)) { + if (scan->collect_untracked) + preload_bulk_invalidate_untracked( + worker); goto next_record; } + if (!has_tracked_descendants) { + if (!scan->collect_untracked || + preload_bulk_untracked_is_invalid( + worker)) + goto next_record; + if (preload_bulk_untracked_root_is_visible( + worker, untracked_root)) + goto next_record; + if (preload_bulk_path_is_excluded( + worker, worker->path.buf, + DT_DIR)) + goto next_record; + if (!untracked_root) + untracked_root = + preload_bulk_untracked_root_new( + worker, + worker->path.buf, + worker->path.len); + } if (((entry.access & S_IFMT) && (entry.access & S_IFMT) != S_IFDIR) || (entry.access & ~(S_IFMT | 07777))) @@ -382,20 +429,50 @@ static int enumerate_directory(struct preload_bulk_worker *worker, preload_bulk_record_tracked_descendants_fallback( worker, worker->path.buf, worker->path.len); + if (scan->collect_untracked) + preload_bulk_invalidate_untracked( + worker); goto next_record; } preload_bulk_schedule_directory( worker, fd, parent_identity, - &child_identity, entry.name, + &child_identity, untracked_root, + entry.name, worker->path.buf, worker->path.len); goto next_record; } if (pos < 0) { - preload_bulk_record_tracked_alias_fallback( - worker, worker->path.buf, - worker->path.len); + int found_alias = + preload_bulk_record_tracked_alias_fallback( + worker, worker->path.buf, + worker->path.len); + + if (scan->collect_untracked && + !preload_bulk_untracked_is_invalid( + worker)) { + int dtype; + + if (found_alias) { + preload_bulk_invalidate_untracked( + worker); + goto next_record; + } + if (entry.type == VREG) + dtype = DT_REG; + else if (entry.type == VLNK) + dtype = DT_LNK; + else + goto next_record; + if (!preload_bulk_path_is_excluded( + worker, worker->path.buf, + dtype)) + preload_bulk_record_untracked( + worker, + task->untracked_root, + worker->path.buf); + } goto next_record; } if (entry.dev != data->root_stat.st_dev) { @@ -457,6 +534,8 @@ static int scan_directory(struct preload_bulk_worker *worker, path_len = strlen(task->path); preload_bulk_record_tracked_descendants_fallback( worker, task->path, path_len); + if (scan->collect_untracked) + preload_bulk_invalidate_untracked(worker); ret = 0; goto out; } @@ -471,7 +550,10 @@ static int scan_directory(struct preload_bulk_worker *worker, goto out; } before_identity = directory_identity(&before); - if (enumerate_directory(worker, task, fd, &before_identity)) + if ((!scan->collect_untracked || + !preload_bulk_untracked_root_is_visible( + worker, task->untracked_root)) && + enumerate_directory(worker, task, fd, &before_identity)) goto out; if (fstat(fd, &after)) goto out; @@ -518,9 +600,11 @@ static const char *finish_scan(struct preload_bulk_scan *scan) } static const struct preload_bulk_backend darwin_backend = { + .collects_untracked = 1, .start = start_scan, .finish = finish_scan, .release = preload_bulk_darwin_release, + .open_proof_parent = preload_bulk_darwin_open_relative, .open_dir_at = preload_bulk_darwin_open_dir_at, .scan_directory = scan_directory, }; diff --git a/preload-index-bulk-index.c b/preload-index-bulk-index.c index bd26b72ccb2986..b09b69582397a8 100644 --- a/preload-index-bulk-index.c +++ b/preload-index-bulk-index.c @@ -1,5 +1,6 @@ #include "git-compat-util.h" #include "name-hash.h" +#include "object.h" #include "preload-index-bulk.h" #include "read-cache-ll.h" @@ -108,6 +109,12 @@ static int size_change_is_definitive(const struct cache_entry *ce, DATA_CHANGED); } +int preload_bulk_index_entry_is_gitlink(struct preload_bulk_scan *scan, + int pos) +{ + return pos >= 0 && S_ISGITLINK(scan->istate->cache[pos]->ce_mode); +} + void preload_bulk_record_tracked( struct preload_bulk_worker *worker, int pos, const struct stat *st) { diff --git a/preload-index-bulk-thread.c b/preload-index-bulk-thread.c index 61702a5a5ba07a..793bb79f6a670a 100644 --- a/preload-index-bulk-thread.c +++ b/preload-index-bulk-thread.c @@ -53,6 +53,7 @@ void preload_bulk_schedule_directory( struct preload_bulk_worker *worker, int parent_fd, const struct preload_bulk_dir_identity *parent_identity, const struct preload_bulk_dir_identity *child_identity, + struct preload_bulk_untracked_root *untracked_root, const char *name, const char *path, size_t path_len) { struct preload_bulk_scan *scan = worker->scan; @@ -67,6 +68,7 @@ void preload_bulk_schedule_directory( task->child_identity = *child_identity; task->has_child_identity = 1; } + task->untracked_root = untracked_root; task->fd = -1; if (reserve_open_fd(&scan->queue)) { task->reserved_fd = 1; @@ -79,6 +81,8 @@ void preload_bulk_schedule_directory( if (saved_errno == EXDEV) { preload_bulk_record_tracked_descendants_fallback( worker, path, path_len); + if (scan->collect_untracked) + preload_bulk_invalidate_untracked(worker); free(task); return; } diff --git a/preload-index-bulk.h b/preload-index-bulk.h index 317a9cb244275d..2934d31b8674c1 100644 --- a/preload-index-bulk.h +++ b/preload-index-bulk.h @@ -19,6 +19,7 @@ struct preload_bulk_task { struct preload_bulk_task *next; struct preload_bulk_dir_identity parent_identity; struct preload_bulk_dir_identity child_identity; + struct preload_bulk_untracked_root *untracked_root; int fd; unsigned reserved_fd : 1; unsigned has_parent_identity : 1; @@ -121,12 +122,15 @@ void preload_bulk_schedule_directory( struct preload_bulk_worker *worker, int parent_fd, const struct preload_bulk_dir_identity *parent_identity, const struct preload_bulk_dir_identity *child_identity, + struct preload_bulk_untracked_root *untracked_root, const char *name, const char *path, size_t path_len); int preload_bulk_index_position(struct preload_bulk_scan *scan, const char *path, size_t path_len); int preload_bulk_index_pos_has_tracked_descendants( struct preload_bulk_scan *scan, const char *path, size_t path_len, int pos); +int preload_bulk_index_entry_is_gitlink(struct preload_bulk_scan *scan, + int pos); void preload_bulk_record_tracked( struct preload_bulk_worker *worker, int pos, const struct stat *st); void preload_bulk_record_tracked_fallback( From 5cdd1d8ed710ccaa075333646399c0ddf2b10211 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 13:27:01 -0500 Subject: [PATCH 089/105] status: reuse complete APFS untracked preload results Normal-mode status walks the worktree for untracked paths even after the opted-in APFS preloader has visited the same directories. Reusing that walk is incorrect if status requests different reporting semantics or the bulk scan cannot prove that its exclusion sources remain valid. Request visible paths only for an expanded index in normal untracked mode without a pathspec, ignored output, or an untracked cache. The bulk path requires core.preloadIndex, core.preloadIndexBulk, and a disabled fsmonitor. Transfer paths only after the bulk scan, worktree-namespace checks, and anchored exclusion-source proof finish successfully. Complete the ordinary tracked refresh, clear the borrowed index destination, and skip the second directory walk only for a complete untracked result. Retain ordinary traversal for unsupported backends, incomplete proofs, case aliases, embedded repositories, changed exclusion sources, and ineligible reporting modes. A failed untracked proof preserves independently valid tracked results; a changed directory instead invalidates the entire bulk scan. Extend the APFS tests to compare output with ordinary status. Cover collapsed directories, ignored-only and empty directories, special files, activation guards, case aliases, nested repositories, tracked submodules, separate Git directories, changed configured and repository exclusions, a newly appearing configured exclusion, bidirectional per-directory changes, hard-linked exclusions, and tracked-file replacement. Signed-off-by: Taylor Blau --- preload-index.c | 2 + read-cache-ll.h | 3 +- t/t7529-preload-index-apfs.sh | 264 ++++++++++++++++++++++++++++++++++ wt-status.c | 21 ++- wt-status.h | 2 + 5 files changed, 289 insertions(+), 3 deletions(-) diff --git a/preload-index.c b/preload-index.c index fbe83f8e7a1a89..d7c7f99896c28e 100644 --- a/preload-index.c +++ b/preload-index.c @@ -327,6 +327,7 @@ static unsigned char *preload_bulk_try(struct index_state *index) } if (result.untracked_complete && index->preload_untracked) { *index->preload_untracked = result.untracked; + index->preload_untracked_complete = 1; result.untracked = (struct string_list)STRING_LIST_INIT_DUP; } @@ -368,6 +369,7 @@ void preload_index(struct index_state *index, int core_preload_index = 1; preload_index_bulk_result_clear(index); + index->preload_untracked_complete = 0; if (index->preload_untracked) string_list_clear(index->preload_untracked, 0); repo_config_get_bool(index->repo, "core.preloadindex", &core_preload_index); diff --git a/read-cache-ll.h b/read-cache-ll.h index 2318999f2e3899..c7e4104715e890 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -190,7 +190,8 @@ struct index_state { fsmonitor_untracked_valid : 1, fsmonitor_untracked_extension_seen : 1, fsmonitor_untracked_extension_invalid : 1, - fsmonitor_pending_token_from_provider : 1; + fsmonitor_pending_token_from_provider : 1, + preload_untracked_complete : 1; enum sparse_index_mode sparse_index; struct hashmap name_hash; struct hashmap dir_hash; diff --git a/t/t7529-preload-index-apfs.sh b/t/t7529-preload-index-apfs.sh index e5c2c5015b0c6d..5ab700650c44df 100755 --- a/t/t7529-preload-index-apfs.sh +++ b/t/t7529-preload-index-apfs.sh @@ -74,6 +74,23 @@ compare_status () { test_cmp expect actual } +compare_fallback_status () { + repo=$1 && + fallback_trace=$TRASH_DIRECTORY/$2 && + shift 2 && + GIT_OPTIONAL_LOCKS=0 \ + git -C "$repo" -c core.preloadIndex=false \ + status --porcelain=v2 "$@" >expect && + rm -f "$fallback_trace" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TRACE2_EVENT="$fallback_trace" \ + git -C "$repo" status --porcelain=v2 "$@" >actual && + test_cmp expect actual && + test_grep "\"category\":\"read_directory\"" "$fallback_trace" +} + configured_bulk_status () { repo=$1 && output=$2 && @@ -182,11 +199,23 @@ finish_raced_status () { test_trace2_data index preload/bulk_applied 0 <"$race_trace" } +finish_raced_untracked_status () { + printf "resume\n" >&9 && + exec 9>&- && + wait "$status_pid" && + status_pid= && + ordinary_status "$1" expect && + test_cmp expect actual && + test_trace2_data index preload/bulk_applied "$2" <"$race_trace" +} + test_expect_success 'clean entries are published without lstat' ' setup_repo clean && bulk_status clean actual clean.trace && test_must_be_empty actual && check_data clean.trace preload/bulk_applied 8 && + check_data clean.trace preload/bulk_untracked_complete 1 && + check_data clean.trace preload/bulk_untracked_count 0 && check_lstat_data clean.trace 0 ' @@ -256,6 +285,229 @@ test_expect_success CASE_INSENSITIVE_FS \ } ' +test_expect_success 'visible paths are returned by the bulk walk' ' + setup_repo visible-output && + test_write_lines "*.ignored" >visible-output/.gitignore && + git -C visible-output add .gitignore && + git -C visible-output commit -m ignore && + test_write_lines root >visible-output/root-untracked && + test_write_lines nested >visible-output/nested/untracked && + mkdir -p visible-output/collapsed/deep \ + visible-output/ignored-only/deep \ + visible-output/empty && + test_write_lines collapsed >visible-output/collapsed/deep/file && + test_write_lines ignored >visible-output/ignored-only/deep/file.ignored && + compare_status visible-output visible-output.trace && + test_grep "^? root-untracked$" actual && + test_grep "^? nested/untracked$" actual && + test_grep "^? collapsed/$" actual && + test_grep ! "ignored-only" actual && + test_grep ! "empty" actual && + check_data visible-output.trace preload/bulk_untracked_complete 1 && + check_data visible-output.trace preload/bulk_untracked_count 3 && + test_grep ! "\"category\":\"read_directory\"" \ + visible-output.trace +' + +test_expect_success PIPE 'special files are ignored' ' + setup_repo special-file && + mkfifo special-file/fifo && + compare_status special-file special-file.trace && + test_must_be_empty actual && + check_data special-file.trace preload/bulk_untracked_complete 1 && + check_data special-file.trace preload/bulk_untracked_count 0 +' + +test_expect_success 'activation guards retain ordinary traversal' ' + setup_repo activation-guards && + test_write_lines "*.ignored" >activation-guards/.gitignore && + git -C activation-guards add .gitignore && + git -C activation-guards commit -m ignore && + test_write_lines visible >activation-guards/visible && + test_write_lines ignored >activation-guards/file.ignored && + compare_fallback_status activation-guards all.trace \ + --untracked-files=all && + compare_fallback_status activation-guards ignored.trace \ + --ignored && + compare_fallback_status activation-guards pathspec.trace \ + -- nested && + git -C activation-guards update-index --untracked-cache && + compare_fallback_status activation-guards untracked-cache.trace +' + +test_expect_success CASE_INSENSITIVE_FS 'case aliases fall back' ' + setup_repo untracked-case-alias && + mv untracked-case-alias/root untracked-case-alias/ROOT && + compare_status untracked-case-alias untracked-case-alias.trace && + check_data untracked-case-alias.trace preload/bulk_untracked_complete 0 && + check_data untracked-case-alias.trace preload/bulk_untracked_count 0 +' + +test_expect_success 'nested repositories fall back' ' + setup_repo nested-repo && + test_write_lines "embedded/**" >nested-repo/.gitignore && + git -C nested-repo add .gitignore && + git -C nested-repo commit -m ignore && + mkdir nested-repo/embedded && + git -C nested-repo/embedded init && + test_write_lines ignored >nested-repo/embedded/file && + compare_status nested-repo nested-repo.trace && + test_grep "^? embedded/$" actual && + check_data nested-repo.trace preload/bulk_untracked_complete 0 +' + +test_expect_success 'tracked submodules retain collected paths' ' + git init submodule-child && + git -C submodule-child commit --allow-empty -m base && + setup_repo submodule-parent && + git -C submodule-parent -c protocol.file.allow=always \ + submodule add ../submodule-child embedded && + git -C submodule-parent commit -m submodule && + git -C submodule-parent update-index --refresh && + test_write_lines visible >submodule-parent/visible && + compare_status submodule-parent submodule-parent.trace && + test_grep "^? visible$" actual && + check_data submodule-parent.trace preload/bulk_untracked_complete 1 +' + +test_expect_success 'exclude changes discard collected paths' ' + setup_repo exclude-race && + exclude=$TRASH_DIRECTORY/exclude-race.patterns && + test_write_lines visible >"$exclude" && + git -C exclude-race config core.excludesFile "$exclude" && + test_write_lines visible >exclude-race/visible && + test_when_finished cleanup_race && + start_raced_status exclude-race "" && + >"$exclude" && + finish_raced_untracked_status exclude-race 8 && + test_grep "^? visible$" actual && + check_data exclude-race.trace preload/bulk_untracked_complete 0 && + check_data exclude-race.trace preload/bulk_untracked_reason exclude-race +' + +test_expect_success 'info exclude changes discard collected paths' ' + setup_repo info-exclude && + test_write_lines visible >info-exclude/.git/info/exclude && + test_write_lines visible >info-exclude/visible && + test_when_finished cleanup_race && + start_raced_status info-exclude "" && + >info-exclude/.git/info/exclude && + finish_raced_untracked_status info-exclude 8 && + test_grep "^? visible$" actual && + check_data info-exclude.trace preload/bulk_untracked_complete 0 && + check_data info-exclude.trace \ + preload/bulk_untracked_reason exclude-race +' + +test_expect_success 'new exclusion source discards collected paths' ' + setup_repo absent-exclude && + exclude_dir=$TRASH_DIRECTORY/absent-exclude-config && + exclude=$exclude_dir/ignore && + rm -rf "$exclude_dir" && + git -C absent-exclude config core.excludesFile "$exclude" && + test_write_lines visible >absent-exclude/visible && + test_when_finished cleanup_race && + test_when_finished "rm -rf \"$exclude_dir\"" && + start_raced_status absent-exclude "" && + mkdir "$exclude_dir" && + test_write_lines visible >"$exclude" && + finish_raced_untracked_status absent-exclude 8 && + test_must_be_empty actual && + check_data absent-exclude.trace preload/bulk_untracked_complete 0 && + check_data absent-exclude.trace \ + preload/bulk_untracked_reason exclude-race +' + +test_expect_success 'separate-git-dir excludes are proven' ' + setup_repo separate-info && + mv separate-info/.git separate-info.git && + printf "gitdir: ../separate-info.git\n" >separate-info/.git && + test_write_lines visible >separate-info.git/info/exclude && + test_write_lines visible >separate-info/visible && + compare_status separate-info separate-info.trace && + test_must_be_empty actual && + check_data separate-info.trace preload/bulk_untracked_complete 1 +' + +test_expect_success 'nested per-directory excludes are closed both ways' ' + test_when_finished cleanup_race && + for direction in visible-to-ignored ignored-to-visible + do + repo=per-dir-$direction && + setup_repo "$repo" && + git -C "$repo" config core.trustctime false && + case "$direction" in + visible-to-ignored) + initial=nomatch && + updated=visible + ;; + ignored-to-visible) + initial=visible && + updated=nomatch + ;; + esac && + test_write_lines "$initial" >"$repo/nested/.gitignore" && + git -C "$repo" add nested/.gitignore && + git -C "$repo" commit -m ignore && + git -C "$repo" update-index --assume-unchanged \ + nested/.gitignore && + test_write_lines visible >"$repo/nested/visible" && + mtime=$(test-tool chmtime --get \ + "$repo/nested/.gitignore") && + start_raced_status "$repo" "" && + test_write_lines "$updated" >"$repo/nested/.gitignore" && + test-tool chmtime "=$mtime" "$repo/nested/.gitignore" && + finish_raced_untracked_status "$repo" 8 && + check_data "$repo.trace" \ + preload/bulk_untracked_complete 0 && + check_data "$repo.trace" \ + preload/bulk_untracked_reason exclude-race && + case "$direction" in + visible-to-ignored) + test_must_be_empty actual + ;; + ignored-to-visible) + test_grep "^? nested/visible$" actual + ;; + esac || + return 1 + done +' + +test_expect_success 'multiply-linked per-directory excludes are proven' ' + setup_repo linked-exclude && + test_write_lines visible >linked-exclude/.gitignore && + git -C linked-exclude add .gitignore && + git -C linked-exclude commit -m ignore && + test_write_lines visible >linked-exclude/visible && + ln linked-exclude/.gitignore linked-exclude-alias && + test_when_finished "rm -f linked-exclude-alias" && + compare_status linked-exclude linked-exclude.trace && + test_must_be_empty actual && + check_data linked-exclude.trace preload/bulk_untracked_complete 1 && + check_data linked-exclude.trace preload/bulk_applied 8 && + check_data linked-exclude.trace preload/bulk_untracked_count 0 +' + +test_expect_success 'multiply-linked exclude changes discard paths' ' + setup_repo linked-exclude-race && + test_write_lines visible >linked-exclude-race/.gitignore && + git -C linked-exclude-race add .gitignore && + git -C linked-exclude-race commit -m ignore && + test_write_lines visible >linked-exclude-race/visible && + ln linked-exclude-race/.gitignore linked-exclude-race-alias && + test_when_finished "rm -f linked-exclude-race-alias" && + test_when_finished cleanup_race && + start_raced_status linked-exclude-race "" && + test_write_lines nomatch >linked-exclude-race-alias && + finish_raced_untracked_status linked-exclude-race 8 && + test_grep "^? visible$" actual && + check_data linked-exclude-race.trace \ + preload/bulk_untracked_complete 0 && + check_data linked-exclude-race.trace \ + preload/bulk_untracked_reason exclude-race +' + test_expect_success ULIMIT_FILE_DESCRIPTORS \ 'bulk preload reopens directories under a low descriptor limit' ' git init low-fd && @@ -313,6 +565,18 @@ test_expect_success SYMLINKS \ test_file_not_empty actual ' +test_expect_success 'tracked-file replacement directories are pruned' ' + setup_repo replacement-dir && + rm replacement-dir/root && + mkdir -p replacement-dir/root/deep/embedded && + test_write_lines hidden >replacement-dir/root/deep/untracked && + git -C replacement-dir/root/deep/embedded init && + compare_status replacement-dir replacement-dir.trace && + test_line_count = 1 actual && + check_data replacement-dir.trace preload/bulk_fallback 1 && + check_data replacement-dir.trace preload/bulk_untracked_complete 1 +' + test_expect_success 'staged and unmerged entries agree' ' setup_repo index-states && test_write_lines staged >index-states/root && diff --git a/wt-status.c b/wt-status.c index 37660a24e6f4cc..bcdf74a004070f 100644 --- a/wt-status.c +++ b/wt-status.c @@ -946,6 +946,10 @@ void wt_status_start_untracked_cache_preload(struct wt_status *s) return; } + if (s->show_untracked_files == SHOW_NORMAL_UNTRACKED_FILES && + !istate->untracked && + istate->sparse_index == INDEX_EXPANDED) + istate->preload_untracked = &s->untracked; /* Restore verified stats before cached excludes inspect them. */ if (fstat_is_reliable() && !istate->split_index && fsm_settings__get_mode(s->repo) == FSMONITOR_MODE_IPC && @@ -996,6 +1000,10 @@ static int wt_status_collect_untracked_1( if (!s->show_untracked_files) return 0; + if (s->untracked_from_preload && + !istate->untracked && + !s->show_ignored_mode) + return 0; if (s->show_untracked_files != SHOW_ALL_UNTRACKED_FILES) dir.flags |= wt_status_untracked_dir_flags(s); @@ -1474,13 +1482,22 @@ int wt_status_refresh_index(struct wt_status *s, unsigned int refresh_flags, int require_untracked) { + struct index_state *istate = s->repo->index; struct semantic_verify_proof *proof; + int ret; wt_status_begin_attr_snapshot(s); - refresh_fsmonitor(s->repo->index); + refresh_fsmonitor(istate); proof = wt_status_prepare_semantic_verify(s); - return wt_status_close_fsmonitor_token( + ret = wt_status_close_fsmonitor_token( s, proof, refresh_flags, require_untracked, 0); + if (istate->preload_untracked == &s->untracked) { + s->untracked_from_preload = + istate->preload_untracked_complete; + istate->preload_untracked = NULL; + istate->preload_untracked_complete = 0; + } + return ret; } static void wt_status_release_attr_snapshot(struct wt_status *s) diff --git a/wt-status.h b/wt-status.h index 0f7104b4c6ac5f..99b005cb8b5cea 100644 --- a/wt-status.h +++ b/wt-status.h @@ -141,6 +141,8 @@ struct wt_status { int committable; int workdir_dirty; unsigned untracked_from_token_closure : 1; + unsigned untracked_from_preload : 1; + unsigned bulk_update_index_stat : 1; const char *index_file; FILE *fp; const char *prefix; From 6fbcbef5dd2e69741233b969847d5988b0c18597 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 13:32:42 -0500 Subject: [PATCH 090/105] preload-index: translate complete Linux statx observations A Linux directory scan cannot substitute its results for lstat() when file identity, timestamps, or mount membership are missing. Depending on libc's statx declarations would also tie the implementation to the age of the installed Linux headers. Define the required statx syscall ABI locally and request complete basic statistics and a mount identifier. Reject invalid nanosecond fields, foreign mounts, and device, inode, link-count, owner, size, or timestamp values that cannot be represented in struct stat. Register the metadata module in the Make, CMake, and Meson Linux builds. The native Linux boundary build compiles it with DEVELOPER=1, but this patch does not select a backend or change the fallback. Signed-off-by: Taylor Blau --- compat/preload-index/bulk-linux-stat.c | 140 +++++++++++++++++++++++++ compat/preload-index/bulk-linux.h | 73 +++++++++++++ config.mak.uname | 1 + contrib/buildsystems/CMakeLists.txt | 6 +- meson.build | 5 +- 5 files changed, 223 insertions(+), 2 deletions(-) create mode 100644 compat/preload-index/bulk-linux-stat.c create mode 100644 compat/preload-index/bulk-linux.h diff --git a/compat/preload-index/bulk-linux-stat.c b/compat/preload-index/bulk-linux-stat.c new file mode 100644 index 00000000000000..a8594d7230f9d0 --- /dev/null +++ b/compat/preload-index/bulk-linux-stat.c @@ -0,0 +1,140 @@ +#include "git-compat-util.h" + +#ifdef __linux__ + +#include +#include + +#include "compat/preload-index/bulk-linux.h" +#include "preload-index-bulk.h" + +#if defined(SYS_getdents64) && defined(SYS_statx) + +int preload_bulk_linux_statx_raw(int dirfd, const char *path, int flags, + struct preload_linux_statx *stx) +{ + memset(stx, 0, sizeof(*stx)); + return syscall(SYS_statx, dirfd, path, flags, + PRELOAD_STATX_BASIC_STATS | PRELOAD_STATX_MNT_ID, stx); +} + +int preload_bulk_linux_statx_complete( + const struct preload_linux_statx *stx) +{ + return (stx->mask & + (PRELOAD_STATX_BASIC_STATS | PRELOAD_STATX_MNT_ID)) == + (PRELOAD_STATX_BASIC_STATS | PRELOAD_STATX_MNT_ID) && + stx->mtime.tv_nsec < 1000000000 && + stx->ctime.tv_nsec < 1000000000; +} + +int preload_bulk_linux_statx_same(const struct preload_linux_statx *a, + const struct preload_linux_statx *b) +{ + return preload_bulk_linux_statx_complete(a) && + preload_bulk_linux_statx_complete(b) && + a->mnt_id == b->mnt_id && + a->dev_major == b->dev_major && + a->dev_minor == b->dev_minor && + a->ino == b->ino && a->mode == b->mode && + a->nlink == b->nlink && a->uid == b->uid && + a->gid == b->gid && a->size == b->size && + a->mtime.tv_sec == b->mtime.tv_sec && + a->mtime.tv_nsec == b->mtime.tv_nsec && + a->ctime.tv_sec == b->ctime.tv_sec && + a->ctime.tv_nsec == b->ctime.tv_nsec; +} + +static int statx_to_stat(const struct preload_linux_statx *stx, + struct stat *st) +{ + dev_t dev; + + if (!preload_bulk_linux_statx_complete(stx)) + return -1; + memset(st, 0, sizeof(*st)); + dev = makedev(stx->dev_major, stx->dev_minor); + if (major(dev) != stx->dev_major || minor(dev) != stx->dev_minor) + return -1; + st->st_dev = dev; + st->st_ino = stx->ino; + if ((uint64_t)st->st_ino != stx->ino) + return -1; + st->st_mode = stx->mode; + st->st_nlink = stx->nlink; + if ((uint64_t)st->st_nlink != stx->nlink) + return -1; + st->st_uid = stx->uid; + st->st_gid = stx->gid; + if ((uint64_t)st->st_uid != stx->uid || + (uint64_t)st->st_gid != stx->gid) + return -1; + st->st_size = stx->size; + if (st->st_size < 0 || (uint64_t)st->st_size != stx->size) + return -1; + st->st_mtim.tv_sec = stx->mtime.tv_sec; + st->st_mtim.tv_nsec = stx->mtime.tv_nsec; + st->st_ctim.tv_sec = stx->ctime.tv_sec; + st->st_ctim.tv_nsec = stx->ctime.tv_nsec; + if ((int64_t)st->st_mtim.tv_sec != stx->mtime.tv_sec || + (int64_t)st->st_ctim.tv_sec != stx->ctime.tv_sec) + return -1; + return 0; +} + +int preload_bulk_linux_entry_stat(struct preload_bulk_worker *worker, + int dirfd, const char *name, + struct preload_linux_statx *stx, + struct stat *st) +{ + struct preload_bulk_linux_data *data = + worker->scan->platform_data; + + if (preload_bulk_linux_statx_raw( + dirfd, name, + PRELOAD_AT_SYMLINK_NOFOLLOW | PRELOAD_AT_NO_AUTOMOUNT, + stx)) + return -1; + if (!preload_bulk_linux_statx_complete(stx)) { + errno = EOPNOTSUPP; + return -1; + } + if (stx->mnt_id != data->root_mnt_id) { + errno = EXDEV; + return -1; + } + if (statx_to_stat(stx, st)) { + errno = EOVERFLOW; + return -1; + } + return 0; +} + +int preload_bulk_linux_fd_stat(struct preload_bulk_worker *worker, int fd, + struct preload_linux_statx *stx, + struct stat *st) +{ + struct preload_bulk_linux_data *data = + worker->scan->platform_data; + + if (preload_bulk_linux_statx_raw(fd, "", PRELOAD_AT_EMPTY_PATH, + stx)) + return -1; + if (!preload_bulk_linux_statx_complete(stx)) { + errno = EOPNOTSUPP; + return -1; + } + if (stx->mnt_id != data->root_mnt_id) { + errno = EXDEV; + return -1; + } + if (statx_to_stat(stx, st)) { + errno = EOVERFLOW; + return -1; + } + return 0; +} + +#endif /* SYS_getdents64 && SYS_statx */ + +#endif /* __linux__ */ diff --git a/compat/preload-index/bulk-linux.h b/compat/preload-index/bulk-linux.h new file mode 100644 index 00000000000000..e4530ac0fda8e6 --- /dev/null +++ b/compat/preload-index/bulk-linux.h @@ -0,0 +1,73 @@ +#ifndef PRELOAD_INDEX_BULK_LINUX_H +#define PRELOAD_INDEX_BULK_LINUX_H + +#ifdef __linux__ + +#include + +#define PRELOAD_AT_NO_AUTOMOUNT 0x800 +#define PRELOAD_AT_EMPTY_PATH 0x1000 +#define PRELOAD_AT_SYMLINK_NOFOLLOW 0x100 +#define PRELOAD_STATX_BASIC_STATS 0x000007ffU +#define PRELOAD_STATX_MNT_ID 0x00001000U + +struct preload_linux_statx_timestamp { + int64_t tv_sec; + uint32_t tv_nsec; + int32_t reserved; +}; + +struct preload_linux_statx { + uint32_t mask; + uint32_t blksize; + uint64_t attributes; + uint32_t nlink; + uint32_t uid; + uint32_t gid; + uint16_t mode; + uint16_t spare0; + uint64_t ino; + uint64_t size; + uint64_t blocks; + uint64_t attributes_mask; + struct preload_linux_statx_timestamp atime; + struct preload_linux_statx_timestamp btime; + struct preload_linux_statx_timestamp ctime; + struct preload_linux_statx_timestamp mtime; + uint32_t rdev_major; + uint32_t rdev_minor; + uint32_t dev_major; + uint32_t dev_minor; + uint64_t mnt_id; + uint32_t dio_mem_align; + uint32_t dio_offset_align; + uint64_t spare3[12]; +}; + +struct preload_bulk_linux_data { + uint64_t root_mnt_id; +}; + +struct preload_bulk_worker; + +#if defined(SYS_getdents64) && defined(SYS_statx) + +int preload_bulk_linux_statx_raw(int dirfd, const char *path, int flags, + struct preload_linux_statx *stx); +int preload_bulk_linux_statx_complete( + const struct preload_linux_statx *stx); +int preload_bulk_linux_statx_same(const struct preload_linux_statx *a, + const struct preload_linux_statx *b); +int preload_bulk_linux_entry_stat(struct preload_bulk_worker *worker, + int dirfd, const char *name, + struct preload_linux_statx *stx, + struct stat *st); +int preload_bulk_linux_fd_stat(struct preload_bulk_worker *worker, int fd, + struct preload_linux_statx *stx, + struct stat *st); + +#endif /* SYS_getdents64 && SYS_statx */ + +#endif /* __linux__ */ + +#endif /* PRELOAD_INDEX_BULK_LINUX_H */ diff --git a/config.mak.uname b/config.mak.uname index 31f4524d1158ab..f59b4bfa4aea4a 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -63,6 +63,7 @@ ifeq ($(uname_S),Linux) PROCFS_EXECUTABLE_PATH = /proc/self/exe HAVE_PLATFORM_PROCINFO = YesPlease COMPAT_OBJS += compat/linux/procinfo.o + COMPAT_OBJS += compat/preload-index/bulk-linux-stat.o EXTLIBS += -ldl # centos7/rhel7 provides gcc 4.8.5 and zlib 1.2.7. ifneq ($(findstring .el7.,$(uname_R)),) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 87152a27fe7cb1..aec2421a64da6b 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -273,7 +273,11 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows") elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_compile_definitions(PROCFS_EXECUTABLE_PATH="/proc/self/exe" HAVE_DEV_TTY ) - list(APPEND compat_SOURCES unix-socket.c unix-stream-server.c compat/linux/procinfo.c) + list(APPEND compat_SOURCES + unix-socket.c + unix-stream-server.c + compat/linux/procinfo.c + compat/preload-index/bulk-linux-stat.c) elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") add_compile_definitions(HAVE_PRELOAD_INDEX_BULK PRECOMPOSE_UNICODE USE_ST_TIMESPEC) diff --git a/meson.build b/meson.build index 63994b45cdf02f..3850ee113e5a93 100644 --- a/meson.build +++ b/meson.build @@ -1359,7 +1359,10 @@ elif host_machine.system() == 'windows' endif if host_machine.system() == 'linux' - compat_sources += 'compat/linux/procinfo.c' + compat_sources += [ + 'compat/linux/procinfo.c', + 'compat/preload-index/bulk-linux-stat.c', + ] elif host_machine.system() == 'windows' compat_sources += 'compat/win32/trace2_win32_process_info.c' elif host_machine.system() == 'darwin' From e83e5cd7780857bc3a28f17505e130ef78dc5cf0 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 13:33:21 -0500 Subject: [PATCH 091/105] preload-index: anchor Linux directory opens to the worktree A directory name observed during enumeration may resolve outside the original worktree after a rename, symlink replacement, magic-link traversal, or mount change. Path-based reopening would then inspect an unverified namespace. Introduce descriptor-relative Linux directory-open helpers. Prefer openat2() with beneath-root resolution and reject symlinks, magic links, and mount crossings when that syscall is available. Otherwise reject empty, absolute, dot-dot, and malformed paths. Open root-relative paths one component at a time and verify each mount. Keep direct child opens descriptor-relative; directory scanning verifies their mounts before enumeration. Register the opening module with Make, CMake, and Meson. The native Linux boundary build compiles it with DEVELOPER=1, but backend selection remains unchanged. Signed-off-by: Taylor Blau --- compat/preload-index/bulk-linux-open.c | 149 +++++++++++++++++++++++++ compat/preload-index/bulk-linux.h | 20 ++++ config.mak.uname | 1 + contrib/buildsystems/CMakeLists.txt | 1 + meson.build | 1 + 5 files changed, 172 insertions(+) create mode 100644 compat/preload-index/bulk-linux-open.c diff --git a/compat/preload-index/bulk-linux-open.c b/compat/preload-index/bulk-linux-open.c new file mode 100644 index 00000000000000..02cd5b55b87f43 --- /dev/null +++ b/compat/preload-index/bulk-linux-open.c @@ -0,0 +1,149 @@ +#include "git-compat-util.h" + +#ifdef __linux__ + +#include + +#include "compat/preload-index/bulk-linux.h" +#include "preload-index-bulk.h" + +#if defined(SYS_getdents64) && defined(SYS_statx) + +static int valid_component(const char *component, size_t len) +{ + return len && + !(len == 1 && component[0] == '.') && + !(len == 2 && component[0] == '.' && component[1] == '.'); +} + +static int verify_mount(struct preload_bulk_scan *scan, int fd) +{ + struct preload_bulk_linux_data *data = scan->platform_data; + struct preload_linux_statx stx; + + if (preload_bulk_linux_statx_raw(fd, "", PRELOAD_AT_EMPTY_PATH, + &stx)) + return -1; + if (!preload_bulk_linux_statx_complete(&stx)) { + errno = EOPNOTSUPP; + return -1; + } + if (stx.mnt_id != data->root_mnt_id) { + errno = EXDEV; + return -1; + } + return 0; +} + +#ifdef SYS_openat2 +int preload_bulk_linux_openat2_raw(int dirfd, const char *path) +{ + struct preload_linux_open_how how = { + .flags = O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC, + .resolve = PRELOAD_RESOLVE_BENEATH | + PRELOAD_RESOLVE_NO_SYMLINKS | + PRELOAD_RESOLVE_NO_MAGICLINKS | + PRELOAD_RESOLVE_NO_XDEV, + }; + + return syscall(SYS_openat2, dirfd, path, &how, sizeof(how)); +} +#endif + +static int open_one_fallback(struct preload_bulk_scan *scan, int parent_fd, + const char *name, int check_mount) +{ + int fd = openat(parent_fd, name, + O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + + if (fd < 0) + return -1; + if (check_mount && verify_mount(scan, fd)) { + int saved_errno = errno; + + close(fd); + errno = saved_errno; + return -1; + } + return fd; +} + +int preload_bulk_linux_open_dir_at( + struct preload_bulk_worker *worker, int parent_fd, + const char *name) +{ + struct preload_bulk_scan *scan = worker->scan; + struct preload_bulk_linux_data *data = scan->platform_data; + + if (!valid_component(name, strlen(name)) || strchr(name, '/')) { + errno = EINVAL; + return -1; + } +#ifdef SYS_openat2 + if (data->use_openat2) + return preload_bulk_linux_openat2_raw(parent_fd, name); +#else + (void)data; +#endif + /* scan_directory() verifies the opened descriptor's mount ID. */ + return open_one_fallback(scan, parent_fd, name, 0); +} + +int preload_bulk_linux_open_relative(struct preload_bulk_scan *scan, + const char *path) +{ + struct preload_bulk_linux_data *data = scan->platform_data; + const char *component = path; + int fd; + + if (!*path || *path == '/' || path[strlen(path) - 1] == '/') { + errno = EINVAL; + return -1; + } +#ifdef SYS_openat2 + if (data->use_openat2) + return preload_bulk_linux_openat2_raw(scan->root_fd, path); +#else + (void)data; +#endif + fd = fcntl(scan->root_fd, F_DUPFD_CLOEXEC, 0); + if (fd < 0) + return -1; + if (verify_mount(scan, fd)) { + int saved_errno = errno; + + close(fd); + errno = saved_errno; + return -1; + } + if (!strcmp(path, ".")) + return fd; + while (*component) { + const char *slash = strchr(component, '/'); + size_t len = slash ? (size_t)(slash - component) : + strlen(component); + char *name; + int next; + + if (!valid_component(component, len)) { + close(fd); + errno = EINVAL; + return -1; + } + name = xmemdupz(component, len); + next = open_one_fallback(scan, fd, name, 1); + free(name); + close(fd); + if (next < 0) + return -1; + fd = next; + if (!slash) + break; + component = slash + 1; + } + return fd; +} + +#endif /* SYS_getdents64 && SYS_statx */ + +#endif /* __linux__ */ diff --git a/compat/preload-index/bulk-linux.h b/compat/preload-index/bulk-linux.h index e4530ac0fda8e6..e26ee469350b23 100644 --- a/compat/preload-index/bulk-linux.h +++ b/compat/preload-index/bulk-linux.h @@ -10,6 +10,10 @@ #define PRELOAD_AT_SYMLINK_NOFOLLOW 0x100 #define PRELOAD_STATX_BASIC_STATS 0x000007ffU #define PRELOAD_STATX_MNT_ID 0x00001000U +#define PRELOAD_RESOLVE_NO_XDEV 0x01 +#define PRELOAD_RESOLVE_NO_MAGICLINKS 0x02 +#define PRELOAD_RESOLVE_NO_SYMLINKS 0x04 +#define PRELOAD_RESOLVE_BENEATH 0x08 struct preload_linux_statx_timestamp { int64_t tv_sec; @@ -44,10 +48,18 @@ struct preload_linux_statx { uint64_t spare3[12]; }; +struct preload_linux_open_how { + uint64_t flags; + uint64_t mode; + uint64_t resolve; +}; + struct preload_bulk_linux_data { uint64_t root_mnt_id; + int use_openat2; }; +struct preload_bulk_scan; struct preload_bulk_worker; #if defined(SYS_getdents64) && defined(SYS_statx) @@ -66,6 +78,14 @@ int preload_bulk_linux_fd_stat(struct preload_bulk_worker *worker, int fd, struct preload_linux_statx *stx, struct stat *st); +#ifdef SYS_openat2 +int preload_bulk_linux_openat2_raw(int dirfd, const char *path); +#endif +int preload_bulk_linux_open_dir_at(struct preload_bulk_worker *worker, + int parent_fd, const char *name); +int preload_bulk_linux_open_relative(struct preload_bulk_scan *scan, + const char *path); + #endif /* SYS_getdents64 && SYS_statx */ #endif /* __linux__ */ diff --git a/config.mak.uname b/config.mak.uname index f59b4bfa4aea4a..293d084e01e318 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -63,6 +63,7 @@ ifeq ($(uname_S),Linux) PROCFS_EXECUTABLE_PATH = /proc/self/exe HAVE_PLATFORM_PROCINFO = YesPlease COMPAT_OBJS += compat/linux/procinfo.o + COMPAT_OBJS += compat/preload-index/bulk-linux-open.o COMPAT_OBJS += compat/preload-index/bulk-linux-stat.o EXTLIBS += -ldl # centos7/rhel7 provides gcc 4.8.5 and zlib 1.2.7. diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index aec2421a64da6b..815b7512af0d64 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -277,6 +277,7 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") unix-socket.c unix-stream-server.c compat/linux/procinfo.c + compat/preload-index/bulk-linux-open.c compat/preload-index/bulk-linux-stat.c) elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") add_compile_definitions(HAVE_PRELOAD_INDEX_BULK PRECOMPOSE_UNICODE diff --git a/meson.build b/meson.build index 3850ee113e5a93..3cf26a3882f2d2 100644 --- a/meson.build +++ b/meson.build @@ -1361,6 +1361,7 @@ endif if host_machine.system() == 'linux' compat_sources += [ 'compat/linux/procinfo.c', + 'compat/preload-index/bulk-linux-open.c', 'compat/preload-index/bulk-linux-stat.c', ] elif host_machine.system() == 'windows' From 3233ddb2406e859b69d049777b179683b65fdb15 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 13:34:12 -0500 Subject: [PATCH 092/105] preload-index: enumerate Linux directories without trusting d_type A getdents64 record supplies a type hint, not proof that a path is a regular file or directory. Trusting that hint can hide a tracked replacement, misapply ignore rules, or report the wrong visible untracked shape. Parse record lengths and names within a bounded 1 MiB worker buffer. Require statx metadata and matching mount identity for tracked paths. When collecting untracked paths, obtain authoritative metadata before classifying a wholly untracked file or directory. Use directory hints only to schedule paths with tracked descendants. Preserve per-entry fallback for special and multiply linked tracked files. Stop an untracked subtree once its normal-status witness is visible, and invalidate uncertain untracked results. Register the module in all three Linux builds; the native DEVELOPER=1 boundary build compiles it without selecting the backend. Signed-off-by: Taylor Blau --- compat/preload-index/bulk-linux-entry.c | 263 ++++++++++++++++++++++++ compat/preload-index/bulk-linux.h | 7 + config.mak.uname | 1 + contrib/buildsystems/CMakeLists.txt | 1 + meson.build | 1 + 5 files changed, 273 insertions(+) create mode 100644 compat/preload-index/bulk-linux-entry.c diff --git a/compat/preload-index/bulk-linux-entry.c b/compat/preload-index/bulk-linux-entry.c new file mode 100644 index 00000000000000..d911c2be137964 --- /dev/null +++ b/compat/preload-index/bulk-linux-entry.c @@ -0,0 +1,263 @@ +#include "git-compat-util.h" + +#ifdef __linux__ + +#include +#include + +#include "compat/preload-index/bulk-linux.h" +#include "dir.h" +#include "preload-index-bulk.h" + +#define PRELOAD_INDEX_BULK_LINUX_BUFFER_SIZE (1024 * 1024) + +struct preload_linux_dirent64 { + uint64_t ino; + int64_t off; + uint16_t reclen; + uint8_t type; + char name[FLEX_ARRAY]; +}; + +#if defined(SYS_getdents64) && defined(SYS_statx) + +static void record_foreign_entry(struct preload_bulk_worker *worker, + const char *path, size_t path_len, + int pos, mode_t mode) +{ + if (pos >= 0) + preload_bulk_record_tracked_fallback(worker, pos); + if (S_ISDIR(mode)) + preload_bulk_record_tracked_descendants_fallback( + worker, path, path_len); + if (worker->scan->collect_untracked) + preload_bulk_invalidate_untracked(worker); +} + +static void handle_directory( + struct preload_bulk_worker *worker, + const struct preload_bulk_task *task, int fd, + const struct preload_bulk_dir_identity *parent_identity, + const char *name, int pos) +{ + struct preload_bulk_scan *scan = worker->scan; + struct preload_bulk_untracked_root *untracked_root = + task->untracked_root; + int has_tracked_descendants; + + if (pos >= 0) { + if (scan->collect_untracked && + preload_bulk_index_entry_is_gitlink(scan, pos)) + return; + preload_bulk_record_tracked_fallback(worker, pos); + return; + } + has_tracked_descendants = + preload_bulk_index_pos_has_tracked_descendants( + scan, worker->path.buf, worker->path.len, pos); + if (!has_tracked_descendants && + preload_bulk_record_tracked_alias_fallback( + worker, worker->path.buf, worker->path.len)) { + if (scan->collect_untracked) + preload_bulk_invalidate_untracked(worker); + return; + } + if (!has_tracked_descendants) { + if (!scan->collect_untracked || + preload_bulk_untracked_is_invalid(worker) || + preload_bulk_untracked_root_is_visible( + worker, untracked_root) || + preload_bulk_path_is_excluded( + worker, worker->path.buf, DT_DIR)) + return; + if (!untracked_root) + untracked_root = preload_bulk_untracked_root_new( + worker, worker->path.buf, worker->path.len); + } + preload_bulk_schedule_directory( + worker, fd, parent_identity, NULL, untracked_root, + name, worker->path.buf, worker->path.len); +} + +static void record_untracked(struct preload_bulk_worker *worker, + const struct preload_bulk_task *task, + int dtype) +{ + struct preload_bulk_scan *scan = worker->scan; + + if (!scan->collect_untracked || + preload_bulk_untracked_is_invalid(worker)) + return; + if (preload_bulk_record_tracked_alias_fallback( + worker, worker->path.buf, worker->path.len)) { + preload_bulk_invalidate_untracked(worker); + return; + } + if (!preload_bulk_path_is_excluded( + worker, worker->path.buf, dtype)) + preload_bulk_record_untracked( + worker, task->untracked_root, worker->path.buf); +} + +int preload_bulk_linux_enumerate( + struct preload_bulk_worker *worker, + struct preload_bulk_task *task, int fd, + const struct preload_bulk_dir_identity *parent_identity) +{ + struct preload_bulk_scan *scan = worker->scan; + char *buf = worker->buffer; + size_t path_prefix_len; + + if (!buf) { + buf = xmalloc(PRELOAD_INDEX_BULK_LINUX_BUFFER_SIZE); + worker->buffer = buf; + } + worker->dirs++; + strbuf_reset(&worker->path); + if (strcmp(task->path, ".")) { + strbuf_addstr(&worker->path, task->path); + strbuf_addch(&worker->path, '/'); + } + path_prefix_len = worker->path.len; + + for (;;) { + long bytes = syscall(SYS_getdents64, fd, buf, + PRELOAD_INDEX_BULK_LINUX_BUFFER_SIZE); + size_t offset = 0; + + worker->bulk_calls++; + if (bytes < 0) + return -1; + if (!bytes) + return 0; + while (offset < (size_t)bytes) { + struct preload_linux_dirent64 *de = + (void *)(buf + offset); + size_t minimum = + offsetof(struct preload_linux_dirent64, name) + 1; + size_t name_space; + struct preload_linux_statx stx; + struct stat st; + char *nul; + unsigned char dtype; + int has_tracked_descendants = 0, pos; + + if (scan->collect_untracked && + preload_bulk_untracked_root_is_visible( + worker, task->untracked_root)) + return 0; + if ((size_t)bytes - offset < minimum || + de->reclen < minimum || + de->reclen > (size_t)bytes - offset) + goto malformed; + name_space = de->reclen - + offsetof(struct preload_linux_dirent64, name); + nul = memchr(de->name, '\0', name_space); + if (!nul || nul == de->name || + memchr(de->name, '/', nul - de->name)) + goto malformed; + offset += de->reclen; + if (is_dot_or_dotdot(de->name)) + continue; + worker->entries++; + if (!fspathcmp(de->name, ".git")) { + if (strcmp(task->path, ".") && + scan->collect_untracked) + preload_bulk_invalidate_untracked( + worker); + continue; + } + + strbuf_setlen(&worker->path, path_prefix_len); + strbuf_addstr(&worker->path, de->name); + if (worker->path.len > INT_MAX) + goto malformed; + pos = preload_bulk_index_position( + scan, worker->path.buf, worker->path.len); + dtype = de->type; + + /* + * Exact tracked paths always reach statx. A directory + * which may contain tracked descendants can be + * scheduled directly: the O_DIRECTORY open and + * descriptor statx remain authoritative. + * + * A hint cannot classify a wholly untracked entry: + * file-versus-directory changes exclude matching and + * result shape. Force those entries through statx before + * taking either shortcut. + */ + if (pos < 0 && dtype != DT_UNKNOWN) + has_tracked_descendants = + preload_bulk_index_pos_has_tracked_descendants( + scan, worker->path.buf, + worker->path.len, pos); + if (scan->collect_untracked && pos < 0 && + !has_tracked_descendants) + dtype = DT_UNKNOWN; + if (dtype == DT_DIR && pos < 0) { + handle_directory(worker, task, fd, + parent_identity, + de->name, pos); + continue; + } + if (pos < 0 && !has_tracked_descendants && + (dtype == DT_REG || dtype == DT_LNK)) { + record_untracked( + worker, task, + dtype == DT_LNK ? DT_LNK : DT_REG); + continue; + } + if (pos < 0 && !has_tracked_descendants && + dtype != DT_UNKNOWN) { + preload_bulk_record_tracked_alias_fallback( + worker, worker->path.buf, + worker->path.len); + continue; + } + if (preload_bulk_linux_entry_stat( + worker, fd, de->name, &stx, &st)) { + if (errno == EXDEV) { + record_foreign_entry( + worker, worker->path.buf, + worker->path.len, pos, + stx.mode); + continue; + } + goto malformed; + } + if (S_ISDIR(st.st_mode)) { + handle_directory(worker, task, fd, + parent_identity, + de->name, pos); + continue; + } + if (pos < 0) { + if (S_ISREG(st.st_mode)) + record_untracked(worker, task, DT_REG); + else if (S_ISLNK(st.st_mode)) + record_untracked(worker, task, DT_LNK); + else + preload_bulk_record_tracked_alias_fallback( + worker, worker->path.buf, + worker->path.len); + continue; + } + if ((!S_ISREG(st.st_mode) && !S_ISLNK(st.st_mode)) || + st.st_nlink != 1) { + preload_bulk_record_tracked_fallback( + worker, pos); + continue; + } + preload_bulk_record_tracked(worker, pos, &st); + } + } + +malformed: + worker->malformed++; + return -1; +} + +#endif /* SYS_getdents64 && SYS_statx */ + +#endif /* __linux__ */ diff --git a/compat/preload-index/bulk-linux.h b/compat/preload-index/bulk-linux.h index e26ee469350b23..f8ec615656d601 100644 --- a/compat/preload-index/bulk-linux.h +++ b/compat/preload-index/bulk-linux.h @@ -60,7 +60,9 @@ struct preload_bulk_linux_data { }; struct preload_bulk_scan; +struct preload_bulk_task; struct preload_bulk_worker; +struct preload_bulk_dir_identity; #if defined(SYS_getdents64) && defined(SYS_statx) @@ -86,6 +88,11 @@ int preload_bulk_linux_open_dir_at(struct preload_bulk_worker *worker, int preload_bulk_linux_open_relative(struct preload_bulk_scan *scan, const char *path); +int preload_bulk_linux_enumerate( + struct preload_bulk_worker *worker, + struct preload_bulk_task *task, int fd, + const struct preload_bulk_dir_identity *parent_identity); + #endif /* SYS_getdents64 && SYS_statx */ #endif /* __linux__ */ diff --git a/config.mak.uname b/config.mak.uname index 293d084e01e318..d17fc31a0b541a 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -63,6 +63,7 @@ ifeq ($(uname_S),Linux) PROCFS_EXECUTABLE_PATH = /proc/self/exe HAVE_PLATFORM_PROCINFO = YesPlease COMPAT_OBJS += compat/linux/procinfo.o + COMPAT_OBJS += compat/preload-index/bulk-linux-entry.o COMPAT_OBJS += compat/preload-index/bulk-linux-open.o COMPAT_OBJS += compat/preload-index/bulk-linux-stat.o EXTLIBS += -ldl diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 815b7512af0d64..4190755222a9d3 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -277,6 +277,7 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") unix-socket.c unix-stream-server.c compat/linux/procinfo.c + compat/preload-index/bulk-linux-entry.c compat/preload-index/bulk-linux-open.c compat/preload-index/bulk-linux-stat.c) elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") diff --git a/meson.build b/meson.build index 3cf26a3882f2d2..a4d11cd7699f64 100644 --- a/meson.build +++ b/meson.build @@ -1361,6 +1361,7 @@ endif if host_machine.system() == 'linux' compat_sources += [ 'compat/linux/procinfo.c', + 'compat/preload-index/bulk-linux-entry.c', 'compat/preload-index/bulk-linux-open.c', 'compat/preload-index/bulk-linux-stat.c', ] From 7d0c5b448b629ed823ff2b550418c35aa3f21952 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 13:34:39 -0500 Subject: [PATCH 093/105] preload-index: detect replaced Linux scan directories Holding a directory descriptor establishes what workers read, but does not prove that the original directory stayed in the worktree. A child can also move under a different parent while queued. Publishing observations from either replacement could hide worktree changes. Capture the complete directory statx observation and converted stat identity before enumeration. Verify the descriptor mount and recheck both identities afterward. For queued children, resolve the parent through the held child descriptor and compare it with the recorded parent identity. Add the mount identifier to the shared directory identity and register the Linux scan module with Make, CMake, and Meson. The native DEVELOPER=1 boundary build compiles it, while recorded directory changes prevent the completed scan from being accepted. Signed-off-by: Taylor Blau --- compat/preload-index/bulk-linux-scan.c | 107 +++++++++++++++++++++++++ compat/preload-index/bulk-linux.h | 2 + config.mak.uname | 1 + contrib/buildsystems/CMakeLists.txt | 1 + meson.build | 1 + preload-index-bulk.h | 1 + 6 files changed, 113 insertions(+) create mode 100644 compat/preload-index/bulk-linux-scan.c diff --git a/compat/preload-index/bulk-linux-scan.c b/compat/preload-index/bulk-linux-scan.c new file mode 100644 index 00000000000000..c6da94aa41564b --- /dev/null +++ b/compat/preload-index/bulk-linux-scan.c @@ -0,0 +1,107 @@ +#include "git-compat-util.h" + +#ifdef __linux__ + +#include + +#include "compat/preload-index/bulk-linux.h" +#include "path-namespace.h" +#include "preload-index-bulk.h" + +#if defined(SYS_getdents64) && defined(SYS_statx) + +static struct preload_bulk_dir_identity directory_identity( + const struct preload_linux_statx *stx, const struct stat *st) +{ + struct preload_bulk_dir_identity result = { + .stat = *st, + .platform_id = stx->mnt_id, + .complete = 1, + }; + + return result; +} + +static int directory_identity_matches( + const struct preload_bulk_dir_identity *before, + const struct preload_linux_statx *stx, const struct stat *after) +{ + return S_ISDIR(after->st_mode) && + path_namespace_stat_equal(&before->stat, after) && + before->platform_id == stx->mnt_id; +} + +int preload_bulk_linux_scan_directory(struct preload_bulk_worker *worker, + struct preload_bulk_task *task) +{ + struct preload_bulk_scan *scan = worker->scan; + struct preload_linux_statx before_stx, after_stx; + struct preload_bulk_dir_identity before_identity; + struct stat before, after; + size_t path_len; + int fd = task->fd; + int ret = -1; + + if (fd < 0) + fd = preload_bulk_linux_open_relative(scan, task->path); + if (fd < 0) + goto out; + if (preload_bulk_test_barrier(scan, task->path)) + goto out; + if (preload_bulk_linux_fd_stat( + worker, fd, &before_stx, &before) || + !S_ISDIR(before.st_mode)) { + if (errno != EXDEV) + goto out; + path_len = strlen(task->path); + preload_bulk_record_tracked_descendants_fallback( + worker, task->path, path_len); + if (scan->collect_untracked) + preload_bulk_invalidate_untracked(worker); + ret = 0; + goto out; + } + before_identity = directory_identity(&before_stx, &before); + if ((!scan->collect_untracked || + !preload_bulk_untracked_root_is_visible( + worker, task->untracked_root)) && + preload_bulk_linux_enumerate( + worker, task, fd, &before_identity)) + goto out; + if (preload_bulk_linux_fd_stat( + worker, fd, &after_stx, &after)) + goto out; + if (!preload_bulk_linux_statx_same(&before_stx, &after_stx) || + !directory_identity_matches( + &before_identity, &after_stx, &after)) + worker->changed_dirs++; + ret = 0; + +out: + if (task->has_parent_identity) { + struct preload_linux_statx parent_stx; + struct stat parent_after; + int parent_changed = fd < 0; + + /* + * Resolve ".." through the held child descriptor so a move + * cannot redirect the parent check to the old path. + */ + if (!parent_changed) + parent_changed = preload_bulk_linux_entry_stat( + worker, fd, "..", &parent_stx, + &parent_after); + if (parent_changed || + !directory_identity_matches( + &task->parent_identity, &parent_stx, + &parent_after)) + worker->changed_dirs++; + } + if (fd >= 0) + close(fd); + return ret; +} + +#endif /* SYS_getdents64 && SYS_statx */ + +#endif /* __linux__ */ diff --git a/compat/preload-index/bulk-linux.h b/compat/preload-index/bulk-linux.h index f8ec615656d601..672a71c13328ff 100644 --- a/compat/preload-index/bulk-linux.h +++ b/compat/preload-index/bulk-linux.h @@ -92,6 +92,8 @@ int preload_bulk_linux_enumerate( struct preload_bulk_worker *worker, struct preload_bulk_task *task, int fd, const struct preload_bulk_dir_identity *parent_identity); +int preload_bulk_linux_scan_directory(struct preload_bulk_worker *worker, + struct preload_bulk_task *task); #endif /* SYS_getdents64 && SYS_statx */ diff --git a/config.mak.uname b/config.mak.uname index d17fc31a0b541a..f4215b95b587c5 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -65,6 +65,7 @@ ifeq ($(uname_S),Linux) COMPAT_OBJS += compat/linux/procinfo.o COMPAT_OBJS += compat/preload-index/bulk-linux-entry.o COMPAT_OBJS += compat/preload-index/bulk-linux-open.o + COMPAT_OBJS += compat/preload-index/bulk-linux-scan.o COMPAT_OBJS += compat/preload-index/bulk-linux-stat.o EXTLIBS += -ldl # centos7/rhel7 provides gcc 4.8.5 and zlib 1.2.7. diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 4190755222a9d3..315cedd4406a85 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -279,6 +279,7 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") compat/linux/procinfo.c compat/preload-index/bulk-linux-entry.c compat/preload-index/bulk-linux-open.c + compat/preload-index/bulk-linux-scan.c compat/preload-index/bulk-linux-stat.c) elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") add_compile_definitions(HAVE_PRELOAD_INDEX_BULK PRECOMPOSE_UNICODE diff --git a/meson.build b/meson.build index a4d11cd7699f64..efb383dcb59d0a 100644 --- a/meson.build +++ b/meson.build @@ -1363,6 +1363,7 @@ if host_machine.system() == 'linux' 'compat/linux/procinfo.c', 'compat/preload-index/bulk-linux-entry.c', 'compat/preload-index/bulk-linux-open.c', + 'compat/preload-index/bulk-linux-scan.c', 'compat/preload-index/bulk-linux-stat.c', ] elif host_machine.system() == 'windows' diff --git a/preload-index-bulk.h b/preload-index-bulk.h index 2934d31b8674c1..b08269dfb3ed59 100644 --- a/preload-index-bulk.h +++ b/preload-index-bulk.h @@ -12,6 +12,7 @@ struct preload_bulk_untracked_root; struct preload_bulk_dir_identity { struct stat stat; + uint64_t platform_id; unsigned complete : 1; }; From 58f3f5cec4ffd9f18de1bfdd9147073655b268f6 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 13:35:16 -0500 Subject: [PATCH 094/105] preload-index: validate Linux mount topology around bulk scans Individually anchored descriptors do not establish that the mount namespace or named worktree root stayed unchanged throughout a scan. A mount replacement can invalidate otherwise consistent directory observations. Accept only ext-family and XFS filesystems with complete root statx and mount-identity data. Capture /proc/self/mountinfo before the scan, compare it at completion, and freshly reopen the named worktree root with O_NOFOLLOW to verify its original complete identity. Probe openat2() without requiring it. Register the topology module in all three Linux builds. The native DEVELOPER=1 boundary build compiles it. Missing namespace proof, unsupported filesystems, changed mount tables, or replaced roots reject the result; the retained mount snapshot adds memory and can reject unrelated namespace changes. Signed-off-by: Taylor Blau --- compat/preload-index/bulk-linux-topology.c | 142 +++++++++++++++++++++ compat/preload-index/bulk-linux.h | 8 ++ config.mak.uname | 1 + contrib/buildsystems/CMakeLists.txt | 3 +- meson.build | 1 + 5 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 compat/preload-index/bulk-linux-topology.c diff --git a/compat/preload-index/bulk-linux-topology.c b/compat/preload-index/bulk-linux-topology.c new file mode 100644 index 00000000000000..abfb80d214d10a --- /dev/null +++ b/compat/preload-index/bulk-linux-topology.c @@ -0,0 +1,142 @@ +#include "git-compat-util.h" + +#ifdef __linux__ + +#include +#include + +#include "compat/preload-index/bulk-linux.h" +#include "preload-index-bulk.h" +#include "repository.h" +#include "trace2.h" + +#ifndef EXT_FAMILY_SUPER_MAGIC +#define EXT_FAMILY_SUPER_MAGIC 0xef53 +#endif +#ifndef XFS_SUPER_MAGIC +#define XFS_SUPER_MAGIC 0x58465342 +#endif + +#if defined(SYS_getdents64) && defined(SYS_statx) + +static int read_mountinfo(struct strbuf *out) +{ + int fd = open("/proc/self/mountinfo", O_RDONLY | O_CLOEXEC); + int ret = -1; + + if (fd < 0) + return -1; + strbuf_reset(out); + if (strbuf_read(out, fd, 0) >= 0) + ret = 0; + if (close(fd)) + ret = -1; + return ret; +} + +const char *preload_bulk_linux_start(struct preload_bulk_scan *scan) +{ + struct preload_bulk_linux_data *data; + struct statfs fs; + const char *fs_name; + + CALLOC_ARRAY(data, 1); + strbuf_init(&data->mountinfo, 0); + scan->platform_data = data; + scan->root_fd = open(repo_get_work_tree(scan->repo), + O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (scan->root_fd < 0 || fstatfs(scan->root_fd, &fs)) + return "unsupported-filesystem"; + if ((unsigned long)fs.f_type == EXT_FAMILY_SUPER_MAGIC) + fs_name = "ext-family"; + else if ((unsigned long)fs.f_type == XFS_SUPER_MAGIC) + fs_name = "xfs"; + else + return "unsupported-filesystem"; + trace2_data_string("index", scan->repo, "preload/bulk_filesystem", + fs_name); + if (preload_bulk_linux_statx_raw( + scan->root_fd, "", PRELOAD_AT_EMPTY_PATH, + &data->root_statx) || + !preload_bulk_linux_statx_complete(&data->root_statx) || + !S_ISDIR(data->root_statx.mode)) + return "statx-unavailable"; + data->root_mnt_id = data->root_statx.mnt_id; + if (read_mountinfo(&data->mountinfo)) + return "namespace-check-unavailable"; +#ifdef SYS_openat2 + { + int fd = preload_bulk_linux_openat2_raw(scan->root_fd, "."); + + if (fd >= 0) { + struct preload_linux_statx probe; + + if (!preload_bulk_linux_statx_raw( + fd, "", PRELOAD_AT_EMPTY_PATH, &probe) && + preload_bulk_linux_statx_complete(&probe) && + probe.mnt_id == data->root_mnt_id) + data->use_openat2 = 1; + close(fd); + } + } +#endif + trace2_data_intmax("index", scan->repo, "preload/bulk_openat2", + data->use_openat2); + return NULL; +} + +const char *preload_bulk_linux_finish(struct preload_bulk_scan *scan) +{ + struct preload_bulk_linux_data *data = scan->platform_data; + struct strbuf after = STRBUF_INIT; + struct preload_linux_statx root_after; + const char *result = NULL; + int fd; + + if (read_mountinfo(&after)) { + result = "namespace-check-unavailable"; + goto out; + } + if (strbuf_cmp(&data->mountinfo, &after)) { + trace2_data_intmax( + "index", scan->repo, + "preload/bulk_namespace_churn", 1); + result = "namespace-churn"; + goto out; + } + fd = open(repo_get_work_tree(scan->repo), + O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (fd < 0) { + result = "namespace-race"; + goto out; + } + if (preload_bulk_linux_statx_raw( + fd, "", PRELOAD_AT_EMPTY_PATH, &root_after) || + !preload_bulk_linux_statx_same( + &data->root_statx, &root_after)) + result = "namespace-race"; + close(fd); + +out: + strbuf_release(&after); + return result; +} + +void preload_bulk_linux_release(struct preload_bulk_scan *scan) +{ + struct preload_bulk_linux_data *data = scan->platform_data; + + if (scan->root_fd >= 0) { + close(scan->root_fd); + scan->root_fd = -1; + } + if (!data) + return; + strbuf_release(&data->mountinfo); + free(data); + scan->platform_data = NULL; +} + +#endif /* SYS_getdents64 && SYS_statx */ + +#endif /* __linux__ */ diff --git a/compat/preload-index/bulk-linux.h b/compat/preload-index/bulk-linux.h index 672a71c13328ff..ac1398216e1662 100644 --- a/compat/preload-index/bulk-linux.h +++ b/compat/preload-index/bulk-linux.h @@ -5,6 +5,8 @@ #include +#include "strbuf.h" + #define PRELOAD_AT_NO_AUTOMOUNT 0x800 #define PRELOAD_AT_EMPTY_PATH 0x1000 #define PRELOAD_AT_SYMLINK_NOFOLLOW 0x100 @@ -55,6 +57,8 @@ struct preload_linux_open_how { }; struct preload_bulk_linux_data { + struct preload_linux_statx root_statx; + struct strbuf mountinfo; uint64_t root_mnt_id; int use_openat2; }; @@ -95,6 +99,10 @@ int preload_bulk_linux_enumerate( int preload_bulk_linux_scan_directory(struct preload_bulk_worker *worker, struct preload_bulk_task *task); +const char *preload_bulk_linux_start(struct preload_bulk_scan *scan); +const char *preload_bulk_linux_finish(struct preload_bulk_scan *scan); +void preload_bulk_linux_release(struct preload_bulk_scan *scan); + #endif /* SYS_getdents64 && SYS_statx */ #endif /* __linux__ */ diff --git a/config.mak.uname b/config.mak.uname index f4215b95b587c5..bc768e714c9ee7 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -67,6 +67,7 @@ ifeq ($(uname_S),Linux) COMPAT_OBJS += compat/preload-index/bulk-linux-open.o COMPAT_OBJS += compat/preload-index/bulk-linux-scan.o COMPAT_OBJS += compat/preload-index/bulk-linux-stat.o + COMPAT_OBJS += compat/preload-index/bulk-linux-topology.o EXTLIBS += -ldl # centos7/rhel7 provides gcc 4.8.5 and zlib 1.2.7. ifneq ($(findstring .el7.,$(uname_R)),) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 315cedd4406a85..e961999f61a0a4 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -280,7 +280,8 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") compat/preload-index/bulk-linux-entry.c compat/preload-index/bulk-linux-open.c compat/preload-index/bulk-linux-scan.c - compat/preload-index/bulk-linux-stat.c) + compat/preload-index/bulk-linux-stat.c + compat/preload-index/bulk-linux-topology.c) elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") add_compile_definitions(HAVE_PRELOAD_INDEX_BULK PRECOMPOSE_UNICODE USE_ST_TIMESPEC) diff --git a/meson.build b/meson.build index efb383dcb59d0a..971fa5ea85c7c8 100644 --- a/meson.build +++ b/meson.build @@ -1365,6 +1365,7 @@ if host_machine.system() == 'linux' 'compat/preload-index/bulk-linux-open.c', 'compat/preload-index/bulk-linux-scan.c', 'compat/preload-index/bulk-linux-stat.c', + 'compat/preload-index/bulk-linux-topology.c', ] elif host_machine.system() == 'windows' compat_sources += 'compat/win32/trace2_win32_process_info.c' From 37c3052fd4fe8f34a358ce9c6209c8079bc566ac Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 13:39:36 -0500 Subject: [PATCH 095/105] preload-index: enable the verified Linux bulk scan backend The separately registered Linux metadata, anchored-open, enumeration, directory-validation, and topology modules cannot safely publish a physical scan by themselves. They must share the existing bulk backend lifecycle so every closing check runs before results are accepted. Assemble those modules into the Linux backend and register the shared and platform objects with Make, CMake, and Meson. Retain the existing requirements that core.preloadIndex and core.preloadIndexBulk are enabled and fsmonitor is disabled. Preserve ordinary preload when a required syscall, filesystem, mount proof, or closing validation is unavailable. Cap Linux scans at 16 workers. Each worker can allocate a 1 MiB directory buffer; mount snapshots and retained scan results add further memory. Document ext-family and XFS support and keep directory-type injection confined to the documented test environment. Add and register t7532-preload-index-linux.sh with 12 Linux-only cases. Native Linux validation passes 12/12 in the threaded build and 12/12 in a separate NO_PTHREADS build; CMake and Meson link Git. The suite compares ordinary status for tracked changes, visible and ignored paths, false type hints, fallback shapes, and a synchronized child replacement. Signed-off-by: Taylor Blau --- Documentation/config/core.adoc | 5 +- compat/preload-index/bulk-linux-entry.c | 4 + compat/preload-index/bulk-linux-topology.c | 24 ++ compat/preload-index/bulk-linux.c | 37 +++ compat/preload-index/bulk-linux.h | 2 + config.mak.uname | 11 +- contrib/buildsystems/CMakeLists.txt | 7 +- meson.build | 8 + preload-index-bulk.c | 3 + preload-index-bulk.h | 1 + t/README | 4 + t/meson.build | 1 + t/t7532-preload-index-linux.sh | 346 +++++++++++++++++++++ 13 files changed, 444 insertions(+), 9 deletions(-) create mode 100644 compat/preload-index/bulk-linux.c create mode 100755 t/t7532-preload-index-linux.sh diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index 5f01b603e5761a..59bc4a818cceb8 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -736,8 +736,9 @@ core.preloadIndexBulk:: This replaces per-entry filesystem lookups with a physical directory scan, but may cost more than normal preload depending on filesystem and cache state. Inconclusive scans are discarded before continuing with the normal -preload. Currently this is supported on APFS and only has an effect when -`core.preloadIndex` is enabled. Defaults to false. +preload. Currently this is supported on APFS, ext-family filesystems, and +XFS, and only has an effect when `core.preloadIndex` is enabled. Defaults +to false. core.unsetenvvars:: Windows-only: comma-separated list of environment variables' diff --git a/compat/preload-index/bulk-linux-entry.c b/compat/preload-index/bulk-linux-entry.c index d911c2be137964..8b63db291e7562 100644 --- a/compat/preload-index/bulk-linux-entry.c +++ b/compat/preload-index/bulk-linux-entry.c @@ -105,6 +105,7 @@ int preload_bulk_linux_enumerate( const struct preload_bulk_dir_identity *parent_identity) { struct preload_bulk_scan *scan = worker->scan; + struct preload_bulk_linux_data *data = scan->platform_data; char *buf = worker->buffer; size_t path_prefix_len; @@ -175,6 +176,9 @@ int preload_bulk_linux_enumerate( pos = preload_bulk_index_position( scan, worker->path.buf, worker->path.len); dtype = de->type; + if (data->test_dirent_path && + !strcmp(data->test_dirent_path, worker->path.buf)) + dtype = data->test_dirent_type; /* * Exact tracked paths always reach statx. A directory diff --git a/compat/preload-index/bulk-linux-topology.c b/compat/preload-index/bulk-linux-topology.c index abfb80d214d10a..523e6c08b909b3 100644 --- a/compat/preload-index/bulk-linux-topology.c +++ b/compat/preload-index/bulk-linux-topology.c @@ -2,10 +2,12 @@ #ifdef __linux__ +#include #include #include #include "compat/preload-index/bulk-linux.h" +#include "parse.h" #include "preload-index-bulk.h" #include "repository.h" #include "trace2.h" @@ -19,6 +21,26 @@ #if defined(SYS_getdents64) && defined(SYS_statx) +static void load_test_dirent_type(struct preload_bulk_linux_data *data) +{ + const char *path, *value; + + if (!git_env_bool("GIT_TEST_PRELOAD_INDEX_BULK", 0)) + return; + value = getenv("GIT_TEST_PRELOAD_INDEX_BULK_DIRENT_TYPE"); + if (!value) + return; + if (skip_prefix(value, "dir:", &path)) + data->test_dirent_type = DT_DIR; + else if (skip_prefix(value, "reg:", &path)) + data->test_dirent_type = DT_REG; + else + die("invalid GIT_TEST_PRELOAD_INDEX_BULK_DIRENT_TYPE"); + if (!*path) + die("GIT_TEST_PRELOAD_INDEX_BULK_DIRENT_TYPE needs a path"); + data->test_dirent_path = xstrdup(path); +} + static int read_mountinfo(struct strbuf *out) { int fd = open("/proc/self/mountinfo", O_RDONLY | O_CLOEXEC); @@ -42,6 +64,7 @@ const char *preload_bulk_linux_start(struct preload_bulk_scan *scan) CALLOC_ARRAY(data, 1); strbuf_init(&data->mountinfo, 0); + load_test_dirent_type(data); scan->platform_data = data; scan->root_fd = open(repo_get_work_tree(scan->repo), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); @@ -133,6 +156,7 @@ void preload_bulk_linux_release(struct preload_bulk_scan *scan) if (!data) return; strbuf_release(&data->mountinfo); + free(data->test_dirent_path); free(data); scan->platform_data = NULL; } diff --git a/compat/preload-index/bulk-linux.c b/compat/preload-index/bulk-linux.c new file mode 100644 index 00000000000000..5c4c120a61840c --- /dev/null +++ b/compat/preload-index/bulk-linux.c @@ -0,0 +1,37 @@ +#include "git-compat-util.h" + +#ifdef __linux__ + +#include + +#include "compat/preload-index/bulk-linux.h" +#include "preload-index-bulk.h" + +#if defined(SYS_getdents64) && defined(SYS_statx) + +static const struct preload_bulk_backend linux_backend = { + .collects_untracked = 1, + .max_threads = 16, + .start = preload_bulk_linux_start, + .finish = preload_bulk_linux_finish, + .release = preload_bulk_linux_release, + .open_proof_parent = preload_bulk_linux_open_relative, + .open_dir_at = preload_bulk_linux_open_dir_at, + .scan_directory = preload_bulk_linux_scan_directory, +}; + +const struct preload_bulk_backend *preload_bulk_platform_backend(void) +{ + return &linux_backend; +} + +#else /* !SYS_getdents64 || !SYS_statx */ + +const struct preload_bulk_backend *preload_bulk_platform_backend(void) +{ + return NULL; +} + +#endif + +#endif /* __linux__ */ diff --git a/compat/preload-index/bulk-linux.h b/compat/preload-index/bulk-linux.h index ac1398216e1662..e25db006115584 100644 --- a/compat/preload-index/bulk-linux.h +++ b/compat/preload-index/bulk-linux.h @@ -60,6 +60,8 @@ struct preload_bulk_linux_data { struct preload_linux_statx root_statx; struct strbuf mountinfo; uint64_t root_mnt_id; + char *test_dirent_path; + unsigned char test_dirent_type; int use_openat2; }; diff --git a/config.mak.uname b/config.mak.uname index bc768e714c9ee7..4d205c4c493939 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -63,11 +63,12 @@ ifeq ($(uname_S),Linux) PROCFS_EXECUTABLE_PATH = /proc/self/exe HAVE_PLATFORM_PROCINFO = YesPlease COMPAT_OBJS += compat/linux/procinfo.o - COMPAT_OBJS += compat/preload-index/bulk-linux-entry.o - COMPAT_OBJS += compat/preload-index/bulk-linux-open.o - COMPAT_OBJS += compat/preload-index/bulk-linux-scan.o - COMPAT_OBJS += compat/preload-index/bulk-linux-stat.o - COMPAT_OBJS += compat/preload-index/bulk-linux-topology.o + PRELOAD_INDEX_BULK_BACKEND = linux + PRELOAD_INDEX_BULK_PLATFORM_OBJS += compat/preload-index/bulk-linux-entry.o + PRELOAD_INDEX_BULK_PLATFORM_OBJS += compat/preload-index/bulk-linux-open.o + PRELOAD_INDEX_BULK_PLATFORM_OBJS += compat/preload-index/bulk-linux-scan.o + PRELOAD_INDEX_BULK_PLATFORM_OBJS += compat/preload-index/bulk-linux-stat.o + PRELOAD_INDEX_BULK_PLATFORM_OBJS += compat/preload-index/bulk-linux-topology.o EXTLIBS += -ldl # centos7/rhel7 provides gcc 4.8.5 and zlib 1.2.7. ifneq ($(findstring .el7.,$(uname_R)),) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index e961999f61a0a4..6440b31a4483a1 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -272,11 +272,13 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(NO_UNIX_SOCKETS 1) elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") - add_compile_definitions(PROCFS_EXECUTABLE_PATH="/proc/self/exe" HAVE_DEV_TTY ) + add_compile_definitions(HAVE_PRELOAD_INDEX_BULK + PROCFS_EXECUTABLE_PATH="/proc/self/exe" HAVE_DEV_TTY) list(APPEND compat_SOURCES unix-socket.c unix-stream-server.c compat/linux/procinfo.c + compat/preload-index/bulk-linux.c compat/preload-index/bulk-linux-entry.c compat/preload-index/bulk-linux-open.c compat/preload-index/bulk-linux-scan.c @@ -679,7 +681,8 @@ include_directories(${CMAKE_BINARY_DIR}) #libgit parse_makefile_for_sources(libgit_SOURCES ${CMAKE_SOURCE_DIR}/Makefile "LIB_OBJS") -if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") +if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" OR + CMAKE_SYSTEM_NAME STREQUAL "Linux") list(APPEND libgit_SOURCES preload-index-bulk-index.c preload-index-bulk-thread.c diff --git a/meson.build b/meson.build index 971fa5ea85c7c8..56e794f7dbe32a 100644 --- a/meson.build +++ b/meson.build @@ -1318,6 +1318,8 @@ if host_machine.system() == 'darwin' libgit_c_args += '-DHAVE_PRELOAD_INDEX_BULK' libgit_c_args += '-DPRECOMPOSE_UNICODE' libgit_c_args += '-DPROTECT_HFS_DEFAULT' +elif host_machine.system() == 'linux' + libgit_c_args += '-DHAVE_PRELOAD_INDEX_BULK' endif # Configure general compatibility wrappers. @@ -1361,12 +1363,18 @@ endif if host_machine.system() == 'linux' compat_sources += [ 'compat/linux/procinfo.c', + 'compat/preload-index/bulk-linux.c', 'compat/preload-index/bulk-linux-entry.c', 'compat/preload-index/bulk-linux-open.c', 'compat/preload-index/bulk-linux-scan.c', 'compat/preload-index/bulk-linux-stat.c', 'compat/preload-index/bulk-linux-topology.c', ] + libgit_sources += [ + 'preload-index-bulk-index.c', + 'preload-index-bulk-thread.c', + 'preload-index-bulk.c', + ] elif host_machine.system() == 'windows' compat_sources += 'compat/win32/trace2_win32_process_info.c' elif host_machine.system() == 'darwin' diff --git a/preload-index-bulk.c b/preload-index-bulk.c index cd53761358af4f..eed468b0d156ca 100644 --- a/preload-index-bulk.c +++ b/preload-index-bulk.c @@ -213,6 +213,9 @@ int preload_bulk_collect(struct index_state *istate, int threads, backend->open_proof_parent; if (istate->preload_untracked && !scan.collect_untracked) untracked_reason = "backend-unsupported"; + if (backend->max_threads > 0 && + scan.threads > backend->max_threads) + scan.threads = backend->max_threads; if (scan.collect_untracked) { scan.exclude_dir = &exclude_dir; #if HAVE_THREADS diff --git a/preload-index-bulk.h b/preload-index-bulk.h index b08269dfb3ed59..c6c862dbb8136c 100644 --- a/preload-index-bulk.h +++ b/preload-index-bulk.h @@ -60,6 +60,7 @@ struct preload_bulk_worker { struct preload_bulk_backend { unsigned collects_untracked : 1; + int max_threads; const char *(*start)(struct preload_bulk_scan *scan); const char *(*finish)(struct preload_bulk_scan *scan); void (*release)(struct preload_bulk_scan *scan); diff --git a/t/README b/t/README index e063266240fda9..31ead13c82aeb5 100644 --- a/t/README +++ b/t/README @@ -425,6 +425,10 @@ by overriding the minimum number of cache entries required per thread. GIT_TEST_PRELOAD_INDEX_BULK= overrides the `core.preloadIndexBulk` setting. +GIT_TEST_PRELOAD_INDEX_BULK_DIRENT_TYPE=:, when +GIT_TEST_PRELOAD_INDEX_BULK is enabled, overrides the Linux directory +entry type for one worktree-relative path. is `dir` or `reg`. + GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_PATH=, GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_READY=, and GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_RESUME=, when diff --git a/t/meson.build b/t/meson.build index ea4a0d2c11380c..909f58e975a0fa 100644 --- a/t/meson.build +++ b/t/meson.build @@ -959,6 +959,7 @@ integration_tests = [ 't7528-signed-commit-ssh.sh', 't7529-preload-index-apfs.sh', 't7531-semantic-verify.sh', + 't7532-preload-index-linux.sh', 't7600-merge.sh', 't7601-merge-pull-config.sh', 't7602-merge-octopus-many.sh', diff --git a/t/t7532-preload-index-linux.sh b/t/t7532-preload-index-linux.sh new file mode 100755 index 00000000000000..2941448326397a --- /dev/null +++ b/t/t7532-preload-index-linux.sh @@ -0,0 +1,346 @@ +#!/bin/sh + +test_description='Linux bulk index preload' + +. ./test-lib.sh + +if test "$(uname -s)" != Linux +then + skip_all='Linux getdents64/statx backend required' + test_done +fi + +case "$(stat -f -c %t "$TRASH_DIRECTORY")" in +ef53) + filesystem=ext-family + ;; +58465342) + filesystem=xfs + ;; +*) + skip_all='tests require an ext-family filesystem or XFS' + test_done + ;; +esac + +setup_repo () { + repo=$1 && + git init "$repo" && + mkdir -p "$repo/nested/deep" && + test_write_lines root >"$repo/root" && + test_write_lines peer >"$repo/peer" && + test_write_lines nested >"$repo/nested/tracked" && + test_write_lines deep >"$repo/nested/deep/tracked" && + git -C "$repo" add . && + git -C "$repo" commit -m base && + git -C "$repo" config core.fsmonitor false && + test-tool chmtime -120 "$repo/root" "$repo/peer" \ + "$repo/nested/tracked" "$repo/nested/deep/tracked" && + git -C "$repo" update-index --refresh +} + +test_lazy_prereq LINUX_BULK_PRELOAD ' + setup_repo linux-bulk-prereq && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/linux-bulk-prereq.trace" \ + git -C linux-bulk-prereq \ + -c core.preloadIndexBulk=true \ + status --porcelain=v2 >actual && + test_must_be_empty actual && + test_trace2_data index preload/bulk_result complete \ + <"$TRASH_DIRECTORY/linux-bulk-prereq.trace" +' + +if ! test_have_prereq LINUX_BULK_PRELOAD +then + skip_all="Linux bulk preload backend unavailable at runtime" + test_done +fi + +ordinary_status () { + GIT_OPTIONAL_LOCKS=0 \ + git -C "$1" -c core.preloadIndex=false \ + status --porcelain=v2 >"$2" +} + +bulk_status () { + repo=$1 && + output=$2 && + bulk_trace=$TRASH_DIRECTORY/$3 && + rm -f "$bulk_trace" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TRACE2_EVENT="$bulk_trace" \ + git -C "$repo" status --porcelain=v2 >"$output" +} + +check_data () { + test_trace2_data index "$2" "$3" <"$TRASH_DIRECTORY/$1" +} + +check_lstat_data () { + test_have_prereq !PTHREADS || + check_data "$1" preload/sum_lstat "$2" +} + +compare_status () { + ordinary_status "$1" expect && + bulk_status "$1" actual "$2" && + test_cmp expect actual +} + +cleanup_race () { + exec 9>&- + if test -n "$status_pid" + then + kill "$status_pid" 2>/dev/null || : + wait "$status_pid" 2>/dev/null || : + fi + status_pid= && + rm -f "$ready" "$resume" +} + +wait_for_ready () { + for i in $(test_seq 1 1000) + do + test "$(cat "$ready" 2>/dev/null)" = ready && return 0 + kill -0 "$status_pid" 2>/dev/null || return 1 + sleep 0.01 + done + return 1 +} + +start_raced_status () { + repo=$1 && + barrier=$2 && + ready=$TRASH_DIRECTORY/$repo.ready && + resume=$TRASH_DIRECTORY/$repo.resume && + race_trace=$TRASH_DIRECTORY/$repo.trace && + status_pid= && + rm -f "$ready" "$resume" "$race_trace" && + mkfifo "$resume" && + exec 9<>"$resume" && + { + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_PATH="$barrier" \ + GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_READY="$ready" \ + GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_RESUME="$resume" \ + GIT_TRACE2_EVENT="$race_trace" \ + git -C "$repo" status --porcelain=v2 >actual 9>&- & + status_pid=$! + } && + wait_for_ready +} + +finish_raced_status () { + printf "resume\n" >&9 && + exec 9>&- && + wait "$status_pid" && + status_pid= && + ordinary_status "$1" expect && + test_cmp expect actual && + test_trace2_data index preload/bulk_applied 0 <"$race_trace" +} + +test_expect_success 'clean entries are published without lstat' ' + setup_repo clean && + bulk_status clean actual clean.trace && + test_must_be_empty actual && + check_data clean.trace preload/bulk_filesystem "$filesystem" && + check_data clean.trace preload/bulk_applied 4 && + check_data clean.trace preload/bulk_untracked_complete 1 && + check_lstat_data clean.trace 0 +' + +test_expect_success 'tracked files ignore a directory type hint' ' + setup_repo dirent-file && + ordinary_status dirent-file expect && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_PRELOAD_INDEX_BULK_DIRENT_TYPE=dir:root \ + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/dirent-file.trace" \ + git -C dirent-file status --porcelain=v2 >actual && + test_cmp expect actual && + check_data dirent-file.trace preload/bulk_result complete && + check_data dirent-file.trace preload/bulk_applied 4 && + check_data dirent-file.trace preload/bulk_fallback 0 +' + +test_expect_success 'tracked subtrees ignore a regular-file type hint' ' + setup_repo dirent-prefix && + test_write_lines visible >dirent-prefix/nested/untracked && + ordinary_status dirent-prefix expect && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_PRELOAD_INDEX_BULK_DIRENT_TYPE=reg:nested \ + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/dirent-prefix.trace" \ + git -C dirent-prefix status --porcelain=v2 >actual && + test_cmp expect actual && + check_data dirent-prefix.trace preload/bulk_result complete && + check_data dirent-prefix.trace preload/bulk_applied 4 && + check_data dirent-prefix.trace preload/bulk_definitive_deleted 0 && + check_data dirent-prefix.trace preload/bulk_fallback 0 && + check_data dirent-prefix.trace preload/bulk_untracked_complete 1 +' + +test_expect_success 'untracked directories ignore a regular-file type hint' ' + setup_repo dirent-untracked-directory && + mkdir -p dirent-untracked-directory/collapsed/deep && + test_write_lines visible \ + >dirent-untracked-directory/collapsed/deep/file && + ordinary_status dirent-untracked-directory expect && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_PRELOAD_INDEX_BULK_DIRENT_TYPE=reg:collapsed \ + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/dirent-untracked-directory.trace" \ + git -C dirent-untracked-directory \ + status --porcelain=v2 >actual && + test_cmp expect actual && + test_grep "^? collapsed/$" actual && + check_data dirent-untracked-directory.trace \ + preload/bulk_result complete && + check_data dirent-untracked-directory.trace \ + preload/bulk_untracked_complete 1 && + check_data dirent-untracked-directory.trace \ + preload/bulk_untracked_count 1 +' + +test_expect_success 'ignored directories ignore a regular-file type hint' ' + setup_repo dirent-ignored-directory && + test_write_lines "*.ignored" >dirent-ignored-directory/.gitignore && + git -C dirent-ignored-directory add .gitignore && + git -C dirent-ignored-directory commit -m ignore && + mkdir dirent-ignored-directory/ignored-only && + test_write_lines ignored \ + >dirent-ignored-directory/ignored-only/file.ignored && + ordinary_status dirent-ignored-directory expect && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_PRELOAD_INDEX_BULK_DIRENT_TYPE=reg:ignored-only \ + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/dirent-ignored-directory.trace" \ + git -C dirent-ignored-directory \ + status --porcelain=v2 >actual && + test_cmp expect actual && + test_must_be_empty actual && + check_data dirent-ignored-directory.trace \ + preload/bulk_result complete && + check_data dirent-ignored-directory.trace \ + preload/bulk_untracked_complete 1 && + check_data dirent-ignored-directory.trace \ + preload/bulk_untracked_count 0 +' + +test_expect_success 'untracked files ignore a directory type hint' ' + setup_repo dirent-untracked-file && + test_write_lines "visible/" >dirent-untracked-file/.gitignore && + git -C dirent-untracked-file add .gitignore && + git -C dirent-untracked-file commit -m ignore && + test_write_lines visible >dirent-untracked-file/visible && + ordinary_status dirent-untracked-file expect && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_PRELOAD_INDEX_BULK_DIRENT_TYPE=dir:visible \ + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/dirent-untracked-file.trace" \ + git -C dirent-untracked-file status --porcelain=v2 >actual && + test_cmp expect actual && + test_grep "^? visible$" actual && + check_data dirent-untracked-file.trace \ + preload/bulk_result complete && + check_data dirent-untracked-file.trace \ + preload/bulk_untracked_complete 1 && + check_data dirent-untracked-file.trace \ + preload/bulk_untracked_count 1 +' + +test_expect_success 'visible and ignored paths match ordinary status' ' + setup_repo visible && + test_write_lines "*.ignored" >visible/.gitignore && + git -C visible add .gitignore && + git -C visible commit -m ignore && + test_write_lines root >visible/untracked && + mkdir -p visible/collapsed/deep visible/ignored-only && + test_write_lines nested >visible/collapsed/deep/file && + test_write_lines ignored >visible/ignored-only/file.ignored && + compare_status visible visible.trace && + test_grep "^? untracked$" actual && + test_grep "^? collapsed/$" actual && + test_grep ! "ignored-only" actual && + check_data visible.trace preload/bulk_untracked_complete 1 && + check_data visible.trace preload/bulk_untracked_count 2 && + test_grep ! "\"category\":\"read_directory\"" visible.trace +' + +test_expect_success 'tracked changes match ordinary status' ' + for mode in modified deleted metadata + do + setup_repo "$mode" || return 1 && + case "$mode" in + modified) test_write_lines changed-content >"$mode/root" ;; + deleted) rm "$mode/nested/tracked" ;; + metadata) test-tool chmtime +60 "$mode/root" ;; + esac && + compare_status "$mode" "$mode.trace" || return 1 + done && + check_data modified.trace preload/bulk_definitive_modified 1 && + check_data modified.trace refresh/sum_lstat 0 && + check_data deleted.trace preload/bulk_definitive_deleted 1 && + check_data deleted.trace refresh/sum_lstat 0 && + check_data metadata.trace preload/bulk_content_check 1 && + check_data metadata.trace refresh/sum_lstat 0 +' + +test_expect_success 'tracked-file replacement directories are pruned' ' + setup_repo replacement-dir && + rm replacement-dir/root && + mkdir -p replacement-dir/root/deep/embedded && + test_write_lines hidden >replacement-dir/root/deep/untracked && + git -C replacement-dir/root/deep/embedded init && + compare_status replacement-dir replacement-dir.trace && + test_line_count = 1 actual && + check_data replacement-dir.trace preload/bulk_fallback 1 && + check_data replacement-dir.trace preload/bulk_untracked_complete 1 +' + +test_expect_success PIPE 'tracked FIFO replacements fall back' ' + setup_repo tracked-fifo && + rm tracked-fifo/root && + mkfifo tracked-fifo/root && + compare_status tracked-fifo tracked-fifo.trace && + test_grep "^1 \\.M .* root$" actual && + check_data tracked-fifo.trace preload/bulk_result complete && + check_data tracked-fifo.trace preload/bulk_applied 3 && + check_data tracked-fifo.trace preload/bulk_fallback 1 && + check_data tracked-fifo.trace preload/bulk_definitive_deleted 0 +' + +test_expect_success PIPE 'queued child replacement discards observations' ' + setup_repo child-race && + test_when_finished cleanup_race && + start_raced_status child-race nested/deep && + mv child-race/nested/deep child-race/deep-away && + mkdir child-race/nested/deep && + test_write_lines dirty >child-race/nested/deep/tracked && + finish_raced_status child-race && + test_file_not_empty actual +' + +test_expect_success PIPE 'fallback shapes retain exact output' ' + setup_repo shapes && + ln shapes/root shapes/linked && + mkfifo shapes/fifo && + git init shapes/embedded && + compare_status shapes shapes.trace && + check_data shapes.trace preload/bulk_fallback 1 && + check_data shapes.trace preload/bulk_untracked_complete 0 +' + +test_done From 1d4aae671f19cb3ff191d530c10d33bec63c4653 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Wed, 29 Jul 2026 16:53:33 -0500 Subject: [PATCH 096/105] status: close bulk content proofs with the provider token The bulk preloader rejected an active fsmonitor provider, so status verified ambiguous tracked entries through a separate semantic scan. Publishing bulk observations or refreshed stat data before the closing provider query would permit a concurrent change to invalidate a clean result. Pass held parent descriptors, basenames, and observed metadata from both platform walkers to semantic_verify_file_at(). Borrow the captured provider proof epoch, hash eligible raw-safe files during the bulk walk, and retain clean states and stat updates provisionally. After the closing provider query confirms the same epoch, validate all pending positions before publishing clean states, refreshed stat data, and fsmonitor-valid bits. Clear provisional state on provider failure, epoch mismatch, or invalid updates, and retain the existing complete-refresh fallback. Choose the provider-backed bulk path from its actual safety conditions, not from whether semantic history is awaiting adoption. This lets a trivial daemon response or daemon restart rebuild and close an ordinary bulk proof, including for a skipHash index, while retaining the complete proof epoch and closing query. Require both preload settings, an expanded index, a pending built-in IPC token, and an eligible whole-worktree request. Keep APFS and Linux within their platform and filesystem limits. Allocate a bounded hash buffer and attribute check per content-verification worker, and retain tracked states and stat updates only until closure. Extend the APFS and Linux tests with same-size, restored-mtime content changes. Cover accepted closure, provider failure, dirty status, daemon token reset with a null-checksum index, and Trace2 evidence of hashing, deferred publication, and token acceptance or rejection. Signed-off-by: Taylor Blau --- compat/preload-index/bulk-darwin.c | 3 +- compat/preload-index/bulk-linux-entry.c | 3 +- fsmonitor-ll.h | 3 +- fsmonitor.c | 13 +- preload-index-bulk-index.c | 61 ++++++- preload-index-bulk-thread.c | 74 ++++++++ preload-index-bulk.c | 14 ++ preload-index-bulk.h | 28 ++- preload-index.c | 221 ++++++++++++++++++++---- preload-index.h | 2 + read-cache-ll.h | 10 +- read-cache.c | 4 + semantic-verify-file.c | 32 +++- semantic-verify-internal.h | 2 +- t/t7519-status-fsmonitor.sh | 31 ++++ t/t7527-builtin-fsmonitor.sh | 86 +++++++++ t/t7529-preload-index-apfs.sh | 114 ++++++++++++ t/t7532-preload-index-linux.sh | 75 ++++++++ wt-status.c | 103 +++++++++-- 19 files changed, 809 insertions(+), 70 deletions(-) diff --git a/compat/preload-index/bulk-darwin.c b/compat/preload-index/bulk-darwin.c index 7e65c5a24c78eb..8aef3a12a61413 100644 --- a/compat/preload-index/bulk-darwin.c +++ b/compat/preload-index/bulk-darwin.c @@ -495,7 +495,8 @@ static int enumerate_directory(struct preload_bulk_worker *worker, entry.uid, entry.gid, entry.access, entry.linkcount, entry.size)) goto malformed_record; - preload_bulk_record_tracked(worker, pos, &st); + preload_bulk_record_tracked( + worker, pos, fd, entry.name, &st, 0); next_record: record += entry.record_len; diff --git a/compat/preload-index/bulk-linux-entry.c b/compat/preload-index/bulk-linux-entry.c index 8b63db291e7562..02cb27882bece3 100644 --- a/compat/preload-index/bulk-linux-entry.c +++ b/compat/preload-index/bulk-linux-entry.c @@ -253,7 +253,8 @@ int preload_bulk_linux_enumerate( worker, pos); continue; } - preload_bulk_record_tracked(worker, pos, &st); + preload_bulk_record_tracked( + worker, pos, fd, de->name, &st, 0); } } diff --git a/fsmonitor-ll.h b/fsmonitor-ll.h index 7268bc18802b76..759775506fc01a 100644 --- a/fsmonitor-ll.h +++ b/fsmonitor-ll.h @@ -72,7 +72,8 @@ int fsmonitor_reopen_token(struct index_state *istate); enum fsmonitor_token_result fsmonitor_query_pending_token( struct index_state *istate, int untracked_ready); void fsmonitor_accept_pending_token(struct index_state *istate, - int untracked_ready); + int untracked_proof_complete, + int untracked_cache_valid); void fsmonitor_reject_pending_token(struct index_state *istate); void fsmonitor_mark_untracked_cache_valid(struct index_state *istate); diff --git a/fsmonitor.c b/fsmonitor.c index 91ee5ccc33ae6f..e96965a66f7861 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -1361,8 +1361,11 @@ enum fsmonitor_token_result fsmonitor_query_pending_token( } void fsmonitor_accept_pending_token(struct index_state *istate, - int untracked_ready) + int untracked_proof_complete, + int untracked_cache_valid) { + if (untracked_cache_valid && !untracked_proof_complete) + BUG("valid untracked cache without a complete proof"); if (!fsmonitor_pending_token_from_provider(istate)) return; FREE_AND_NULL(istate->fsmonitor_last_update); @@ -1370,15 +1373,15 @@ void fsmonitor_accept_pending_token(struct index_state *istate, istate->fsmonitor_last_update_pending = NULL; istate->fsmonitor_pending_token_from_provider = 0; istate->fsmonitor_token_valid = 1; - istate->fsmonitor_untracked_valid = !!untracked_ready; + istate->fsmonitor_untracked_valid = !!untracked_cache_valid; if (istate->untracked) - istate->untracked->use_fsmonitor = !!untracked_ready; + istate->untracked->use_fsmonitor = !!untracked_cache_valid; istate->cache_changed |= FSMONITOR_CHANGED; FREE_AND_NULL(istate->fsmonitor_untracked_token); - if (untracked_ready) + if (untracked_cache_valid) istate->fsmonitor_untracked_token = xstrdup(istate->fsmonitor_last_update); - else { + else if (!untracked_proof_complete) { /* * Keep a query anchored at the accepted tracked token. A * later in-process status may need to close work done after diff --git a/preload-index-bulk-index.c b/preload-index-bulk-index.c index b09b69582397a8..a4edb54230a3d7 100644 --- a/preload-index-bulk-index.c +++ b/preload-index-bulk-index.c @@ -3,6 +3,9 @@ #include "object.h" #include "preload-index-bulk.h" #include "read-cache-ll.h" +#include "repository.h" +#include "semantic-verify.h" +#include "semantic-verify-internal.h" #ifndef __has_builtin #define __has_builtin(x) 0 @@ -78,6 +81,18 @@ static int record_tracked_state(struct preload_bulk_worker *worker, int pos, return recorded; } +static void record_stat_update(struct preload_bulk_worker *worker, int pos, + const struct stat_data *stat_data) +{ + struct preload_bulk_stat_update *update; + + ALLOC_GROW(worker->stat_updates, worker->stat_updates_nr + 1, + worker->stat_updates_alloc); + update = &worker->stat_updates[worker->stat_updates_nr++]; + update->cache_pos = pos; + memcpy(&update->stat_data, stat_data, sizeof(update->stat_data)); +} + static int tracked_entry_is_eligible(const struct cache_entry *ce) { return !ce_stage(ce) && @@ -109,6 +124,38 @@ static int size_change_is_definitive(const struct cache_entry *ce, DATA_CHANGED); } +static unsigned char verify_content_at( + struct preload_bulk_worker *worker, int pos, int parent_fd, + const char *basename, const struct stat *st, + int observed_has_platform_identity, struct stat_data *stat_data, + int *has_stat_update) +{ + struct preload_bulk_scan *scan = worker->scan; + struct cache_entry *ce = scan->istate->cache[pos]; + struct semantic_verify_file_result file; + + *has_stat_update = 0; + if (!scan->verify_content || + !semantic_verify_classify_entry( + scan->istate, ce, worker->attr_check, &file)) + return PRELOAD_BULK_TRACKED_CONTENT_CHECK; + semantic_verify_file_at( + parent_fd, basename, st, observed_has_platform_identity, + scan->root_dev, ce, scan->istate->repo, + worker->hash_buffer, &file); + worker->bytes_hashed += file.bytes_hashed; + if (file.kind == SEMANTIC_VERIFY_RAW_MODIFIED) + return PRELOAD_BULK_TRACKED_DEFINITIVE_MODIFIED; + if (file.kind != SEMANTIC_VERIFY_RAW_CLEAN || !file.persistable) + return PRELOAD_BULK_TRACKED_CONTENT_CHECK; + if (memcmp(&file.stat_data, &ce->ce_stat_data, + sizeof(file.stat_data))) { + memcpy(stat_data, &file.stat_data, sizeof(*stat_data)); + *has_stat_update = 1; + } + return PRELOAD_BULK_TRACKED_CLEAN; +} + int preload_bulk_index_entry_is_gitlink(struct preload_bulk_scan *scan, int pos) { @@ -116,12 +163,16 @@ int preload_bulk_index_entry_is_gitlink(struct preload_bulk_scan *scan, } void preload_bulk_record_tracked( - struct preload_bulk_worker *worker, int pos, const struct stat *st) + struct preload_bulk_worker *worker, int pos, int parent_fd, + const char *basename, const struct stat *st, + int observed_has_platform_identity) { struct preload_bulk_scan *scan = worker->scan; struct cache_entry *ce = scan->istate->cache[pos]; + struct stat_data stat_data; unsigned int changed; unsigned char state; + int has_stat_update = 0; if (!tracked_entry_is_eligible(ce)) return; @@ -133,8 +184,12 @@ void preload_bulk_record_tracked( else if (size_change_is_definitive(ce, st, changed)) state = PRELOAD_BULK_TRACKED_DEFINITIVE_MODIFIED; else - state = PRELOAD_BULK_TRACKED_CONTENT_CHECK; - record_tracked_state(worker, pos, state); + state = verify_content_at( + worker, pos, parent_fd, basename, st, + observed_has_platform_identity, &stat_data, + &has_stat_update); + if (record_tracked_state(worker, pos, state) && has_stat_update) + record_stat_update(worker, pos, &stat_data); } void preload_bulk_record_tracked_fallback( diff --git a/preload-index-bulk-thread.c b/preload-index-bulk-thread.c index 793bb79f6a670a..6cfc4f8409c45a 100644 --- a/preload-index-bulk-thread.c +++ b/preload-index-bulk-thread.c @@ -2,7 +2,13 @@ #include +#include "attr.h" +#include "clean-status.h" +#include "convert.h" #include "preload-index-bulk.h" +#include "read-cache-ll.h" +#include "semantic-verify-internal.h" +#include "trace2.h" #define PRELOAD_INDEX_BULK_OPEN_FD_CAP 128 #define PRELOAD_INDEX_BULK_OPEN_FD_RESERVE 16 @@ -185,12 +191,77 @@ static void *preload_bulk_worker_main(void *data) static void release_workers(struct preload_bulk_scan *scan) { for (int i = 0; i < scan->threads; i++) { + attr_check_free(scan->workers[i].attr_check); free(scan->workers[i].buffer); + free(scan->workers[i].hash_buffer); + free(scan->workers[i].stat_updates); strbuf_release(&scan->workers[i].path); } FREE_AND_NULL(scan->workers); } +static void prepare_content_verification(struct preload_bulk_scan *scan) +{ + if (!scan->proof_epoch) + return; + + convert_attrs_prepare(scan->istate); + for (int i = 0; i < scan->threads; i++) { + scan->workers[i].attr_check = convert_attrs_check_alloc(); + git_check_attr( + scan->istate, "", scan->workers[i].attr_check); + } + if (!clean_status_proof_epoch_prime_matches( + scan->istate, scan->proof_epoch)) { + for (int i = 0; i < scan->threads; i++) { + attr_check_free(scan->workers[i].attr_check); + scan->workers[i].attr_check = NULL; + } + git_attr_invalidate_all(); + return; + } + for (int i = 0; i < scan->threads; i++) + scan->workers[i].hash_buffer = + xmalloc(SEMANTIC_VERIFY_HASH_BUFFER_SIZE); + scan->verify_content = 1; + trace2_data_intmax("index", scan->repo, + "preload/bulk_content_verify", 1); +} + +static void collect_stat_updates(struct preload_bulk_scan *scan) +{ + size_t nr = 0; + + for (int i = 0; i < scan->threads; i++) { + struct preload_bulk_worker *worker = &scan->workers[i]; + + for (size_t j = 0; j < worker->stat_updates_nr; j++) { + struct preload_bulk_stat_update *update = + &worker->stat_updates[j]; + + if (update->cache_pos >= scan->istate->cache_nr) + BUG("bulk stat update position out of range"); + if (scan->tracked_state[update->cache_pos] == + PRELOAD_BULK_TRACKED_CLEAN) + nr++; + } + } + ALLOC_ARRAY(scan->stat_updates, nr); + for (int i = 0; i < scan->threads; i++) { + struct preload_bulk_worker *worker = &scan->workers[i]; + + for (size_t j = 0; j < worker->stat_updates_nr; j++) { + struct preload_bulk_stat_update *update = + &worker->stat_updates[j]; + + if (scan->tracked_state[update->cache_pos] != + PRELOAD_BULK_TRACKED_CLEAN) + continue; + scan->stat_updates[scan->stat_updates_nr++] = *update; + } + } +} + int preload_bulk_run_scan(struct preload_bulk_scan *scan, struct preload_bulk_run_result *result) { @@ -207,6 +278,7 @@ int preload_bulk_run_scan(struct preload_bulk_scan *scan, scan->workers[i].scan = scan; strbuf_init(&scan->workers[i].path, 0); } + prepare_content_verification(scan); FLEX_ALLOC_STR(root_task, path, "."); if (!reserve_open_fd(&scan->queue)) @@ -244,9 +316,11 @@ int preload_bulk_run_scan(struct preload_bulk_scan *scan, result->dirs += worker->dirs; result->entries += worker->entries; result->bulk_calls += worker->bulk_calls; + result->bytes_hashed += worker->bytes_hashed; result->changed_dirs += worker->changed_dirs; result->malformed += worker->malformed; } + collect_stat_updates(scan); result->threads = started_threads; result->untracked_complete = scan->collect_untracked && !scan->queue.untracked_invalid; diff --git a/preload-index-bulk.c b/preload-index-bulk.c index eed468b0d156ca..5756df1faceb90 100644 --- a/preload-index-bulk.c +++ b/preload-index-bulk.c @@ -6,6 +6,7 @@ #include "parse.h" #include "preload-index-bulk.h" #include "read-cache-ll.h" +#include "repository.h" #include "trace2.h" struct preload_bulk_untracked_root { @@ -179,11 +180,13 @@ int preload_bulk_collect(struct index_state *istate, int threads, .repo = istate->repo, .istate = istate, .backend = backend, + .proof_epoch = istate->preload_bulk_proof_epoch, .root_fd = -1, .threads = threads, .untracked = STRING_LIST_INIT_DUP, }; struct preload_bulk_run_result run_result = { 0 }; + struct stat root_stat; const char *start_error, *finish_error = NULL; const char *untracked_reason = NULL; int scan_error = -1; @@ -236,6 +239,11 @@ int preload_bulk_collect(struct index_state *istate, int threads, CALLOC_ARRAY(scan.tracked_state, istate->cache_nr); start_error = backend->start(&scan); + if (!start_error && scan.proof_epoch && + (scan.root_fd < 0 || fstat(scan.root_fd, &root_stat))) + start_error = "root-stat"; + if (!start_error && scan.proof_epoch) + scan.root_dev = root_stat.st_dev; if (!start_error) { if (scan.collect_untracked) { exclude_proof = exclude_source_proof_create( @@ -296,11 +304,15 @@ int preload_bulk_collect(struct index_state *istate, int threads, } if (clean) { result->tracked_state = scan.tracked_state; + result->stat_updates = scan.stat_updates; + result->stat_updates_nr = scan.stat_updates_nr; result->nr = istate->cache_nr; result->can_skip_unseen_preload = scan.can_skip_unseen_preload; result->untracked_complete = run_result.untracked_complete; scan.tracked_state = NULL; + scan.stat_updates = NULL; + scan.stat_updates_nr = 0; } backend->release(&scan); @@ -320,12 +332,14 @@ int preload_bulk_collect(struct index_state *istate, int threads, exclude_source_proof_release(exclude_proof); } free(scan.tracked_state); + free(scan.stat_updates); return clean ? 0 : -1; } void preload_bulk_result_release(struct preload_bulk_result *result) { FREE_AND_NULL(result->tracked_state); + FREE_AND_NULL(result->stat_updates); string_list_clear(&result->untracked, 0); memset(result, 0, sizeof(*result)); } diff --git a/preload-index-bulk.h b/preload-index-bulk.h index c6c862dbb8136c..3a7d0ef84c1c05 100644 --- a/preload-index-bulk.h +++ b/preload-index-bulk.h @@ -3,11 +3,16 @@ #include "git-compat-util.h" #include "preload-index.h" +#include "statinfo.h" #include "strbuf.h" #include "string-list.h" #include "thread-utils.h" struct dir_struct; +struct attr_check; +struct clean_status_proof_epoch; +struct index_state; +struct repository; struct preload_bulk_untracked_root; struct preload_bulk_dir_identity { @@ -49,10 +54,16 @@ struct preload_bulk_worker { struct preload_bulk_scan *scan; pthread_t thread; void *buffer; + void *hash_buffer; + struct attr_check *attr_check; + struct preload_bulk_stat_update *stat_updates; + size_t stat_updates_nr; + size_t stat_updates_alloc; struct strbuf path; uint64_t dirs; uint64_t entries; uint64_t bulk_calls; + uint64_t bytes_hashed; uint64_t changed_dirs; uint64_t malformed; unsigned started : 1; @@ -87,13 +98,18 @@ struct preload_bulk_scan { struct preload_bulk_queue queue; struct preload_bulk_worker *workers; unsigned char *tracked_state; + struct preload_bulk_stat_update *stat_updates; + size_t stat_updates_nr; + struct clean_status_proof_epoch *proof_epoch; struct dir_struct *exclude_dir; pthread_mutex_t exclude_mutex; struct preload_bulk_untracked_root *untracked_roots; struct string_list untracked; int root_fd; int threads; + dev_t root_dev; unsigned collect_untracked : 1; + unsigned verify_content : 1; unsigned case_insensitive : 1; unsigned can_skip_unseen_preload : 1; }; @@ -104,12 +120,15 @@ struct preload_bulk_run_result { uint64_t bulk_calls; uint64_t changed_dirs; uint64_t malformed; + uint64_t bytes_hashed; int threads; unsigned untracked_complete : 1; }; struct preload_bulk_result { unsigned char *tracked_state; + struct preload_bulk_stat_update *stat_updates; + size_t stat_updates_nr; size_t nr; const char *outcome; const char *reason; @@ -120,6 +139,11 @@ struct preload_bulk_result { unsigned untracked_complete : 1; }; +struct preload_bulk_stat_update { + uint32_t cache_pos; + struct stat_data stat_data; +}; + void preload_bulk_schedule_directory( struct preload_bulk_worker *worker, int parent_fd, const struct preload_bulk_dir_identity *parent_identity, @@ -134,7 +158,9 @@ int preload_bulk_index_pos_has_tracked_descendants( int preload_bulk_index_entry_is_gitlink(struct preload_bulk_scan *scan, int pos); void preload_bulk_record_tracked( - struct preload_bulk_worker *worker, int pos, const struct stat *st); + struct preload_bulk_worker *worker, int pos, int parent_fd, + const char *basename, const struct stat *st, + int observed_has_platform_identity); void preload_bulk_record_tracked_fallback( struct preload_bulk_worker *worker, int pos); void preload_bulk_record_tracked_descendants_fallback( diff --git a/preload-index.c b/preload-index.c index d7c7f99896c28e..a82063e8fd4149 100644 --- a/preload-index.c +++ b/preload-index.c @@ -47,6 +47,7 @@ struct thread_data { struct progress_data *progress; #ifdef HAVE_PRELOAD_INDEX_BULK const unsigned char *bulk_state; + unsigned bulk_provider_pending : 1; #endif int offset, nr; int t2_nr_lstat; @@ -90,7 +91,9 @@ static void *preload_thread(void *_data) #ifdef HAVE_PRELOAD_INDEX_BULK if (state == PRELOAD_BULK_TRACKED_CONTENT_CHECK || state == PRELOAD_BULK_TRACKED_DEFINITIVE_MODIFIED || - state == PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED) + state == PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED || + (p->bulk_provider_pending && + state == PRELOAD_BULK_TRACKED_CLEAN)) continue; #endif if (p->progress && !(nr & 31)) { @@ -127,6 +130,13 @@ static void *preload_thread(void *_data) } #ifdef HAVE_PRELOAD_INDEX_BULK +struct preload_bulk_pending { + unsigned char *tracked_state; + struct preload_bulk_stat_update *stat_updates; + size_t stat_updates_nr; + unsigned provider : 1; +}; + static int stat_data_is_zero(const struct stat_data *sd) { return !sd->sd_ctime.sec && @@ -140,21 +150,24 @@ static int stat_data_is_zero(const struct stat_data *sd) !sd->sd_size; } -static int preload_bulk_entry_is_useful(const struct cache_entry *ce) +static int preload_bulk_entry_is_useful(const struct cache_entry *ce, + int allow_zero_stat) { return preload_entry_needs_stat(ce) && !ce_intent_to_add(ce) && !(ce->ce_flags & (CE_VALID | CE_REMOVE)) && (S_ISREG(ce->ce_mode) || S_ISLNK(ce->ce_mode)) && - !stat_data_is_zero(&ce->ce_stat_data); + (allow_zero_stat || !stat_data_is_zero(&ce->ce_stat_data)); } -static size_t preload_bulk_useful_candidates(struct index_state *index) +static size_t preload_bulk_useful_candidates(struct index_state *index, + int allow_zero_stat) { size_t useful = 0; for (size_t i = 0; i < index->cache_nr; i++) - if (preload_bulk_entry_is_useful(index->cache[i])) + if (preload_bulk_entry_is_useful( + index->cache[i], allow_zero_stat)) useful++; return useful; } @@ -162,6 +175,7 @@ static size_t preload_bulk_useful_candidates(struct index_state *index) static size_t preload_bulk_apply_result( struct index_state *index, struct preload_bulk_result *result, + int defer_all, int *has_deferred) { size_t applied = 0; @@ -181,7 +195,7 @@ static size_t preload_bulk_apply_result( */ if (result->can_skip_unseen_preload && state == PRELOAD_BULK_TRACKED_UNSEEN && - preload_bulk_entry_is_useful(ce)) { + preload_bulk_entry_is_useful(ce, defer_all)) { state = PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED; result->tracked_state[i] = state; } @@ -190,9 +204,14 @@ static size_t preload_bulk_apply_result( state == PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED) && preload_entry_needs_stat(ce)) *has_deferred = 1; + if (defer_all && state == PRELOAD_BULK_TRACKED_CLEAN && + preload_entry_needs_stat(ce)) + *has_deferred = 1; if (state != PRELOAD_BULK_TRACKED_CLEAN) continue; - if (!preload_bulk_entry_is_useful(ce)) + if (!preload_bulk_entry_is_useful(ce, defer_all)) + continue; + if (defer_all) continue; ce_mark_uptodate(ce); mark_fsmonitor_valid(index, ce); @@ -263,6 +282,9 @@ static void preload_bulk_trace_result( result->run.entries); trace2_data_intmax("index", index->repo, "preload/bulk_calls", result->run.bulk_calls); + trace2_data_intmax("index", index->repo, + "preload/bulk_bytes_hashed", + result->run.bytes_hashed); trace2_data_intmax("index", index->repo, "preload/bulk_workers", result->run.threads); trace2_data_intmax("index", index->repo, @@ -283,47 +305,72 @@ static void preload_bulk_trace_result( result->untracked.nr); } -static unsigned char *preload_bulk_try(struct index_state *index) +static int preload_bulk_config_enabled(struct index_state *index) +{ + int enabled = 0; + int control; + + control = git_env_bool("GIT_TEST_PRELOAD_INDEX_BULK", -1); + if (control < 0) + repo_config_get_bool(index->repo, "core.preloadindexbulk", + &enabled); + else + enabled = control; + return enabled; +} + +static void preload_bulk_try(struct index_state *index, + unsigned int refresh_flags, + struct preload_bulk_pending *pending) { struct preload_bulk_result result = { 0 }; - unsigned char *tracked_state = NULL; size_t useful; size_t applied = 0; + int provider = !!index->preload_bulk_proof_epoch; int has_deferred = 0; - int enabled = 0; - int control, threads; + int threads; /* * Let the test variable override configuration without bypassing * any of the proof checks. */ - control = git_env_bool("GIT_TEST_PRELOAD_INDEX_BULK", -1); - if (control < 0) - repo_config_get_bool(index->repo, "core.preloadindexbulk", - &enabled); - else - enabled = control; - if (!enabled || - fsm_settings__get_mode(index->repo) != FSMONITOR_MODE_DISABLED || + if (!preload_bulk_config_enabled(index) || !preload_bulk_available()) - return NULL; - useful = preload_bulk_useful_candidates(index); + return; + if (provider) { + if (!(refresh_flags & REFRESH_DEFER_BULK_DIRTY) || + fsm_settings__get_mode(index->repo) != FSMONITOR_MODE_IPC || + !fsmonitor_pending_token_from_provider(index)) + return; + } else if (fsm_settings__get_mode(index->repo) != + FSMONITOR_MODE_DISABLED) { + return; + } + useful = preload_bulk_useful_candidates(index, provider); trace2_data_intmax("index", index->repo, "preload/bulk_useful", useful); trace2_data_intmax("index", index->repo, "preload/bulk_cache_nr", index->cache_nr); - if (!useful) - return NULL; + if (!useful && !index->preload_untracked) + return; threads = preload_bulk_threads(useful); trace2_region_enter("index", "preload/bulk", index->repo); if (!preload_bulk_collect(index, threads, &result)) { applied = preload_bulk_apply_result(index, &result, + provider, &has_deferred); } preload_bulk_trace_result(index, &result, applied); if (has_deferred) { - tracked_state = result.tracked_state; + pending->tracked_state = result.tracked_state; result.tracked_state = NULL; + pending->provider = provider; + if (provider) { + pending->stat_updates = result.stat_updates; + pending->stat_updates_nr = result.stat_updates_nr; + result.stat_updates = NULL; + result.stat_updates_nr = 0; + } } if (result.untracked_complete && index->preload_untracked) { *index->preload_untracked = result.untracked; @@ -333,26 +380,126 @@ static unsigned char *preload_bulk_try(struct index_state *index) } trace2_region_leave("index", "preload/bulk", index->repo); preload_bulk_result_release(&result); - return tracked_state; } static void preload_bulk_finish_state(struct index_state *index, - unsigned char **state, + struct preload_bulk_pending *pending, unsigned int refresh_flags) { - if ((refresh_flags & REFRESH_DEFER_BULK_DIRTY) && *state) { - index->preload_bulk_tracked_state = *state; + if ((refresh_flags & REFRESH_DEFER_BULK_DIRTY) && + pending->tracked_state) { + index->preload_bulk_tracked_state = pending->tracked_state; index->preload_bulk_tracked_nr = index->cache_nr; - *state = NULL; + index->preload_bulk_stat_updates = pending->stat_updates; + index->preload_bulk_stat_updates_nr = + pending->stat_updates_nr; + index->preload_bulk_provider_pending = pending->provider; + memset(pending, 0, sizeof(*pending)); } - FREE_AND_NULL(*state); + free(pending->tracked_state); + free(pending->stat_updates); +} + +static int compare_stat_update(const void *va, const void *vb) +{ + const struct preload_bulk_stat_update *a = va; + const struct preload_bulk_stat_update *b = vb; + + return a->cache_pos < b->cache_pos ? -1 : + a->cache_pos > b->cache_pos ? 1 : 0; } #endif void preload_index_bulk_result_clear(struct index_state *index) { FREE_AND_NULL(index->preload_bulk_tracked_state); + FREE_AND_NULL(index->preload_bulk_stat_updates); index->preload_bulk_tracked_nr = 0; + index->preload_bulk_stat_updates_nr = 0; + index->preload_bulk_provider_pending = 0; +} + +int preload_index_bulk_can_close_provider(struct index_state *index) +{ +#ifdef HAVE_PRELOAD_INDEX_BULK + int core_preload_index = 1; + + repo_config_get_bool(index->repo, "core.preloadindex", + &core_preload_index); + return core_preload_index && + preload_bulk_config_enabled(index) && + preload_bulk_available() && + index->sparse_index == INDEX_EXPANDED && + fsm_settings__get_mode(index->repo) == FSMONITOR_MODE_IPC && + fsmonitor_pending_token_from_provider(index) && + (preload_bulk_useful_candidates(index, 1) || + index->preload_untracked); +#else + (void)index; + return 0; +#endif +} + +int preload_index_bulk_result_accept(struct index_state *index) +{ +#ifdef HAVE_PRELOAD_INDEX_BULK + size_t update_nr = 0; + int applied = 0; + + if (!index->preload_bulk_provider_pending) + return 0; + if (!index->preload_bulk_tracked_state || + index->preload_bulk_tracked_nr != index->cache_nr) + return -1; + + QSORT(index->preload_bulk_stat_updates, + index->preload_bulk_stat_updates_nr, compare_stat_update); + for (size_t i = 0; i < index->preload_bulk_stat_updates_nr; i++) { + struct preload_bulk_stat_update *update = + &index->preload_bulk_stat_updates[i]; + + if (update->cache_pos >= index->cache_nr || + index->preload_bulk_tracked_state[update->cache_pos] != + PRELOAD_BULK_TRACKED_CLEAN || + (i && update[-1].cache_pos == update->cache_pos)) + return -1; + } + + for (size_t i = 0; i < index->cache_nr; i++) { + struct cache_entry *ce = index->cache[i]; + struct preload_bulk_stat_update *update = NULL; + + if (index->preload_bulk_tracked_state[i] != + PRELOAD_BULK_TRACKED_CLEAN) + continue; + if (update_nr < index->preload_bulk_stat_updates_nr && + index->preload_bulk_stat_updates[update_nr].cache_pos == i) + update = + &index->preload_bulk_stat_updates[update_nr++]; + if (update && + memcmp(&ce->ce_stat_data, &update->stat_data, + sizeof(ce->ce_stat_data))) { + memcpy(&ce->ce_stat_data, &update->stat_data, + sizeof(ce->ce_stat_data)); + ce->ce_flags |= CE_UPDATE_IN_BASE; + index->cache_changed |= CE_ENTRY_CHANGED; + } + ce_mark_uptodate(ce); + mark_fsmonitor_valid(index, ce); + applied++; + } + if (update_nr != index->preload_bulk_stat_updates_nr) + BUG("validated bulk stat update was not applied"); + + FREE_AND_NULL(index->preload_bulk_stat_updates); + index->preload_bulk_stat_updates_nr = 0; + index->preload_bulk_provider_pending = 0; + trace2_data_intmax("index", index->repo, + "preload/bulk_provider_applied", applied); +#else + (void)index; +#endif + return 0; } void preload_index(struct index_state *index, @@ -363,7 +510,7 @@ void preload_index(struct index_state *index, struct thread_data data[MAX_PARALLEL]; struct progress_data pd; #ifdef HAVE_PRELOAD_INDEX_BULK - unsigned char *bulk_state = NULL; + struct preload_bulk_pending bulk = { 0 }; #endif int t2_sum_lstat = 0; int core_preload_index = 1; @@ -379,11 +526,11 @@ void preload_index(struct index_state *index, #ifdef HAVE_PRELOAD_INDEX_BULK if (!pathspec || !pathspec->nr) - bulk_state = preload_bulk_try(index); + preload_bulk_try(index, refresh_flags, &bulk); #endif if (!HAVE_THREADS) { #ifdef HAVE_PRELOAD_INDEX_BULK - preload_bulk_finish_state(index, &bulk_state, refresh_flags); + preload_bulk_finish_state(index, &bulk, refresh_flags); #endif return; } @@ -393,7 +540,7 @@ void preload_index(struct index_state *index, threads = 2; if (threads < 2) { #ifdef HAVE_PRELOAD_INDEX_BULK - preload_bulk_finish_state(index, &bulk_state, refresh_flags); + preload_bulk_finish_state(index, &bulk, refresh_flags); #endif return; } @@ -421,7 +568,9 @@ void preload_index(struct index_state *index, p->index = index; #ifdef HAVE_PRELOAD_INDEX_BULK - p->bulk_state = bulk_state ? bulk_state + offset : NULL; + p->bulk_state = bulk.tracked_state ? + bulk.tracked_state + offset : NULL; + p->bulk_provider_pending = bulk.provider; #endif if (pathspec) copy_pathspec(&p->pathspec, pathspec); @@ -443,7 +592,7 @@ void preload_index(struct index_state *index, } stop_progress(&pd.progress); #ifdef HAVE_PRELOAD_INDEX_BULK - preload_bulk_finish_state(index, &bulk_state, refresh_flags); + preload_bulk_finish_state(index, &bulk, refresh_flags); #endif if (pathspec) { diff --git a/preload-index.h b/preload-index.h index bb6deb6130cc2f..7f7fdcca28acb2 100644 --- a/preload-index.h +++ b/preload-index.h @@ -21,5 +21,7 @@ int repo_read_index_preload(struct repository *, const struct pathspec *pathspec, unsigned refresh_flags); void preload_index_bulk_result_clear(struct index_state *index); +int preload_index_bulk_can_close_provider(struct index_state *index); +int preload_index_bulk_result_accept(struct index_state *index); #endif /* PRELOAD_INDEX_H */ diff --git a/read-cache-ll.h b/read-cache-ll.h index c7e4104715e890..f32da7ad018aea 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -31,6 +31,9 @@ struct cache_entry { char name[FLEX_ARRAY]; /* more */ }; +struct clean_status_proof_epoch; +struct preload_bulk_stat_update; + #define CE_STAGEMASK (0x3000) #define CE_EXTENDED (0x4000) #define CE_VALID (0x8000) @@ -191,7 +194,8 @@ struct index_state { fsmonitor_untracked_extension_seen : 1, fsmonitor_untracked_extension_invalid : 1, fsmonitor_pending_token_from_provider : 1, - preload_untracked_complete : 1; + preload_untracked_complete : 1, + preload_bulk_provider_pending : 1; enum sparse_index_mode sparse_index; struct hashmap name_hash; struct hashmap dir_hash; @@ -199,6 +203,10 @@ struct index_state { struct untracked_cache *untracked; unsigned char *preload_bulk_tracked_state; size_t preload_bulk_tracked_nr; + struct preload_bulk_stat_update *preload_bulk_stat_updates; + size_t preload_bulk_stat_updates_nr; + /* Borrowed only while refresh_index() performs a provider scan. */ + struct clean_status_proof_epoch *preload_bulk_proof_epoch; /* Borrowed for the duration of preload_index(). */ struct string_list *preload_untracked; char *fsmonitor_last_update; diff --git a/read-cache.c b/read-cache.c index 6683fe5953e1df..10fc20b7224651 100644 --- a/read-cache.c +++ b/read-cache.c @@ -1601,6 +1601,9 @@ int refresh_index(struct index_state *istate, unsigned int flags, unsigned char state = istate->preload_bulk_tracked_state[i]; + if (istate->preload_bulk_provider_pending && + state == PRELOAD_BULK_TRACKED_CLEAN) + continue; if (state == PRELOAD_BULK_TRACKED_CONTENT_CHECK) { ce->ce_flags |= CE_CONTENT_CHECK_REQUIRED; continue; @@ -2507,6 +2510,7 @@ void release_index(struct index_state *istate) free(istate->fsmonitor_untracked_token); clean_status_release(istate); free(istate->preload_bulk_tracked_state); + free(istate->preload_bulk_stat_updates); free(istate->cache); discard_split_index(istate); free_untracked_cache(istate->untracked); diff --git a/semantic-verify-file.c b/semantic-verify-file.c index 72389ac10c065c..e2cfa3ca94d1e0 100644 --- a/semantic-verify-file.c +++ b/semantic-verify-file.c @@ -103,9 +103,32 @@ int semantic_verify_classify_entry(struct index_state *istate, } #if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +static int observed_stat_equal(const struct stat *a, const struct stat *b, + int has_platform_identity) +{ + if (a->st_dev != b->st_dev || a->st_ino != b->st_ino || + a->st_mode != b->st_mode || a->st_nlink != b->st_nlink || + a->st_uid != b->st_uid || a->st_gid != b->st_gid || + a->st_size != b->st_size || a->st_mtime != b->st_mtime || + ST_MTIME_NSEC(*a) != ST_MTIME_NSEC(*b) || + a->st_ctime != b->st_ctime || + ST_CTIME_NSEC(*a) != ST_CTIME_NSEC(*b)) + return 0; +#ifdef __APPLE__ + if (has_platform_identity && + (a->st_birthtimespec.tv_sec != b->st_birthtimespec.tv_sec || + a->st_birthtimespec.tv_nsec != b->st_birthtimespec.tv_nsec || + a->st_gen != b->st_gen)) + return 0; +#else + (void)has_platform_identity; +#endif + return 1; +} + void semantic_verify_file_at(int parent_fd, const char *basename, const struct stat *observed, - dev_t root_dev, + int observed_has_platform_identity, dev_t root_dev, const struct cache_entry *ce, struct repository *repo, void *buffer, struct semantic_verify_file_result *result) @@ -138,7 +161,8 @@ void semantic_verify_file_at(int parent_fd, const char *basename, if (fstat(fd, &fd_before)) goto unstable; if (fd_before.st_dev != root_dev || - !path_namespace_stat_equal(&path_before, &fd_before)) { + !observed_stat_equal(&path_before, &fd_before, + observed_has_platform_identity)) { errno = EAGAIN; goto unstable; } @@ -199,7 +223,7 @@ void semantic_verify_file(struct semantic_verify_root *root, SEMANTIC_VERIFY_RAW_MODIFIED : SEMANTIC_VERIFY_ERROR; return; } - semantic_verify_file_at(parent_fd, basename, &path_before, + semantic_verify_file_at(parent_fd, basename, &path_before, 1, root->stat.st_dev, ce, repo, buffer, result); } #else @@ -214,7 +238,7 @@ static void semantic_verify_file_unavailable( void semantic_verify_file_at( int parent_fd UNUSED, const char *basename UNUSED, const struct stat *observed UNUSED, - dev_t root_dev UNUSED, + int observed_has_platform_identity UNUSED, dev_t root_dev UNUSED, const struct cache_entry *ce UNUSED, struct repository *repo UNUSED, void *buffer UNUSED, struct semantic_verify_file_result *result) diff --git a/semantic-verify-internal.h b/semantic-verify-internal.h index de1003226b1ff2..8d48ce92a83100 100644 --- a/semantic-verify-internal.h +++ b/semantic-verify-internal.h @@ -80,7 +80,7 @@ void semantic_verify_file(struct semantic_verify_root *root, struct semantic_verify_file_result *result); void semantic_verify_file_at(int parent_fd, const char *basename, const struct stat *observed, - dev_t root_dev, + int observed_has_platform_identity, dev_t root_dev, const struct cache_entry *ce, struct repository *repo, void *buffer, struct semantic_verify_file_result *result); diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 002d5f1a83c1fc..66c5a3a28bfdf3 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -594,6 +594,37 @@ prepare_builtin_closure_repo () { ) } +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'builtin closure initializes a new untracked cache' ' + test_when_finished "rm -rf builtin-closure-new-uc" && + test_create_repo builtin-closure-new-uc && + ( + cd builtin-closure-new-uc && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + test_grep UNTR .git/index && + test_grep ! FSUC .git/index && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual && + test_grep \ + "\"event\":\"region_enter\".*\"category\":\"dir\",\"label\":\"read_directory\"" \ + .git/status.trace >.git/read-directory && + test_line_count = 1 .git/read-directory && + test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep FSUC .git/index + ) +' + test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ 'builtin clean closure publishes its proof' ' test_when_finished "rm -rf builtin-closure-clean" && diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index 0ff924e2828c9b..417efb5989739a 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -1562,6 +1562,92 @@ test_expect_success 'bound query replaces a legacy daemon' ' ) ' +test_expect_success MACOS 'daemon token reset closes a skipHash index' ' + test_when_finished \ + "stop_daemon_delete_repo daemon-token-reset" && + test_create_repo daemon-token-reset && + ( + cd daemon-token-reset && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test_commit remove removed && + test_commit keep clean && + git config core.preloadIndexBulk true && + git config core.untrackedCache true && + git config index.skipHash true && + test-tool chmtime =-60 tracked removed clean && + git update-index --refresh && + git config core.fsmonitor true && + start_daemon && + + git update-index --force-write-index && + git status --porcelain=v2 >.git/prime.out && + test_must_be_empty .git/prime.out && + test_grep FSMN .git/index && + test_grep FSCF .git/index && + test_grep FSUC .git/index && + test_trailing_hash .git/index >.git/index.hash && + test_oid zero >.git/zero && + test_cmp .git/zero .git/index.hash && + test-tool dump-fsmonitor >.git/token.before && + token_before=$(sed -n \ + "s/^fsmonitor last update //p" .git/token.before) && + + git fsmonitor--daemon stop && + start_daemon && + GIT_TRACE2_EVENT="$PWD/.git/reset.trace" \ + git status --porcelain=v2 >.git/reset.out && + test_must_be_empty .git/reset.out && + test_trace2_data fsm_client query/trivial-response 1 \ + <.git/reset.trace && + test_trace2_data index preload/bulk_provider_applied \ + "[1-9][0-9]*" \ + <.git/reset.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/reset.trace && + test-tool dump-fsmonitor >.git/token.after && + token_after=$(sed -n \ + "s/^fsmonitor last update //p" .git/token.after) && + test -n "$token_before" && + test -n "$token_after" && + test "$token_before" != "$token_after" && + + GIT_TRACE2_EVENT="$PWD/.git/warm.trace" \ + git status --porcelain=v2 >.git/warm.out && + test_must_be_empty .git/warm.out && + test_trace2_data index refresh/sum_lstat 0 \ + <.git/warm.trace && + ! test_trace2_data index refresh/sum_lstat \ + "[1-9][0-9]*" <.git/warm.trace && + ! test_trace2_data fsm_client query/trivial-response 1 \ + <.git/warm.trace && + + git fsmonitor--daemon stop && + echo changed >>tracked && + rm removed && + start_daemon && + GIT_TRACE2_EVENT="$PWD/.git/dirty-reset.trace" \ + git status --porcelain=v2 >.git/dirty-reset.out && + test_line_count = 2 .git/dirty-reset.out && + test_grep "^1 \.M .* tracked$" .git/dirty-reset.out && + test_grep "^1 \.D .* removed$" .git/dirty-reset.out && + test_trace2_data fsm_client query/trivial-response 1 \ + <.git/dirty-reset.trace && + test_trace2_data index preload/bulk_provider_applied 1 \ + <.git/dirty-reset.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/dirty-reset.trace && + + GIT_TRACE2_EVENT="$PWD/.git/dirty-warm.trace" \ + git status --porcelain=v2 >.git/dirty-warm.out && + test_cmp .git/dirty-reset.out .git/dirty-warm.out && + test_trace2_data index refresh/sum_lstat 2 \ + <.git/dirty-warm.trace && + ! test_trace2_data fsm_client query/trivial-response 1 \ + <.git/dirty-warm.trace + ) +' + test_expect_success 'bound query accepts a capability superset' ' test_when_finished \ "stop_daemon_delete_repo capability-superset" && diff --git a/t/t7529-preload-index-apfs.sh b/t/t7529-preload-index-apfs.sh index 5ab700650c44df..0c0446aba27219 100755 --- a/t/t7529-preload-index-apfs.sh +++ b/t/t7529-preload-index-apfs.sh @@ -107,6 +107,47 @@ configured_bulk_status () { status --porcelain=v2 >"$output" } +setup_provider_proof_repo () { + setup_repo "$1" && + ( + cd "$1" && + git config core.trustctime false && + git config core.checkStat minimal && + mtime=$(test-tool chmtime --get root) && + printf "dirt\n" >root && + test-tool chmtime =$mtime root && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid root && + test_grep ! FSCF .git/index + ) +} + +setup_provider_proof_repo_with_untracked_cache () { + setup_repo "$1" && + ( + cd "$1" && + git config core.untrackedCache true && + git status --porcelain=2 >.git/prime && + test_must_be_empty .git/prime && + test_grep UNTR .git/index && + git config core.trustctime false && + git config core.checkStat minimal && + mtime=$(test-tool chmtime --get root) && + printf "dirt\n" >root && + test-tool chmtime =$mtime root && + test_write_lines visible >visible && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid root && + test_grep ! FSCF .git/index + ) +} + test_expect_success 'bulk preload follows its configuration' ' setup_repo opt-in && GIT_OPTIONAL_LOCKS=0 \ @@ -144,6 +185,79 @@ test_expect_success 'bulk preload waits for fsmonitor provider closure' ' test_grep ! "\"key\":\"preload/bulk_result\"" fsmonitor.trace ' +test_expect_success 'provider closure accepts bulk content proofs' ' + setup_provider_proof_repo provider-proof && + ( + cd provider-proof && + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_grep "^1 \.M .* root$" .git/actual && + test_trace2_data status semantic_verify/bulk_scan 1 \ + <.git/status.trace && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/status.trace && + test_trace2_data index preload/bulk_content_verify 1 \ + <.git/status.trace && + test_trace2_data index preload/bulk_bytes_hashed \ + "[1-9][0-9]*" \ + <.git/status.trace && + test_trace2_data index preload/bulk_provider_applied 7 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep FSCF .git/index + ) +' + +test_expect_success \ + 'provider bulk preserves an existing untracked-cache binding' ' + setup_provider_proof_repo_with_untracked_cache provider-proof-uc && + ( + cd provider-proof-uc && + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=2 >.git/actual && + test_grep "^1 \.M .* root$" .git/actual && + test_grep "^? visible$" .git/actual && + ! test_trace2_data index preload/bulk_untracked_complete 1 \ + <.git/status.trace && + test_grep \ + "\"event\":\"region_enter\".*\"category\":\"dir\",\"label\":\"read_directory\"" \ + .git/status.trace >.git/read-directory && + test_line_count = 1 .git/read-directory && + test_grep FSUC .git/index + ) +' + +test_expect_success 'provider failure discards bulk content proofs' ' + setup_provider_proof_repo provider-failure && + ( + cd provider-failure && + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CE \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_grep "^1 \.M .* root$" .git/actual && + test_trace2_data status semantic_verify/bulk_scan 1 \ + <.git/status.trace && + test_trace2_data index preload/bulk_content_verify 1 \ + <.git/status.trace && + ! test_trace2_data index preload/bulk_provider_applied \ + "[0-9][0-9]*" <.git/status.trace && + test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep ! FSCF .git/index + ) +' + cleanup_race () { exec 9>&- if test -n "$status_pid" diff --git a/t/t7532-preload-index-linux.sh b/t/t7532-preload-index-linux.sh index 2941448326397a..4d4063809f884c 100755 --- a/t/t7532-preload-index-linux.sh +++ b/t/t7532-preload-index-linux.sh @@ -39,6 +39,27 @@ setup_repo () { git -C "$repo" update-index --refresh } +setup_provider_proof_repo () { + setup_repo "$1" && + ( + cd "$1" && + git config core.trustctime false && + git config core.checkStat minimal && + size=$(test_file_size root) && + mtime=$(test-tool chmtime --get root) && + printf "dirt\n" >root && + test "$(test_file_size root)" = "$size" && + test-tool chmtime =$mtime root && + test "$(test-tool chmtime --get root)" = "$mtime" && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid root && + test_grep ! FSCF .git/index + ) +} + test_lazy_prereq LINUX_BULK_PRELOAD ' setup_repo linux-bulk-prereq && GIT_OPTIONAL_LOCKS=0 \ @@ -156,6 +177,60 @@ test_expect_success 'clean entries are published without lstat' ' check_lstat_data clean.trace 0 ' +test_expect_success 'provider closure accepts bulk content proofs' ' + ( + sane_unset GIT_TEST_SPLIT_INDEX && + setup_provider_proof_repo provider-proof && + cd provider-proof && + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_grep "^1 \.M .* root$" .git/actual && + test_trace2_data status semantic_verify/bulk_scan 1 \ + <.git/status.trace && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/status.trace && + test_trace2_data index preload/bulk_content_verify 1 \ + <.git/status.trace && + test_trace2_data index preload/bulk_bytes_hashed \ + "[1-9][0-9]*" <.git/status.trace && + test_trace2_data index preload/bulk_provider_applied 3 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep FSCF .git/index + ) +' + +test_expect_success 'provider failure discards bulk content proofs' ' + ( + sane_unset GIT_TEST_SPLIT_INDEX && + setup_provider_proof_repo provider-failure && + cd provider-failure && + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CE \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_grep "^1 \.M .* root$" .git/actual && + test_trace2_data status semantic_verify/bulk_scan 1 \ + <.git/status.trace && + test_trace2_data index preload/bulk_content_verify 1 \ + <.git/status.trace && + test_trace2_data index preload/bulk_bytes_hashed \ + "[1-9][0-9]*" <.git/status.trace && + ! test_trace2_data index preload/bulk_provider_applied \ + "[0-9][0-9]*" <.git/status.trace && + test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep ! FSCF .git/index + ) +' + test_expect_success 'tracked files ignore a directory type hint' ' setup_repo dirent-file && ordinary_status dirent-file expect && diff --git a/wt-status.c b/wt-status.c index bcdf74a004070f..9c285d714958a1 100644 --- a/wt-status.c +++ b/wt-status.c @@ -1048,8 +1048,16 @@ static int wt_status_collect_untracked_1( return used_untracked_cache; } +static int wt_status_can_use_bulk_provider( + struct wt_status *s, unsigned int refresh_flags) +{ + return !s->show_ignored_mode && !s->pathspec.nr && + (refresh_flags & REFRESH_DEFER_BULK_DIRTY) && + preload_index_bulk_can_close_provider(s->repo->index); +} + static struct semantic_verify_proof *wt_status_prepare_semantic_verify( - struct wt_status *s) + struct wt_status *s, unsigned int refresh_flags) { struct index_state *istate = s->repo->index; struct semantic_verify_options options = SEMANTIC_VERIFY_OPTIONS_INIT; @@ -1064,6 +1072,11 @@ static struct semantic_verify_proof *wt_status_prepare_semantic_verify( !fsmonitor_pending_token_from_provider(istate) || !clean_status_fsmonitor_semantic_adoption_needed(istate)) return NULL; + if (wt_status_can_use_bulk_provider(s, refresh_flags)) { + trace2_data_intmax("status", s->repo, + "semantic_verify/bulk_scan", 1); + return NULL; + } options.require_proof_epoch = 1; options.attr_snapshot = s->attr_source_snapshot; @@ -1094,7 +1107,9 @@ struct wt_status_token_closure { unsigned int refresh_flags; int require_untracked; int can_prime; + int use_bulk_provider; int untracked_ready; + int untracked_proof_complete; struct string_list staged_untracked; struct string_list staged_ignored; int staged_untracked_ready; @@ -1189,17 +1204,37 @@ static void wt_status_discard_semantic_verify( static void wt_status_refresh_for_token( struct wt_status *s, unsigned int refresh_flags, - struct clean_status_proof_epoch **epoch, int *refresh_result) + struct clean_status_proof_epoch **epoch, int use_bulk_provider, + int *refresh_result) { struct index_state *istate = s->repo->index; clean_status_release_proof_epoch(*epoch); *epoch = clean_status_capture_proof_epoch( istate, s->attr_source_snapshot); + if (*epoch && use_bulk_provider) + istate->preload_bulk_proof_epoch = *epoch; if (*epoch) { *refresh_result |= refresh_index( istate, refresh_flags, &s->pathspec, NULL, NULL); } + istate->preload_bulk_proof_epoch = NULL; +} + +static int wt_status_untracked_cache_valid( + const struct wt_status_token_closure *closure) +{ + const struct index_state *istate = closure->status->repo->index; + + return closure->untracked_ready && + istate->untracked && istate->untracked->root; +} + +static void wt_status_record_bulk_untracked( + struct wt_status_token_closure *closure) +{ + if (closure->status->repo->index->preload_untracked_complete) + closure->untracked_proof_complete = 1; } static int wt_status_close_ordinary_fsmonitor_token( @@ -1219,6 +1254,7 @@ static int wt_status_close_ordinary_fsmonitor_token( if (reliable_stat) { wt_status_refresh_for_token( s, closure->refresh_flags, &scan_epoch, + closure->use_bulk_provider, &closure->refresh_result); if (!scan_epoch) return 0; @@ -1227,8 +1263,12 @@ static int wt_status_close_ordinary_fsmonitor_token( istate, closure->refresh_flags, &s->pathspec, NULL, NULL); } - if (!closure->untracked_ready && closure->can_prime) { - closure->untracked_ready = wt_status_stage_untracked(closure); + wt_status_record_bulk_untracked(closure); + if (!closure->untracked_proof_complete && closure->can_prime) { + closure->untracked_ready = + wt_status_stage_untracked(closure); + closure->untracked_proof_complete = + closure->untracked_ready; if (closure->queries) trace2_data_intmax( "status", s->repo, @@ -1245,7 +1285,8 @@ static int wt_status_close_ordinary_fsmonitor_token( break; closure->queries++; result = fsmonitor_query_pending_token( - istate, closure->untracked_ready); + istate, + wt_status_untracked_cache_valid(closure)); if (result == FSMONITOR_TOKEN_CLEAN) { if (reliable_stat && !clean_status_proof_epoch_matches( @@ -1253,19 +1294,27 @@ static int wt_status_close_ordinary_fsmonitor_token( wt_status_reset_attr_snapshot_if_changed(s); break; } - if (closure->untracked_ready || + if (closure->untracked_proof_complete || !closure->require_untracked) { + if (preload_index_bulk_result_accept(istate) < 0) + break; if (reliable_stat) clean_status_mark_fsmonitor_config_valid( istate, istate->fsmonitor_last_update_pending); clean_status_release_proof_epoch(scan_epoch); fsmonitor_accept_pending_token( - istate, closure->untracked_ready); + istate, + closure->untracked_proof_complete, + wt_status_untracked_cache_valid( + closure)); return 1; } break; } + wt_status_discard_staged_untracked(closure); + closure->untracked_proof_complete = + !closure->require_untracked || !istate->untracked; clean_status_release_proof_epoch(scan_epoch); scan_epoch = NULL; if (!fsmonitor_token_requires_rescan(result)) @@ -1278,6 +1327,7 @@ static int wt_status_close_ordinary_fsmonitor_token( if (reliable_stat) { wt_status_refresh_for_token( s, closure->refresh_flags, &scan_epoch, + closure->use_bulk_provider, &closure->refresh_result); if (!scan_epoch) break; @@ -1286,9 +1336,14 @@ static int wt_status_close_ordinary_fsmonitor_token( istate, closure->refresh_flags, &s->pathspec, NULL, NULL); } - if (closure->can_prime) + wt_status_record_bulk_untracked(closure); + if (!closure->untracked_proof_complete && + closure->can_prime) { closure->untracked_ready = wt_status_stage_untracked(closure); + closure->untracked_proof_complete = + closure->untracked_ready; + } } clean_status_release_proof_epoch(scan_epoch); return 0; @@ -1309,7 +1364,8 @@ wt_status_close_semantic_fsmonitor_token( struct index_state *istate = s->repo->index; enum fsmonitor_token_result result; int defer_untracked = - closure->can_prime && !closure->untracked_ready; + closure->can_prime && + !closure->untracked_proof_complete; int applied; if (!semantic_verify_start_token_is_current(istate, *proof)) { @@ -1321,7 +1377,8 @@ wt_status_close_semantic_fsmonitor_token( /* The first query closes the tracked scan and its semantic proof. */ closure->queries++; result = fsmonitor_query_pending_token( - istate, defer_untracked ? 0 : closure->untracked_ready); + istate, defer_untracked ? 0 : + wt_status_untracked_cache_valid(closure)); if (result != FSMONITOR_TOKEN_CLEAN) { wt_status_discard_semantic_verify( s, proof, "token-reset"); @@ -1349,7 +1406,10 @@ wt_status_close_semantic_fsmonitor_token( } if (defer_untracked) { - closure->untracked_ready = wt_status_stage_untracked(closure); + closure->untracked_ready = + wt_status_stage_untracked(closure); + closure->untracked_proof_complete = + closure->untracked_ready; trace2_data_intmax( "status", s->repo, "fsmonitor_token/untracked-after-semantic", @@ -1361,11 +1421,14 @@ wt_status_close_semantic_fsmonitor_token( /* A second query closes the subsequent untracked scan. */ closure->queries++; result = fsmonitor_query_pending_token( - istate, closure->untracked_ready); + istate, + wt_status_untracked_cache_valid(closure)); if (result != FSMONITOR_TOKEN_CLEAN) { + wt_status_discard_staged_untracked(closure); untracked_cache_invalidate_all(istate); fsmonitor_invalidate_semantics(istate); closure->untracked_ready = 0; + closure->untracked_proof_complete = 0; wt_status_discard_semantic_verify( s, proof, "token-reset"); if (fsmonitor_token_requires_rescan(result)) @@ -1384,7 +1447,9 @@ wt_status_close_semantic_fsmonitor_token( istate, istate->fsmonitor_last_update_pending); semantic_verify_proof_clear(*proof); *proof = NULL; - fsmonitor_accept_pending_token(istate, closure->untracked_ready); + fsmonitor_accept_pending_token( + istate, closure->untracked_proof_complete, + wt_status_untracked_cache_valid(closure)); return WT_STATUS_TOKEN_CLOSURE_ACCEPTED; } @@ -1430,11 +1495,15 @@ static int wt_status_close_fsmonitor_token( } closure.can_prime = require_untracked && - istate->untracked && istate->untracked->root && + istate->untracked && s->show_untracked_files != SHOW_NO_UNTRACKED_FILES && !s->show_ignored_mode; + closure.use_bulk_provider = + wt_status_can_use_bulk_provider(s, refresh_flags); closure.untracked_ready = !istate->untracked || !istate->untracked->root; + closure.untracked_proof_complete = + !require_untracked || !istate->untracked; if (require_untracked && !closure.can_prime && !closure.untracked_ready) BUG("cannot close required untracked scan"); @@ -1463,6 +1532,7 @@ static int wt_status_close_fsmonitor_token( fallback: wt_status_discard_semantic_verify(s, &proof, "fallback"); wt_status_discard_staged_untracked(&closure); + preload_index_bulk_result_clear(istate); fsmonitor_reject_pending_token(istate); if (fstat_is_reliable()) { if (closure.can_prime) @@ -1488,7 +1558,7 @@ int wt_status_refresh_index(struct wt_status *s, wt_status_begin_attr_snapshot(s); refresh_fsmonitor(istate); - proof = wt_status_prepare_semantic_verify(s); + proof = wt_status_prepare_semantic_verify(s, refresh_flags); ret = wt_status_close_fsmonitor_token( s, proof, refresh_flags, require_untracked, 0); if (istate->preload_untracked == &s->untracked) { @@ -1573,7 +1643,8 @@ void wt_status_collect(struct wt_status *s) (used_untracked_cache || !s->repo->index->untracked || !s->repo->index->untracked->root)) { if (fsmonitor_pending_token_from_provider(s->repo->index)) - fsmonitor_accept_pending_token(s->repo->index, 1); + fsmonitor_accept_pending_token( + s->repo->index, 1, used_untracked_cache); else fsmonitor_reject_pending_token(s->repo->index); } From 176af6bec64fdb2149a3c7bf4ffa7d292e61a3fd Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 00:13:48 -0500 Subject: [PATCH 097/105] exclude: compute stable source-proof digests A live exclude-source proof uses filesystem identity to keep one observation coherent. That identity cannot compare equivalent ignore sources captured by separate status processes: replacing a file with the same contents changes its identity without changing ignore semantics. Hash the existing, validated observations in first-observation order. Frame the digest with its version, source object format, unique source count, path, lookup policy, presence, and content identity. Exclude transient stat identity so an equivalent replacement retains the same semantic digest. Extend the existing exclude-proof unit tests to capture independent proofs across a same-content replacement and repeated observation. The digest is independently testable without issuing a sidecar or changing normal exclude-source validation. Signed-off-by: Taylor Blau --- exclude-source-proof.c | 46 +++++++++++++++++++++++++-- exclude-source-proof.h | 10 ++++++ t/unit-tests/u-exclude-source-proof.c | 32 +++++++++++++++++++ 3 files changed, 85 insertions(+), 3 deletions(-) diff --git a/exclude-source-proof.c b/exclude-source-proof.c index 022aa5a7ba1d5a..1a2ebd5f87190b 100644 --- a/exclude-source-proof.c +++ b/exclude-source-proof.c @@ -1,5 +1,6 @@ #include "git-compat-util.h" #include "exclude-source-proof.h" +#include "hash-framing.h" #include "object-file.h" #include "path-namespace.h" #include "read-cache-ll.h" @@ -8,9 +9,9 @@ #include "trace2.h" /* - * Each entry describes one path/policy observation. Filesystem identities - * are used only to make capture and validation coherent; the durable - * observation is the source's existence and bytes. + * Each entry describes one path/policy observation for replay or a stable + * digest. Filesystem identities make capture and validation coherent; the + * durable observation is the source's existence and bytes. */ struct exclude_source_proof_entry { char *path; @@ -415,6 +416,45 @@ int exclude_source_proof_validate(struct exclude_source_proof *proof) return valid; } +int exclude_source_proof_digest( + struct exclude_source_proof *proof, + const struct git_hash_algo *algo, + struct object_id *oid) +{ + static const char domain[] = "git-exclude-source-proof-digest-v1"; + const struct git_hash_algo *source_algo; + struct git_hash_ctx ctx; + unsigned char count[sizeof(uint64_t)]; + unsigned char format[sizeof(uint32_t)]; + + if (!proof || !algo || !oid || + !exclude_source_proof_validate(proof)) + return -1; + source_algo = proof->istate->repo->hash_algo; + git_hash_init(&ctx, algo); + hash_length_delimited(&ctx, domain, sizeof(domain) - 1); + put_be32(format, source_algo->format_id); + hash_length_delimited(&ctx, format, sizeof(format)); + put_be64(count, proof->nr); + hash_length_delimited(&ctx, count, sizeof(count)); + for (size_t i = 0; i < proof->nr; i++) { + const struct exclude_source_proof_entry *entry = + &proof->entries[i]; + unsigned char policy[] = { + entry->nofollow, + entry->exists, + }; + + hash_length_delimited(&ctx, entry->path, + strlen(entry->path)); + hash_length_delimited(&ctx, policy, sizeof(policy)); + hash_length_delimited(&ctx, entry->oid.hash, + entry->exists ? source_algo->rawsz : 0); + } + git_hash_final_oid(oid, &ctx); + return 0; +} + void exclude_source_proof_release(struct exclude_source_proof *proof) { if (!proof) diff --git a/exclude-source-proof.h b/exclude-source-proof.h index e2932f535fb7a4..f1c03b4a3cbf5f 100644 --- a/exclude-source-proof.h +++ b/exclude-source-proof.h @@ -12,7 +12,9 @@ struct exclude_source_capture; struct exclude_source_proof; +struct git_hash_algo; struct index_state; +struct object_id; struct stat; typedef int (*exclude_source_open_parent_fn)(void *data, const char *path); @@ -33,6 +35,14 @@ void exclude_source_capture_record( void exclude_source_capture_error(struct exclude_source_capture *capture); void exclude_source_capture_release(struct exclude_source_capture *capture); int exclude_source_proof_validate(struct exclude_source_proof *proof); +/* + * Hash unique observations in first-observation order. The caller must + * capture them in a deterministic order when comparing across processes. + */ +int exclude_source_proof_digest( + struct exclude_source_proof *proof, + const struct git_hash_algo *algo, + struct object_id *oid); void exclude_source_proof_release(struct exclude_source_proof *proof); #endif /* EXCLUDE_SOURCE_PROOF_H */ diff --git a/t/unit-tests/u-exclude-source-proof.c b/t/unit-tests/u-exclude-source-proof.c index 26d5f16e6f2e9c..e579dbcd51247a 100644 --- a/t/unit-tests/u-exclude-source-proof.c +++ b/t/unit-tests/u-exclude-source-proof.c @@ -198,6 +198,37 @@ void test_exclude_source_proof__rejects_conflicting_observations(void) free(parent); } +void test_exclude_source_proof__digest_deduplicates_and_ignores_identity(void) +{ + struct exclude_source_proof *first_proof = + exclude_source_proof_create(&istate, NULL, open_parent); + struct exclude_source_proof *second_proof; + struct object_id first, second; + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(first_proof, source); + cl_must_pass(exclude_source_proof_digest( + first_proof, repo.hash_algo, &first)); + + cl_must_pass(unlink(source)); + write_file_buf(source, "content", 7); + second_proof = + exclude_source_proof_create(&istate, NULL, open_parent); + record_file(second_proof, source); + record_file(second_proof, source); + cl_must_pass(exclude_source_proof_digest( + second_proof, repo.hash_algo, &second)); + cl_assert(oideq(&first, &second)); + + exclude_source_proof_release(second_proof); + exclude_source_proof_release(first_proof); + free(source); + free(parent); +} + void test_exclude_source_proof__rejects_open_failure(void) { struct exclude_source_proof *proof = new_proof(); @@ -391,6 +422,7 @@ SKIP_TEST(test_exclude_source_proof__rejects_different_content_replacement) SKIP_TEST(test_exclude_source_proof__accepts_repeated_observation) SKIP_TEST(test_exclude_source_proof__rejects_missing_source_buffer) SKIP_TEST(test_exclude_source_proof__rejects_conflicting_observations) +SKIP_TEST(test_exclude_source_proof__digest_deduplicates_and_ignores_identity) SKIP_TEST(test_exclude_source_proof__rejects_open_failure) SKIP_TEST(test_exclude_source_proof__fails_closed_without_parent_opener) SKIP_TEST(test_exclude_source_proof__honors_nofollow) From e884a179f79420977e177db960d07f1f7132afe1 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 14:04:25 -0500 Subject: [PATCH 098/105] status: add a bounded clean-status sidecar format A later status invocation cannot safely reuse an empty result unless its persistent record identifies the exact index and semantic inputs that the original scan proved. Accepting truncated, ambiguous, or forward-versioned records would turn a cache miss into a false clean result. Define the version-one CSTS encoding and serialize index identity in fixed-width network-byte-order fields. Bind the index format, entry count, checksum, HEAD tree, configuration and repository hashes, one exclude digest, and a bounded builtin-provider token. Protect the complete record with the repository's object-format checksum. Reject unknown flags, unsupported index formats, null required object IDs, invalid token bounds or prefixes, bad checksums, truncation, and trailing payload. Add fixed-width identity and sidecar unit coverage for both SHA-1 and SHA-256. Register the new source and unit suite in both Make and Meson. This patch defines and tests the format; it neither writes a sidecar nor changes status dispatch. Signed-off-by: Taylor Blau --- Makefile | 2 + clean-status-identity.c | 29 +++ clean-status-identity.h | 9 + clean-status-sidecar.c | 128 +++++++++++ clean-status-sidecar.h | 35 +++ meson.build | 1 + t/meson.build | 1 + t/unit-tests/u-clean-status-identity.c | 38 ++++ t/unit-tests/u-clean-status-sidecar.c | 287 +++++++++++++++++++++++++ 9 files changed, 530 insertions(+) create mode 100644 clean-status-sidecar.c create mode 100644 clean-status-sidecar.h create mode 100644 t/unit-tests/u-clean-status-sidecar.c diff --git a/Makefile b/Makefile index 1c3d6e1520a9c1..28d62e31922170 100644 --- a/Makefile +++ b/Makefile @@ -1135,6 +1135,7 @@ LIB_OBJS += clean-status-history.o LIB_OBJS += clean-status-identity.o LIB_OBJS += clean-status-index.o LIB_OBJS += clean-status-manifest.o +LIB_OBJS += clean-status-sidecar.o LIB_OBJS += color.o LIB_OBJS += column.o LIB_OBJS += combine-diff.o @@ -1574,6 +1575,7 @@ CLAR_TEST_SUITES += u-clean-status-history CLAR_TEST_SUITES += u-clean-status-identity CLAR_TEST_SUITES += u-clean-status-index CLAR_TEST_SUITES += u-clean-status-manifest +CLAR_TEST_SUITES += u-clean-status-sidecar CLAR_TEST_SUITES += u-ctype CLAR_TEST_SUITES += u-dir CLAR_TEST_SUITES += u-example-decorate diff --git a/clean-status-identity.c b/clean-status-identity.c index 415ecab19a64f2..98d5a7e51b9a73 100644 --- a/clean-status-identity.c +++ b/clean-status-identity.c @@ -1,5 +1,6 @@ #include "git-compat-util.h" #include "clean-status-identity.h" +#include "strbuf.h" int clean_status_identity_from_stat(struct clean_status_identity *identity, const struct stat *st) @@ -25,3 +26,31 @@ int clean_status_identity_equal(const struct clean_status_identity *a, { return path_stat_identity_equal(&a->stat, &b->stat); } + +void clean_status_identity_write(struct strbuf *out, + const struct clean_status_identity *identity) +{ + uint64_t value; + size_t i; + + for (i = 0; i < ARRAY_SIZE(identity->stat.fields); i++) { + put_be64(&value, identity->stat.fields[i]); + strbuf_add(out, &value, sizeof(value)); + } +} + +int clean_status_identity_read(const unsigned char **p, + const unsigned char *end, + struct clean_status_identity *identity) +{ + size_t i; + + memset(identity, 0, sizeof(*identity)); + for (i = 0; i < ARRAY_SIZE(identity->stat.fields); i++) { + if ((size_t)(end - *p) < sizeof(uint64_t)) + return -1; + identity->stat.fields[i] = get_be64(*p); + *p += sizeof(uint64_t); + } + return 0; +} diff --git a/clean-status-identity.h b/clean-status-identity.h index 68e459a0349a1f..a22f8438f50874 100644 --- a/clean-status-identity.h +++ b/clean-status-identity.h @@ -3,16 +3,25 @@ #include "path-namespace.h" +struct strbuf; struct stat; struct clean_status_identity { struct path_stat_identity stat; }; +#define CLEAN_STATUS_IDENTITY_SIZE \ + (PATH_STAT_IDENTITY_FIELDS * sizeof(uint64_t)) + int clean_status_identity_from_stat(struct clean_status_identity *identity, const struct stat *st); int clean_status_identity_is_durable(void); int clean_status_identity_equal(const struct clean_status_identity *a, const struct clean_status_identity *b); +void clean_status_identity_write(struct strbuf *out, + const struct clean_status_identity *identity); +int clean_status_identity_read(const unsigned char **p, + const unsigned char *end, + struct clean_status_identity *identity); #endif /* CLEAN_STATUS_IDENTITY_H */ diff --git a/clean-status-sidecar.c b/clean-status-sidecar.c new file mode 100644 index 00000000000000..b850f27e6a6d57 --- /dev/null +++ b/clean-status-sidecar.c @@ -0,0 +1,128 @@ +#include "git-compat-util.h" +#include "clean-status-sidecar.h" +#include "fsmonitor-clean-proof.h" +#include "hash-framing.h" +#include "strbuf.h" + +#define CLEAN_STATUS_SIDECAR_MAGIC "CSTS" + +static int checksum_valid(const void *data, size_t len, + const struct git_hash_algo *algo) +{ + const unsigned char *bytes = data; + unsigned char actual[GIT_MAX_RAWSZ]; + + if (len < algo->rawsz) + return 0; + hash_buffer_digest(algo, data, len - algo->rawsz, actual); + return !memcmp(actual, bytes + len - algo->rawsz, algo->rawsz); +} + +static int token_valid(const unsigned char *token, size_t token_len) +{ + static const char prefix[] = "builtin:"; + + return token && token_len && + token_len <= FSMONITOR_CLEAN_PROOF_TOKEN_MAX && + !memchr(token, '\0', token_len) && + token_len >= sizeof(prefix) - 1 && + !memcmp(token, prefix, sizeof(prefix) - 1); +} + +static int proof_valid(const struct clean_status_proof *proof, + const struct git_hash_algo *algo) +{ + return proof->index_version >= 2 && proof->index_version <= 4 && + !is_null_oid(&proof->index_checksum) && + !is_null_oid(&proof->head_tree) && + !is_null_oid(&proof->exclude_source_digest) && + proof->index_checksum.algo == hash_algo_by_ptr(algo) && + proof->head_tree.algo == hash_algo_by_ptr(algo) && + proof->exclude_source_digest.algo == hash_algo_by_ptr(algo); +} + +int clean_status_sidecar_parse(struct clean_status_sidecar *sidecar, + const void *data, size_t len, + const struct git_hash_algo *algo) +{ + const unsigned char *p = data; + const unsigned char *end; + size_t minimum = 4 + 2 * sizeof(uint32_t) + + CLEAN_STATUS_IDENTITY_SIZE + 3 * sizeof(uint32_t) + + 6 * algo->rawsz + 1; + uint32_t flags, token_len; + + memset(sidecar, 0, sizeof(*sidecar)); + if (len < minimum || memcmp(p, CLEAN_STATUS_SIDECAR_MAGIC, 4) || + !checksum_valid(data, len, algo)) + return -1; + end = p + len - algo->rawsz; + p += 4; + if (get_be32(p) != CLEAN_STATUS_SIDECAR_VERSION) + return -1; + p += sizeof(uint32_t); + flags = get_be32(p); + p += sizeof(uint32_t); + if (flags) + return -1; + if (clean_status_identity_read(&p, end, &sidecar->identity)) + return -1; + sidecar->proof.index_version = get_be32(p); + p += sizeof(uint32_t); + sidecar->proof.cache_nr = get_be32(p); + p += sizeof(uint32_t); + oidread(&sidecar->proof.index_checksum, p, algo); + p += algo->rawsz; + oidread(&sidecar->proof.head_tree, p, algo); + p += algo->rawsz; + memcpy(sidecar->proof.config_hash, p, algo->rawsz); + p += algo->rawsz; + memcpy(sidecar->proof.repo_hash, p, algo->rawsz); + p += algo->rawsz; + oidread(&sidecar->proof.exclude_source_digest, p, algo); + p += algo->rawsz; + token_len = get_be32(p); + p += sizeof(uint32_t); + if (!proof_valid(&sidecar->proof, algo) || + (size_t)(end - p) != token_len || + !token_valid(p, token_len)) + return -1; + sidecar->token = p; + sidecar->token_len = token_len; + return 0; +} + +int clean_status_sidecar_write(struct strbuf *out, + const struct clean_status_sidecar *sidecar, + const struct git_hash_algo *algo) +{ + uint32_t value; + + strbuf_reset(out); + if (!proof_valid(&sidecar->proof, algo) || + sidecar->token_len > UINT32_MAX || + !token_valid(sidecar->token, sidecar->token_len)) + return -1; + + strbuf_add(out, CLEAN_STATUS_SIDECAR_MAGIC, 4); + put_be32(&value, CLEAN_STATUS_SIDECAR_VERSION); + strbuf_add(out, &value, sizeof(value)); + put_be32(&value, 0); + strbuf_add(out, &value, sizeof(value)); + clean_status_identity_write(out, &sidecar->identity); + put_be32(&value, sidecar->proof.index_version); + strbuf_add(out, &value, sizeof(value)); + put_be32(&value, sidecar->proof.cache_nr); + strbuf_add(out, &value, sizeof(value)); + strbuf_add(out, sidecar->proof.index_checksum.hash, algo->rawsz); + strbuf_add(out, sidecar->proof.head_tree.hash, algo->rawsz); + strbuf_add(out, sidecar->proof.config_hash, algo->rawsz); + strbuf_add(out, sidecar->proof.repo_hash, algo->rawsz); + strbuf_add(out, sidecar->proof.exclude_source_digest.hash, + algo->rawsz); + put_be32(&value, sidecar->token_len); + strbuf_add(out, &value, sizeof(value)); + strbuf_add(out, sidecar->token, sidecar->token_len); + hash_append_checksum(out, algo); + return 0; +} diff --git a/clean-status-sidecar.h b/clean-status-sidecar.h new file mode 100644 index 00000000000000..1b99434f1f7045 --- /dev/null +++ b/clean-status-sidecar.h @@ -0,0 +1,35 @@ +#ifndef CLEAN_STATUS_SIDECAR_H +#define CLEAN_STATUS_SIDECAR_H + +#include "clean-status-identity.h" +#include "hash.h" + +struct strbuf; + +#define CLEAN_STATUS_SIDECAR_VERSION 1 + +struct clean_status_proof { + uint32_t index_version; + uint32_t cache_nr; + struct object_id index_checksum; + struct object_id head_tree; + unsigned char config_hash[GIT_MAX_RAWSZ]; + unsigned char repo_hash[GIT_MAX_RAWSZ]; + struct object_id exclude_source_digest; +}; + +struct clean_status_sidecar { + struct clean_status_identity identity; + struct clean_status_proof proof; + const unsigned char *token; + size_t token_len; +}; + +int clean_status_sidecar_parse(struct clean_status_sidecar *sidecar, + const void *data, size_t len, + const struct git_hash_algo *algo); +int clean_status_sidecar_write(struct strbuf *out, + const struct clean_status_sidecar *sidecar, + const struct git_hash_algo *algo); + +#endif /* CLEAN_STATUS_SIDECAR_H */ diff --git a/meson.build b/meson.build index 56e794f7dbe32a..5fd2b58f93f241 100644 --- a/meson.build +++ b/meson.build @@ -340,6 +340,7 @@ libgit_sources = [ 'clean-status-identity.c', 'clean-status-index.c', 'clean-status-manifest.c', + 'clean-status-sidecar.c', 'color.c', 'column.c', 'combine-diff.c', diff --git a/t/meson.build b/t/meson.build index 909f58e975a0fa..8be6cc6bd98ad1 100644 --- a/t/meson.build +++ b/t/meson.build @@ -6,6 +6,7 @@ clar_test_suites = [ 'unit-tests/u-clean-status-identity.c', 'unit-tests/u-clean-status-index.c', 'unit-tests/u-clean-status-manifest.c', + 'unit-tests/u-clean-status-sidecar.c', 'unit-tests/u-ctype.c', 'unit-tests/u-dir.c', 'unit-tests/u-example-decorate.c', diff --git a/t/unit-tests/u-clean-status-identity.c b/t/unit-tests/u-clean-status-identity.c index 33e7b80fcfc7e7..6875d594c58c5c 100644 --- a/t/unit-tests/u-clean-status-identity.c +++ b/t/unit-tests/u-clean-status-identity.c @@ -1,5 +1,43 @@ #include "unit-test.h" #include "clean-status-identity.h" +#include "strbuf.h" + +void test_clean_status_identity__round_trips_fixed_width_encoding(void) +{ + struct clean_status_identity expected = { + .stat.fields = { + 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, + }, + }; + struct clean_status_identity actual; + struct strbuf encoded = STRBUF_INIT; + const unsigned char *p; + + clean_status_identity_write(&encoded, &expected); + cl_assert_equal_i(encoded.len, CLEAN_STATUS_IDENTITY_SIZE); + p = (const unsigned char *)encoded.buf; + cl_assert_equal_i(clean_status_identity_read( + &p, (const unsigned char *)encoded.buf + encoded.len, &actual), 0); + cl_assert_equal_i(p - (const unsigned char *)encoded.buf, encoded.len); + cl_assert(clean_status_identity_equal(&expected, &actual)); + strbuf_release(&encoded); +} + +void test_clean_status_identity__rejects_every_truncation(void) +{ + struct clean_status_identity identity = { 0 }, parsed; + struct strbuf encoded = STRBUF_INIT; + + clean_status_identity_write(&encoded, &identity); + for (size_t len = 0; len < encoded.len; len++) { + const unsigned char *p = (const unsigned char *)encoded.buf; + + cl_assert_equal_i(clean_status_identity_read( + &p, (const unsigned char *)encoded.buf + len, &parsed), -1); + } + strbuf_release(&encoded); +} void test_clean_status_identity__requires_a_single_link_regular_file(void) { diff --git a/t/unit-tests/u-clean-status-sidecar.c b/t/unit-tests/u-clean-status-sidecar.c new file mode 100644 index 00000000000000..56ea5581b2a059 --- /dev/null +++ b/t/unit-tests/u-clean-status-sidecar.c @@ -0,0 +1,287 @@ +#include "unit-test.h" +#include "clean-status-sidecar.h" +#include "fsmonitor-clean-proof.h" +#include "hash-framing.h" +#include "strbuf.h" + +struct sidecar_fixture { + struct clean_status_sidecar sidecar; + struct strbuf encoded; +}; + +static void fill_oid(struct object_id *oid, unsigned char value, + const struct git_hash_algo *algo) +{ + unsigned char hash[GIT_MAX_RAWSZ]; + + memset(hash, value, algo->rawsz); + oidread(oid, hash, algo); +} + +static void fixture_init(struct sidecar_fixture *fixture, + const struct git_hash_algo *algo) +{ + static const unsigned char token[] = "builtin:1:2"; + struct clean_status_proof *proof; + + memset(fixture, 0, sizeof(*fixture)); + fixture->encoded = (struct strbuf)STRBUF_INIT; + fixture->sidecar.identity.stat.fields[0] = 1; + fixture->sidecar.identity.stat.fields[1] = 2; + proof = &fixture->sidecar.proof; + proof->index_version = 4; + proof->cache_nr = 5; + fill_oid(&proof->index_checksum, 2, algo); + fill_oid(&proof->head_tree, 3, algo); + memset(proof->config_hash, 4, algo->rawsz); + memset(proof->repo_hash, 5, algo->rawsz); + fill_oid(&proof->exclude_source_digest, 6, algo); + fixture->sidecar.token = token; + fixture->sidecar.token_len = sizeof(token) - 1; +} + +static void fixture_encode(struct sidecar_fixture *fixture, + const struct git_hash_algo *algo) +{ + cl_assert_equal_i(clean_status_sidecar_write( + &fixture->encoded, &fixture->sidecar, algo), 0); +} + +static void fixture_release(struct sidecar_fixture *fixture) +{ + strbuf_release(&fixture->encoded); +} + +static void replace_checksum(struct strbuf *encoded, + const struct git_hash_algo *algo) +{ + strbuf_setlen(encoded, encoded->len - algo->rawsz); + hash_append_checksum(encoded, algo); +} + +static size_t flags_offset(void) +{ + return 4 + sizeof(uint32_t); +} + +static size_t proof_offset(void) +{ + return 4 + 2 * sizeof(uint32_t) + CLEAN_STATUS_IDENTITY_SIZE; +} + +static size_t index_checksum_offset(void) +{ + return proof_offset() + 2 * sizeof(uint32_t); +} + +static size_t head_tree_offset(const struct git_hash_algo *algo) +{ + return index_checksum_offset() + algo->rawsz; +} + +static size_t exclude_digest_offset(const struct git_hash_algo *algo) +{ + return index_checksum_offset() + 4 * algo->rawsz; +} + +static size_t token_length_offset(const struct git_hash_algo *algo) +{ + return index_checksum_offset() + 5 * algo->rawsz; +} + +static size_t token_offset(const struct git_hash_algo *algo) +{ + return token_length_offset(algo) + sizeof(uint32_t); +} + +static void assert_parse_fails(struct sidecar_fixture *fixture, + const struct git_hash_algo *algo) +{ + struct clean_status_sidecar parsed; + + replace_checksum(&fixture->encoded, algo); + cl_assert_equal_i(clean_status_sidecar_parse( + &parsed, fixture->encoded.buf, fixture->encoded.len, algo), -1); +} + +static void assert_round_trip(const struct git_hash_algo *algo) +{ + struct sidecar_fixture fixture; + struct clean_status_sidecar parsed; + + fixture_init(&fixture, algo); + fixture_encode(&fixture, algo); + cl_assert_equal_i(clean_status_sidecar_parse( + &parsed, fixture.encoded.buf, fixture.encoded.len, algo), 0); + cl_assert(clean_status_identity_equal(&parsed.identity, + &fixture.sidecar.identity)); + cl_assert_equal_i(parsed.proof.index_version, + fixture.sidecar.proof.index_version); + cl_assert_equal_i(parsed.proof.cache_nr, + fixture.sidecar.proof.cache_nr); + cl_assert(oideq(&parsed.proof.index_checksum, + &fixture.sidecar.proof.index_checksum)); + cl_assert(oideq(&parsed.proof.head_tree, + &fixture.sidecar.proof.head_tree)); + cl_assert(!memcmp(parsed.proof.config_hash, + fixture.sidecar.proof.config_hash, algo->rawsz)); + cl_assert(!memcmp(parsed.proof.repo_hash, + fixture.sidecar.proof.repo_hash, algo->rawsz)); + cl_assert(oideq(&parsed.proof.exclude_source_digest, + &fixture.sidecar.proof.exclude_source_digest)); + cl_assert_equal_i(parsed.token_len, fixture.sidecar.token_len); + cl_assert(!memcmp(parsed.token, fixture.sidecar.token, + parsed.token_len)); + fixture_release(&fixture); +} + +void test_clean_status_sidecar__round_trips_both_object_formats(void) +{ + assert_round_trip(&hash_algos[GIT_HASH_SHA1]); + assert_round_trip(&hash_algos[GIT_HASH_SHA256]); +} + +void test_clean_status_sidecar__rejects_bad_envelopes(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct sidecar_fixture fixture; + struct clean_status_sidecar parsed; + + fixture_init(&fixture, algo); + fixture_encode(&fixture, algo); + + cl_assert_equal_i(clean_status_sidecar_parse( + &parsed, fixture.encoded.buf, fixture.encoded.len - 1, algo), -1); + + fixture.encoded.buf[0] ^= 1; + assert_parse_fails(&fixture, algo); + fixture.encoded.buf[0] ^= 1; + + put_be32(fixture.encoded.buf + 4, CLEAN_STATUS_SIDECAR_VERSION + 1); + assert_parse_fails(&fixture, algo); + put_be32(fixture.encoded.buf + 4, CLEAN_STATUS_SIDECAR_VERSION); + + put_be32(fixture.encoded.buf + flags_offset(), 1); + assert_parse_fails(&fixture, algo); + put_be32(fixture.encoded.buf + flags_offset(), 0); + + fixture.encoded.buf[fixture.encoded.len - 1] ^= 1; + cl_assert_equal_i(clean_status_sidecar_parse( + &parsed, fixture.encoded.buf, fixture.encoded.len, algo), -1); + fixture_release(&fixture); +} + +void test_clean_status_sidecar__rejects_invalid_proofs(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct sidecar_fixture fixture; + + fixture_init(&fixture, algo); + fixture_encode(&fixture, algo); + + put_be32(fixture.encoded.buf + proof_offset(), 1); + assert_parse_fails(&fixture, algo); + put_be32(fixture.encoded.buf + proof_offset(), 4); + + memset(fixture.encoded.buf + index_checksum_offset(), 0, algo->rawsz); + assert_parse_fails(&fixture, algo); + memset(fixture.encoded.buf + index_checksum_offset(), 2, algo->rawsz); + + memset(fixture.encoded.buf + head_tree_offset(algo), 0, algo->rawsz); + assert_parse_fails(&fixture, algo); + memset(fixture.encoded.buf + head_tree_offset(algo), 3, algo->rawsz); + + memset(fixture.encoded.buf + exclude_digest_offset(algo), 0, + algo->rawsz); + assert_parse_fails(&fixture, algo); + fixture_release(&fixture); +} + +void test_clean_status_sidecar__rejects_invalid_tokens(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct sidecar_fixture fixture; + size_t token_len_offset = token_length_offset(algo); + size_t token_start = token_offset(algo); + + fixture_init(&fixture, algo); + fixture_encode(&fixture, algo); + + put_be32(fixture.encoded.buf + token_len_offset, 0); + assert_parse_fails(&fixture, algo); + put_be32(fixture.encoded.buf + token_len_offset, + fixture.sidecar.token_len); + + fixture.encoded.buf[token_start] = 'x'; + assert_parse_fails(&fixture, algo); + fixture.encoded.buf[token_start] = 'b'; + + fixture.encoded.buf[token_start + fixture.sidecar.token_len - 1] = '\0'; + assert_parse_fails(&fixture, algo); + fixture.encoded.buf[token_start + fixture.sidecar.token_len - 1] = '2'; + + put_be32(fixture.encoded.buf + token_len_offset, + FSMONITOR_CLEAN_PROOF_TOKEN_MAX + 1); + assert_parse_fails(&fixture, algo); + fixture_release(&fixture); +} + +void test_clean_status_sidecar__accepts_the_maximum_token(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct sidecar_fixture fixture; + struct clean_status_sidecar parsed; + unsigned char *token; + + fixture_init(&fixture, algo); + token = xmalloc(FSMONITOR_CLEAN_PROOF_TOKEN_MAX); + memset(token, 'x', FSMONITOR_CLEAN_PROOF_TOKEN_MAX); + memcpy(token, "builtin:", strlen("builtin:")); + fixture.sidecar.token = token; + fixture.sidecar.token_len = FSMONITOR_CLEAN_PROOF_TOKEN_MAX; + fixture_encode(&fixture, algo); + cl_assert_equal_i(clean_status_sidecar_parse( + &parsed, fixture.encoded.buf, fixture.encoded.len, algo), 0); + cl_assert_equal_i(parsed.token_len, FSMONITOR_CLEAN_PROOF_TOKEN_MAX); + free(token); + fixture_release(&fixture); +} + +void test_clean_status_sidecar__rejects_trailing_payload(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct sidecar_fixture fixture; + + fixture_init(&fixture, algo); + fixture_encode(&fixture, algo); + strbuf_setlen(&fixture.encoded, fixture.encoded.len - algo->rawsz); + strbuf_addch(&fixture.encoded, 'x'); + hash_append_checksum(&fixture.encoded, algo); + cl_assert_equal_i(clean_status_sidecar_parse( + &fixture.sidecar, fixture.encoded.buf, fixture.encoded.len, algo), + -1); + fixture_release(&fixture); +} + +void test_clean_status_sidecar__rejects_invalid_writes(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct sidecar_fixture fixture; + unsigned char *token; + + fixture_init(&fixture, algo); + oidclr(&fixture.sidecar.proof.index_checksum, algo); + cl_assert_equal_i(clean_status_sidecar_write( + &fixture.encoded, &fixture.sidecar, algo), -1); + fill_oid(&fixture.sidecar.proof.index_checksum, 2, algo); + + token = xmalloc(FSMONITOR_CLEAN_PROOF_TOKEN_MAX + 1); + memset(token, 'x', FSMONITOR_CLEAN_PROOF_TOKEN_MAX + 1); + memcpy(token, "builtin:", strlen("builtin:")); + fixture.sidecar.token = token; + fixture.sidecar.token_len = FSMONITOR_CLEAN_PROOF_TOKEN_MAX + 1; + cl_assert_equal_i(clean_status_sidecar_write( + &fixture.encoded, &fixture.sidecar, algo), -1); + free(token); + fixture_release(&fixture); +} From a7ebc9ea493171817b4c38aaeca69d546e4b89eb Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 00:57:06 -0500 Subject: [PATCH 099/105] status: install sidecars against a pinned index A valid sidecar encoding is not sufficient if its named index can be replaced between proof capture and publication. Publishing that record would let a later reader associate one clean result with another index. Expose the existing index-snapshot open and named-path revalidation helpers at their first store consumer. Require a durable index identity on local APFS, a matching index format, entry count, and checksum, and agreement between the held descriptor and the named index. Encode the sidecar under its own lockfile and repeat the index checks before committing that lock. Register the store unit suite with Make and Meson. Its local-APFS tests cover successful installation for SHA-1 and SHA-256 and rejection when the source index is replaced after pinning. Other filesystems fail closed. Signed-off-by: Taylor Blau --- Makefile | 1 + clean-status-index.c | 9 +- clean-status-index.h | 6 ++ clean-status-sidecar.c | 104 ++++++++++++++++++ clean-status-sidecar.h | 9 ++ t/meson.build | 1 + t/unit-tests/u-clean-status-index.c | 2 + t/unit-tests/u-clean-status-store.c | 161 ++++++++++++++++++++++++++++ 8 files changed, 292 insertions(+), 1 deletion(-) create mode 100644 t/unit-tests/u-clean-status-store.c diff --git a/Makefile b/Makefile index 28d62e31922170..e6e992e9efd29b 100644 --- a/Makefile +++ b/Makefile @@ -1576,6 +1576,7 @@ CLAR_TEST_SUITES += u-clean-status-identity CLAR_TEST_SUITES += u-clean-status-index CLAR_TEST_SUITES += u-clean-status-manifest CLAR_TEST_SUITES += u-clean-status-sidecar +CLAR_TEST_SUITES += u-clean-status-store CLAR_TEST_SUITES += u-ctype CLAR_TEST_SUITES += u-dir CLAR_TEST_SUITES += u-example-decorate diff --git a/clean-status-index.c b/clean-status-index.c index ba38d4411b9bbb..c0cdc61d873ae4 100644 --- a/clean-status-index.c +++ b/clean-status-index.c @@ -73,7 +73,14 @@ static int snapshot_open( return -1; } -static int clean_status_index_snapshot_still_matches_path( +int clean_status_index_snapshot_open( + struct clean_status_index_snapshot *snapshot, const char *path, + const struct git_hash_algo *algo) +{ + return snapshot_open(snapshot, path, algo, 0); +} + +int clean_status_index_snapshot_still_matches_path( const struct clean_status_index_snapshot *snapshot, const char *path, const struct git_hash_algo *algo) { diff --git a/clean-status-index.h b/clean-status-index.h index 02dea99f71ae96..aac29b20057bf1 100644 --- a/clean-status-index.h +++ b/clean-status-index.h @@ -14,6 +14,12 @@ struct clean_status_index_snapshot { int fd; }; +int clean_status_index_snapshot_open( + struct clean_status_index_snapshot *snapshot, const char *path, + const struct git_hash_algo *algo); +int clean_status_index_snapshot_still_matches_path( + const struct clean_status_index_snapshot *snapshot, const char *path, + const struct git_hash_algo *algo); int clean_status_index_snapshot_pin( struct clean_status_index_snapshot *snapshot, struct index_state *istate); diff --git a/clean-status-sidecar.c b/clean-status-sidecar.c index b850f27e6a6d57..e97873f82fa929 100644 --- a/clean-status-sidecar.c +++ b/clean-status-sidecar.c @@ -1,10 +1,23 @@ #include "git-compat-util.h" + +#ifdef __APPLE__ +#include +#endif + +#include "clean-status-index.h" #include "clean-status-sidecar.h" #include "fsmonitor-clean-proof.h" #include "hash-framing.h" +#include "lockfile.h" #include "strbuf.h" +#include "wrapper.h" #define CLEAN_STATUS_SIDECAR_MAGIC "CSTS" +#define CLEAN_STATUS_FILESYSTEM_ID_SIZE 16 + +struct clean_status_filesystem_id { + unsigned char value[CLEAN_STATUS_FILESYSTEM_ID_SIZE]; +}; static int checksum_valid(const void *data, size_t len, const struct git_hash_algo *algo) @@ -126,3 +139,94 @@ int clean_status_sidecar_write(struct strbuf *out, hash_append_checksum(out, algo); return 0; } + +static char *sidecar_path(const char *index_path) +{ + return xstrfmt("%s.csts", index_path); +} + +static int local_apfs_id(int fd MAYBE_UNUSED, + struct clean_status_filesystem_id *id) +{ +#ifdef __APPLE__ + struct statfs fs; +#endif + + memset(id, 0, sizeof(*id)); +#ifdef __APPLE__ + if (fstatfs(fd, &fs) || !(fs.f_flags & MNT_LOCAL) || + strcmp(fs.f_fstypename, "apfs") || + sizeof(fs.f_fsid) > sizeof(id->value)) + return -1; + memcpy(id->value, &fs.f_fsid, sizeof(fs.f_fsid)); + return 0; +#else + return -1; +#endif +} + +static int sidecar_matches_snapshot( + const char *index_path, const struct clean_status_sidecar *sidecar, + const struct clean_status_index_snapshot *snapshot, + const struct git_hash_algo *algo) +{ + struct clean_status_filesystem_id fsid; + + return clean_status_identity_is_durable() && + snapshot && snapshot->fd >= 0 && + !local_apfs_id(snapshot->fd, &fsid) && + clean_status_identity_equal(&snapshot->identity, + &sidecar->identity) && + snapshot->version == sidecar->proof.index_version && + snapshot->cache_nr == sidecar->proof.cache_nr && + oideq(&snapshot->checksum, + &sidecar->proof.index_checksum) && + clean_status_index_snapshot_still_matches_path( + snapshot, index_path, algo); +} + +int clean_status_sidecar_pin_source( + const char *index_path, const struct clean_status_sidecar *sidecar, + const struct git_hash_algo *algo, + struct clean_status_index_snapshot *snapshot) +{ + if (clean_status_index_snapshot_open(snapshot, index_path, algo)) + return -1; + if (sidecar_matches_snapshot( + index_path, sidecar, snapshot, algo)) + return 0; + clean_status_index_snapshot_release(snapshot); + return -1; +} + +int clean_status_sidecar_install( + const char *index_path, const struct clean_status_sidecar *sidecar, + const struct clean_status_index_snapshot *snapshot, + const struct git_hash_algo *algo) +{ + struct strbuf encoded = STRBUF_INIT; + struct lock_file lock = LOCK_INIT; + char *path = sidecar_path(index_path); + int sidecar_fd = -1, ret = -1; + + if (!sidecar_matches_snapshot( + index_path, sidecar, snapshot, algo) || + clean_status_sidecar_write(&encoded, sidecar, algo)) + goto done; + sidecar_fd = hold_lock_file_for_update(&lock, path, 0); + if (sidecar_fd < 0 || + (size_t)write_in_full(sidecar_fd, encoded.buf, encoded.len) != + encoded.len || + !sidecar_matches_snapshot( + index_path, sidecar, snapshot, algo) || + commit_lock_file(&lock)) + goto done; + ret = 0; + +done: + if (ret) + rollback_lock_file(&lock); + free(path); + strbuf_release(&encoded); + return ret; +} diff --git a/clean-status-sidecar.h b/clean-status-sidecar.h index 1b99434f1f7045..05e03c952022e5 100644 --- a/clean-status-sidecar.h +++ b/clean-status-sidecar.h @@ -4,6 +4,7 @@ #include "clean-status-identity.h" #include "hash.h" +struct clean_status_index_snapshot; struct strbuf; #define CLEAN_STATUS_SIDECAR_VERSION 1 @@ -31,5 +32,13 @@ int clean_status_sidecar_parse(struct clean_status_sidecar *sidecar, int clean_status_sidecar_write(struct strbuf *out, const struct clean_status_sidecar *sidecar, const struct git_hash_algo *algo); +int clean_status_sidecar_pin_source( + const char *index_path, const struct clean_status_sidecar *sidecar, + const struct git_hash_algo *algo, + struct clean_status_index_snapshot *snapshot); +int clean_status_sidecar_install( + const char *index_path, const struct clean_status_sidecar *sidecar, + const struct clean_status_index_snapshot *snapshot, + const struct git_hash_algo *algo); #endif /* CLEAN_STATUS_SIDECAR_H */ diff --git a/t/meson.build b/t/meson.build index 8be6cc6bd98ad1..02cc78fbcea251 100644 --- a/t/meson.build +++ b/t/meson.build @@ -7,6 +7,7 @@ clar_test_suites = [ 'unit-tests/u-clean-status-index.c', 'unit-tests/u-clean-status-manifest.c', 'unit-tests/u-clean-status-sidecar.c', + 'unit-tests/u-clean-status-store.c', 'unit-tests/u-ctype.c', 'unit-tests/u-dir.c', 'unit-tests/u-example-decorate.c', diff --git a/t/unit-tests/u-clean-status-index.c b/t/unit-tests/u-clean-status-index.c index 979f67634d03d3..a83ca0676467d1 100644 --- a/t/unit-tests/u-clean-status-index.c +++ b/t/unit-tests/u-clean-status-index.c @@ -135,6 +135,8 @@ static void assert_rejects_null_checksum(const struct git_hash_algo *algo) istate.cache_nr = 7; fixture_clear_checksum(&fixture, algo); oidcpy(&istate.oid, &fixture.checksum); + cl_assert_equal_i(clean_status_index_snapshot_open( + &snapshot, fixture.path, algo), -1); cl_assert_equal_i(clean_status_index_snapshot_pin( &snapshot, &istate), -1); fixture_release(&fixture); diff --git a/t/unit-tests/u-clean-status-store.c b/t/unit-tests/u-clean-status-store.c new file mode 100644 index 00000000000000..ac25bb225e3c2e --- /dev/null +++ b/t/unit-tests/u-clean-status-store.c @@ -0,0 +1,161 @@ +#include "unit-test.h" + +#ifdef __APPLE__ +#include +#endif + +#include "clean-status-index.h" +#include "clean-status-sidecar.h" +#include "dir.h" +#include "strbuf.h" + +struct store_fixture { + char *directory; + struct strbuf index_path; + struct clean_status_sidecar sidecar; +}; + +static void fill_oid(struct object_id *oid, unsigned char value, + const struct git_hash_algo *algo) +{ + unsigned char hash[GIT_MAX_RAWSZ]; + + memset(hash, value, algo->rawsz); + oidread(oid, hash, algo); +} + +static void fixture_init(struct store_fixture *fixture, + const struct git_hash_algo *algo) +{ + static const unsigned char token[] = "builtin:1:2"; + struct strbuf index = STRBUF_INIT; + struct clean_status_proof *proof; + struct stat st; + const char *tmp = getenv("TMPDIR"); + uint32_t value; + + memset(fixture, 0, sizeof(*fixture)); + fixture->index_path = (struct strbuf)STRBUF_INIT; + fixture->directory = xstrfmt("%s/status-store.XXXXXX", + tmp ? tmp : "/tmp"); + cl_assert(mkdtemp(fixture->directory) != NULL); + strbuf_addf(&fixture->index_path, "%s/index", fixture->directory); + strbuf_addstr(&index, "DIRC"); + put_be32(&value, 4); + strbuf_add(&index, &value, sizeof(value)); + put_be32(&value, 5); + strbuf_add(&index, &value, sizeof(value)); + strbuf_addchars(&index, 2, algo->rawsz); + write_file_buf(fixture->index_path.buf, index.buf, index.len); + cl_assert_equal_i(stat(fixture->index_path.buf, &st), 0); + cl_assert_equal_i(clean_status_identity_from_stat( + &fixture->sidecar.identity, &st), 0); + proof = &fixture->sidecar.proof; + proof->index_version = 4; + proof->cache_nr = 5; + fill_oid(&proof->index_checksum, 2, algo); + fill_oid(&proof->head_tree, 3, algo); + memset(proof->config_hash, 4, algo->rawsz); + memset(proof->repo_hash, 5, algo->rawsz); + fill_oid(&proof->exclude_source_digest, 6, algo); + fixture->sidecar.token = token; + fixture->sidecar.token_len = sizeof(token) - 1; + strbuf_release(&index); +} + +static void fixture_release(struct store_fixture *fixture) +{ + struct strbuf cleanup = STRBUF_INIT; + + strbuf_addstr(&cleanup, fixture->directory); + cl_assert_equal_i(remove_dir_recursively(&cleanup, 0), 0); + strbuf_release(&cleanup); + strbuf_release(&fixture->index_path); + free(fixture->directory); +} + +static struct strbuf sidecar_path(struct store_fixture *fixture) +{ + struct strbuf path = STRBUF_INIT; + + strbuf_addf(&path, "%s.csts", fixture->index_path.buf); + return path; +} + +static void require_local_apfs(const char *path MAYBE_UNUSED) +{ +#ifdef __APPLE__ + struct statfs fs; + int fd = git_open_cloexec(path, O_RDONLY); + + if (fd < 0 || fstatfs(fd, &fs) || !(fs.f_flags & MNT_LOCAL) || + strcmp(fs.f_fstypename, "apfs")) { + if (fd >= 0) + close(fd); + cl_skip(); + } + close(fd); +#else + cl_skip(); +#endif +} + +static void assert_installs_against_source(const struct git_hash_algo *algo) +{ + struct clean_status_sidecar parsed; + struct clean_status_index_snapshot snapshot; + struct store_fixture fixture; + struct strbuf encoded = STRBUF_INIT; + struct strbuf path; + + fixture_init(&fixture, algo); + path = sidecar_path(&fixture); + cl_assert_equal_i(clean_status_sidecar_pin_source( + fixture.index_path.buf, &fixture.sidecar, algo, &snapshot), 0); + cl_assert_equal_i(clean_status_sidecar_install( + fixture.index_path.buf, &fixture.sidecar, &snapshot, algo), 0); + cl_assert(strbuf_read_file(&encoded, path.buf, 0) > 0); + cl_assert_equal_i(clean_status_sidecar_parse( + &parsed, encoded.buf, encoded.len, algo), 0); + cl_assert(clean_status_identity_equal( + &parsed.identity, &fixture.sidecar.identity)); + + clean_status_index_snapshot_release(&snapshot); + strbuf_release(&path); + strbuf_release(&encoded); + fixture_release(&fixture); +} + +void test_clean_status_store__installs_both_object_formats(void) +{ + require_local_apfs(getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); + assert_installs_against_source(&hash_algos[GIT_HASH_SHA1]); + assert_installs_against_source(&hash_algos[GIT_HASH_SHA256]); +} + +static void assert_rejects_replaced_source(const struct git_hash_algo *algo) +{ + struct clean_status_index_snapshot snapshot; + struct store_fixture fixture; + struct strbuf replacement = STRBUF_INIT; + + fixture_init(&fixture, algo); + cl_assert_equal_i(clean_status_sidecar_pin_source( + fixture.index_path.buf, &fixture.sidecar, algo, &snapshot), 0); + strbuf_addf(&replacement, "%s/replacement", fixture.directory); + write_file(replacement.buf, "replacement"); + cl_assert_equal_i(rename(replacement.buf, fixture.index_path.buf), 0); + cl_assert_equal_i(clean_status_sidecar_install( + fixture.index_path.buf, &fixture.sidecar, &snapshot, algo), -1); + + clean_status_index_snapshot_release(&snapshot); + strbuf_release(&replacement); + fixture_release(&fixture); +} + +void test_clean_status_store__rejects_a_replaced_source_index(void) +{ + require_local_apfs(getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); + assert_rejects_replaced_source(&hash_algos[GIT_HASH_SHA1]); + assert_rejects_replaced_source(&hash_algos[GIT_HASH_SHA256]); +} From 6be2be5b2c1bb7595da1b70d1b7da788b693eaf9 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 00:57:16 -0500 Subject: [PATCH 100/105] status: classify indexes that clean proofs may certify A clean provider response does not establish that every index entry can be represented by an empty status result. Conflicted entries, submodules, sparse entries, intent-to-add entries, and independently trusted stat state can all require ordinary index processing. Introduce a single conservative certifiability check. Require a non-null index checksum and provider-valid ordinary entries. Reject gitlinks, nonzero stages, intent-to-add, skip-worktree, CE_VALID, and unrecognized entry flags while allowing the explicitly supported in-memory flags. Extend the existing index unit suite to exercise accepted ordinary entries and each unsupported entry shape. The classifier does not issue a proof or change status behavior by itself. Signed-off-by: Taylor Blau --- clean-status-index.c | 25 +++++++++++++++ clean-status-index.h | 3 ++ t/unit-tests/u-clean-status-index.c | 49 +++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+) diff --git a/clean-status-index.c b/clean-status-index.c index c0cdc61d873ae4..48250acc533e79 100644 --- a/clean-status-index.c +++ b/clean-status-index.c @@ -2,6 +2,7 @@ #include "clean-status.h" #include "clean-status-index.h" #include "clean-status-internal.h" +#include "object.h" #include "read-cache-ll.h" #include "repository.h" #include "wrapper.h" @@ -146,6 +147,30 @@ void clean_status_index_snapshot_release( snapshot->fd = -1; } +int clean_status_index_entries_are_certifiable( + const struct index_state *istate) +{ + for (size_t i = 0; i < istate->cache_nr; i++) { + const struct cache_entry *ce = istate->cache[i]; + + if (S_ISGITLINK(ce->ce_mode) || ce_stage(ce) || + ce_intent_to_add(ce) || ce_skip_worktree(ce) || + (ce->ce_flags & CE_VALID) || + !(ce->ce_flags & CE_FSMONITOR_VALID) || + (ce->ce_flags & ~(CE_UPTODATE | CE_HASHED | + CE_FSMONITOR_VALID | + CE_UPDATE_IN_BASE))) + return 0; + } + return 1; +} + +int clean_status_index_is_certifiable(const struct index_state *istate) +{ + return !is_null_oid(&istate->oid) && + clean_status_index_entries_are_certifiable(istate); +} + void clean_status_record_source_identity(struct index_state *istate, const struct stat *st) { diff --git a/clean-status-index.h b/clean-status-index.h index aac29b20057bf1..def687ac534026 100644 --- a/clean-status-index.h +++ b/clean-status-index.h @@ -28,5 +28,8 @@ int clean_status_index_snapshot_still_matches( const struct index_state *istate); void clean_status_index_snapshot_release( struct clean_status_index_snapshot *snapshot); +int clean_status_index_entries_are_certifiable( + const struct index_state *istate); +int clean_status_index_is_certifiable(const struct index_state *istate); #endif /* CLEAN_STATUS_INDEX_H */ diff --git a/t/unit-tests/u-clean-status-index.c b/t/unit-tests/u-clean-status-index.c index a83ca0676467d1..145d06175cbf0b 100644 --- a/t/unit-tests/u-clean-status-index.c +++ b/t/unit-tests/u-clean-status-index.c @@ -3,6 +3,7 @@ #include "clean-status-index.h" #include "clean-status-internal.h" #include "dir.h" +#include "object.h" #include "read-cache-ll.h" #include "repository.h" #include "strbuf.h" @@ -331,3 +332,51 @@ void test_clean_status_index__binds_the_parsed_source(void) strbuf_release(&path); free(worktree); } + +void test_clean_status_index__recognizes_certifiable_entries(void) +{ + struct repository repo = { .hash_algo = &hash_algos[GIT_HASH_SHA1] }; + struct index_state istate = INDEX_STATE_INIT(&repo); + struct cache_entry *ce; + static const char path[] = "tracked"; + + CALLOC_ARRAY(istate.cache, 1); + istate.cache_alloc = istate.cache_nr = 1; + memset(istate.oid.hash, 1, repo.hash_algo->rawsz); + oid_set_algo(&istate.oid, repo.hash_algo); + ce = make_empty_cache_entry(&istate, sizeof(path) - 1); + ce->ce_mode = S_IFREG | 0644; + ce->ce_namelen = sizeof(path) - 1; + memcpy(ce->name, path, sizeof(path)); + istate.cache[0] = ce; + + ce->ce_flags = CE_FSMONITOR_VALID; + cl_assert(clean_status_index_is_certifiable(&istate)); + ce->ce_flags |= CE_UPTODATE | CE_HASHED; + cl_assert(clean_status_index_is_certifiable(&istate)); + + oidclr(&istate.oid, repo.hash_algo); + cl_assert(!clean_status_index_is_certifiable(&istate)); + cl_assert(clean_status_index_entries_are_certifiable(&istate)); + memset(istate.oid.hash, 1, repo.hash_algo->rawsz); + + ce->ce_flags = 0; + cl_assert(!clean_status_index_is_certifiable(&istate)); + ce->ce_flags = CE_FSMONITOR_VALID | CE_VALID; + cl_assert(!clean_status_index_is_certifiable(&istate)); + ce->ce_flags = CE_FSMONITOR_VALID | CE_UPDATE_IN_BASE; + cl_assert(clean_status_index_is_certifiable(&istate)); + ce->ce_flags = CE_FSMONITOR_VALID | create_ce_flags(1); + cl_assert(!clean_status_index_is_certifiable(&istate)); + ce->ce_flags = CE_FSMONITOR_VALID | CE_INTENT_TO_ADD; + cl_assert(!clean_status_index_is_certifiable(&istate)); + ce->ce_flags = CE_FSMONITOR_VALID | CE_SKIP_WORKTREE; + cl_assert(!clean_status_index_is_certifiable(&istate)); + ce->ce_flags = CE_FSMONITOR_VALID | CE_WT_REMOVE; + cl_assert(!clean_status_index_is_certifiable(&istate)); + ce->ce_flags = CE_FSMONITOR_VALID; + ce->ce_mode = S_IFGITLINK; + cl_assert(!clean_status_index_is_certifiable(&istate)); + + release_index(&istate); +} From a6993ad2f30a23e4367913f8e794046588a1e30b Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 28 Jul 2026 02:28:35 -0500 Subject: [PATCH 101/105] status: issue sidecars after a verified full scan An empty status result cannot certify the next invocation unless its tracked and untracked observations, ignore sources, provider token, configuration, repository, HEAD, and named index belong to one completed scan. Publishing a digest before provider-token closure, or reusing cached replacement-ref state, could issue a false clean proof. Retain the standard-exclude digest produced by the complete bulk scan. Keep provider-originated digest state pending until token closure accepts it, and preserve the accepted digest when consuming single-use tracked results. Inspect a fresh, uncached ref store and reject effective replacement refs. Then fingerprint the held local-APFS index and worktree, repository paths, locale, and external attribute state. Issue a sidecar only for the literal, top-level, empty porcelain-v2 command after persistent semantic history, an eligible expanded index, a complete untracked scan, and the HEAD cache tree all agree. Install against the pinned index before rolling back its held index lock; otherwise retain ordinary index-update behavior. Add the focused sidecar integration suite and register its source and production code with the relevant Make and Meson builds. Cover prior semantic history, unchanged index contents, exact command shape, and rejection of external attributes, untracked-cache results, and alternate indexes. No early status answer is introduced here. Signed-off-by: Taylor Blau --- Makefile | 1 + builtin/commit.c | 8 ++ clean-status-sidecar-issue.c | 169 ++++++++++++++++++++++++++ clean-status-sidecar.c | 120 +++++++++++++++++++ clean-status-sidecar.h | 9 ++ clean-status.h | 7 ++ dir.c | 31 ++++- dir.h | 1 + meson.build | 1 + preload-index-bulk.c | 18 ++- preload-index-bulk.h | 4 + preload-index.c | 56 ++++++++- preload-index.h | 6 + read-cache-ll.h | 6 +- refs.c | 17 +++ refs.h | 6 + replace-object.c | 13 ++ replace-object.h | 7 ++ t/meson.build | 1 + t/t7508-status.sh | 40 +++++++ t/t7530-status-clean-sidecar.sh | 136 +++++++++++++++++++++ wt-status.c | 206 ++++++++++++++++++++++++++++---- wt-status.h | 13 ++ 23 files changed, 844 insertions(+), 32 deletions(-) create mode 100644 clean-status-sidecar-issue.c create mode 100755 t/t7530-status-clean-sidecar.sh diff --git a/Makefile b/Makefile index e6e992e9efd29b..2dd5c29fc07e95 100644 --- a/Makefile +++ b/Makefile @@ -1136,6 +1136,7 @@ LIB_OBJS += clean-status-identity.o LIB_OBJS += clean-status-index.o LIB_OBJS += clean-status-manifest.o LIB_OBJS += clean-status-sidecar.o +LIB_OBJS += clean-status-sidecar-issue.o LIB_OBJS += color.o LIB_OBJS += column.o LIB_OBJS += combine-diff.o diff --git a/builtin/commit.c b/builtin/commit.c index 452d56c20a4abd..6ced81bbf4f32b 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1614,6 +1614,9 @@ struct repository *repo UNUSED) struct clean_status_config_digest clean_digest; unsigned int progress_flag = 0; int fd; + int default_status_command = argc == 1 && (!prefix || !*prefix); + int exact_clean_command = argc == 2 && + !strcmp(argv[1], "--porcelain=v2") && (!prefix || !*prefix); struct object_id oid; static struct option builtin_status_options[] = { OPT__VERBOSE(&verbose, N_("be verbose")), @@ -1696,6 +1699,8 @@ struct repository *repo UNUSED) parse_pathspec(&s.pathspec, 0, PATHSPEC_PREFER_FULL, prefix, argv); + s.allow_clean_status_shortcuts = + default_status_command && !s.pathspec.nr; if (status_format != STATUS_FORMAT_PORCELAIN && status_format != STATUS_FORMAT_PORCELAIN_V2) @@ -1731,6 +1736,9 @@ struct repository *repo UNUSED) wt_status_collect(&s); + if (exact_clean_command && 0 <= fd && + clean_status_issue_sidecar(&s, &clean_digest, &index_lock)) + fd = -1; if (0 <= fd) repo_update_index_if_able(the_repository, &index_lock); diff --git a/clean-status-sidecar-issue.c b/clean-status-sidecar-issue.c new file mode 100644 index 00000000000000..93fa79c2cbb540 --- /dev/null +++ b/clean-status-sidecar-issue.c @@ -0,0 +1,169 @@ +#include "git-compat-util.h" +#include "cache-tree.h" +#include "clean-status.h" +#include "clean-status-index.h" +#include "clean-status-internal.h" +#include "clean-status-sidecar.h" +#include "environment.h" +#include "fsmonitor-clean-proof.h" +#include "fsmonitor-ll.h" +#include "fsmonitor-settings.h" +#include "lockfile.h" +#include "object-name.h" +#include "preload-index.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "trace2.h" +#include "wt-status.h" + +static void trace_miss(struct repository *repo, const char *reason) +{ + trace2_data_string("status", repo, "clean-proof/miss", reason); +} + +static int issue_test_barrier(void) +{ + const char *ready = + getenv("GIT_TEST_STATUS_CLEAN_SIDECAR_ISSUE_BARRIER_READY"); + const char *resume = + getenv("GIT_TEST_STATUS_CLEAN_SIDECAR_ISSUE_BARRIER_RESUME"); + struct strbuf buf = STRBUF_INIT; + int ret; + + if (!ready && !resume) + return 0; + if (!ready || !resume) + return -1; + write_file(ready, "ready"); + ret = strbuf_read_file(&buf, resume, 1) > 0 ? 0 : -1; + strbuf_release(&buf); + return ret; +} + +static int output_is_certifiable(const struct wt_status *status) +{ + return status->status_format == STATUS_FORMAT_PORCELAIN_V2 && + !status->pathspec.nr && !status->show_branch && + !status->show_stash && !status->show_ignored_mode && + !status->null_termination && !status->verbose && + status->show_untracked_files == SHOW_NORMAL_UNTRACKED_FILES && + !status->change.nr && !status->untracked.nr && + !status->ignored.nr; +} + +static int history_is_certifiable(const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + + return state && + clean_status_has_persistent_fsmonitor_semantic_history(istate) && + clean_status_revalidated_token_matches(istate) && + state->manifest.current_valid && + (state->manifest.current_flags & FSMONITOR_CLEAN_PROOF_ALL) == + FSMONITOR_CLEAN_PROOF_ALL; +} + +static int fsmonitor_state_is_certifiable( + struct repository *repo, const struct index_state *istate) +{ + return !istate->split_index && + istate->sparse_index == INDEX_EXPANDED && + fsm_settings__get_mode(repo) == FSMONITOR_MODE_IPC && + !fsmonitor_has_pending_token(istate) && + istate->fsmonitor_token_valid && + istate->fsmonitor_last_update && + strlen(istate->fsmonitor_last_update) <= + FSMONITOR_CLEAN_PROOF_TOKEN_MAX && + clean_status_index_is_certifiable(istate); +} + +static int untracked_scan_is_certifiable( + struct wt_status *status, struct object_id *exclude_digest, + struct stat *scanned_worktree) +{ + if (status->untracked_from_preload) + return !preload_index_bulk_standard_excludes_digest( + status->repo->index, exclude_digest, + scanned_worktree); + if (status->untracked_from_token_closure) + return !wt_status_certified_excludes_digest( + status, exclude_digest, scanned_worktree); + return 0; +} + +int clean_status_issue_sidecar( + struct wt_status *status, + const struct clean_status_config_digest *config, + struct lock_file *index_lock) +{ + struct repository *repo = status->repo; + struct index_state *istate = repo->index; + struct clean_status_index_snapshot index = { .fd = -1 }; + struct clean_status_sidecar sidecar = { 0 }; + struct object_id exclude_digest, head_tree; + struct stat scanned_worktree; + unsigned char repo_hash[GIT_MAX_RAWSZ]; + int installed = 0; + + if (!is_lock_file_locked(index_lock) || + !config->finalized || config->unsafe_filter || + !output_is_certifiable(status)) { + trace_miss(repo, "issue-command-or-output"); + goto done; + } + if (!history_is_certifiable(istate)) { + trace_miss(repo, "issue-coherent-history"); + goto done; + } + if (getenv(INDEX_ENVIRONMENT) || + !fsmonitor_state_is_certifiable(repo, istate) || + !untracked_scan_is_certifiable( + status, &exclude_digest, &scanned_worktree)) { + trace_miss(repo, "issue-scan-or-index-shape"); + goto done; + } + if (issue_test_barrier()) { + trace_miss(repo, "issue-test-barrier"); + goto done; + } + if (!status->attr_source_snapshot || + clean_status_index_snapshot_pin(&index, istate) || + clean_status_repository_fingerprint( + repo, status->attr_source_snapshot, &index, + &scanned_worktree, repo_hash)) { + trace_miss(repo, "issue-pinned-inputs"); + goto done; + } + if (repo_get_oid_tree(repo, "HEAD^{tree}", &head_tree) || + !istate->cache_tree || istate->cache_tree->entry_count < 0 || + !oideq(&head_tree, &istate->cache_tree->oid)) { + trace_miss(repo, "issue-head-cache-tree"); + goto done; + } + + sidecar.identity = index.identity; + sidecar.proof.index_version = index.version; + sidecar.proof.cache_nr = index.cache_nr; + oidcpy(&sidecar.proof.index_checksum, &index.checksum); + oidcpy(&sidecar.proof.head_tree, &head_tree); + memcpy(sidecar.proof.config_hash, config->hash, + repo->hash_algo->rawsz); + memcpy(sidecar.proof.repo_hash, repo_hash, + repo->hash_algo->rawsz); + oidcpy(&sidecar.proof.exclude_source_digest, &exclude_digest); + sidecar.token = (const unsigned char *)istate->fsmonitor_last_update; + sidecar.token_len = strlen(istate->fsmonitor_last_update); + + if (clean_status_sidecar_install( + repo->index_file, &sidecar, &index, repo->hash_algo)) { + trace_miss(repo, "issue-sidecar-write"); + goto done; + } + rollback_lock_file(index_lock); + trace2_data_intmax("status", repo, "clean-proof/sidecar", 1); + installed = 1; + +done: + clean_status_index_snapshot_release(&index); + return installed; +} diff --git a/clean-status-sidecar.c b/clean-status-sidecar.c index e97873f82fa929..a68d3dc8059367 100644 --- a/clean-status-sidecar.c +++ b/clean-status-sidecar.c @@ -4,12 +4,18 @@ #include #endif +#include "abspath.h" +#include "attr-fingerprint.h" #include "clean-status-index.h" #include "clean-status-sidecar.h" #include "fsmonitor-clean-proof.h" #include "hash-framing.h" #include "lockfile.h" +#include "path.h" +#include "repository.h" +#include "replace-object.h" #include "strbuf.h" +#include "worktree.h" #include "wrapper.h" #define CLEAN_STATUS_SIDECAR_MAGIC "CSTS" @@ -145,6 +151,18 @@ static char *sidecar_path(const char *index_path) return xstrfmt("%s.csts", index_path); } +static int open_nofollow_nonblocking(const char *path, int flags) +{ +#ifdef O_NONBLOCK + return open_nofollow(path, flags | O_NONBLOCK); +#else + (void)path; + (void)flags; + errno = ENOSYS; + return -1; +#endif +} + static int local_apfs_id(int fd MAYBE_UNUSED, struct clean_status_filesystem_id *id) { @@ -230,3 +248,105 @@ int clean_status_sidecar_install( strbuf_release(&encoded); return ret; } + +static int current_worktree_is_main(struct repository *repo) +{ + struct worktree *worktree = get_current_worktree(repo); + int ret = worktree && is_main_worktree(worktree); + + free_worktree(worktree); + return ret; +} + +static int worktree_root_identity( + const struct stat *st, uint64_t *identity MAYBE_UNUSED) +{ + if (!S_ISDIR(st->st_mode)) + return -1; +#ifdef __APPLE__ + identity[0] = st->st_dev; + identity[1] = st->st_ino; + identity[2] = st->st_birthtimespec.tv_sec; + identity[3] = st->st_birthtimespec.tv_nsec; + identity[4] = st->st_gen; + return 0; +#else + return -1; +#endif +} + +int clean_status_repository_fingerprint( + struct repository *repo, + const struct attr_source_snapshot *attrs, + const struct clean_status_index_snapshot *index, + const struct stat *scanned_worktree, + unsigned char *out) +{ + static const char domain[] = "git-clean-status-repository-v1"; + const struct attr_fingerprint *attr_fingerprint = + attr_source_snapshot_fingerprint(attrs); + struct clean_status_filesystem_id index_fsid, worktree_fsid; + struct git_hash_ctx ctx; + struct stat st; + char *worktree = NULL, *gitdir = NULL, *commondir = NULL; + uint64_t root_identity[5]; + uint64_t scanned_root_identity[5]; + uint64_t value; + int worktree_fd = -1, ret = -1; + + if (!attr_fingerprint || attr_fingerprint->sources_present || + !index || index->fd < 0 || !scanned_worktree || + is_bare_repository(repo) || + !repo_get_work_tree(repo) || + !current_worktree_is_main(repo) || + repo_has_replace_refs_uncached(repo)) + goto done; + + worktree = real_pathdup(repo_get_work_tree(repo), 0); + gitdir = real_pathdup(repo_get_git_dir(repo), 0); + commondir = real_pathdup(repo_get_common_dir(repo), 0); + if (!worktree || !gitdir || !commondir) + goto done; + worktree_fd = open_nofollow_nonblocking( + worktree, O_RDONLY | O_CLOEXEC); + if (worktree_fd < 0 || + local_apfs_id(worktree_fd, &worktree_fsid) || + local_apfs_id(index->fd, &index_fsid) || + fstat(worktree_fd, &st) || + worktree_root_identity(&st, root_identity) || + worktree_root_identity( + scanned_worktree, scanned_root_identity) || + memcmp(root_identity, scanned_root_identity, + sizeof(root_identity))) + goto done; + + git_hash_init(&ctx, repo->hash_algo); + hash_length_delimited(&ctx, domain, sizeof(domain) - 1); + hash_length_delimited(&ctx, worktree, strlen(worktree)); + hash_length_delimited(&ctx, gitdir, strlen(gitdir)); + hash_length_delimited(&ctx, commondir, strlen(commondir)); + for (size_t i = 0; i < ARRAY_SIZE(root_identity); i++) { + put_be64(&value, root_identity[i]); + hash_length_delimited(&ctx, &value, sizeof(value)); + } + hash_length_delimited(&ctx, worktree_fsid.value, + sizeof(worktree_fsid.value)); + hash_length_delimited(&ctx, index_fsid.value, + sizeof(index_fsid.value)); + hash_length_delimited(&ctx, attr_fingerprint->content_hash, + repo->hash_algo->rawsz); + hash_optional_cstring(&ctx, setlocale(LC_CTYPE, NULL)); + hash_optional_cstring(&ctx, getenv("LC_ALL")); + hash_optional_cstring(&ctx, getenv("LC_CTYPE")); + hash_optional_cstring(&ctx, getenv("LANG")); + git_hash_final(out, &ctx); + ret = 0; + +done: + if (worktree_fd >= 0) + close(worktree_fd); + free(worktree); + free(gitdir); + free(commondir); + return ret; +} diff --git a/clean-status-sidecar.h b/clean-status-sidecar.h index 05e03c952022e5..963d6bccc6b65a 100644 --- a/clean-status-sidecar.h +++ b/clean-status-sidecar.h @@ -5,7 +5,10 @@ #include "hash.h" struct clean_status_index_snapshot; +struct attr_source_snapshot; +struct repository; struct strbuf; +struct stat; #define CLEAN_STATUS_SIDECAR_VERSION 1 @@ -40,5 +43,11 @@ int clean_status_sidecar_install( const char *index_path, const struct clean_status_sidecar *sidecar, const struct clean_status_index_snapshot *snapshot, const struct git_hash_algo *algo); +int clean_status_repository_fingerprint( + struct repository *repo, + const struct attr_source_snapshot *attrs, + const struct clean_status_index_snapshot *index, + const struct stat *scanned_worktree, + unsigned char *out); #endif /* CLEAN_STATUS_SIDECAR_H */ diff --git a/clean-status.h b/clean-status.h index 488810ae6a7d11..1a5b62b3924537 100644 --- a/clean-status.h +++ b/clean-status.h @@ -6,9 +6,11 @@ struct index_state; struct attr_source_snapshot; struct clean_status_proof_epoch; +struct lock_file; struct repository; struct stat; struct strbuf; +struct wt_status; enum clean_status_attr_change { CLEAN_STATUS_ATTR_CONTENT_CHANGED = 1 << 0, @@ -69,6 +71,11 @@ void clean_status_record_source_identity(struct index_state *istate, int clean_status_verify_null_index(const struct index_state *istate, const struct stat *st); +int clean_status_issue_sidecar( + struct wt_status *status, + const struct clean_status_config_digest *config, + struct lock_file *index_lock); + int clean_status_read_fsmonitor_config(struct index_state *istate, const void *data, unsigned long size); void clean_status_prepare_fsmonitor_config(struct index_state *istate); diff --git a/dir.c b/dir.c index 4e2e474e083871..c1057362ea573f 100644 --- a/dir.c +++ b/dir.c @@ -933,6 +933,8 @@ static enum path_treatment read_directory_recursive(struct dir_struct *dir, struct index_state *istate, const char *path, int len, struct untracked_cache_dir *untracked, int check_only, int stop_at_first_file, const struct pathspec *pathspec); +static int resolve_dtype_with_error(int dtype, struct index_state *istate, + const char *path, int len, int *failed); static int resolve_dtype(int dtype, struct index_state *istate, const char *path, int len); struct dirent *readdir_skip_dot_and_dotdot(DIR *dirp) @@ -3296,6 +3298,12 @@ unsigned char get_dtype(struct dirent *e, struct strbuf *path, static int resolve_dtype(int dtype, struct index_state *istate, const char *path, int len) +{ + return resolve_dtype_with_error(dtype, istate, path, len, NULL); +} + +static int resolve_dtype_with_error(int dtype, struct index_state *istate, + const char *path, int len, int *failed) { struct stat st; @@ -3304,8 +3312,11 @@ static int resolve_dtype(int dtype, struct index_state *istate, dtype = get_index_dtype(istate, path, len); if (dtype != DT_UNKNOWN) return dtype; - if (lstat(path, &st)) + if (lstat(path, &st)) { + if (failed && !is_missing_file_error(errno)) + *failed = 1; return dtype; + } if (S_ISREG(st.st_mode)) return DT_REG; if (S_ISDIR(st.st_mode)) @@ -3361,6 +3372,7 @@ static enum path_treatment treat_path(struct dir_struct *dir, const struct pathspec *pathspec) { int has_path_in_index, dtype, excluded; + int dtype_failed = 0; if (!cdir->d_name) return treat_path_fast(dir, cdir, istate, path, @@ -3372,7 +3384,11 @@ static enum path_treatment treat_path(struct dir_struct *dir, if (simplify_away(path->buf, path->len, pathspec)) return path_none; - dtype = resolve_dtype(cdir->d_type, istate, path->buf, path->len); + dtype = resolve_dtype_with_error( + cdir->d_type, istate, path->buf, path->len, + &dtype_failed); + if (dtype_failed) + dir->internal.traversal_failed = 1; /* Always exclude indexed files */ has_path_in_index = !!index_file_exists(istate, path->buf, path->len, @@ -3523,8 +3539,10 @@ static int open_cached_dir(struct cached_dir *cdir, return 0; c_path = path->len ? path->buf : "."; cdir->fdir = opendir(c_path); - if (!cdir->fdir) + if (!cdir->fdir) { + dir->internal.traversal_failed = 1; warning_errno(_("could not open directory '%s'"), c_path); + } if (dir->untracked) { invalidate_directory(dir->untracked, untracked); dir->untracked->dir_opened++; @@ -3534,13 +3552,16 @@ static int open_cached_dir(struct cached_dir *cdir, return 0; } -static int read_cached_dir(struct cached_dir *cdir) +static int read_cached_dir(struct cached_dir *cdir, struct dir_struct *dir) { struct dirent *de; if (cdir->fdir) { + errno = 0; de = readdir_skip_dot_and_dotdot(cdir->fdir); if (!de) { + if (errno) + dir->internal.traversal_failed = 1; cdir->d_name = NULL; cdir->d_type = DT_UNKNOWN; return -1; @@ -3698,7 +3719,7 @@ static enum path_treatment read_directory_recursive(struct dir_struct *dir, if (untracked) untracked->check_only = !!check_only; - while (!read_cached_dir(&cdir)) { + while (!read_cached_dir(&cdir, dir)) { /* check how the file or directory should be treated */ state = treat_path(dir, untracked, &cdir, istate, &path, baselen, pathspec); diff --git a/dir.h b/dir.h index 23eed870a0e235..088e06c1ada4d8 100644 --- a/dir.h +++ b/dir.h @@ -365,6 +365,7 @@ struct dir_struct { unsigned visited_paths; unsigned visited_directories; unsigned untracked_cache_preloaded : 1; + unsigned traversal_failed : 1; /* * Optional borrowed proof that covers every exclusion source diff --git a/meson.build b/meson.build index 5fd2b58f93f241..c7797e0ac790d7 100644 --- a/meson.build +++ b/meson.build @@ -341,6 +341,7 @@ libgit_sources = [ 'clean-status-index.c', 'clean-status-manifest.c', 'clean-status-sidecar.c', + 'clean-status-sidecar-issue.c', 'color.c', 'column.c', 'combine-diff.c', diff --git a/preload-index-bulk.c b/preload-index-bulk.c index 5756df1faceb90..9aa15920b530b0 100644 --- a/preload-index-bulk.c +++ b/preload-index-bulk.c @@ -186,9 +186,11 @@ int preload_bulk_collect(struct index_state *istate, int threads, .untracked = STRING_LIST_INIT_DUP, }; struct preload_bulk_run_result run_result = { 0 }; + struct object_id standard_excludes_digest; struct stat root_stat; const char *start_error, *finish_error = NULL; const char *untracked_reason = NULL; + int standard_excludes_digest_valid = 0; int scan_error = -1; int clean; @@ -239,7 +241,7 @@ int preload_bulk_collect(struct index_state *istate, int threads, CALLOC_ARRAY(scan.tracked_state, istate->cache_nr); start_error = backend->start(&scan); - if (!start_error && scan.proof_epoch && + if (!start_error && (scan.proof_epoch || scan.collect_untracked) && (scan.root_fd < 0 || fstat(scan.root_fd, &root_stat))) start_error = "root-stat"; if (!start_error && scan.proof_epoch) @@ -251,6 +253,11 @@ int preload_bulk_collect(struct index_state *istate, int threads, exclude_dir.internal.exclude_source_proof = exclude_proof; setup_standard_excludes(&exclude_dir); + standard_excludes_digest_valid = + !exclude_source_proof_digest( + exclude_proof, + istate->repo->hash_algo, + &standard_excludes_digest); } scan_error = preload_bulk_run_scan(&scan, &run_result); if (!scan_error) @@ -264,7 +271,8 @@ int preload_bulk_collect(struct index_state *istate, int threads, "index", "preload/bulk_excludes", istate->repo); exclude_proof_valid = exclude_source_proof_validate(exclude_proof); - if (!exclude_proof_valid) { + if (!standard_excludes_digest_valid || + !exclude_proof_valid) { run_result.untracked_complete = 0; untracked_reason = "exclude-race"; } @@ -310,6 +318,12 @@ int preload_bulk_collect(struct index_state *istate, int threads, result->can_skip_unseen_preload = scan.can_skip_unseen_preload; result->untracked_complete = run_result.untracked_complete; + if (result->untracked_complete) { + result->standard_excludes_digest_valid = 1; + oidcpy(&result->standard_excludes_digest, + &standard_excludes_digest); + result->scanned_worktree = root_stat; + } scan.tracked_state = NULL; scan.stat_updates = NULL; scan.stat_updates_nr = 0; diff --git a/preload-index-bulk.h b/preload-index-bulk.h index 3a7d0ef84c1c05..ff5583438fd07a 100644 --- a/preload-index-bulk.h +++ b/preload-index-bulk.h @@ -2,6 +2,7 @@ #define PRELOAD_INDEX_BULK_H #include "git-compat-util.h" +#include "hash.h" #include "preload-index.h" #include "statinfo.h" #include "strbuf.h" @@ -137,6 +138,9 @@ struct preload_bulk_result { unsigned can_skip_unseen_preload : 1; struct string_list untracked; unsigned untracked_complete : 1; + unsigned standard_excludes_digest_valid : 1; + struct object_id standard_excludes_digest; + struct stat scanned_worktree; }; struct preload_bulk_stat_update { diff --git a/preload-index.c b/preload-index.c index a82063e8fd4149..892f1204676829 100644 --- a/preload-index.c +++ b/preload-index.c @@ -134,7 +134,10 @@ struct preload_bulk_pending { unsigned char *tracked_state; struct preload_bulk_stat_update *stat_updates; size_t stat_updates_nr; + struct object_id standard_excludes_digest; + struct stat scanned_worktree; unsigned provider : 1; + unsigned standard_excludes_digest_valid : 1; }; static int stat_data_is_zero(const struct stat_data *sd) @@ -372,6 +375,13 @@ static void preload_bulk_try(struct index_state *index, result.stat_updates_nr = 0; } } + if (result.standard_excludes_digest_valid) { + pending->provider = provider; + pending->standard_excludes_digest_valid = 1; + oidcpy(&pending->standard_excludes_digest, + &result.standard_excludes_digest); + pending->scanned_worktree = result.scanned_worktree; + } if (result.untracked_complete && index->preload_untracked) { *index->preload_untracked = result.untracked; index->preload_untracked_complete = 1; @@ -394,7 +404,19 @@ static void preload_bulk_finish_state(struct index_state *index, index->preload_bulk_stat_updates_nr = pending->stat_updates_nr; index->preload_bulk_provider_pending = pending->provider; - memset(pending, 0, sizeof(*pending)); + pending->tracked_state = NULL; + pending->stat_updates = NULL; + pending->stat_updates_nr = 0; + } + if (pending->standard_excludes_digest_valid) { + oidcpy(&index->preload_bulk_standard_excludes_digest, + &pending->standard_excludes_digest); + index->preload_bulk_scanned_worktree = + pending->scanned_worktree; + if (pending->provider) + index->preload_bulk_excludes_digest_pending = 1; + else + index->preload_bulk_excludes_digest_valid = 1; } free(pending->tracked_state); free(pending->stat_updates); @@ -410,13 +432,22 @@ static int compare_stat_update(const void *va, const void *vb) } #endif -void preload_index_bulk_result_clear(struct index_state *index) +void preload_index_bulk_result_consume(struct index_state *index) { FREE_AND_NULL(index->preload_bulk_tracked_state); FREE_AND_NULL(index->preload_bulk_stat_updates); index->preload_bulk_tracked_nr = 0; index->preload_bulk_stat_updates_nr = 0; index->preload_bulk_provider_pending = 0; + index->preload_bulk_excludes_digest_pending = 0; +} + +void preload_index_bulk_result_clear(struct index_state *index) +{ + preload_index_bulk_result_consume(index); + index->preload_bulk_excludes_digest_valid = 0; + oidclr(&index->preload_bulk_standard_excludes_digest, + index->repo->hash_algo); } int preload_index_bulk_can_close_provider(struct index_state *index) @@ -446,8 +477,11 @@ int preload_index_bulk_result_accept(struct index_state *index) size_t update_nr = 0; int applied = 0; - if (!index->preload_bulk_provider_pending) + if (!index->preload_bulk_provider_pending && + !index->preload_bulk_excludes_digest_pending) return 0; + if (!index->preload_bulk_provider_pending) + goto accept_digest; if (!index->preload_bulk_tracked_state || index->preload_bulk_tracked_nr != index->cache_nr) return -1; @@ -494,6 +528,11 @@ int preload_index_bulk_result_accept(struct index_state *index) FREE_AND_NULL(index->preload_bulk_stat_updates); index->preload_bulk_stat_updates_nr = 0; index->preload_bulk_provider_pending = 0; +accept_digest: + if (index->preload_bulk_excludes_digest_pending) { + index->preload_bulk_excludes_digest_pending = 0; + index->preload_bulk_excludes_digest_valid = 1; + } trace2_data_intmax("index", index->repo, "preload/bulk_provider_applied", applied); #else @@ -502,6 +541,17 @@ int preload_index_bulk_result_accept(struct index_state *index) return 0; } +int preload_index_bulk_standard_excludes_digest( + const struct index_state *index, struct object_id *digest, + struct stat *scanned_worktree) +{ + if (!index->preload_bulk_excludes_digest_valid) + return -1; + oidcpy(digest, &index->preload_bulk_standard_excludes_digest); + *scanned_worktree = index->preload_bulk_scanned_worktree; + return 0; +} + void preload_index(struct index_state *index, const struct pathspec *pathspec, unsigned int refresh_flags) diff --git a/preload-index.h b/preload-index.h index 7f7fdcca28acb2..87e6d0b89276a7 100644 --- a/preload-index.h +++ b/preload-index.h @@ -2,8 +2,10 @@ #define PRELOAD_INDEX_H struct index_state; +struct object_id; struct pathspec; struct repository; +struct stat; enum preload_bulk_tracked_state { PRELOAD_BULK_TRACKED_UNSEEN = 0, @@ -21,7 +23,11 @@ int repo_read_index_preload(struct repository *, const struct pathspec *pathspec, unsigned refresh_flags); void preload_index_bulk_result_clear(struct index_state *index); +void preload_index_bulk_result_consume(struct index_state *index); int preload_index_bulk_can_close_provider(struct index_state *index); int preload_index_bulk_result_accept(struct index_state *index); +int preload_index_bulk_standard_excludes_digest( + const struct index_state *index, struct object_id *digest, + struct stat *scanned_worktree); #endif /* PRELOAD_INDEX_H */ diff --git a/read-cache-ll.h b/read-cache-ll.h index f32da7ad018aea..3d6a77c51edf92 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -195,7 +195,9 @@ struct index_state { fsmonitor_untracked_extension_invalid : 1, fsmonitor_pending_token_from_provider : 1, preload_untracked_complete : 1, - preload_bulk_provider_pending : 1; + preload_bulk_provider_pending : 1, + preload_bulk_excludes_digest_pending : 1, + preload_bulk_excludes_digest_valid : 1; enum sparse_index_mode sparse_index; struct hashmap name_hash; struct hashmap dir_hash; @@ -205,6 +207,8 @@ struct index_state { size_t preload_bulk_tracked_nr; struct preload_bulk_stat_update *preload_bulk_stat_updates; size_t preload_bulk_stat_updates_nr; + struct object_id preload_bulk_standard_excludes_digest; + struct stat preload_bulk_scanned_worktree; /* Borrowed only while refresh_index() performs a provider scan. */ struct clean_status_proof_epoch *preload_bulk_proof_epoch; /* Borrowed for the duration of preload_index(). */ diff --git a/refs.c b/refs.c index 92d5df5b71fa4b..845a5fd3f7587a 100644 --- a/refs.c +++ b/refs.c @@ -2350,6 +2350,23 @@ static struct ref_store *ref_store_init(struct repository *repo, return refs; } +int refs_for_each_replace_ref_uncached(struct repository *repo, + refs_for_each_cb cb, void *cb_data) +{ + struct ref_store *refs; + int ret; + + if (!repo->gitdir) + BUG("attempting to get uncached refs outside of repository"); + + refs = ref_store_init(repo, repo->ref_storage_format, repo->gitdir, + REF_STORE_READ); + ret = refs_for_each_replace_ref(refs, cb, cb_data); + ref_store_release(refs); + free(refs); + return ret; +} + void ref_store_release(struct ref_store *ref_store) { ref_store->be->release(ref_store); diff --git a/refs.h b/refs.h index 9979446d15fd3b..1f80233c5fc472 100644 --- a/refs.h +++ b/refs.h @@ -520,6 +520,12 @@ int refs_for_each_remote_ref(struct ref_store *refs, refs_for_each_cb fn, void *cb_data); int refs_for_each_replace_ref(struct ref_store *refs, refs_for_each_cb fn, void *cb_data); +/* + * Iterate replacement refs through a fresh read-only ref store, without + * consulting caches held by the repository's main ref store. + */ +int refs_for_each_replace_ref_uncached(struct repository *repo, + refs_for_each_cb fn, void *cb_data); /** * Iterate all refs in "prefixes" by partitioning prefixes into disjoint sets diff --git a/replace-object.c b/replace-object.c index 03d0f1f083bed9..29f7c4903e20de 100644 --- a/replace-object.c +++ b/replace-object.c @@ -107,3 +107,16 @@ int replace_refs_enabled(struct repository *r) /* repository has no objects or refs. */ return 0; } + +static int has_replace_ref(const struct reference *ref UNUSED, + void *data UNUSED) +{ + return 1; +} + +int repo_has_replace_refs_uncached(struct repository *r) +{ + if (!replace_refs_enabled(r)) + return 0; + return refs_for_each_replace_ref_uncached(r, has_replace_ref, NULL) != 0; +} diff --git a/replace-object.h b/replace-object.h index 4c9f2a2383d577..595b4598e7e629 100644 --- a/replace-object.h +++ b/replace-object.h @@ -31,6 +31,13 @@ const struct object_id *do_lookup_replace_object(struct repository *r, */ int replace_refs_enabled(struct repository *r); +/* + * Return whether the repository currently has any replacement objects that + * would be honored by lookup_replace_object(). Do not consult the cached + * replacement map. + */ +int repo_has_replace_refs_uncached(struct repository *r); + /* * If object sha1 should be replaced, return the replacement object's * name (replaced recursively, if necessary). The return value is diff --git a/t/meson.build b/t/meson.build index 02cc78fbcea251..6beb78851c887f 100644 --- a/t/meson.build +++ b/t/meson.build @@ -960,6 +960,7 @@ integration_tests = [ 't7527-builtin-fsmonitor.sh', 't7528-signed-commit-ssh.sh', 't7529-preload-index-apfs.sh', + 't7530-status-clean-sidecar.sh', 't7531-semantic-verify.sh', 't7532-preload-index-linux.sh', 't7600-merge.sh', diff --git a/t/t7508-status.sh b/t/t7508-status.sh index 0fd7c79911572e..8059c64940f165 100755 --- a/t/t7508-status.sh +++ b/t/t7508-status.sh @@ -1789,4 +1789,44 @@ test_expect_success EXPENSIVE,SIZE_T_IS_64BIT 'status does not re-read unchanged ) ' +test_expect_success 'status uses only a matching effective cache-tree' ' + test_when_finished "rm -rf cache-tree-status" && + test_create_repo cache-tree-status && + ( + cd cache-tree-status && + sane_unset GIT_TEST_SPLIT_INDEX && + test_write_lines base >tracked && + git add tracked && + git commit -m base && + + GIT_TRACE2_EVENT="$PWD/.git/clean.trace" \ + git status >.git/clean && + test_grep "nothing to commit, working tree clean" \ + .git/clean && + test_trace2_data status index/cache-tree-match 1 \ + <.git/clean.trace && + + test_write_lines staged >tracked && + git add tracked && + git write-tree >.git/staged-tree && + GIT_TRACE2_EVENT="$PWD/.git/staged.trace" \ + git status >.git/staged && + test_grep "Changes to be committed:" .git/staged && + test_grep "modified:.*tracked" .git/staged && + ! test_trace2_data status index/cache-tree-match 1 \ + <.git/staged.trace && + + replacement_tree=$(cat .git/staged-tree) && + git reset --hard HEAD && + head_tree=$(git rev-parse HEAD^{tree}) && + git replace "$head_tree" "$replacement_tree" && + GIT_TRACE2_EVENT="$PWD/.git/replaced.trace" \ + git status >.git/replaced && + test_grep "Changes to be committed:" .git/replaced && + test_grep "modified:.*tracked" .git/replaced && + ! test_trace2_data status index/cache-tree-match 1 \ + <.git/replaced.trace + ) +' + test_done diff --git a/t/t7530-status-clean-sidecar.sh b/t/t7530-status-clean-sidecar.sh new file mode 100755 index 00000000000000..f904c278b888b9 --- /dev/null +++ b/t/t7530-status-clean-sidecar.sh @@ -0,0 +1,136 @@ +#!/bin/sh + +test_description='exact clean status sidecars' + +. ./test-lib.sh + +test_lazy_prereq LOCAL_APFS ' + test_have_prereq MACOS && + /bin/df -t apfs "$TRASH_DIRECTORY" >/dev/null +' + +if ! test_have_prereq FSMONITOR_DAEMON,LOCAL_APFS,MACOS +then + skip_all='clean status sidecars require local APFS and the macOS fsmonitor daemon' + test_done +fi + +test_lazy_prereq DURABLE_FSMONITOR ' + test_create_repo durable-fsmonitor-probe || return 1 + ( + cd durable-fsmonitor-probe && + test_commit base tracked && + git config core.fsmonitor true && + git fsmonitor--daemon start --start-timeout=10 && + git status --porcelain=v2 >/dev/null && + test-tool dump-fsmonitor >token && + grep "^fsmonitor last update builtin:" token + result=$? + git fsmonitor--daemon stop >/dev/null 2>&1 || : + exit $result + ) +' + +stop_daemon () { + git -C "$1" fsmonitor--daemon stop 2>/dev/null || : +} + +setup_repo () { + repo=$1 && + test_create_repo "$repo" && + test_commit -C "$repo" base tracked && + test-tool chmtime -120 "$repo/tracked" && + git -C "$repo" update-index --refresh && + git -C "$repo" config core.fsmonitor true && + git -C "$repo" fsmonitor--daemon start --start-timeout=10 +} + +bulk_status () { + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + git "$@" +} + +prime_semantic_history () { + repo=$1 && + bulk_status -C "$repo" status --porcelain=2 >actual.1 && + test_must_be_empty actual.1 && + bulk_status -C "$repo" status --porcelain=2 >actual.2 && + test_must_be_empty actual.2 && + test_grep FSCF "$repo/.git/index" +} + +test_expect_success DURABLE_FSMONITOR \ + 'exact clean status installs a sidecar without rewriting the index' ' + test_when_finished "stop_daemon sidecar-issue" && + setup_repo sidecar-issue && + test_env GIT_TRACE2_EVENT="$PWD/first-scan.trace" \ + bulk_status -C sidecar-issue status --porcelain=v2 \ + >actual.first && + test_must_be_empty actual.first && + test_path_is_missing sidecar-issue/.git/index.csts && + test_grep "\"value\":\"issue-coherent-history\"" first-scan.trace && + + prime_semantic_history sidecar-issue && + git -C sidecar-issue config core.autocrlf false && + cp sidecar-issue/.git/index index.before && + + test_env GIT_TRACE2_EVENT="$PWD/issue.trace" \ + bulk_status -C sidecar-issue status --porcelain=v2 >actual && + test_must_be_empty actual && + test_cmp index.before sidecar-issue/.git/index && + test_path_is_file sidecar-issue/.git/index.csts && + test_grep \ + "\"key\":\"preload/bulk_untracked_complete\",\"value\":\"1\"" \ + issue.trace && + test_grep "\"key\":\"preload/bulk_provider_applied\"" issue.trace && + test_grep "\"key\":\"clean-proof/sidecar\"" issue.trace && + test_grep ! "\"label\":\"do_write_index\"" issue.trace +' + +test_expect_success DURABLE_FSMONITOR \ + 'only an exact empty output installs a sidecar' ' + test_when_finished "stop_daemon sidecar-shape" && + setup_repo sidecar-shape && + prime_semantic_history sidecar-shape && + + bulk_status -C sidecar-shape status --porcelain=2 >actual && + test_must_be_empty actual && + test_path_is_missing sidecar-shape/.git/index.csts && + + bulk_status -C sidecar-shape status --porcelain=v2 --branch >actual && + test_grep "^# branch.oid " actual && + test_path_is_missing sidecar-shape/.git/index.csts && + + echo changed >sidecar-shape/tracked && + bulk_status -C sidecar-shape status --porcelain=v2 >actual && + test_grep "^1 .M " actual && + test_path_is_missing sidecar-shape/.git/index.csts +' + +test_expect_success DURABLE_FSMONITOR \ + 'external attributes, untracked cache, and alternate indexes are rejected' ' + test_when_finished "stop_daemon sidecar-inputs" && + setup_repo sidecar-inputs && + prime_semantic_history sidecar-inputs && + + test_write_lines "tracked -text" \ + >sidecar-inputs/.git/info/attributes && + bulk_status -C sidecar-inputs status --porcelain=v2 >actual && + test_must_be_empty actual && + test_path_is_missing sidecar-inputs/.git/index.csts && + + rm sidecar-inputs/.git/info/attributes && + git -C sidecar-inputs config core.untrackedCache true && + bulk_status -C sidecar-inputs status --porcelain=v2 >actual && + test_must_be_empty actual && + test_path_is_missing sidecar-inputs/.git/index.csts && + + cp sidecar-inputs/.git/index sidecar-inputs/.git/alternate-index && + test_env GIT_INDEX_FILE="$PWD/sidecar-inputs/.git/alternate-index" \ + bulk_status -C sidecar-inputs status --porcelain=v2 >actual && + test_must_be_empty actual && + test_path_is_missing sidecar-inputs/.git/alternate-index.csts +' + +test_done diff --git a/wt-status.c b/wt-status.c index 9c285d714958a1..cfecafbde2adc7 100644 --- a/wt-status.c +++ b/wt-status.c @@ -2,22 +2,26 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "git-compat-util.h" +#include "abspath.h" #include "advice.h" #include "attr.h" #include "attr-fingerprint.h" #include "wt-status.h" +#include "cache-tree.h" #include "object.h" #include "dir.h" #include "commit.h" #include "clean-status.h" #include "diff.h" #include "environment.h" +#include "exclude-source-proof.h" #include "gettext.h" #include "hash.h" #include "hex.h" #include "object-name.h" #include "path.h" #include "preload-index.h" +#include "replace-object.h" #include "revision.h" #include "diffcore.h" #include "quote.h" @@ -41,6 +45,7 @@ #include "sequencer.h" #include "fsmonitor.h" #include "fsmonitor-settings.h" +#include "wrapper.h" #define AB_DELAY_WARNING_IN_MS (2 * 1000) #define UF_DELAY_WARNING_IN_MS (2 * 1000) @@ -583,7 +588,11 @@ static struct cache_entry **wt_status_collect_preload_changes( deleted); clear: - preload_index_bulk_result_clear(istate); + /* + * The tracked result is single-use. Keep a closed excludes digest + * available until the index writer has consumed it. + */ + preload_index_bulk_result_consume(istate); return direct; } @@ -732,11 +741,51 @@ static void wt_status_collect_changes_worktree(struct wt_status *s) release_revisions(&rev); } +static int wt_status_cache_tree_matches_reference(struct wt_status *s) +{ + struct index_state *istate = s->repo->index; + struct object_id reference_tree; + struct strbuf reference = STRBUF_INIT; + int matches = 0; + + if (!s->allow_clean_status_shortcuts || s->is_initial || + getenv(INDEX_ENVIRONMENT) || istate->split_index || + istate->sparse_index != INDEX_EXPANDED || + !istate->cache_tree || + istate->cache_tree->entry_count < 0 || + (unsigned int)istate->cache_tree->entry_count != + istate->cache_nr) + return 0; + + /* + * A replacement below the root can change the effective tree without + * changing the root object name stored in the commit. + */ + if (replace_refs_enabled(s->repo)) { + prepare_replace_object(s->repo); + if (oidmap_get_size(&s->repo->objects->replace_map)) + return 0; + } + + strbuf_addf(&reference, "%s^{tree}", s->reference); + if (!repo_get_oid_tree(s->repo, reference.buf, &reference_tree) && + oideq(&istate->cache_tree->oid, &reference_tree)) + matches = 1; + strbuf_release(&reference); + return matches; +} + static void wt_status_collect_changes_index(struct wt_status *s) { struct rev_info rev; struct setup_revision_opt opt; + if (wt_status_cache_tree_matches_reference(s)) { + trace2_data_intmax("status", s->repo, + "index/cache-tree-match", 1); + return; + } + repo_init_revisions(s->repo, &rev, NULL); memset(&opt, 0, sizeof(opt)); opt.def = s->is_initial ? empty_tree_oid_hex(s->repo->hash_algo) : s->reference; @@ -879,6 +928,109 @@ static unsigned int wt_status_untracked_dir_flags(const struct wt_status *s) return DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES; } +struct wt_status_exclude_context { + int root_fd; +}; + +static void wt_status_release_exclude_proof(struct wt_status *s) +{ + exclude_source_proof_release(s->certify_exclude_proof); + s->certify_exclude_proof = NULL; + if (s->certify_exclude_context) { + if (s->certify_exclude_context->root_fd >= 0) + close(s->certify_exclude_context->root_fd); + FREE_AND_NULL(s->certify_exclude_context); + } + oidclr(&s->certify_exclude_digest, s->repo->hash_algo); + s->certify_exclude_digest_valid = 0; + s->certify_untracked_scan_failed = 0; +} + +#if EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN + +static int wt_status_open_exclude_parent(void *data, const char *path) +{ + struct wt_status_exclude_context *context = data; + int flags = O_RDONLY | O_NONBLOCK | O_DIRECTORY | O_CLOEXEC | + O_NOFOLLOW; + + if (is_absolute_path(path)) + return open(path, flags); + return openat(context->root_fd, path, flags); +} + +static void wt_status_prepare_exclude_proof( + struct wt_status *s, struct dir_struct *dir) +{ + struct wt_status_exclude_context *context; + const char *worktree; + + if (!s->certify_clean_status) + return; + if (!s->certify_exclude_proof) { + worktree = repo_get_work_tree(s->repo); + if (!worktree) + return; + CALLOC_ARRAY(context, 1); + context->root_fd = open_nofollow( + worktree, + O_RDONLY | O_NONBLOCK | O_DIRECTORY | O_CLOEXEC); + if (context->root_fd < 0) { + free(context); + return; + } + s->certify_exclude_context = context; + s->certify_exclude_proof = exclude_source_proof_create( + s->repo->index, context, + wt_status_open_exclude_parent); + } + dir->internal.exclude_source_proof = + s->certify_exclude_proof; +} + +static void wt_status_record_exclude_digest(struct wt_status *s) +{ + if (s->certify_exclude_digest_valid || + !s->certify_exclude_proof) + return; + s->certify_exclude_digest_valid = + !exclude_source_proof_digest( + s->certify_exclude_proof, + s->repo->hash_algo, + &s->certify_exclude_digest); +} + +#else + +static void wt_status_prepare_exclude_proof( + struct wt_status *s UNUSED, struct dir_struct *dir UNUSED) +{ +} + +static void wt_status_record_exclude_digest(struct wt_status *s UNUSED) +{ +} + +#endif + +int wt_status_certified_excludes_digest( + struct wt_status *s, struct object_id *digest, + struct stat *scanned_worktree) +{ + if (s->certify_untracked_scan_failed || + !s->certify_exclude_digest_valid || + !s->certify_exclude_proof || + !s->certify_exclude_context || + s->certify_exclude_context->root_fd < 0 || + !exclude_source_proof_validate( + s->certify_exclude_proof) || + fstat(s->certify_exclude_context->root_fd, + scanned_worktree)) + return -1; + oidcpy(digest, &s->certify_exclude_digest); + return 0; +} + static int wt_status_begin_attr_snapshot(struct wt_status *s) { int ret; @@ -931,6 +1083,9 @@ void wt_status_start_untracked_cache_preload(struct wt_status *s) wt_status_begin_attr_snapshot(s); /* Record the provider token before either filesystem traversal. */ refresh_fsmonitor(istate); + if (s->certify_clean_status && + !fsmonitor_has_pending_token(istate)) + fsmonitor_reopen_token(istate); if (s->pathspec.nr || s->show_untracked_files == SHOW_NO_UNTRACKED_FILES || s->show_ignored_mode) @@ -1000,10 +1155,6 @@ static int wt_status_collect_untracked_1( if (!s->show_untracked_files) return 0; - if (s->untracked_from_preload && - !istate->untracked && - !s->show_ignored_mode) - return 0; if (s->show_untracked_files != SHOW_ALL_UNTRACKED_FILES) dir.flags |= wt_status_untracked_dir_flags(s); @@ -1016,12 +1167,16 @@ static int wt_status_collect_untracked_1( dir.untracked = istate->untracked; } + wt_status_prepare_exclude_proof(s, &dir); setup_standard_excludes(&dir); + wt_status_record_exclude_digest(s); wt_status_finish_untracked_cache_preload(s); dir.internal.untracked_cache_preloaded = s->untracked_cache_preloaded; fill_directory(&dir, istate, &s->pathspec); + if (s->certify_clean_status && dir.internal.traversal_failed) + s->certify_untracked_scan_failed = 1; used_untracked_cache = dir.untracked && dir.untracked == istate->untracked; @@ -1096,6 +1251,8 @@ static int wt_status_collect_untracked(struct wt_status *s) { if (s->untracked_from_token_closure && !s->show_ignored_mode) return 1; + if (s->untracked_from_preload && !s->show_ignored_mode) + return 0; return wt_status_collect_untracked_1( s, &s->untracked, &s->ignored); } @@ -1154,6 +1311,22 @@ static void wt_status_publish_staged_untracked( closure->staged_untracked_ready = 0; } +static int wt_status_untracked_cache_valid( + const struct wt_status_token_closure *closure) +{ + const struct index_state *istate = closure->status->repo->index; + + return closure->untracked_ready && + istate->untracked && istate->untracked->root; +} + +static void wt_status_record_bulk_untracked( + struct wt_status_token_closure *closure) +{ + if (closure->status->repo->index->preload_untracked_complete) + closure->untracked_proof_complete = 1; +} + static int fsmonitor_token_requires_rescan(enum fsmonitor_token_result result) { return result == FSMONITOR_TOKEN_CHANGED || @@ -1221,22 +1394,6 @@ static void wt_status_refresh_for_token( istate->preload_bulk_proof_epoch = NULL; } -static int wt_status_untracked_cache_valid( - const struct wt_status_token_closure *closure) -{ - const struct index_state *istate = closure->status->repo->index; - - return closure->untracked_ready && - istate->untracked && istate->untracked->root; -} - -static void wt_status_record_bulk_untracked( - struct wt_status_token_closure *closure) -{ - if (closure->status->repo->index->preload_untracked_complete) - closure->untracked_proof_complete = 1; -} - static int wt_status_close_ordinary_fsmonitor_token( struct wt_status_token_closure *closure, int refreshed_before_closure) @@ -1583,6 +1740,12 @@ void wt_status_invalidate_refresh(struct wt_status *s) { struct index_state *istate = s->repo->index; + if (s->untracked_from_token_closure) { + string_list_clear(&s->untracked, 0); + string_list_clear(&s->ignored, 0); + s->untracked_from_token_closure = 0; + } + wt_status_release_exclude_proof(s); wt_status_release_attr_snapshot(s); if (!s->pathspec.nr && !istate->split_index && fsmonitor_reopen_token(istate)) @@ -1658,6 +1821,7 @@ void wt_status_collect_free_buffers(struct wt_status *s) { untracked_cache_preload_release(s->untracked_cache_preload); s->untracked_cache_preload = NULL; + wt_status_release_exclude_proof(s); wt_status_release_attr_snapshot(s); wt_status_state_free_buffers(&s->state); } diff --git a/wt-status.h b/wt-status.h index 99b005cb8b5cea..afee3b4ad9840c 100644 --- a/wt-status.h +++ b/wt-status.h @@ -7,7 +7,10 @@ #include "remote.h" struct repository; +struct stat; struct attr_source_snapshot; +struct exclude_source_proof; +struct wt_status_exclude_context; struct worktree; struct untracked_cache_preload; @@ -140,6 +143,8 @@ struct wt_status { /* These are computed during processing of the individual sections */ int committable; int workdir_dirty; + unsigned allow_clean_status_shortcuts : 1; + unsigned certify_clean_status : 1; unsigned untracked_from_token_closure : 1; unsigned untracked_from_preload : 1; unsigned bulk_update_index_stat : 1; @@ -152,8 +157,13 @@ struct wt_status { uint32_t untracked_in_ms; struct untracked_cache_preload *untracked_cache_preload; struct attr_source_snapshot *attr_source_snapshot; + struct exclude_source_proof *certify_exclude_proof; + struct wt_status_exclude_context *certify_exclude_context; + struct object_id certify_exclude_digest; unsigned untracked_cache_preloaded : 1; unsigned attr_snapshot_failed : 1; + unsigned certify_exclude_digest_valid : 1; + unsigned certify_untracked_scan_failed : 1; }; size_t wt_status_locate_end(const char *s, size_t len); @@ -171,6 +181,9 @@ int wt_status_refresh_index(struct wt_status *s, unsigned int refresh_flags, int require_untracked); void wt_status_invalidate_refresh(struct wt_status *s); +int wt_status_certified_excludes_digest( + struct wt_status *s, struct object_id *digest, + struct stat *scanned_worktree); /* * Collect all changes between the two trees. Changes will be displayed as if From 086edcbf8bcbc126c6f01bfd91ce8cf62c772dc5 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Fri, 24 Jul 2026 01:16:38 -0500 Subject: [PATCH 102/105] status: load bounded clean-status sidecars An installed sidecar cannot be inspected safely by opening an untrusted adjacent path without bounds. A symbolic link, named pipe, oversized record, or growing file could redirect the read, block status, or consume unbounded memory. Open the named sidecar without following symbolic links and request a nonblocking descriptor. Accept only a regular file of at most 8192 bytes, read exactly its recorded size, reject an additional byte, and parse its checksummed contents into caller-owned storage. Clear failed records and release storage explicitly. Platforms without nonblocking support fail closed. Extend the registered store unit suite to cover owned token storage under both object formats, symbolic links, FIFOs, and an oversized 8193-byte record. The loader is testable at this boundary; it does not yet bypass index deserialization. Signed-off-by: Taylor Blau --- clean-status-sidecar.c | 44 +++++++++++ clean-status-sidecar.h | 16 +++- t/unit-tests/u-clean-status-store.c | 116 ++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+), 1 deletion(-) diff --git a/clean-status-sidecar.c b/clean-status-sidecar.c index a68d3dc8059367..95db486a47b289 100644 --- a/clean-status-sidecar.c +++ b/clean-status-sidecar.c @@ -19,6 +19,7 @@ #include "wrapper.h" #define CLEAN_STATUS_SIDECAR_MAGIC "CSTS" +#define CLEAN_STATUS_SIDECAR_MAX_SIZE 8192 #define CLEAN_STATUS_FILESYSTEM_ID_SIZE 16 struct clean_status_filesystem_id { @@ -163,6 +164,49 @@ static int open_nofollow_nonblocking(const char *path, int flags) #endif } +int clean_status_sidecar_load( + const char *index_path, const struct git_hash_algo *algo, + struct clean_status_sidecar_record *record) +{ + struct stat st; + char extra; + char *path = sidecar_path(index_path); + int fd = -1, ret = -1; + size_t size; + + memset(&record->sidecar, 0, sizeof(record->sidecar)); + strbuf_reset(&record->storage); + fd = open_nofollow_nonblocking(path, O_RDONLY | O_CLOEXEC); + if (fd < 0 || fstat(fd, &st) || !S_ISREG(st.st_mode) || + st.st_size < 0 || st.st_size > CLEAN_STATUS_SIDECAR_MAX_SIZE) + goto done; + size = xsize_t(st.st_size); + strbuf_grow(&record->storage, size); + strbuf_setlen(&record->storage, size); + if ((size_t)read_in_full(fd, record->storage.buf, size) != size || + read(fd, &extra, 1) != 0 || + clean_status_sidecar_parse(&record->sidecar, + record->storage.buf, + record->storage.len, algo)) + goto done; + ret = 0; + +done: + if (ret) + strbuf_reset(&record->storage); + if (fd >= 0) + close(fd); + free(path); + return ret; +} + +void clean_status_sidecar_record_release( + struct clean_status_sidecar_record *record) +{ + strbuf_release(&record->storage); + memset(&record->sidecar, 0, sizeof(record->sidecar)); +} + static int local_apfs_id(int fd MAYBE_UNUSED, struct clean_status_filesystem_id *id) { diff --git a/clean-status-sidecar.h b/clean-status-sidecar.h index 963d6bccc6b65a..8149acbe5b86bb 100644 --- a/clean-status-sidecar.h +++ b/clean-status-sidecar.h @@ -3,11 +3,11 @@ #include "clean-status-identity.h" #include "hash.h" +#include "strbuf.h" struct clean_status_index_snapshot; struct attr_source_snapshot; struct repository; -struct strbuf; struct stat; #define CLEAN_STATUS_SIDECAR_VERSION 1 @@ -29,12 +29,26 @@ struct clean_status_sidecar { size_t token_len; }; +struct clean_status_sidecar_record { + struct clean_status_sidecar sidecar; + struct strbuf storage; +}; + +#define CLEAN_STATUS_SIDECAR_RECORD_INIT { \ + .storage = STRBUF_INIT, \ +} + int clean_status_sidecar_parse(struct clean_status_sidecar *sidecar, const void *data, size_t len, const struct git_hash_algo *algo); int clean_status_sidecar_write(struct strbuf *out, const struct clean_status_sidecar *sidecar, const struct git_hash_algo *algo); +int clean_status_sidecar_load( + const char *index_path, const struct git_hash_algo *algo, + struct clean_status_sidecar_record *record); +void clean_status_sidecar_record_release( + struct clean_status_sidecar_record *record); int clean_status_sidecar_pin_source( const char *index_path, const struct clean_status_sidecar *sidecar, const struct git_hash_algo *algo, diff --git a/t/unit-tests/u-clean-status-store.c b/t/unit-tests/u-clean-status-store.c index ac25bb225e3c2e..4c2e5739aaceea 100644 --- a/t/unit-tests/u-clean-status-store.c +++ b/t/unit-tests/u-clean-status-store.c @@ -82,6 +82,122 @@ static struct strbuf sidecar_path(struct store_fixture *fixture) return path; } +#ifdef O_NONBLOCK +static void write_fixture_sidecar(struct store_fixture *fixture, + const struct git_hash_algo *algo) +{ + struct strbuf encoded = STRBUF_INIT; + struct strbuf path = sidecar_path(fixture); + + cl_assert_equal_i(clean_status_sidecar_write( + &encoded, &fixture->sidecar, algo), 0); + write_file_buf(path.buf, encoded.buf, encoded.len); + strbuf_release(&path); + strbuf_release(&encoded); +} + +static void assert_loads_sidecar(const struct git_hash_algo *algo) +{ + struct clean_status_sidecar_record record = + CLEAN_STATUS_SIDECAR_RECORD_INIT; + struct store_fixture fixture; + + fixture_init(&fixture, algo); + write_fixture_sidecar(&fixture, algo); + cl_assert_equal_i(clean_status_sidecar_load( + fixture.index_path.buf, algo, &record), 0); + cl_assert(clean_status_identity_equal( + &record.sidecar.identity, &fixture.sidecar.identity)); + cl_assert_equal_i(record.sidecar.proof.index_version, + fixture.sidecar.proof.index_version); + cl_assert_equal_i(record.sidecar.proof.cache_nr, + fixture.sidecar.proof.cache_nr); + cl_assert(oideq(&record.sidecar.proof.index_checksum, + &fixture.sidecar.proof.index_checksum)); + cl_assert(oideq(&record.sidecar.proof.head_tree, + &fixture.sidecar.proof.head_tree)); + cl_assert(!memcmp(record.sidecar.proof.config_hash, + fixture.sidecar.proof.config_hash, algo->rawsz)); + cl_assert(!memcmp(record.sidecar.proof.repo_hash, + fixture.sidecar.proof.repo_hash, algo->rawsz)); + cl_assert(oideq(&record.sidecar.proof.exclude_source_digest, + &fixture.sidecar.proof.exclude_source_digest)); + cl_assert_equal_i(record.sidecar.token_len, fixture.sidecar.token_len); + cl_assert(!memcmp(record.sidecar.token, fixture.sidecar.token, + record.sidecar.token_len)); + cl_assert(record.sidecar.token >= + (const unsigned char *)record.storage.buf); + cl_assert(record.sidecar.token + record.sidecar.token_len <= + (const unsigned char *)record.storage.buf + + record.storage.len); + + clean_status_sidecar_record_release(&record); + fixture_release(&fixture); +} +#endif + +void test_clean_status_store__loads_owned_sidecars_in_both_object_formats(void) +{ +#ifdef O_NONBLOCK + assert_loads_sidecar(&hash_algos[GIT_HASH_SHA1]); + assert_loads_sidecar(&hash_algos[GIT_HASH_SHA256]); +#else + cl_skip(); +#endif +} + +void test_clean_status_store__rejects_nonregular_sidecars(void) +{ +#if defined(O_NONBLOCK) && !defined(GIT_WINDOWS_NATIVE) + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct clean_status_sidecar_record record = + CLEAN_STATUS_SIDECAR_RECORD_INIT; + struct store_fixture fixture; + struct strbuf path, target = STRBUF_INIT; + + fixture_init(&fixture, algo); + path = sidecar_path(&fixture); + strbuf_addf(&target, "%s/target", fixture.directory); + write_file(target.buf, "target"); + cl_assert_equal_i(symlink(target.buf, path.buf), 0); + cl_assert_equal_i(clean_status_sidecar_load( + fixture.index_path.buf, algo, &record), -1); + cl_assert_equal_i(unlink(path.buf), 0); + cl_assert_equal_i(mkfifo(path.buf, 0600), 0); + cl_assert_equal_i(clean_status_sidecar_load( + fixture.index_path.buf, algo, &record), -1); + + clean_status_sidecar_record_release(&record); + strbuf_release(&target); + strbuf_release(&path); + fixture_release(&fixture); +#else + cl_skip(); +#endif +} + +void test_clean_status_store__rejects_oversized_sidecars(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct clean_status_sidecar_record record = + CLEAN_STATUS_SIDECAR_RECORD_INIT; + struct store_fixture fixture; + struct strbuf oversized = STRBUF_INIT; + struct strbuf path; + + fixture_init(&fixture, algo); + path = sidecar_path(&fixture); + strbuf_addchars(&oversized, 'x', 8193); + write_file_buf(path.buf, oversized.buf, oversized.len); + cl_assert_equal_i(clean_status_sidecar_load( + fixture.index_path.buf, algo, &record), -1); + + clean_status_sidecar_record_release(&record); + strbuf_release(&oversized); + strbuf_release(&path); + fixture_release(&fixture); +} + static void require_local_apfs(const char *path MAYBE_UNUSED) { #ifdef __APPLE__ From 9db038b23fd6bee396d2a51218a3612ea0b6c21a Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 27 Jul 2026 16:07:12 -0500 Subject: [PATCH 103/105] exclude: support nonblocking proof captures Validating a sidecar must recapture standard excludes before status can trust an empty result. Opening an exclude source that has become a named pipe may otherwise block the supposedly cheap validation. Add an explicit nonblocking flag to exclude-source proof creation and carry it into the existing anchored source-open operation. Reject unknown flags, request nonblocking captures for sidecar issuance, and update the existing bulk-scan and unit-test callers to pass zero, preserving their current blocking and symbolic-link policies. Add a focused FIFO unit test showing that an opted-in proof captures and validates an empty pipe without waiting. The later early-status consumer can reuse nonblocking capture without changing ordinary exclude handling. Signed-off-by: Taylor Blau --- exclude-source-proof.c | 11 ++++++++--- exclude-source-proof.h | 6 +++++- preload-index-bulk.c | 2 +- t/unit-tests/u-exclude-source-proof.c | 28 +++++++++++++++++++++++---- wt-status.c | 3 ++- 5 files changed, 40 insertions(+), 10 deletions(-) diff --git a/exclude-source-proof.c b/exclude-source-proof.c index 1a2ebd5f87190b..83ee1626d93746 100644 --- a/exclude-source-proof.c +++ b/exclude-source-proof.c @@ -29,6 +29,7 @@ struct exclude_source_proof { struct strintmap entries_by_path[2]; size_t nr; size_t alloc; + unsigned nonblocking : 1; unsigned invalid : 1; }; @@ -187,7 +188,7 @@ static struct exclude_source_capture *capture_begin( struct exclude_source_proof *exclude_source_proof_create( struct index_state *istate, void *open_data, - exclude_source_open_parent_fn open_parent) + exclude_source_open_parent_fn open_parent, unsigned flags) { struct exclude_source_proof *proof; @@ -195,13 +196,16 @@ struct exclude_source_proof *exclude_source_proof_create( proof->istate = istate; proof->open_data = open_data; proof->open_parent = open_parent; + proof->nonblocking = + !!(flags & EXCLUDE_SOURCE_PROOF_NONBLOCKING); strintmap_init_with_options(&proof->entries_by_path[0], -1, NULL, 0); strintmap_init_with_options(&proof->entries_by_path[1], -1, NULL, 0); if (!EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN || !istate || !istate->repo || !istate->repo->hash_algo || - !open_parent) + !open_parent || + (flags & ~EXCLUDE_SOURCE_PROOF_NONBLOCKING)) proof->invalid = 1; return proof; } @@ -220,7 +224,8 @@ int exclude_source_capture_open(struct exclude_source_capture *capture) return -1; } return open_source_at(capture->parent_fd, capture->relative, - capture->nofollow, 0); + capture->nofollow, + capture->proof->nonblocking); } int exclude_source_capture_absent(struct exclude_source_capture *capture) diff --git a/exclude-source-proof.h b/exclude-source-proof.h index f1c03b4a3cbf5f..ab9b626c664380 100644 --- a/exclude-source-proof.h +++ b/exclude-source-proof.h @@ -19,9 +19,13 @@ struct stat; typedef int (*exclude_source_open_parent_fn)(void *data, const char *path); +enum exclude_source_proof_flags { + EXCLUDE_SOURCE_PROOF_NONBLOCKING = (1 << 0), +}; + struct exclude_source_proof *exclude_source_proof_create( struct index_state *istate, void *open_data, - exclude_source_open_parent_fn open_parent); + exclude_source_open_parent_fn open_parent, unsigned flags); struct exclude_source_capture *exclude_source_capture_begin( struct exclude_source_proof *proof, const char *path, int nofollow); diff --git a/preload-index-bulk.c b/preload-index-bulk.c index 9aa15920b530b0..92ce36e8fe4850 100644 --- a/preload-index-bulk.c +++ b/preload-index-bulk.c @@ -249,7 +249,7 @@ int preload_bulk_collect(struct index_state *istate, int threads, if (!start_error) { if (scan.collect_untracked) { exclude_proof = exclude_source_proof_create( - istate, &scan, open_exclude_parent); + istate, &scan, open_exclude_parent, 0); exclude_dir.internal.exclude_source_proof = exclude_proof; setup_standard_excludes(&exclude_dir); diff --git a/t/unit-tests/u-exclude-source-proof.c b/t/unit-tests/u-exclude-source-proof.c index e579dbcd51247a..be1f2d046599b1 100644 --- a/t/unit-tests/u-exclude-source-proof.c +++ b/t/unit-tests/u-exclude-source-proof.c @@ -29,7 +29,7 @@ static int open_parent(void *data UNUSED, const char *path) static struct exclude_source_proof *new_proof(void) { return exclude_source_proof_create( - &istate, NULL, open_parent); + &istate, NULL, open_parent, 0); } static char *make_path(const char *name) @@ -201,7 +201,7 @@ void test_exclude_source_proof__rejects_conflicting_observations(void) void test_exclude_source_proof__digest_deduplicates_and_ignores_identity(void) { struct exclude_source_proof *first_proof = - exclude_source_proof_create(&istate, NULL, open_parent); + exclude_source_proof_create(&istate, NULL, open_parent, 0); struct exclude_source_proof *second_proof; struct object_id first, second; char *parent = make_path("parent"); @@ -216,7 +216,7 @@ void test_exclude_source_proof__digest_deduplicates_and_ignores_identity(void) cl_must_pass(unlink(source)); write_file_buf(source, "content", 7); second_proof = - exclude_source_proof_create(&istate, NULL, open_parent); + exclude_source_proof_create(&istate, NULL, open_parent, 0); record_file(second_proof, source); record_file(second_proof, source); cl_must_pass(exclude_source_proof_digest( @@ -249,7 +249,7 @@ void test_exclude_source_proof__rejects_open_failure(void) void test_exclude_source_proof__fails_closed_without_parent_opener(void) { struct exclude_source_proof *proof = - exclude_source_proof_create(&istate, NULL, NULL); + exclude_source_proof_create(&istate, NULL, NULL, 0); cl_assert(!exclude_source_capture_begin(proof, "/dev/null", 0)); cl_assert(!exclude_source_proof_validate(proof)); @@ -410,6 +410,25 @@ void test_exclude_source_proof__rejects_nonempty_fifo_replacement(void) free(parent); } +void test_exclude_source_proof__captures_fifo_without_blocking(void) +{ + struct exclude_source_proof *proof = + exclude_source_proof_create( + &istate, NULL, open_parent, + EXCLUDE_SOURCE_PROOF_NONBLOCKING); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + cl_must_pass(mkfifo(source, 0600)); + record_file(proof, source); + cl_assert(exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + #else #define EMPTY_TEST(name) void name(void) {} @@ -432,5 +451,6 @@ SKIP_TEST(test_exclude_source_proof__reresolves_absent_source_parent) SKIP_TEST(test_exclude_source_proof__accepts_dev_null) SKIP_TEST(test_exclude_source_proof__accepts_empty_fifo_replacement) SKIP_TEST(test_exclude_source_proof__rejects_nonempty_fifo_replacement) +SKIP_TEST(test_exclude_source_proof__captures_fifo_without_blocking) #endif diff --git a/wt-status.c b/wt-status.c index cfecafbde2adc7..329d3ca4410e81 100644 --- a/wt-status.c +++ b/wt-status.c @@ -982,7 +982,8 @@ static void wt_status_prepare_exclude_proof( s->certify_exclude_context = context; s->certify_exclude_proof = exclude_source_proof_create( s->repo->index, context, - wt_status_open_exclude_parent); + wt_status_open_exclude_parent, + EXCLUDE_SOURCE_PROOF_NONBLOCKING); } dir->internal.exclude_source_proof = s->certify_exclude_proof; From ac58e113b9ccb1739dc5903491f0d6bfa576f56b Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 27 Jul 2026 16:07:26 -0500 Subject: [PATCH 104/105] status: answer exact clean status before index deserialization An issued clean-status sidecar has no latency benefit while status still deserializes the index before checking it. Moving the check earlier is safe only if the recorded proof is revalidated around an empty builtin-fsmonitor delta. Attempt the sidecar only for the literal top-level porcelain-v2 command on an eligible main worktree. Load the bounded record, pin the named local-APFS index, recapture excludes without blocking, and check configuration, attributes, repository identity, HEAD, and provider mode. Query the builtin provider directly from the stored token. Keep the attribute and exclude proofs alive across that query. Recheck configuration, HEAD, fresh replacement-ref and repository state, attribute contents and namespace, exclude-source identity, and both the held and named index before accepting an empty delta. Return without deserializing index entries only when every check succeeds; otherwise continue through ordinary status. Unsupported anchored-open platforms take that ordinary path. Register the fast-path source with Make and Meson. Extend the existing sidecar integration suite for read-only hits, dirty worktree shapes, loose, packed, and custom replacement refs, sidecar and exclude FIFOs, changed configuration, attributes, HEAD, null-checksum indexes, and post-query replacement or exclude races. Signed-off-by: Taylor Blau --- Makefile | 1 + builtin/commit.c | 12 + clean-status-fast.c | 252 ++++++++++++++++ clean-status-internal.h | 2 - clean-status.h | 5 + fsmonitor.c | 2 +- fsmonitor.h | 2 + meson.build | 1 + t/t7519-status-fsmonitor.sh | 70 +++++ t/t7527-builtin-fsmonitor.sh | 8 +- t/t7530-status-clean-sidecar.sh | 507 +++++++++++++++++++++++++++++++- wt-status.c | 43 +++ wt-status.h | 1 + 13 files changed, 892 insertions(+), 14 deletions(-) create mode 100644 clean-status-fast.c diff --git a/Makefile b/Makefile index 2dd5c29fc07e95..42e303b4c71502 100644 --- a/Makefile +++ b/Makefile @@ -1136,6 +1136,7 @@ LIB_OBJS += clean-status-identity.o LIB_OBJS += clean-status-index.o LIB_OBJS += clean-status-manifest.o LIB_OBJS += clean-status-sidecar.o +LIB_OBJS += clean-status-fast.o LIB_OBJS += clean-status-sidecar-issue.o LIB_OBJS += color.o LIB_OBJS += column.o diff --git a/builtin/commit.c b/builtin/commit.c index 6ced81bbf4f32b..e9c4b42f81a4fd 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1617,6 +1617,7 @@ struct repository *repo UNUSED) int default_status_command = argc == 1 && (!prefix || !*prefix); int exact_clean_command = argc == 2 && !strcmp(argv[1], "--porcelain=v2") && (!prefix || !*prefix); + int exact_clean_query; struct object_id oid; static struct option builtin_status_options[] = { OPT__VERBOSE(&verbose, N_("be verbose")), @@ -1701,6 +1702,17 @@ struct repository *repo UNUSED) prefix, argv); s.allow_clean_status_shortcuts = default_status_command && !s.pathspec.nr; + exact_clean_query = exact_clean_command && + status_format == STATUS_FORMAT_PORCELAIN_V2 && + !s.pathspec.nr && !s.show_branch && !s.show_stash && + !s.show_ignored_mode && !s.null_termination && !s.verbose && + s.show_untracked_files == SHOW_NORMAL_UNTRACKED_FILES; + s.certify_clean_status = exact_clean_query; + if (exact_clean_query && + clean_status_try_sidecar(the_repository, &clean_digest)) { + wt_status_collect_free_buffers(&s); + return 0; + } if (status_format != STATUS_FORMAT_PORCELAIN && status_format != STATUS_FORMAT_PORCELAIN_V2) diff --git a/clean-status-fast.c b/clean-status-fast.c new file mode 100644 index 00000000000000..8b2ba3e36d0117 --- /dev/null +++ b/clean-status-fast.c @@ -0,0 +1,252 @@ +#include "git-compat-util.h" +#include "abspath.h" +#include "attr-fingerprint.h" +#include "clean-status.h" +#include "clean-status-index.h" +#include "clean-status-sidecar.h" +#include "dir.h" +#include "environment.h" +#include "exclude-source-proof.h" +#include "fsmonitor.h" +#include "fsmonitor-settings.h" +#include "object-name.h" +#include "repository.h" +#include "trace2.h" +#include "worktree.h" +#include "wrapper.h" + +#if !EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN + +int clean_status_try_sidecar( + struct repository *repo UNUSED, + const struct clean_status_config_digest *config UNUSED) +{ + return 0; +} + +#else + +struct fast_exclude_context { + int root_fd; +}; + +static void trace_miss(struct repository *repo, const char *reason) +{ + trace2_data_string("status", repo, "clean-proof/miss", reason); +} + +static int open_exclude_parent(void *data, const char *path) +{ + struct fast_exclude_context *context = data; + int flags = O_RDONLY | O_NONBLOCK | O_DIRECTORY | O_CLOEXEC; + +#ifdef O_NOFOLLOW + flags |= O_NOFOLLOW; +#endif + if (is_absolute_path(path)) + return open(path, flags); + return openat(context->root_fd, path, flags); +} + +static int capture_standard_excludes( + struct repository *repo, struct fast_exclude_context *context, + struct exclude_source_proof **proof, struct object_id *digest) +{ + struct dir_struct dir = DIR_INIT; + int ret; + + *proof = exclude_source_proof_create( + repo->index, context, open_exclude_parent, + EXCLUDE_SOURCE_PROOF_NONBLOCKING); + dir.internal.exclude_source_proof = *proof; + setup_standard_excludes(&dir); + ret = exclude_source_proof_digest(*proof, repo->hash_algo, digest); + dir_clear(&dir); + return ret; +} + +static int attr_snapshot_still_matches( + struct repository *repo, const struct attr_source_snapshot *snapshot) +{ + const struct attr_fingerprint *expected = + attr_source_snapshot_fingerprint(snapshot); + struct attr_fingerprint current; + + return expected && + !attr_fingerprint_repository(repo, ¤t) && + current.sources_present == expected->sources_present && + !memcmp(current.content_hash, expected->content_hash, + repo->hash_algo->rawsz) && + !memcmp(current.namespace_hash, expected->namespace_hash, + repo->hash_algo->rawsz); +} + +static int fast_path_test_barrier(void) +{ + const char *ready = + getenv("GIT_TEST_STATUS_CLEAN_SIDECAR_BARRIER_READY"); + const char *resume = + getenv("GIT_TEST_STATUS_CLEAN_SIDECAR_BARRIER_RESUME"); + struct strbuf buf = STRBUF_INIT; + int fd; + int ret; + + if (!ready && !resume) + return 0; + if (!ready || !resume) + return -1; + fd = open(resume, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + write_file(ready, "ready"); + ret = strbuf_read(&buf, fd, 1) > 0 ? 0 : -1; + close(fd); + strbuf_release(&buf); + return ret; +} + +static int current_worktree_is_main(struct repository *repo) +{ + struct worktree *worktree = get_current_worktree(repo); + int ret = worktree && is_main_worktree(worktree); + + free_worktree(worktree); + return ret; +} + +int clean_status_try_sidecar( + struct repository *repo, + const struct clean_status_config_digest *config) +{ + struct clean_status_sidecar_record record = + CLEAN_STATUS_SIDECAR_RECORD_INIT; + struct clean_status_index_snapshot index = { .fd = -1 }; + struct attr_source_snapshot *attrs = NULL; + struct exclude_source_proof *excludes = NULL; + struct fast_exclude_context exclude_context = { .root_fd = -1 }; + struct fsmonitor_query_result query = FSMONITOR_QUERY_RESULT_INIT; + struct clean_status_config_digest fresh_config; + struct object_id exclude_digest, head_tree; + struct stat scanned_worktree; + unsigned char repo_hash[GIT_MAX_RAWSZ]; + char *query_token = NULL; + int ret = 0; + + if (!config->finalized || config->unsafe_filter || + getenv(INDEX_ENVIRONMENT) || is_bare_repository(repo) || + !repo_get_work_tree(repo) || + !current_worktree_is_main(repo) || + fsm_settings__get_mode(repo) != FSMONITOR_MODE_IPC) { + trace_miss(repo, "fast-repository-shape"); + goto done; + } + if (clean_status_sidecar_load( + repo->index_file, repo->hash_algo, &record)) { + trace_miss(repo, "fast-sidecar-missing-or-corrupt"); + goto done; + } + if (clean_status_sidecar_pin_source( + repo->index_file, &record.sidecar, repo->hash_algo, + &index)) { + trace_miss(repo, "fast-index-mismatch"); + goto done; + } + if (memcmp(config->hash, record.sidecar.proof.config_hash, + repo->hash_algo->rawsz)) { + trace_miss(repo, "fast-config-changed"); + goto done; + } + if (attr_source_snapshot_repository(repo, &attrs)) { + trace_miss(repo, "fast-attributes"); + goto done; + } + exclude_context.root_fd = open_nofollow( + repo_get_work_tree(repo), + O_RDONLY | O_NONBLOCK | O_DIRECTORY | O_CLOEXEC); + if (exclude_context.root_fd < 0 || + fstat(exclude_context.root_fd, &scanned_worktree) || + capture_standard_excludes( + repo, &exclude_context, &excludes, &exclude_digest) || + !oideq(&exclude_digest, + &record.sidecar.proof.exclude_source_digest)) { + trace_miss(repo, "fast-excludes"); + goto done; + } + if (clean_status_repository_fingerprint( + repo, attrs, &index, &scanned_worktree, repo_hash)) { + trace_miss(repo, "fast-repository-unavailable"); + goto done; + } + if (memcmp(repo_hash, record.sidecar.proof.repo_hash, + repo->hash_algo->rawsz)) { + trace_miss(repo, "fast-repository-input"); + goto done; + } + if (repo_get_oid_tree(repo, "HEAD^{tree}", &head_tree) || + !oideq(&head_tree, &record.sidecar.proof.head_tree)) { + trace_miss(repo, "fast-head-changed"); + goto done; + } + + query_token = xmemdupz( + record.sidecar.token, record.sidecar.token_len); + if (query_builtin_fsmonitor(query_token, &query) != + FSMONITOR_QUERY_DELTA || + query.paths.len) { + trace_miss(repo, "fast-provider-changed"); + goto done; + } + if (fast_path_test_barrier()) { + trace_miss(repo, "fast-test-barrier"); + goto done; + } + + if (clean_status_config_read_repository(repo, &fresh_config) || + fresh_config.unsafe_filter || + memcmp(fresh_config.hash, config->hash, + repo->hash_algo->rawsz)) { + trace_miss(repo, "fast-config-raced"); + goto done; + } + if (repo_get_oid_tree(repo, "HEAD^{tree}", &head_tree) || + !oideq(&head_tree, &record.sidecar.proof.head_tree)) { + trace_miss(repo, "fast-head-raced"); + goto done; + } + if (clean_status_repository_fingerprint( + repo, attrs, &index, &scanned_worktree, repo_hash) || + memcmp(repo_hash, record.sidecar.proof.repo_hash, + repo->hash_algo->rawsz)) { + trace_miss(repo, "fast-repository-raced"); + goto done; + } + if (!attr_snapshot_still_matches(repo, attrs)) { + trace_miss(repo, "fast-attributes-raced"); + goto done; + } + if (!exclude_source_proof_validate(excludes)) { + trace_miss(repo, "fast-excludes-raced"); + goto done; + } + if (!clean_status_index_snapshot_still_matches_path( + &index, repo->index_file, repo->hash_algo)) { + trace_miss(repo, "fast-index-raced"); + goto done; + } + + trace2_data_intmax("status", repo, "clean-proof/hit", 1); + ret = 1; + +done: + free(query_token); + fsmonitor_query_result_release(&query); + if (exclude_context.root_fd >= 0) + close(exclude_context.root_fd); + exclude_source_proof_release(excludes); + attr_source_snapshot_free(attrs); + clean_status_index_snapshot_release(&index); + clean_status_sidecar_record_release(&record); + return ret; +} + +#endif /* EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN */ diff --git a/clean-status-internal.h b/clean-status-internal.h index d4dc54e6616f61..000c7ce0077379 100644 --- a/clean-status-internal.h +++ b/clean-status-internal.h @@ -40,7 +40,5 @@ struct clean_status_state { }; struct clean_status_state *clean_status_get_state(struct index_state *istate); -int clean_status_revalidated_token_matches( - const struct index_state *istate); #endif /* CLEAN_STATUS_INTERNAL_H */ diff --git a/clean-status.h b/clean-status.h index 1a5b62b3924537..036e3a9c592f2c 100644 --- a/clean-status.h +++ b/clean-status.h @@ -44,6 +44,8 @@ void clean_status_release_proof_epoch( int clean_status_fsmonitor_config_mismatch(const struct index_state *istate); int clean_status_fsmonitor_strong_mismatch(const struct index_state *istate); +int clean_status_revalidated_token_matches( + const struct index_state *istate); int clean_status_has_persistent_fsmonitor_semantic_history( const struct index_state *istate); @@ -75,6 +77,9 @@ int clean_status_issue_sidecar( struct wt_status *status, const struct clean_status_config_digest *config, struct lock_file *index_lock); +int clean_status_try_sidecar( + struct repository *repo, + const struct clean_status_config_digest *config); int clean_status_read_fsmonitor_config(struct index_state *istate, const void *data, unsigned long size); diff --git a/fsmonitor.c b/fsmonitor.c index e96965a66f7861..66db89c6c07447 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -815,7 +815,7 @@ enum fsmonitor_query_outcome fsmonitor_parse_builtin_response( return FSMONITOR_QUERY_ERROR; } -static enum fsmonitor_query_outcome query_builtin_fsmonitor( +enum fsmonitor_query_outcome query_builtin_fsmonitor( const char *since_token, struct fsmonitor_query_result *result) { const char *test_sequence = diff --git a/fsmonitor.h b/fsmonitor.h index e6c617bec77f04..136f4769c36fc2 100644 --- a/fsmonitor.h +++ b/fsmonitor.h @@ -37,6 +37,8 @@ struct fsmonitor_query_result { void fsmonitor_query_result_release(struct fsmonitor_query_result *result); enum fsmonitor_query_outcome fsmonitor_parse_builtin_response( const struct strbuf *raw, struct fsmonitor_query_result *result); +enum fsmonitor_query_outcome query_builtin_fsmonitor( + const char *since_token, struct fsmonitor_query_result *result); /* * A pathname monitor cannot prove that every name for a multiply-linked diff --git a/meson.build b/meson.build index c7797e0ac790d7..f510267b8270c4 100644 --- a/meson.build +++ b/meson.build @@ -341,6 +341,7 @@ libgit_sources = [ 'clean-status-index.c', 'clean-status-manifest.c', 'clean-status-sidecar.c', + 'clean-status-fast.c', 'clean-status-sidecar-issue.c', 'color.c', 'column.c', diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 66c5a3a28bfdf3..a82dd36019f7f6 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -594,6 +594,76 @@ prepare_builtin_closure_repo () { ) } +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'bare status reuses a current tracked fsmonitor proof' ' + test_when_finished "rm -rf builtin-tracked-clean" && + prepare_builtin_closure_repo builtin-tracked-clean && + ( + cd builtin-tracked-clean && + sane_unset GIT_TEST_SPLIT_INDEX && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain >.git/prime && + test_must_be_empty .git/prime && + + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + GIT_TRACE2_EVENT="$PWD/.git/clean.trace" \ + git status >.git/clean && + test_grep "nothing to commit, working tree clean" \ + .git/clean && + test_trace2_data status fsmonitor/tracked-clean 1 \ + <.git/clean.trace && + test_trace2_data status index/cache-tree-match 1 \ + <.git/clean.trace && + + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/exact.trace" \ + git status --porcelain=v2 >.git/exact && + test_must_be_empty .git/exact && + ! test_trace2_data status fsmonitor/tracked-clean 1 \ + <.git/exact.trace && + test_grep \ + "\"category\":\"index\",\"label\":\"refresh\"" \ + .git/exact.trace && + + test_write_lines changed >tracked && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$PWD/.git/dirty.trace" \ + git status >.git/dirty && + test_grep "modified:.*tracked" .git/dirty && + ! test_trace2_data status fsmonitor/tracked-clean 1 \ + <.git/dirty.trace && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git checkout -- tracked && + test_write_lines staged >tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git add tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git write-tree >.git/staged-tree && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain >.git/staged-prime && + test_grep "^M tracked$" .git/staged-prime && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + GIT_TRACE2_EVENT="$PWD/.git/staged.trace" \ + git status >.git/staged && + test_grep "Changes to be committed:" .git/staged && + test_grep "modified:.*tracked" .git/staged && + test_trace2_data status fsmonitor/tracked-clean 1 \ + <.git/staged.trace && + ! test_trace2_data status index/cache-tree-match 1 \ + <.git/staged.trace + ) +' + test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ 'builtin closure initializes a new untracked cache' ' test_when_finished "rm -rf builtin-closure-new-uc" && diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index 417efb5989739a..a9d75b061a17ac 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -1615,8 +1615,6 @@ test_expect_success MACOS 'daemon token reset closes a skipHash index' ' GIT_TRACE2_EVENT="$PWD/.git/warm.trace" \ git status --porcelain=v2 >.git/warm.out && test_must_be_empty .git/warm.out && - test_trace2_data index refresh/sum_lstat 0 \ - <.git/warm.trace && ! test_trace2_data index refresh/sum_lstat \ "[1-9][0-9]*" <.git/warm.trace && ! test_trace2_data fsm_client query/trivial-response 1 \ @@ -1641,7 +1639,9 @@ test_expect_success MACOS 'daemon token reset closes a skipHash index' ' GIT_TRACE2_EVENT="$PWD/.git/dirty-warm.trace" \ git status --porcelain=v2 >.git/dirty-warm.out && test_cmp .git/dirty-reset.out .git/dirty-warm.out && - test_trace2_data index refresh/sum_lstat 2 \ + test_trace2_data index preload/bulk_provider_applied 1 \ + <.git/dirty-warm.trace && + test_trace2_data index refresh/sum_lstat 0 \ <.git/dirty-warm.trace && ! test_trace2_data fsm_client query/trivial-response 1 \ <.git/dirty-warm.trace @@ -1753,7 +1753,7 @@ test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ test_trace2_data fsmonitor config/token-advanced 1 \ <.git/deleted.trace && - GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCC \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DDCC \ GIT_TEST_FSMONITOR_QUERY_PATH=.gitattributes \ GIT_TRACE2_EVENT="$PWD/.git/attributes.trace" \ git status --porcelain=v2 >.git/attributes && diff --git a/t/t7530-status-clean-sidecar.sh b/t/t7530-status-clean-sidecar.sh index f904c278b888b9..1616d5faf9a7db 100755 --- a/t/t7530-status-clean-sidecar.sh +++ b/t/t7530-status-clean-sidecar.sh @@ -60,6 +60,138 @@ prime_semantic_history () { test_grep FSCF "$repo/.git/index" } +issue_sidecar () { + repo=$1 && + prime_semantic_history "$repo" && + git -C "$repo" config core.autocrlf false && + bulk_status -C "$repo" status --porcelain=v2 >actual.issue && + test_must_be_empty actual.issue && + test_path_is_file "$repo/.git/index.csts" +} + +assert_fallback_matches_oracle () { + repo=$1 && + sidecar_trace=$2 && + GIT_OPTIONAL_LOCKS=0 git -c core.fsmonitor=false \ + -c core.untrackedCache=false -C "$repo" \ + status --porcelain=v2 >expect && + GIT_OPTIONAL_LOCKS=0 GIT_TRACE2_EVENT="$PWD/$sidecar_trace" \ + git -C "$repo" status --porcelain=v2 >actual && + test_cmp expect actual && + test_grep ! "\"key\":\"clean-proof/hit\"" "$sidecar_trace" +} + +assert_custom_replace_fallback_matches_oracle () { + repo=$1 && + sidecar_trace=$2 && + GIT_REPLACE_REF_BASE=refs/status-replace/ \ + GIT_OPTIONAL_LOCKS=0 git -c core.fsmonitor=false \ + -c core.untrackedCache=false -C "$repo" \ + status --porcelain=v2 >expect && + GIT_REPLACE_REF_BASE=refs/status-replace/ \ + GIT_OPTIONAL_LOCKS=0 GIT_TRACE2_EVENT="$PWD/$sidecar_trace" \ + git -C "$repo" status --porcelain=v2 >actual && + test_cmp expect actual && + test_grep ! "\"key\":\"clean-proof/hit\"" "$sidecar_trace" +} + +replacement_tree () { + repo=$1 && + blob=$(printf "replacement\n" | + git -C "$repo" hash-object -w --stdin) && + printf "100644 blob %s\ttracked\n" "$blob" | + git -C "$repo" mktree +} + +cleanup_fast_race () { + if test -n "$status_pid" + then + kill "$status_pid" 2>/dev/null || : + wait "$status_pid" 2>/dev/null || : + fi && + status_pid= && + exec 9>&- && + rm -f "$ready" "$resume" +} + +wait_for_fast_ready () { + for i in $(test_seq 1 1000) + do + test "$(cat "$ready" 2>/dev/null)" = ready && return 0 + kill -0 "$status_pid" 2>/dev/null || return 1 + sleep 0.01 + done + return 1 +} + +start_fast_raced_status () { + repo=$1 && + ready=$TRASH_DIRECTORY/$repo.fast-ready && + resume=$TRASH_DIRECTORY/$repo.fast-resume && + race_trace=$TRASH_DIRECTORY/$repo.fast-trace && + status_pid= && + rm -f "$ready" "$resume" "$race_trace" && + : >"$ready" && + mkfifo "$resume" && + exec 9<>"$resume" && + { + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_STATUS_CLEAN_SIDECAR_BARRIER_READY="$ready" \ + GIT_TEST_STATUS_CLEAN_SIDECAR_BARRIER_RESUME="$resume" \ + GIT_TRACE2_EVENT="$race_trace" \ + git -C "$repo" status --porcelain=v2 \ + >raced.actual 9>&- & + status_pid=$! + } && + wait_for_fast_ready +} + +start_issue_raced_status () { + repo=$1 && + ready=$TRASH_DIRECTORY/$repo.issue-ready && + resume=$TRASH_DIRECTORY/$repo.issue-resume && + race_trace=$TRASH_DIRECTORY/$repo.issue-trace && + status_pid= && + rm -f "$ready" "$resume" "$race_trace" && + : >"$ready" && + mkfifo "$resume" && + exec 9<>"$resume" && + { + test_env \ + GIT_TEST_STATUS_CLEAN_SIDECAR_ISSUE_BARRIER_READY="$ready" \ + GIT_TEST_STATUS_CLEAN_SIDECAR_ISSUE_BARRIER_RESUME="$resume" \ + GIT_TRACE2_EVENT="$race_trace" \ + bulk_status -C "$repo" status --porcelain=v2 \ + >raced.actual 9>&- & + status_pid=$! + } && + wait_for_fast_ready +} + +stop_after_fast_fallback () { + for i in $(test_seq 1 1000) + do + if grep -q "\"value\":\"fast-excludes-raced\"" \ + "$race_trace" + then + kill "$status_pid" 2>/dev/null || return 1 + wait "$status_pid" 2>/dev/null || : + status_pid= + return 0 + fi + kill -0 "$status_pid" 2>/dev/null || return 1 + sleep 0.01 + done + return 1 +} + +finish_fast_raced_status () { + printf "resume\n" >&9 && + exec 9>&- && + wait "$status_pid" && + status_pid= +} + test_expect_success DURABLE_FSMONITOR \ 'exact clean status installs a sidecar without rewriting the index' ' test_when_finished "stop_daemon sidecar-issue" && @@ -85,7 +217,160 @@ test_expect_success DURABLE_FSMONITOR \ issue.trace && test_grep "\"key\":\"preload/bulk_provider_applied\"" issue.trace && test_grep "\"key\":\"clean-proof/sidecar\"" issue.trace && - test_grep ! "\"label\":\"do_write_index\"" issue.trace + test_grep ! "\"label\":\"do_write_index\"" issue.trace && + + GIT_TRACE2_EVENT="$PWD/hit.trace" \ + git -C sidecar-issue status --porcelain=v2 >actual && + test_must_be_empty actual && + test_grep "\"key\":\"clean-proof/hit\"" hit.trace && + test_grep ! "\"label\":\"do_read_index\"" hit.trace +' + +test_expect_success DURABLE_FSMONITOR \ + 'exact clean status certifies an existing untracked cache' ' + test_when_finished "stop_daemon sidecar-untracked-cache" && + setup_repo sidecar-untracked-cache && + git -C sidecar-untracked-cache config core.untrackedCache true && + git -C sidecar-untracked-cache config core.autocrlf false && + bulk_status -C sidecar-untracked-cache status --porcelain=2 \ + >actual.1 && + test_must_be_empty actual.1 && + bulk_status -C sidecar-untracked-cache status --porcelain=2 \ + >actual.2 && + test_must_be_empty actual.2 && + test_grep UNTR sidecar-untracked-cache/.git/index && + + test_env GIT_TRACE2_EVENT="$PWD/untracked-cache-issue.trace" \ + bulk_status -C sidecar-untracked-cache \ + status --porcelain=v2 >actual && + test_must_be_empty actual && + test_path_is_file sidecar-untracked-cache/.git/index.csts && + test_grep ! \ + "\"key\":\"preload/bulk_useful\"" \ + untracked-cache-issue.trace && + test_grep \ + "\"event\":\"region_enter\".*\"category\":\"dir\",\"label\":\"read_directory\"" \ + untracked-cache-issue.trace >untracked-cache-read-directory && + test_line_count = 1 untracked-cache-read-directory && + test_grep "\"key\":\"proof_valid\",\"value\":\"1\"" \ + untracked-cache-issue.trace && + test_grep FSUC sidecar-untracked-cache/.git/index && + + GIT_TRACE2_EVENT="$PWD/untracked-cache-hit.trace" \ + git -C sidecar-untracked-cache status --porcelain=v2 \ + >actual && + test_must_be_empty actual && + test_grep "\"key\":\"clean-proof/hit\"" \ + untracked-cache-hit.trace && + test_grep ! "\"label\":\"do_read_index\"" \ + untracked-cache-hit.trace +' + +test_expect_success POSIXPERM,DURABLE_FSMONITOR \ + 'an incomplete untracked traversal cannot issue a sidecar' ' + test_when_finished "stop_daemon sidecar-unreadable" && + setup_repo sidecar-unreadable && + git -C sidecar-unreadable config core.untrackedCache true && + git -C sidecar-unreadable config core.autocrlf false && + prime_semantic_history sidecar-unreadable && + mkdir sidecar-unreadable/hidden && + test_when_finished "chmod u+rwx sidecar-unreadable/hidden" && + test_write_lines untracked >sidecar-unreadable/hidden/untracked && + chmod a-r sidecar-unreadable/hidden && + + test_env GIT_TRACE2_EVENT="$PWD/unreadable.trace" \ + bulk_status -C sidecar-unreadable \ + status --porcelain=v2 >actual 2>err && + test_must_be_empty actual && + test_grep "could not open directory .hidden/." err && + test_path_is_missing sidecar-unreadable/.git/index.csts && + test_grep "\"value\":\"issue-scan-or-index-shape\"" \ + unreadable.trace && + + chmod u+rwx sidecar-unreadable/hidden && + git -C sidecar-unreadable status --porcelain=v2 >actual && + test_grep "^? hidden/" actual +' + +test_expect_success DURABLE_FSMONITOR \ + 'a replaced worktree root cannot inherit a sidecar' ' + test_when_finished "stop_daemon sidecar-root-race" && + test_when_finished "stop_daemon sidecar-root-race.scanned" && + test_when_finished "cleanup_fast_race" && + setup_repo sidecar-root-race && + git -C sidecar-root-race config core.untrackedCache true && + prime_semantic_history sidecar-root-race && + git -C sidecar-root-race config core.autocrlf false && + cp -R sidecar-root-race sidecar-root-race.replacement && + test_write_lines replacement-only \ + >sidecar-root-race.replacement/replacement-only && + rm -f sidecar-root-race.replacement/.git/index.csts && + + start_issue_raced_status sidecar-root-race && + mv sidecar-root-race sidecar-root-race.scanned && + mv sidecar-root-race.replacement sidecar-root-race && + finish_fast_raced_status && + + test_must_be_empty raced.actual && + test_path_is_missing sidecar-root-race/.git/index.csts && + test_grep \ + "\"category\":\"dir\",\"label\":\"read_directory\"" \ + "$race_trace" && + test_grep ! \ + "\"key\":\"preload/bulk_untracked_complete\",\"value\":\"1\"" \ + "$race_trace" && + test_grep "\"key\":\"proof_valid\",\"value\":\"1\"" "$race_trace" && + test_grep "\"value\":\"issue-pinned-inputs\"" "$race_trace" +' + +test_expect_success PIPE,DURABLE_FSMONITOR \ + 'an existing per-directory exclude FIFO cannot block sidecar issuance' ' + test_when_finished "stop_daemon sidecar-issue-fifo" && + setup_repo sidecar-issue-fifo && + test_when_finished "rm -f sidecar-issue-fifo/.gitignore" && + git -C sidecar-issue-fifo config core.untrackedCache true && + git -C sidecar-issue-fifo config core.autocrlf false && + prime_semantic_history sidecar-issue-fifo && + mkfifo sidecar-issue-fifo/.gitignore && + + test_env GIT_TRACE2_EVENT="$PWD/issue-fifo.trace" \ + bulk_status -C sidecar-issue-fifo \ + status --porcelain=v2 >actual && + test_must_be_empty actual && + test_path_is_file sidecar-issue-fifo/.git/index.csts && + test_grep "\"key\":\"clean-proof/sidecar\"" issue-fifo.trace +' + +test_expect_success DURABLE_FSMONITOR \ + 'a clean exact status replaces a stale sidecar' ' + test_when_finished "stop_daemon sidecar-reissue" && + setup_repo sidecar-reissue && + git -C sidecar-reissue config core.untrackedCache true && + issue_sidecar sidecar-reissue && + test_write_lines untracked >sidecar-reissue/untracked && + git -C sidecar-reissue status --porcelain=2 >dirty && + test_grep "^? untracked$" dirty && + rm sidecar-reissue/untracked && + + test_env GIT_TRACE2_EVENT="$PWD/reissue.trace" \ + bulk_status -C sidecar-reissue status --porcelain=v2 \ + >actual && + test_must_be_empty actual && + test_grep "\"key\":\"clean-proof/sidecar\"" reissue.trace && + test_grep ! \ + "\"key\":\"preload/bulk_useful\"" \ + reissue.trace && + test_grep \ + "\"event\":\"region_enter\".*\"category\":\"dir\",\"label\":\"read_directory\"" \ + reissue.trace >reissue-read-directory && + test_line_count = 1 reissue-read-directory && + test_grep ! "\"label\":\"do_write_index\"" reissue.trace && + + GIT_TRACE2_EVENT="$PWD/reissued-hit.trace" \ + git -C sidecar-reissue status --porcelain=v2 >actual && + test_must_be_empty actual && + test_grep "\"key\":\"clean-proof/hit\"" reissued-hit.trace && + test_grep ! "\"label\":\"do_read_index\"" reissued-hit.trace ' test_expect_success DURABLE_FSMONITOR \ @@ -109,7 +394,7 @@ test_expect_success DURABLE_FSMONITOR \ ' test_expect_success DURABLE_FSMONITOR \ - 'external attributes, untracked cache, and alternate indexes are rejected' ' + 'external attributes and alternate indexes are rejected' ' test_when_finished "stop_daemon sidecar-inputs" && setup_repo sidecar-inputs && prime_semantic_history sidecar-inputs && @@ -121,11 +406,6 @@ test_expect_success DURABLE_FSMONITOR \ test_path_is_missing sidecar-inputs/.git/index.csts && rm sidecar-inputs/.git/info/attributes && - git -C sidecar-inputs config core.untrackedCache true && - bulk_status -C sidecar-inputs status --porcelain=v2 >actual && - test_must_be_empty actual && - test_path_is_missing sidecar-inputs/.git/index.csts && - cp sidecar-inputs/.git/index sidecar-inputs/.git/alternate-index && test_env GIT_INDEX_FILE="$PWD/sidecar-inputs/.git/alternate-index" \ bulk_status -C sidecar-inputs status --porcelain=v2 >actual && @@ -133,4 +413,217 @@ test_expect_success DURABLE_FSMONITOR \ test_path_is_missing sidecar-inputs/.git/alternate-index.csts ' +test_expect_success DURABLE_FSMONITOR \ + 'a fast hit remains read-only without optional locks' ' + test_when_finished "stop_daemon sidecar-read-only" && + setup_repo sidecar-read-only && + issue_sidecar sidecar-read-only && + cp sidecar-read-only/.git/index.csts sidecar.before && + + GIT_OPTIONAL_LOCKS=0 GIT_TRACE2_EVENT="$PWD/read-only.trace" \ + git -C sidecar-read-only status --porcelain=v2 >actual && + test_must_be_empty actual && + test_grep "\"key\":\"clean-proof/hit\"" read-only.trace && + test_grep ! "\"label\":\"do_read_index\"" read-only.trace && + test_cmp sidecar.before sidecar-read-only/.git/index.csts +' + +test_expect_success DURABLE_FSMONITOR \ + 'provider changes fall back for each dirty worktree shape' ' + test_when_finished "stop_daemon sidecar-modified" && + test_when_finished "stop_daemon sidecar-deleted" && + test_when_finished "stop_daemon sidecar-renamed" && + test_when_finished "stop_daemon sidecar-untracked" && + + setup_repo sidecar-modified && + issue_sidecar sidecar-modified && + echo changed >sidecar-modified/tracked && + assert_fallback_matches_oracle sidecar-modified modified.trace && + test_grep "^1 .M " actual && + + setup_repo sidecar-deleted && + issue_sidecar sidecar-deleted && + rm sidecar-deleted/tracked && + assert_fallback_matches_oracle sidecar-deleted deleted.trace && + test_grep "^1 .D " actual && + + setup_repo sidecar-renamed && + issue_sidecar sidecar-renamed && + mv sidecar-renamed/tracked sidecar-renamed/renamed && + assert_fallback_matches_oracle sidecar-renamed renamed.trace && + test_grep "^1 .D " actual && + test_grep "^? renamed" actual && + + setup_repo sidecar-untracked && + issue_sidecar sidecar-untracked && + echo untracked >sidecar-untracked/new-file && + assert_fallback_matches_oracle sidecar-untracked untracked.trace && + test_grep "^? new-file" actual +' + +test_expect_success DURABLE_FSMONITOR \ + 'loose and packed replace refs invalidate a sidecar' ' + test_when_finished "stop_daemon sidecar-replace" && + setup_repo sidecar-replace && + issue_sidecar sidecar-replace && + old_tree=$(git -C sidecar-replace rev-parse "HEAD^{tree}") && + new_tree=$(replacement_tree sidecar-replace) && + git -C sidecar-replace replace "$old_tree" "$new_tree" && + + assert_fallback_matches_oracle sidecar-replace replace-loose.trace && + test_grep "^1 M. " actual && + git -C sidecar-replace pack-refs --all && + test_path_is_missing \ + "sidecar-replace/.git/refs/replace/$old_tree" && + assert_fallback_matches_oracle sidecar-replace replace-packed.trace && + test_grep "^1 M. " actual +' + +test_expect_success DURABLE_FSMONITOR \ + 'a custom replace namespace invalidates a sidecar' ' + test_when_finished "stop_daemon sidecar-custom-replace" && + setup_repo sidecar-custom-replace && + issue_sidecar sidecar-custom-replace && + old_tree=$(git -C sidecar-custom-replace rev-parse "HEAD^{tree}") && + new_tree=$(replacement_tree sidecar-custom-replace) && + GIT_REPLACE_REF_BASE=refs/status-replace/ \ + git -C sidecar-custom-replace update-ref \ + "refs/status-replace/$old_tree" "$new_tree" && + + assert_custom_replace_fallback_matches_oracle \ + sidecar-custom-replace replace-custom.trace && + test_grep "^1 M. " actual +' + +test_expect_success PIPE,DURABLE_FSMONITOR \ + 'a sidecar FIFO cannot block or supply a hit' ' + test_when_finished "stop_daemon sidecar-fifo" && + test_when_finished "rm -f sidecar-fifo/.git/index.csts" && + setup_repo sidecar-fifo && + issue_sidecar sidecar-fifo && + rm sidecar-fifo/.git/index.csts && + mkfifo sidecar-fifo/.git/index.csts && + + GIT_OPTIONAL_LOCKS=0 GIT_TRACE2_EVENT="$PWD/fifo.trace" \ + git -C sidecar-fifo status --porcelain=v2 >actual && + test_must_be_empty actual && + test_grep ! "\"key\":\"clean-proof/hit\"" fifo.trace && + test_grep "\"value\":\"fast-sidecar-missing-or-corrupt\"" fifo.trace +' + +test_expect_success DURABLE_FSMONITOR \ + 'non-provider proof inputs invalidate a sidecar' ' + test_when_finished "stop_daemon sidecar-metadata" && + setup_repo sidecar-metadata && + issue_sidecar sidecar-metadata && + + git -C sidecar-metadata config status.relativePaths false && + assert_fallback_matches_oracle sidecar-metadata config.trace && + test_grep "\"value\":\"fast-config-changed\"" config.trace && + git -C sidecar-metadata config --unset status.relativePaths && + + cp sidecar-metadata/.git/info/exclude info-exclude && + test_write_lines ignored >sidecar-metadata/.git/info/exclude && + assert_fallback_matches_oracle sidecar-metadata exclude.trace && + test_grep "\"value\":\"fast-excludes\"" exclude.trace && + mv info-exclude sidecar-metadata/.git/info/exclude && + + test_write_lines "tracked ident" \ + >sidecar-metadata/.git/info/attributes && + assert_fallback_matches_oracle sidecar-metadata attributes.trace && + test_grep "\"value\":\"fast-repository-unavailable\"" attributes.trace && + rm sidecar-metadata/.git/info/attributes && + + blob=$(printf "different\n" | + git -C sidecar-metadata hash-object -w --stdin) && + tree=$(printf "100644 blob %s\ttracked\n" "$blob" | + git -C sidecar-metadata mktree) && + commit=$(printf "different tree\n" | + git -C sidecar-metadata commit-tree "$tree" -p HEAD) && + git -C sidecar-metadata update-ref HEAD "$commit" && + assert_fallback_matches_oracle sidecar-metadata head.trace && + test_grep "\"value\":\"fast-head-changed\"" head.trace && + test_grep "^1 M. " actual +' + +test_expect_success DURABLE_FSMONITOR \ + 'a v4 skipHash index is not certified' ' + test_when_finished "stop_daemon sidecar-v4" && + setup_repo sidecar-v4 && + prime_semantic_history sidecar-v4 && + git -C sidecar-v4 config index.version 4 && + git -C sidecar-v4 config index.skipHash true && + git -C sidecar-v4 update-index --force-write-index && + git -C sidecar-v4 config core.autocrlf false && + + dd if=/dev/zero of=zeros bs=20 count=1 2>/dev/null && + tail -c 20 sidecar-v4/.git/index >trailer && + test_cmp_bin zeros trailer && + test_env GIT_TRACE2_EVENT="$PWD/v4.trace" \ + bulk_status -C sidecar-v4 status --porcelain=v2 >actual && + test_must_be_empty actual && + test_path_is_missing sidecar-v4/.git/index.csts && + test_grep ! "\"key\":\"clean-proof/hit\"" v4.trace +' + +test_expect_success PIPE,DURABLE_FSMONITOR \ + 'an existing exclude FIFO cannot block fast-path capture' ' + test_when_finished "stop_daemon sidecar-exclude-fifo" && + setup_repo sidecar-exclude-fifo && + exclude_file=$(mktemp \ + "${TMPDIR:-/tmp}/git-status-exclude-fifo.XXXXXX") && + test_when_finished "rm -f \"$exclude_file\"" && + git -C sidecar-exclude-fifo config core.excludesFile \ + "$exclude_file" && + issue_sidecar sidecar-exclude-fifo && + rm "$exclude_file" && + mkfifo "$exclude_file" && + + GIT_OPTIONAL_LOCKS=0 GIT_TRACE2_EVENT="$PWD/exclude-fifo.trace" \ + git -C sidecar-exclude-fifo status --porcelain=v2 >actual && + test_must_be_empty actual && + test_grep "\"key\":\"clean-proof/hit\"" exclude-fifo.trace +' + +test_expect_success PIPE,DURABLE_FSMONITOR \ + 'a raced exclude FIFO cannot block sidecar validation' ' + test_when_finished "stop_daemon sidecar-exclude-race" && + test_when_finished "cleanup_fast_race" && + setup_repo sidecar-exclude-race && + exclude_file=$(mktemp \ + "${TMPDIR:-/tmp}/git-status-exclude-race.XXXXXX") && + test_when_finished "rm -f \"$exclude_file\"" && + test_write_lines ignored >"$exclude_file" && + git -C sidecar-exclude-race config core.excludesFile \ + "$exclude_file" && + issue_sidecar sidecar-exclude-race && + + start_fast_raced_status sidecar-exclude-race && + rm "$exclude_file" && + mkfifo "$exclude_file" && + printf "resume\n" >&9 && + exec 9>&- && + stop_after_fast_fallback && + test_must_be_empty raced.actual && + test_grep ! "\"key\":\"clean-proof/hit\"" "$race_trace" && + test_grep "\"value\":\"fast-excludes-raced\"" "$race_trace" +' + +test_expect_success PIPE,DURABLE_FSMONITOR \ + 'a replace ref created after the provider query prevents a hit' ' + test_when_finished "stop_daemon sidecar-replace-race" && + test_when_finished "cleanup_fast_race" && + setup_repo sidecar-replace-race && + issue_sidecar sidecar-replace-race && + old_tree=$(git -C sidecar-replace-race rev-parse "HEAD^{tree}") && + new_tree=$(replacement_tree sidecar-replace-race) && + + start_fast_raced_status sidecar-replace-race && + git -C sidecar-replace-race update-ref \ + "refs/replace/$old_tree" "$new_tree" && + finish_fast_raced_status && + test_grep ! "\"key\":\"clean-proof/hit\"" "$race_trace" && + test_grep "\"value\":\"fast-repository-raced\"" "$race_trace" +' + test_done diff --git a/wt-status.c b/wt-status.c index 329d3ca4410e81..a77fd27f553b26 100644 --- a/wt-status.c +++ b/wt-status.c @@ -12,6 +12,7 @@ #include "dir.h" #include "commit.h" #include "clean-status.h" +#include "clean-status-index.h" #include "diff.h" #include "environment.h" #include "exclude-source-proof.h" @@ -716,6 +717,11 @@ static void wt_status_collect_changes_worktree(struct wt_status *s) size_t direct_nr; struct rev_info rev; + if (s->tracked_from_fsmonitor) { + preload_index_bulk_result_consume(s->repo->index); + return; + } + direct = wt_status_collect_preload_changes(s, &direct_nr); repo_init_revisions(s->repo, &rev, NULL); setup_revisions(0, NULL, &rev, NULL); @@ -1137,6 +1143,7 @@ static void wt_status_finish_untracked_cache_preload(struct wt_status *s) if (!index_invalidated) return; + s->tracked_from_fsmonitor = 0; preload_index_bulk_result_clear(istate); trace2_data_intmax("status", s->repo, "fsmonitor/exclude-index-invalidated", @@ -1611,6 +1618,26 @@ wt_status_close_semantic_fsmonitor_token( return WT_STATUS_TOKEN_CLOSURE_ACCEPTED; } +static int wt_status_tracked_fsmonitor_state_is_current( + struct wt_status *s) +{ + struct index_state *istate = s->repo->index; + + return s->allow_clean_status_shortcuts && + !s->certify_clean_status && !s->pathspec.nr && + !getenv(INDEX_ENVIRONMENT) && !istate->split_index && + istate->sparse_index == INDEX_EXPANDED && + fsm_settings__get_mode(s->repo) == FSMONITOR_MODE_IPC && + is_fsmonitor_refreshed(istate) && + !fsmonitor_has_pending_token(istate) && + istate->fsmonitor_token_valid && + istate->fsmonitor_last_update && + *istate->fsmonitor_last_update && + clean_status_revalidated_token_matches(istate) && + !clean_status_manifest_global_fallback(istate) && + !clean_status_worktree_manifest_needs_refresh(istate); +} + static int wt_status_close_fsmonitor_token( struct wt_status *s, struct semantic_verify_proof *proof, unsigned int refresh_flags, int require_untracked, @@ -1635,6 +1662,20 @@ static int wt_status_close_fsmonitor_token( wt_status_discard_semantic_verify( s, &proof, "provider-unavailable"); + if (!refreshed_before_closure && attr_inputs_match && + wt_status_tracked_fsmonitor_state_is_current(s) && + clean_status_index_entries_are_certifiable(istate)) { + s->tracked_from_fsmonitor = 1; + trace2_data_intmax( + "status", s->repo, + "fsmonitor/tracked-clean", 1); + return 0; + } + if (refreshed_before_closure && attr_inputs_match && + s->tracked_from_fsmonitor && + wt_status_tracked_fsmonitor_state_is_current(s)) + return closure.refresh_result; + s->tracked_from_fsmonitor = 0; if (!refreshed_before_closure && attr_inputs_match) return refresh_index( istate, refresh_flags, &s->pathspec, @@ -1652,6 +1693,7 @@ static int wt_status_close_fsmonitor_token( return closure.refresh_result; } + s->tracked_from_fsmonitor = 0; closure.can_prime = require_untracked && istate->untracked && s->show_untracked_files != SHOW_NO_UNTRACKED_FILES && @@ -1741,6 +1783,7 @@ void wt_status_invalidate_refresh(struct wt_status *s) { struct index_state *istate = s->repo->index; + s->tracked_from_fsmonitor = 0; if (s->untracked_from_token_closure) { string_list_clear(&s->untracked, 0); string_list_clear(&s->ignored, 0); diff --git a/wt-status.h b/wt-status.h index afee3b4ad9840c..6f5300fe8e5481 100644 --- a/wt-status.h +++ b/wt-status.h @@ -145,6 +145,7 @@ struct wt_status { int workdir_dirty; unsigned allow_clean_status_shortcuts : 1; unsigned certify_clean_status : 1; + unsigned tracked_from_fsmonitor : 1; unsigned untracked_from_token_closure : 1; unsigned untracked_from_preload : 1; unsigned bulk_update_index_stat : 1; From 3b04bff7e427dc98a45cbc0da8aa51cc5b22ce4f Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Tue, 21 Jul 2026 17:20:24 -0500 Subject: [PATCH 105/105] Documentation: describe status clean-proof sidecars A clean-status sidecar is a narrowly scoped proof, not an alternate index or a general cache. Documenting only its serialized bytes would hide the full-scan issuance requirement and the revalidation needed before an empty provider response can answer status. Document the adjacent sidecar path, local-APFS and main-worktree eligibility, fixed-width version-one CSTS fields, a checksum using the repository object-format hash, the separate repository-identity hash, a bounded builtin-provider token, and the 8192-byte read limit. Explain why the source index, configuration, repository, HEAD, attributes, and standard excludes must remain coherent. Describe completed-scan issuance, persistent provider history, held index locks, the post-query race fence, nonblocking source opens, and read-only hits. State that every missing, unsupported, stale, malformed, or raced proof falls back to ordinary status. Register the technical document in both the documentation Makefile and Meson. Signed-off-by: Taylor Blau --- Documentation/Makefile | 1 + Documentation/technical/meson.build | 1 + .../technical/status-clean-proof.adoc | 103 ++++++++++++++++++ 3 files changed, 105 insertions(+) create mode 100644 Documentation/technical/status-clean-proof.adoc diff --git a/Documentation/Makefile b/Documentation/Makefile index 2699f0b24af192..ff7cee368e3f7d 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -142,6 +142,7 @@ TECH_DOCS += technical/send-pack-pipeline TECH_DOCS += technical/shallow TECH_DOCS += technical/sparse-checkout TECH_DOCS += technical/sparse-index +TECH_DOCS += technical/status-clean-proof TECH_DOCS += technical/trivial-merge TECH_DOCS += technical/unambiguous-types TECH_DOCS += technical/unit-tests diff --git a/Documentation/technical/meson.build b/Documentation/technical/meson.build index ec07088c57617f..665977299cdab3 100644 --- a/Documentation/technical/meson.build +++ b/Documentation/technical/meson.build @@ -31,6 +31,7 @@ articles = [ 'shallow.adoc', 'sparse-checkout.adoc', 'sparse-index.adoc', + 'status-clean-proof.adoc', 'trivial-merge.adoc', 'unambiguous-types.adoc', 'unit-tests.adoc', diff --git a/Documentation/technical/status-clean-proof.adoc b/Documentation/technical/status-clean-proof.adoc new file mode 100644 index 00000000000000..f598d36c3353c3 --- /dev/null +++ b/Documentation/technical/status-clean-proof.adoc @@ -0,0 +1,103 @@ +Status clean-proof sidecar +========================== + +The status clean-proof sidecar is an optional cache for one exact clean +`git status --porcelain=v2` result. It is never a source of repository +state. A missing, stale, malformed, unsupported, or raced sidecar makes +status read the index and scan the worktree normally. + +Location and scope +------------------ + +The sidecar for an index at `` is stored at `.csts`. This +keeps a proof for one index from being applied to an alternate index. +Sidecars are currently limited to the main worktree, the builtin file +system monitor, and an index and worktree root on local APFS file +systems. + +Readers pass `O_NONBLOCK` and do not follow symbolic links when opening +the sidecar, then accept only regular files no larger than 8192 bytes. +An open or validation failure falls back to ordinary status. Writers use +the normal lockfile protocol. + +Binary format +------------- + +All integers are stored in network byte order. Hashes use the +repository's object-format hash algorithm. Version 1 consists of: + +* Four-byte magic `CSTS`. + +* A 32-bit version number (currently 1). + +* 32-bit flags (currently zero). + +* Fourteen 64-bit fields describing the source index: device, inode, + mode, link count, uid, gid, size, mtime seconds and nanoseconds, ctime + seconds and nanoseconds, birth time seconds and nanoseconds, and + generation. + +* The 32-bit index format version and 32-bit cache-entry count. + +* The index checksum and `HEAD` tree object ID. + +* Hashes of status-relevant configuration and repository identity. The + repository identity covers the worktree and Git directory paths, the + worktree root identity, the local APFS identifiers of the held index + and worktree root, external-attribute contents, and locale inputs. + +* One object ID digesting the unique standard-exclude observations in + first-observation order. Each observation contains the source path, + symbolic-link lookup policy, presence, and contents. Transient file + system identities used to make an observation coherent are not part + of the digest. + +* A 32-bit provider-token length followed by the token without a + terminating NUL. Version 1 accepts only builtin-fsmonitor tokens. + +* A hash over all preceding bytes in the sidecar. + +Issuance +-------- + +Only a literal, top-level `status --porcelain=v2` invocation can issue a +sidecar. Status first completes the tracked and untracked scan, +semantic-conversion checks, and provider-token closure. The result must +be empty and use the bulk scanner's complete standard-exclude result. +The index must also contain persistent semantic history from an earlier +scan, so the first scan cannot certify itself. + +Status then uses the held attribute snapshot and the scanner-sourced +standard-exclude digest, pins the named index, and checks its ordinary +expanded entries, `HEAD`, and the cache tree. External attribute +sources, effective replacement refs, untracked-cache results, null +index checksums, and other unsupported repository or index shapes +prevent issuance. + +The sidecar is installed while the index lock remains held and after +the pinned index is rechecked. Status then rolls back the index lock, so +issuing a sidecar does not itself rewrite the index. With optional locks +disabled, status does not issue a sidecar. + +Validation and races +-------------------- + +For the same literal command, status attempts validation before loading +the index entries. It checks the bounded sidecar, pins the named index, +and recreates the configuration, attribute, standard-exclude, +repository, and `HEAD` inputs. It then queries the builtin file system +monitor from the recorded token and accepts only an empty delta +response. + +The attribute and exclude snapshots remain held across that query. +Before returning a clean result, status freshly checks configuration, +`HEAD`, uncached replacement refs, the attribute contents and namespace, +the exclude sources, and both the opened and named index identities. +The sidecar's exclude-source opens pass `O_NONBLOCK` and fail closed +when the source cannot be opened and validated; standard-exclude +symbolic links retain their normal lookup behavior. + +A hit produces no output and reads only the 12-byte index header and +the 20- or 32-byte checksum trailer; it does not deserialize cache +entries, write the index, replace the sidecar, or advance its provider +token. Any failed check falls through to ordinary status.