status: preserve the clean-status preview DAG

Documentation/Makefile

@@ -142,6 +142,7 @@ TECH_DOCS += technical/send-pack-pipeline
 TECH_DOCS += technical/shallow
 TECH_DOCS += technical/sparse-checkout
 TECH_DOCS += technical/sparse-index
+TECH_DOCS += technical/status-clean-proof
 TECH_DOCS += technical/trivial-merge
 TECH_DOCS += technical/unambiguous-types
 TECH_DOCS += technical/unit-tests

Documentation/config/core.adoc

@@ -729,6 +729,17 @@ relatively high IO latencies.  When enabled, Git will do the
 index comparison to the filesystem data in parallel, allowing
 overlapping IO's.  Defaults to true.
 
+core.preloadIndexBulk::
+	On supported filesystems, scan working tree directories in bulk before
+	the parallel index preload.
++
+This replaces per-entry filesystem lookups with a physical directory scan,
+but may cost more than normal preload depending on filesystem and cache
+state. Inconclusive scans are discarded before continuing with the normal
+preload. Currently this is supported on APFS, ext-family filesystems, and
+XFS, and only has an effect when `core.preloadIndex` is enabled. Defaults
+to false.
+
 core.unsetenvvars::
 	Windows-only: comma-separated list of environment variables'
 	names that need to be unset before spawning any other process.

Documentation/gitformat-index.adoc

@@ -366,6 +366,49 @@ The remaining data of each directory block is grouped by type:
   - An ewah bitmap, the n-th bit indicates whether the n-th index entry
     is not CE_FSMONITOR_VALID.
 
+== File System Monitor untracked-cache token
+
+  The file system monitor untracked-cache token records the provider
+  token associated with an untracked-cache snapshot. The signature for
+  this extension is { 'F', 'S', 'U', 'C' }.
+
+  The extension consists of:
+
+  - 32-bit version number: the current version is 1.
+
+  - A NUL-terminated string containing the opaque file system monitor
+    token associated with the untracked-cache data.
+
+== File System Monitor semantic proof
+
+  The file system monitor semantic proof records the configuration and
+  attribute inputs for a completed worktree-content verification. Its
+  signature is { 'F', 'S', 'C', 'F' }.
+
+  The extension consists of:
+
+  - 32-bit version number (currently 1).
+
+  - 32-bit magic number identifying version 1 records (`FSC1`).
+
+  - 32-bit flags. The low four bits respectively indicate a complete
+    attribute manifest, a provider-token binding, a stat-data binding, and
+    coverage of the full index. All other bits must be zero.
+
+  - 32-bit length of the provider token.
+
+  - 32-bit length of the attribute manifest.
+
+  - The provider token, without a terminating NUL.
+
+  - Three hashes, using the index hash algorithm, over the relevant Git
+    configuration, semantic-conversion configuration, and attribute state.
+
+  - The attribute manifest described by its length above.
+
+  - A hash over all preceding bytes in this extension, using the index hash
+    algorithm.
+
 == End of Index Entry
 
   The End of Index Entry (EOIE) is used to locate the end of the variable

Documentation/technical/meson.build

@@ -31,6 +31,7 @@ articles = [
   'shallow.adoc',
   'sparse-checkout.adoc',
   'sparse-index.adoc',
+  'status-clean-proof.adoc',
   'trivial-merge.adoc',
   'unambiguous-types.adoc',
   'unit-tests.adoc',

Documentation/technical/status-clean-proof.adoc

@@ -0,0 +1,103 @@
+Status clean-proof sidecar
+==========================
+
+The status clean-proof sidecar is an optional cache for one exact clean
+`git status --porcelain=v2` result. It is never a source of repository
+state. A missing, stale, malformed, unsupported, or raced sidecar makes
+status read the index and scan the worktree normally.
+
+Location and scope
+------------------
+
+The sidecar for an index at `<index>` is stored at `<index>.csts`. This
+keeps a proof for one index from being applied to an alternate index.
+Sidecars are currently limited to the main worktree, the builtin file
+system monitor, and an index and worktree root on local APFS file
+systems.
+
+Readers pass `O_NONBLOCK` and do not follow symbolic links when opening
+the sidecar, then accept only regular files no larger than 8192 bytes.
+An open or validation failure falls back to ordinary status. Writers use
+the normal lockfile protocol.
+
+Binary format
+-------------
+
+All integers are stored in network byte order. Hashes use the
+repository's object-format hash algorithm. Version 1 consists of:
+
+* Four-byte magic `CSTS`.
+
+* A 32-bit version number (currently 1).
+
+* 32-bit flags (currently zero).
+
+* Fourteen 64-bit fields describing the source index: device, inode,
+  mode, link count, uid, gid, size, mtime seconds and nanoseconds, ctime
+  seconds and nanoseconds, birth time seconds and nanoseconds, and
+  generation.
+
+* The 32-bit index format version and 32-bit cache-entry count.
+
+* The index checksum and `HEAD` tree object ID.
+
+* Hashes of status-relevant configuration and repository identity. The
+  repository identity covers the worktree and Git directory paths, the
+  worktree root identity, the local APFS identifiers of the held index
+  and worktree root, external-attribute contents, and locale inputs.
+
+* One object ID digesting the unique standard-exclude observations in
+  first-observation order. Each observation contains the source path,
+  symbolic-link lookup policy, presence, and contents. Transient file
+  system identities used to make an observation coherent are not part
+  of the digest.
+
+* A 32-bit provider-token length followed by the token without a
+  terminating NUL. Version 1 accepts only builtin-fsmonitor tokens.
+
+* A hash over all preceding bytes in the sidecar.
+
+Issuance
+--------
+
+Only a literal, top-level `status --porcelain=v2` invocation can issue a
+sidecar. Status first completes the tracked and untracked scan,
+semantic-conversion checks, and provider-token closure. The result must
+be empty and use the bulk scanner's complete standard-exclude result.
+The index must also contain persistent semantic history from an earlier
+scan, so the first scan cannot certify itself.
+
+Status then uses the held attribute snapshot and the scanner-sourced
+standard-exclude digest, pins the named index, and checks its ordinary
+expanded entries, `HEAD`, and the cache tree. External attribute
+sources, effective replacement refs, untracked-cache results, null
+index checksums, and other unsupported repository or index shapes
+prevent issuance.
+
+The sidecar is installed while the index lock remains held and after
+the pinned index is rechecked. Status then rolls back the index lock, so
+issuing a sidecar does not itself rewrite the index. With optional locks
+disabled, status does not issue a sidecar.
+
+Validation and races
+--------------------
+
+For the same literal command, status attempts validation before loading
+the index entries. It checks the bounded sidecar, pins the named index,
+and recreates the configuration, attribute, standard-exclude,
+repository, and `HEAD` inputs. It then queries the builtin file system
+monitor from the recorded token and accepts only an empty delta
+response.
+
+The attribute and exclude snapshots remain held across that query.
+Before returning a clean result, status freshly checks configuration,
+`HEAD`, uncached replacement refs, the attribute contents and namespace,
+the exclude sources, and both the opened and named index identities.
+The sidecar's exclude-source opens pass `O_NONBLOCK` and fail closed
+when the source cannot be opened and validated; standard-exclude
+symbolic links retain their normal lookup behavior.
+
+A hit produces no output and reads only the 12-byte index header and
+the 20- or 32-byte checksum trailer; it does not deserialize cache
+entries, write the index, replace the sidecar, or advance its provider
+token. Any failed check falls through to ordinary status.

Makefile

@@ -413,6 +413,9 @@ include shared.mak
 # `compat/fsmonitor/fsm-health-<name>.c` files
 # that implement the `fsm_listen__*()` and `fsm_health__*()` routines.
 #
+# If a platform supports bulk worktree scans during index preload, set
+# PRELOAD_INDEX_BULK_BACKEND to the name of its backend.
+#
 # If your platform has OS-specific ways to tell if a repo is incompatible with
 # fsmonitor (whether the hook or IPC daemon version), set FSMONITOR_OS_SETTINGS
 # to the "<name>" of the corresponding `compat/fsmonitor/fsm-settings-<name>.c`
@@ -866,6 +869,7 @@ TEST_BUILTINS_OBJS += test-repository.o
 TEST_BUILTINS_OBJS += test-revision-walking.o
 TEST_BUILTINS_OBJS += test-run-command.o
 TEST_BUILTINS_OBJS += test-scrap-cache-tree.o
+TEST_BUILTINS_OBJS += test-semantic-verify.o
 TEST_BUILTINS_OBJS += test-serve-v2.o
 TEST_BUILTINS_OBJS += test-sha1.o
 TEST_BUILTINS_OBJS += test-sha256.o
@@ -1109,6 +1113,8 @@ LIB_OBJS += archive-tar.o
 LIB_OBJS += archive-zip.o
 LIB_OBJS += archive.o
 LIB_OBJS += attr.o
+LIB_OBJS += attr-fingerprint.o
+LIB_OBJS += attr-manifest.o
 LIB_OBJS += base85.o
 LIB_OBJS += bisect.o
 LIB_OBJS += blame.o
@@ -1122,6 +1128,16 @@ LIB_OBJS += cbtree.o
 LIB_OBJS += chdir-notify.o
 LIB_OBJS += checkout.o
 LIB_OBJS += chunk-format.o
+LIB_OBJS += clean-status.o
+LIB_OBJS += clean-status-config.o
+LIB_OBJS += clean-status-epoch.o
+LIB_OBJS += clean-status-history.o
+LIB_OBJS += clean-status-identity.o
+LIB_OBJS += clean-status-index.o
+LIB_OBJS += clean-status-manifest.o
+LIB_OBJS += clean-status-sidecar.o
+LIB_OBJS += clean-status-fast.o
+LIB_OBJS += clean-status-sidecar-issue.o
 LIB_OBJS += color.o
 LIB_OBJS += column.o
 LIB_OBJS += combine-diff.o
@@ -1167,12 +1183,14 @@ LIB_OBJS += ewah/bitmap.o
 LIB_OBJS += ewah/ewah_bitmap.o
 LIB_OBJS += ewah/ewah_io.o
 LIB_OBJS += ewah/ewah_rlw.o
+LIB_OBJS += exclude-source-proof.o
 LIB_OBJS += exec-cmd.o
 LIB_OBJS += fetch-negotiator.o
 LIB_OBJS += fetch-object-info.o
 LIB_OBJS += fetch-pack.o
 LIB_OBJS += fmt-merge-msg.o
 LIB_OBJS += fsck.o
+LIB_OBJS += fsmonitor-clean-proof.o
 LIB_OBJS += fsmonitor.o
 LIB_OBJS += fsmonitor-ipc.o
 LIB_OBJS += fsmonitor-settings.o
@@ -1255,6 +1273,7 @@ LIB_OBJS += parse-options.o
 LIB_OBJS += patch-delta.o
 LIB_OBJS += patch-ids.o
 LIB_OBJS += path.o
+LIB_OBJS += path-namespace.o
 LIB_OBJS += path-walk.o
 LIB_OBJS += pathspec.o
 LIB_OBJS += pkt-line.o
@@ -1316,6 +1335,11 @@ LIB_OBJS += resolve-undo.o
 LIB_OBJS += revision.o
 LIB_OBJS += run-command.o
 LIB_OBJS += send-pack.o
+LIB_OBJS += semantic-verify-file.o
+LIB_OBJS += semantic-verify-path.o
+LIB_OBJS += semantic-verify-root.o
+LIB_OBJS += semantic-verify-worker.o
+LIB_OBJS += semantic-verify.o
 LIB_OBJS += sequencer.o
 LIB_OBJS += serve.o
 LIB_OBJS += server-info.o
@@ -1374,9 +1398,20 @@ LIB_OBJS += versioncmp.o
 LIB_OBJS += walker.o
 LIB_OBJS += wildmatch.o
 LIB_OBJS += worktree.o
+LIB_OBJS += worktree-attr-manifest.o
+LIB_OBJS += worktree-attr-source.o
 LIB_OBJS += wrapper.o
 LIB_OBJS += write-or-die.o
 LIB_OBJS += ws.o
+ifdef PRELOAD_INDEX_BULK_BACKEND
+BASIC_CFLAGS += -DHAVE_PRELOAD_INDEX_BULK
+PRELOAD_INDEX_BULK_OBJS += preload-index-bulk-index.o
+PRELOAD_INDEX_BULK_OBJS += preload-index-bulk-thread.o
+PRELOAD_INDEX_BULK_OBJS += preload-index-bulk.o
+PRELOAD_INDEX_BULK_OBJS += compat/preload-index/bulk-$(PRELOAD_INDEX_BULK_BACKEND).o
+PRELOAD_INDEX_BULK_OBJS += $(PRELOAD_INDEX_BULK_PLATFORM_OBJS)
+endif
+LIB_OBJS += $(PRELOAD_INDEX_BULK_OBJS)
 LIB_OBJS += wt-status.o
 LIB_OBJS += xdiff-interface.o
 LIB_OBJS += xdiff/xdiffi.o
@@ -1535,9 +1570,22 @@ THIRD_PARTY_SOURCES += sha1dc/%
 THIRD_PARTY_SOURCES += $(UNIT_TEST_DIR)/clar/%
 THIRD_PARTY_SOURCES += $(UNIT_TEST_DIR)/clar/clar/%
 
+CLAR_TEST_SUITES += u-attr-fingerprint
+CLAR_TEST_SUITES += u-attr-manifest
+CLAR_TEST_SUITES += u-clean-status-config
+CLAR_TEST_SUITES += u-clean-status-history
+CLAR_TEST_SUITES += u-clean-status-identity
+CLAR_TEST_SUITES += u-clean-status-index
+CLAR_TEST_SUITES += u-clean-status-manifest
+CLAR_TEST_SUITES += u-clean-status-sidecar
+CLAR_TEST_SUITES += u-clean-status-store
 CLAR_TEST_SUITES += u-ctype
 CLAR_TEST_SUITES += u-dir
 CLAR_TEST_SUITES += u-example-decorate
+CLAR_TEST_SUITES += u-exclude-source-proof
+CLAR_TEST_SUITES += u-fsmonitor-attributes
+CLAR_TEST_SUITES += u-fsmonitor-clean-proof
+CLAR_TEST_SUITES += u-fsmonitor-response
 CLAR_TEST_SUITES += u-hash
 CLAR_TEST_SUITES += u-hashmap
 CLAR_TEST_SUITES += u-list-objects-filter-options
@@ -1546,7 +1594,9 @@ CLAR_TEST_SUITES += u-odb-inmemory
 CLAR_TEST_SUITES += u-oid-array
 CLAR_TEST_SUITES += u-oidmap
 CLAR_TEST_SUITES += u-oidtree
+CLAR_TEST_SUITES += u-path-namespace
 CLAR_TEST_SUITES += u-prio-queue
+CLAR_TEST_SUITES += u-preload-index-bulk-darwin
 CLAR_TEST_SUITES += u-reftable-basics
 CLAR_TEST_SUITES += u-reftable-block
 CLAR_TEST_SUITES += u-reftable-merged
@@ -1562,6 +1612,7 @@ CLAR_TEST_SUITES += u-strvec
 CLAR_TEST_SUITES += u-trailer
 CLAR_TEST_SUITES += u-urlmatch-normalization
 CLAR_TEST_SUITES += u-utf8-width
+CLAR_TEST_SUITES += u-worktree-attr-source
 CLAR_TEST_PROG = $(UNIT_TEST_BIN)/unit-tests$(X)
 CLAR_TEST_OBJS = $(patsubst %,$(UNIT_TEST_DIR)/%.o,$(CLAR_TEST_SUITES))
 CLAR_TEST_OBJS += $(UNIT_TEST_DIR)/clar/clar.o

apply.c

@@ -3539,8 +3539,9 @@ static int verify_index_match(struct apply_state *state,
 			return -1;
 		return 0;
 	}
-	return ie_match_stat(state->repo->index, ce, st,
-			     CE_MATCH_IGNORE_VALID | CE_MATCH_IGNORE_SKIP_WORKTREE);
+	return ie_match_stat_with_content_check(
+		state->repo->index, ce, st,
+		CE_MATCH_IGNORE_VALID | CE_MATCH_IGNORE_SKIP_WORKTREE);
 }
 
 #define SUBMODULE_PATCH_WITHOUT_INDEX 1