diff --git a/.github/workflows/codex-pgo-training.sh b/.github/workflows/codex-pgo-training.sh new file mode 100644 index 00000000000000..070d8b54cae486 --- /dev/null +++ b/.github/workflows/codex-pgo-training.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +# Keep this workload short and biased toward the local Git operations Codex +# invokes frequently. The full Git test suite is too slow for each release +# target and would weight test-harness paths more heavily than status, diff, +# clone, fetch, and repository maintenance. + +set -eu + +git_bin="$PWD/bin-wrappers/git" +training_dir=$(mktemp -d "${TMPDIR:-/tmp}/codex-git-pgo.XXXXXX") +repo="$training_dir/repo" +clone="$training_dir/clone" + +cleanup () { + rm -rf "$training_dir" +} +trap cleanup EXIT HUP INT TERM + +mkdir -p "$training_dir/home" +export HOME="$training_dir/home" +export GIT_CONFIG_NOSYSTEM=1 +export GIT_TERMINAL_PROMPT=0 + +"$git_bin" clone --quiet --no-local "$PWD" "$repo" +"$git_bin" -C "$repo" config user.name "Codex Git PGO" +"$git_bin" -C "$repo" config user.email "codex-git-pgo@openai.com" + +i=0 +while test "$i" -lt 256 +do + dir="$repo/training/$((i % 16))" + mkdir -p "$dir" + printf '%s\n' "$i" >"$dir/file-$i" + i=$((i + 1)) +done + +"$git_bin" -C "$repo" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$repo" status --porcelain=v2 --branch --untracked-files=all >/dev/null +"$git_bin" -C "$repo" ls-files --others --exclude-standard >/dev/null +"$git_bin" -C "$repo" add training +"$git_bin" -C "$repo" diff --cached --stat >/dev/null +"$git_bin" -C "$repo" commit --quiet -m "add training files" + +printf 'changed\n' >>"$repo/training/0/file-0" +rm "$repo/training/1/file-1" +mkdir -p "$repo/untracked" +printf 'new\n' >"$repo/untracked/file" + +"$git_bin" -C "$repo" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$repo" diff --stat >/dev/null +"$git_bin" -C "$repo" diff --name-status >/dev/null +"$git_bin" -C "$repo" ls-files --stage >/dev/null +"$git_bin" -C "$repo" log --oneline --decorate -20 >/dev/null +"$git_bin" -C "$repo" rev-list --objects --all >/dev/null +"$git_bin" -C "$repo" for-each-ref --format='%(refname) %(objectname)' >/dev/null +"$git_bin" -C "$repo" repack -ad +"$git_bin" clone --quiet --no-local "$repo" "$clone" +"$git_bin" -C "$clone" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$clone" fetch --quiet "$repo" diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml index 623c6e09aeb281..e624489adcc05b 100644 --- a/.github/workflows/codex-release.yml +++ b/.github/workflows/codex-release.yml @@ -104,7 +104,7 @@ jobs: name: ${{ matrix.name }} needs: version runs-on: ${{ matrix.os }} - timeout-minutes: 30 + timeout-minutes: 60 strategy: fail-fast: false matrix: @@ -117,6 +117,9 @@ jobs: binary: /tmp/build/git/bin/git file_pattern: Mach-O 64-bit executable arm64 has_gcm: false + lto: thin + profile_format: LLVM + llvm_profdata: xcrun llvm-profdata max_tar_bytes: 67108864 - name: macOS x64 os: macos-15-intel @@ -126,15 +129,21 @@ jobs: binary: /tmp/build/git/bin/git file_pattern: Mach-O 64-bit executable x86_64 has_gcm: false + lto: thin + profile_format: LLVM + llvm_profdata: xcrun llvm-profdata max_tar_bytes: 67108864 + # Keep arm64 builds native so release smoke tests can execute them. - name: Linux arm64 - os: ubuntu-22.04 + os: ubuntu-22.04-arm target_platform: ubuntu asset_platform: ubuntu arch: arm64 binary: /tmp/build/git/bin/git file_pattern: ELF 64-bit.*ARM aarch64 has_gcm: false + lto: auto + profile_format: GCC max_tar_bytes: 67108864 - name: Linux x64 os: ubuntu-22.04 @@ -144,6 +153,8 @@ jobs: binary: /tmp/build/git/bin/git file_pattern: ELF 64-bit.*x86-64 has_gcm: false + lto: auto + profile_format: GCC max_tar_bytes: 67108864 - name: Windows arm64 os: windows-11-arm @@ -153,6 +164,9 @@ jobs: binary: /tmp/build/git/clangarm64/bin/git.exe file_pattern: PE32\+.*ARM64 has_gcm: true + lto: thin + profile_format: LLVM + llvm_profdata: llvm-profdata max_tar_bytes: 134217728 sdk_arch: aarch64 sdk_flavor: full @@ -169,6 +183,8 @@ jobs: binary: /tmp/build/git/mingw64/bin/git.exe file_pattern: PE32\+.*x86-64 has_gcm: true + lto: auto + profile_format: GCC max_tar_bytes: 134217728 sdk_arch: x86_64 sdk_flavor: full @@ -184,8 +200,8 @@ jobs: - name: Check out Dugite Native uses: actions/checkout@v6 with: - repository: desktop/dugite-native - ref: f97e50add48cdcff053a69d95aa343a4a4a258c2 + repository: dreynaud-oai/dugite-native + ref: b6f4473557acb85433fdf9deffe0854a34fd9cc5 path: dugite-native fetch-depth: 0 persist-credentials: false @@ -209,6 +225,11 @@ jobs: -c 'user.email=41898282+github-actions[bot]@users.noreply.github.com' \ tag -a "$VERSION" -m "$VERSION" + - name: Install OpenAI build configuration + shell: bash + working-directory: dugite-native/git + run: cp config.mak.openai config.mak + # Match Dugite Native's compatibility choice for its macOS x64 build. - name: Select Xcode 16.4 if: matrix.target_platform == 'macOS' && matrix.arch == 'x64' @@ -243,19 +264,13 @@ jobs: - name: Install Linux arm64 build dependencies if: matrix.target_platform == 'ubuntu' && matrix.arch == 'arm64' run: | - sudo sed -i "s/^deb/deb [arch=amd64,i386]/g" /etc/apt/sources.list - release="$(lsb_release -s -c)" - echo "deb [arch=arm64,armhf] http://azure.ports.ubuntu.com/ ${release} main universe multiverse restricted" | sudo tee -a /etc/apt/sources.list - echo "deb [arch=arm64,armhf] http://azure.ports.ubuntu.com/ ${release}-updates main universe multiverse restricted" | sudo tee -a /etc/apt/sources.list - sudo dpkg --add-architecture arm64 - sudo apt-get update sudo apt-get install -y \ binutils-aarch64-linux-gnu \ gcc-aarch64-linux-gnu \ - libcurl4-gnutls-dev:arm64 \ - libexpat1-dev:arm64 \ - libssl-dev:arm64 \ - zlib1g-dev:arm64 + libcurl4-gnutls-dev \ + libexpat1-dev \ + libssl-dev \ + zlib1g-dev # Dugite Native currently pins MinGit 2.53. Keep its build script and # dependency schema, but match the runtime to the Git series we compile. @@ -303,13 +318,18 @@ jobs: dependencies.json >"$updated" mv "$updated" dependencies.json - # Dugite cross-compiles several targets. Keep Git's optional Rust - # library disabled until its Makefile can direct Cargo at those targets. + # Build an instrumented Git, run a representative local workload, then + # rebuild with its profile. Keep Git's optional Rust library disabled + # until its Makefile can direct Cargo at these targets. - name: Build the Dugite Native distribution shell: bash working-directory: dugite-native env: NO_RUST: 1 + OPENAI_LLVM_PROFDATA: ${{ matrix.llvm_profdata }} + OPENAI_LTO: ${{ matrix.lto }} + OPENAI_PROFILE: BUILD + OPENAI_PROFILE_FORMAT: ${{ matrix.profile_format }} TARGET_PLATFORM: ${{ matrix.target_platform }} TARGET_ARCH: ${{ matrix.arch }} run: | @@ -335,6 +355,9 @@ jobs: working-directory: dugite-native/git env: MINGW_DIR: ${{ matrix.mingw_dir }} + OPENAI_LLVM_PROFDATA: ${{ matrix.llvm_profdata }} + OPENAI_LTO: ${{ matrix.lto }} + OPENAI_PROFILE_FORMAT: ${{ matrix.profile_format }} run: | set -euo pipefail @@ -349,8 +372,8 @@ jobs: SKIP_DASHED_BUILT_INS=YesPlease ) jobs="${NUMBER_OF_PROCESSORS:-2}" - make -j"$jobs" "${make_args[@]}" all - make "${make_args[@]}" DESTDIR=/tmp/build/git strip install + make -j"$jobs" "${make_args[@]}" OPENAI_PROFILE=BUILD all + make "${make_args[@]}" OPENAI_PROFILE=USE DESTDIR=/tmp/build/git strip install - name: Verify distribution layout and provenance shell: bash @@ -360,6 +383,8 @@ jobs: GIT_BINARY: ${{ matrix.binary }} FILE_PATTERN: ${{ matrix.file_pattern }} HAS_GCM: ${{ matrix.has_gcm }} + LTO: ${{ matrix.lto }} + PROFILE_FORMAT: ${{ matrix.profile_format }} run: | set -euo pipefail if test "$TARGET_PLATFORM" = win32 @@ -387,9 +412,15 @@ jobs: file "$GIT_BINARY" | tee /tmp/git-file-type grep -E "$FILE_PATTERN" /tmp/git-file-type strings "$GIT_BINARY" | grep -F "$GITHUB_SHA" + grep -F -- "-flto=$LTO" dugite-native/git/GIT-CFLAGS + if test "$PROFILE_FORMAT" = LLVM + then + grep -F -- "-fprofile-instr-use=" dugite-native/git/GIT-CFLAGS + else + grep -F -- "-fprofile-use=" dugite-native/git/GIT-CFLAGS + fi - name: Smoke-test the native distribution - if: matrix.arch == 'x64' || matrix.target_platform == 'win32' shell: bash env: TARGET_PLATFORM: ${{ matrix.target_platform }} diff --git a/.github/workflows/codex.yml b/.github/workflows/codex.yml new file mode 100644 index 00000000000000..db3895a0aa4bba --- /dev/null +++ b/.github/workflows/codex.yml @@ -0,0 +1,48 @@ +name: Refresh codex + +on: + workflow_dispatch: + pull_request: + branches: + - codex + - codex-unstable + types: + - opened + - reopened + - synchronize + - ready_for_review + merge_group: + types: + - checks_requested + +permissions: + actions: read + contents: read + pull-requests: read + +jobs: + refresh: + if: github.event_name == 'workflow_dispatch' + uses: openai/git/.github/workflows/codex.yml@meta + admission: + name: Codex admission + if: >- + (github.event_name == 'pull_request' && + github.event.pull_request.base.ref == 'codex') || + (github.event_name == 'merge_group' && + github.event.merge_group.base_ref == 'refs/heads/codex') + permissions: + contents: read + pull-requests: read + uses: openai/git/.github/workflows/codex-admission.yml@meta + unstable_admission: + name: Codex unstable admission + if: >- + (github.event_name == 'pull_request' && + github.event.pull_request.base.ref == 'codex-unstable') || + (github.event_name == 'merge_group' && + github.event.merge_group.base_ref == 'refs/heads/codex-unstable') + permissions: + contents: read + pull-requests: read + uses: openai/git/.github/workflows/codex-admission.yml@meta diff --git a/Documentation/Makefile b/Documentation/Makefile index 2699f0b24af192..ff7cee368e3f7d 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -142,6 +142,7 @@ TECH_DOCS += technical/send-pack-pipeline TECH_DOCS += technical/shallow TECH_DOCS += technical/sparse-checkout TECH_DOCS += technical/sparse-index +TECH_DOCS += technical/status-clean-proof TECH_DOCS += technical/trivial-merge TECH_DOCS += technical/unambiguous-types TECH_DOCS += technical/unit-tests diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index 340329edc38143..59bc4a818cceb8 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -729,6 +729,17 @@ relatively high IO latencies. When enabled, Git will do the index comparison to the filesystem data in parallel, allowing overlapping IO's. Defaults to true. +core.preloadIndexBulk:: + On supported filesystems, scan working tree directories in bulk before + the parallel index preload. ++ +This replaces per-entry filesystem lookups with a physical directory scan, +but may cost more than normal preload depending on filesystem and cache +state. Inconclusive scans are discarded before continuing with the normal +preload. Currently this is supported on APFS, ext-family filesystems, and +XFS, and only has an effect when `core.preloadIndex` is enabled. Defaults +to false. + core.unsetenvvars:: Windows-only: comma-separated list of environment variables' names that need to be unset before spawning any other process. diff --git a/Documentation/gitformat-index.adoc b/Documentation/gitformat-index.adoc index f6a427cb495990..047310ec26d105 100644 --- a/Documentation/gitformat-index.adoc +++ b/Documentation/gitformat-index.adoc @@ -366,6 +366,49 @@ The remaining data of each directory block is grouped by type: - An ewah bitmap, the n-th bit indicates whether the n-th index entry is not CE_FSMONITOR_VALID. +== File System Monitor untracked-cache token + + The file system monitor untracked-cache token records the provider + token associated with an untracked-cache snapshot. The signature for + this extension is { 'F', 'S', 'U', 'C' }. + + The extension consists of: + + - 32-bit version number: the current version is 1. + + - A NUL-terminated string containing the opaque file system monitor + token associated with the untracked-cache data. + +== File System Monitor semantic proof + + The file system monitor semantic proof records the configuration and + attribute inputs for a completed worktree-content verification. Its + signature is { 'F', 'S', 'C', 'F' }. + + The extension consists of: + + - 32-bit version number (currently 1). + + - 32-bit magic number identifying version 1 records (`FSC1`). + + - 32-bit flags. The low four bits respectively indicate a complete + attribute manifest, a provider-token binding, a stat-data binding, and + coverage of the full index. All other bits must be zero. + + - 32-bit length of the provider token. + + - 32-bit length of the attribute manifest. + + - The provider token, without a terminating NUL. + + - Three hashes, using the index hash algorithm, over the relevant Git + configuration, semantic-conversion configuration, and attribute state. + + - The attribute manifest described by its length above. + + - A hash over all preceding bytes in this extension, using the index hash + algorithm. + == End of Index Entry The End of Index Entry (EOIE) is used to locate the end of the variable diff --git a/Documentation/technical/meson.build b/Documentation/technical/meson.build index ec07088c57617f..665977299cdab3 100644 --- a/Documentation/technical/meson.build +++ b/Documentation/technical/meson.build @@ -31,6 +31,7 @@ articles = [ 'shallow.adoc', 'sparse-checkout.adoc', 'sparse-index.adoc', + 'status-clean-proof.adoc', 'trivial-merge.adoc', 'unambiguous-types.adoc', 'unit-tests.adoc', diff --git a/Documentation/technical/status-clean-proof.adoc b/Documentation/technical/status-clean-proof.adoc new file mode 100644 index 00000000000000..f598d36c3353c3 --- /dev/null +++ b/Documentation/technical/status-clean-proof.adoc @@ -0,0 +1,103 @@ +Status clean-proof sidecar +========================== + +The status clean-proof sidecar is an optional cache for one exact clean +`git status --porcelain=v2` result. It is never a source of repository +state. A missing, stale, malformed, unsupported, or raced sidecar makes +status read the index and scan the worktree normally. + +Location and scope +------------------ + +The sidecar for an index at `` is stored at `.csts`. This +keeps a proof for one index from being applied to an alternate index. +Sidecars are currently limited to the main worktree, the builtin file +system monitor, and an index and worktree root on local APFS file +systems. + +Readers pass `O_NONBLOCK` and do not follow symbolic links when opening +the sidecar, then accept only regular files no larger than 8192 bytes. +An open or validation failure falls back to ordinary status. Writers use +the normal lockfile protocol. + +Binary format +------------- + +All integers are stored in network byte order. Hashes use the +repository's object-format hash algorithm. Version 1 consists of: + +* Four-byte magic `CSTS`. + +* A 32-bit version number (currently 1). + +* 32-bit flags (currently zero). + +* Fourteen 64-bit fields describing the source index: device, inode, + mode, link count, uid, gid, size, mtime seconds and nanoseconds, ctime + seconds and nanoseconds, birth time seconds and nanoseconds, and + generation. + +* The 32-bit index format version and 32-bit cache-entry count. + +* The index checksum and `HEAD` tree object ID. + +* Hashes of status-relevant configuration and repository identity. The + repository identity covers the worktree and Git directory paths, the + worktree root identity, the local APFS identifiers of the held index + and worktree root, external-attribute contents, and locale inputs. + +* One object ID digesting the unique standard-exclude observations in + first-observation order. Each observation contains the source path, + symbolic-link lookup policy, presence, and contents. Transient file + system identities used to make an observation coherent are not part + of the digest. + +* A 32-bit provider-token length followed by the token without a + terminating NUL. Version 1 accepts only builtin-fsmonitor tokens. + +* A hash over all preceding bytes in the sidecar. + +Issuance +-------- + +Only a literal, top-level `status --porcelain=v2` invocation can issue a +sidecar. Status first completes the tracked and untracked scan, +semantic-conversion checks, and provider-token closure. The result must +be empty and use the bulk scanner's complete standard-exclude result. +The index must also contain persistent semantic history from an earlier +scan, so the first scan cannot certify itself. + +Status then uses the held attribute snapshot and the scanner-sourced +standard-exclude digest, pins the named index, and checks its ordinary +expanded entries, `HEAD`, and the cache tree. External attribute +sources, effective replacement refs, untracked-cache results, null +index checksums, and other unsupported repository or index shapes +prevent issuance. + +The sidecar is installed while the index lock remains held and after +the pinned index is rechecked. Status then rolls back the index lock, so +issuing a sidecar does not itself rewrite the index. With optional locks +disabled, status does not issue a sidecar. + +Validation and races +-------------------- + +For the same literal command, status attempts validation before loading +the index entries. It checks the bounded sidecar, pins the named index, +and recreates the configuration, attribute, standard-exclude, +repository, and `HEAD` inputs. It then queries the builtin file system +monitor from the recorded token and accepts only an empty delta +response. + +The attribute and exclude snapshots remain held across that query. +Before returning a clean result, status freshly checks configuration, +`HEAD`, uncached replacement refs, the attribute contents and namespace, +the exclude sources, and both the opened and named index identities. +The sidecar's exclude-source opens pass `O_NONBLOCK` and fail closed +when the source cannot be opened and validated; standard-exclude +symbolic links retain their normal lookup behavior. + +A hit produces no output and reads only the 12-byte index header and +the 20- or 32-byte checksum trailer; it does not deserialize cache +entries, write the index, replace the sidecar, or advance its provider +token. Any failed check falls through to ordinary status. diff --git a/Makefile b/Makefile index fac3e8879c8377..42e303b4c71502 100644 --- a/Makefile +++ b/Makefile @@ -413,6 +413,9 @@ include shared.mak # `compat/fsmonitor/fsm-health-.c` files # that implement the `fsm_listen__*()` and `fsm_health__*()` routines. # +# If a platform supports bulk worktree scans during index preload, set +# PRELOAD_INDEX_BULK_BACKEND to the name of its backend. +# # If your platform has OS-specific ways to tell if a repo is incompatible with # fsmonitor (whether the hook or IPC daemon version), set FSMONITOR_OS_SETTINGS # to the "" of the corresponding `compat/fsmonitor/fsm-settings-.c` @@ -866,6 +869,7 @@ TEST_BUILTINS_OBJS += test-repository.o TEST_BUILTINS_OBJS += test-revision-walking.o TEST_BUILTINS_OBJS += test-run-command.o TEST_BUILTINS_OBJS += test-scrap-cache-tree.o +TEST_BUILTINS_OBJS += test-semantic-verify.o TEST_BUILTINS_OBJS += test-serve-v2.o TEST_BUILTINS_OBJS += test-sha1.o TEST_BUILTINS_OBJS += test-sha256.o @@ -1109,6 +1113,8 @@ LIB_OBJS += archive-tar.o LIB_OBJS += archive-zip.o LIB_OBJS += archive.o LIB_OBJS += attr.o +LIB_OBJS += attr-fingerprint.o +LIB_OBJS += attr-manifest.o LIB_OBJS += base85.o LIB_OBJS += bisect.o LIB_OBJS += blame.o @@ -1122,6 +1128,16 @@ LIB_OBJS += cbtree.o LIB_OBJS += chdir-notify.o LIB_OBJS += checkout.o LIB_OBJS += chunk-format.o +LIB_OBJS += clean-status.o +LIB_OBJS += clean-status-config.o +LIB_OBJS += clean-status-epoch.o +LIB_OBJS += clean-status-history.o +LIB_OBJS += clean-status-identity.o +LIB_OBJS += clean-status-index.o +LIB_OBJS += clean-status-manifest.o +LIB_OBJS += clean-status-sidecar.o +LIB_OBJS += clean-status-fast.o +LIB_OBJS += clean-status-sidecar-issue.o LIB_OBJS += color.o LIB_OBJS += column.o LIB_OBJS += combine-diff.o @@ -1167,12 +1183,14 @@ LIB_OBJS += ewah/bitmap.o LIB_OBJS += ewah/ewah_bitmap.o LIB_OBJS += ewah/ewah_io.o LIB_OBJS += ewah/ewah_rlw.o +LIB_OBJS += exclude-source-proof.o LIB_OBJS += exec-cmd.o LIB_OBJS += fetch-negotiator.o LIB_OBJS += fetch-object-info.o LIB_OBJS += fetch-pack.o LIB_OBJS += fmt-merge-msg.o LIB_OBJS += fsck.o +LIB_OBJS += fsmonitor-clean-proof.o LIB_OBJS += fsmonitor.o LIB_OBJS += fsmonitor-ipc.o LIB_OBJS += fsmonitor-settings.o @@ -1255,6 +1273,7 @@ LIB_OBJS += parse-options.o LIB_OBJS += patch-delta.o LIB_OBJS += patch-ids.o LIB_OBJS += path.o +LIB_OBJS += path-namespace.o LIB_OBJS += path-walk.o LIB_OBJS += pathspec.o LIB_OBJS += pkt-line.o @@ -1316,6 +1335,11 @@ LIB_OBJS += resolve-undo.o LIB_OBJS += revision.o LIB_OBJS += run-command.o LIB_OBJS += send-pack.o +LIB_OBJS += semantic-verify-file.o +LIB_OBJS += semantic-verify-path.o +LIB_OBJS += semantic-verify-root.o +LIB_OBJS += semantic-verify-worker.o +LIB_OBJS += semantic-verify.o LIB_OBJS += sequencer.o LIB_OBJS += serve.o LIB_OBJS += server-info.o @@ -1374,9 +1398,20 @@ LIB_OBJS += versioncmp.o LIB_OBJS += walker.o LIB_OBJS += wildmatch.o LIB_OBJS += worktree.o +LIB_OBJS += worktree-attr-manifest.o +LIB_OBJS += worktree-attr-source.o LIB_OBJS += wrapper.o LIB_OBJS += write-or-die.o LIB_OBJS += ws.o +ifdef PRELOAD_INDEX_BULK_BACKEND +BASIC_CFLAGS += -DHAVE_PRELOAD_INDEX_BULK +PRELOAD_INDEX_BULK_OBJS += preload-index-bulk-index.o +PRELOAD_INDEX_BULK_OBJS += preload-index-bulk-thread.o +PRELOAD_INDEX_BULK_OBJS += preload-index-bulk.o +PRELOAD_INDEX_BULK_OBJS += compat/preload-index/bulk-$(PRELOAD_INDEX_BULK_BACKEND).o +PRELOAD_INDEX_BULK_OBJS += $(PRELOAD_INDEX_BULK_PLATFORM_OBJS) +endif +LIB_OBJS += $(PRELOAD_INDEX_BULK_OBJS) LIB_OBJS += wt-status.o LIB_OBJS += xdiff-interface.o LIB_OBJS += xdiff/xdiffi.o @@ -1535,9 +1570,22 @@ THIRD_PARTY_SOURCES += sha1dc/% THIRD_PARTY_SOURCES += $(UNIT_TEST_DIR)/clar/% THIRD_PARTY_SOURCES += $(UNIT_TEST_DIR)/clar/clar/% +CLAR_TEST_SUITES += u-attr-fingerprint +CLAR_TEST_SUITES += u-attr-manifest +CLAR_TEST_SUITES += u-clean-status-config +CLAR_TEST_SUITES += u-clean-status-history +CLAR_TEST_SUITES += u-clean-status-identity +CLAR_TEST_SUITES += u-clean-status-index +CLAR_TEST_SUITES += u-clean-status-manifest +CLAR_TEST_SUITES += u-clean-status-sidecar +CLAR_TEST_SUITES += u-clean-status-store CLAR_TEST_SUITES += u-ctype CLAR_TEST_SUITES += u-dir CLAR_TEST_SUITES += u-example-decorate +CLAR_TEST_SUITES += u-exclude-source-proof +CLAR_TEST_SUITES += u-fsmonitor-attributes +CLAR_TEST_SUITES += u-fsmonitor-clean-proof +CLAR_TEST_SUITES += u-fsmonitor-response CLAR_TEST_SUITES += u-hash CLAR_TEST_SUITES += u-hashmap CLAR_TEST_SUITES += u-list-objects-filter-options @@ -1546,7 +1594,9 @@ CLAR_TEST_SUITES += u-odb-inmemory CLAR_TEST_SUITES += u-oid-array CLAR_TEST_SUITES += u-oidmap CLAR_TEST_SUITES += u-oidtree +CLAR_TEST_SUITES += u-path-namespace CLAR_TEST_SUITES += u-prio-queue +CLAR_TEST_SUITES += u-preload-index-bulk-darwin CLAR_TEST_SUITES += u-reftable-basics CLAR_TEST_SUITES += u-reftable-block CLAR_TEST_SUITES += u-reftable-merged @@ -1562,6 +1612,7 @@ CLAR_TEST_SUITES += u-strvec CLAR_TEST_SUITES += u-trailer CLAR_TEST_SUITES += u-urlmatch-normalization CLAR_TEST_SUITES += u-utf8-width +CLAR_TEST_SUITES += u-worktree-attr-source CLAR_TEST_PROG = $(UNIT_TEST_BIN)/unit-tests$(X) CLAR_TEST_OBJS = $(patsubst %,$(UNIT_TEST_DIR)/%.o,$(CLAR_TEST_SUITES)) CLAR_TEST_OBJS += $(UNIT_TEST_DIR)/clar/clar.o diff --git a/apply.c b/apply.c index f00b7ba4d3a7e6..1f5dda3b6f3fcc 100644 --- a/apply.c +++ b/apply.c @@ -3539,8 +3539,9 @@ static int verify_index_match(struct apply_state *state, return -1; return 0; } - return ie_match_stat(state->repo->index, ce, st, - CE_MATCH_IGNORE_VALID | CE_MATCH_IGNORE_SKIP_WORKTREE); + return ie_match_stat_with_content_check( + state->repo->index, ce, st, + CE_MATCH_IGNORE_VALID | CE_MATCH_IGNORE_SKIP_WORKTREE); } #define SUBMODULE_PATCH_WITHOUT_INDEX 1 diff --git a/attr-fingerprint.c b/attr-fingerprint.c new file mode 100644 index 00000000000000..d0152d6fe23963 --- /dev/null +++ b/attr-fingerprint.c @@ -0,0 +1,265 @@ +#include "git-compat-util.h" +#include "abspath.h" +#include "attr-fingerprint.h" +#include "attr.h" +#include "environment.h" +#include "hash-framing.h" +#include "path.h" +#include "path-namespace.h" +#include "repository.h" +#include "strbuf.h" +#include "wrapper.h" + +struct attr_source_snapshot_entry { + char *path; + char *buf; + size_t len; +}; + +struct attr_source_snapshot { + struct attr_fingerprint fingerprint; + struct attr_source_snapshot_entry sources[ATTR_SOURCE_SNAPSHOT_NR]; +}; + +static int open_attr_source(const char *path) +{ +#ifdef O_NONBLOCK + return git_open_cloexec(path, O_RDONLY | O_NONBLOCK); +#else + (void)path; + errno = ENOSYS; + return -1; +#endif +} + +static int hash_source(struct git_hash_ctx *content_ctx, + struct git_hash_ctx *namespace_ctx, + const struct attr_fingerprint_source *source, + int *present, + struct attr_source_snapshot_entry *snapshot) +{ + struct path_namespace_snapshot *before = NULL, *after = NULL; + struct stat opened_before, opened_after, named; + struct strbuf normalized = STRBUF_INIT; + char *absolute = NULL; + char *buf = NULL; + ssize_t got; + size_t size; + uint32_t state; + int fd = -1, ret = -1; + char extra; + + hash_optional_cstring(content_ctx, source->path); + hash_optional_cstring(namespace_ctx, source->path); + put_be32(&state, source->enabled); + hash_length_delimited(content_ctx, &state, sizeof(state)); + hash_length_delimited(namespace_ctx, &state, sizeof(state)); + *present = 0; + if (!source->enabled || !source->path) + return 0; + + absolute = absolute_pathdup(source->path); + strbuf_addstr(&normalized, absolute); + if (strbuf_normalize_path(&normalized) || + path_namespace_capture(normalized.buf, &before)) + goto done; + *present = path_namespace_target_present(before); + if (!*present) { + if (path_namespace_capture(normalized.buf, &after) || + !path_namespace_equal(before, after)) + goto done; + state = 0; + hash_length_delimited(content_ctx, &state, sizeof(state)); + path_namespace_hash(namespace_ctx, before); + ret = 0; + goto done; + } + + fd = open_attr_source(normalized.buf); + if (fd < 0 || fstat(fd, &opened_before) || + !S_ISREG(opened_before.st_mode) || opened_before.st_nlink != 1 || + opened_before.st_size < 0 || + opened_before.st_size >= ATTR_MAX_FILE_SIZE) + goto done; + size = xsize_t(opened_before.st_size); + buf = xmalloc(size ? size : 1); + got = read_in_full(fd, buf, size); + if (got < 0 || (size_t)got != size || read(fd, &extra, 1) != 0 || + fstat(fd, &opened_after) || stat(normalized.buf, &named) || + !path_namespace_stat_equal(&opened_before, &opened_after) || + !path_namespace_stat_equal(&opened_after, &named) || + path_namespace_capture(normalized.buf, &after) || + !path_namespace_equal(before, after)) + goto done; + state = 1; + hash_length_delimited(content_ctx, &state, sizeof(state)); + path_namespace_hash(namespace_ctx, before); + path_namespace_hash_stat(namespace_ctx, &opened_after); + hash_length_delimited(content_ctx, buf, size); + if (snapshot) { + snapshot->path = xstrdup(source->path); + snapshot->buf = buf; + snapshot->len = size; + buf = NULL; + } + ret = 0; +done: + if (fd >= 0) + close(fd); + free(buf); + free(absolute); + path_namespace_clear(before); + path_namespace_clear(after); + strbuf_release(&normalized); + return ret; +} + +static int fingerprint_sources( + const struct attr_fingerprint_source *sources, size_t nr, + const struct git_hash_algo *algo, struct attr_fingerprint *result, + struct attr_source_snapshot *snapshot) +{ + struct git_hash_ctx content_ctx, namespace_ctx; + uint32_t count; + + if (snapshot && nr != ARRAY_SIZE(snapshot->sources)) + BUG("attribute snapshot source count mismatch"); + memset(result, 0, sizeof(*result)); + git_hash_init(&content_ctx, algo); + git_hash_init(&namespace_ctx, algo); + hash_optional_cstring(&content_ctx, "attribute-source-content-v1"); + hash_optional_cstring(&namespace_ctx, + "attribute-source-namespace-v1"); + if (nr > UINT32_MAX) + return -1; + put_be32(&count, nr); + hash_length_delimited(&content_ctx, &count, sizeof(count)); + hash_length_delimited(&namespace_ctx, &count, sizeof(count)); + for (size_t i = 0; i < nr; i++) { + int present; + struct attr_source_snapshot_entry *entry = + snapshot ? &snapshot->sources[i] : NULL; + + if (hash_source(&content_ctx, &namespace_ctx, &sources[i], + &present, entry)) + return -1; + result->sources_present |= present; + } + git_hash_final(result->content_hash, &content_ctx); + git_hash_final(result->namespace_hash, &namespace_ctx); + return 0; +} + +int attr_fingerprint_sources( + const struct attr_fingerprint_source *sources, size_t nr, + const struct git_hash_algo *algo, struct attr_fingerprint *result) +{ + return fingerprint_sources(sources, nr, algo, result, NULL); +} + +static int repository_sources(struct repository *repo, + struct attr_fingerprint_source *sources, + char **info_attributes) +{ + if (getenv(GIT_ATTR_SOURCE_ENVIRONMENT)) + return -1; + sources[0].path = git_attr_system_file(); + sources[0].enabled = git_attr_system_is_enabled(); + sources[1].path = git_attr_global_file(); + sources[1].enabled = 1; + *info_attributes = repo_git_path(repo, INFOATTRIBUTES_FILE); + sources[2].path = *info_attributes; + sources[2].enabled = 1; + return 0; +} + +int attr_fingerprint_repository(struct repository *repo, + struct attr_fingerprint *result) +{ + struct attr_fingerprint_source sources[ATTR_SOURCE_SNAPSHOT_NR]; + char *info_attributes = NULL; + int ret; + + memset(result, 0, sizeof(*result)); + if (repository_sources(repo, sources, &info_attributes)) + return -1; + ret = attr_fingerprint_sources(sources, ARRAY_SIZE(sources), + repo->hash_algo, result); + free(info_attributes); + return ret; +} + +int attr_source_snapshot_repository(struct repository *repo, + struct attr_source_snapshot **result) +{ + struct attr_fingerprint_source sources[ATTR_SOURCE_SNAPSHOT_NR]; + struct attr_source_snapshot *snapshot; + char *info_attributes = NULL; + + if (!result) + BUG("attr_source_snapshot_repository requires an output"); + *result = NULL; + if (repository_sources(repo, sources, &info_attributes)) + return -1; + CALLOC_ARRAY(snapshot, 1); + if (fingerprint_sources(sources, ARRAY_SIZE(sources), repo->hash_algo, + &snapshot->fingerprint, snapshot)) { + attr_source_snapshot_free(snapshot); + free(info_attributes); + return -1; + } + free(info_attributes); + *result = snapshot; + return 0; +} + +int attr_source_snapshot_matches_repository( + struct repository *repo, + const struct attr_source_snapshot *snapshot) +{ + struct attr_fingerprint current; + + return snapshot && + !attr_fingerprint_repository(repo, ¤t) && + current.sources_present == + snapshot->fingerprint.sources_present && + !memcmp(current.content_hash, + snapshot->fingerprint.content_hash, + repo->hash_algo->rawsz); +} + +const struct attr_fingerprint *attr_source_snapshot_fingerprint( + const struct attr_source_snapshot *snapshot) +{ + return snapshot ? &snapshot->fingerprint : NULL; +} + +int attr_source_snapshot_read( + const struct attr_source_snapshot *snapshot, + enum attr_source_snapshot_kind kind, + const char **path, const char **buf, size_t *len) +{ + const struct attr_source_snapshot_entry *source; + + if (!snapshot || kind >= ATTR_SOURCE_SNAPSHOT_NR || + !path || !buf || !len) + BUG("invalid attribute snapshot read"); + source = &snapshot->sources[kind]; + if (!source->buf) + return 0; + *path = source->path; + *buf = source->buf; + *len = source->len; + return 1; +} + +void attr_source_snapshot_free(struct attr_source_snapshot *snapshot) +{ + if (!snapshot) + return; + for (size_t i = 0; i < ARRAY_SIZE(snapshot->sources); i++) { + free(snapshot->sources[i].path); + free(snapshot->sources[i].buf); + } + free(snapshot); +} diff --git a/attr-fingerprint.h b/attr-fingerprint.h new file mode 100644 index 00000000000000..518a5b31b4e485 --- /dev/null +++ b/attr-fingerprint.h @@ -0,0 +1,46 @@ +#ifndef ATTR_FINGERPRINT_H +#define ATTR_FINGERPRINT_H + +#include "hash.h" + +struct repository; + +struct attr_fingerprint_source { + const char *path; + unsigned int enabled : 1; +}; + +struct attr_fingerprint { + unsigned char content_hash[GIT_MAX_RAWSZ]; + unsigned char namespace_hash[GIT_MAX_RAWSZ]; + unsigned int sources_present : 1; +}; + +enum attr_source_snapshot_kind { + ATTR_SOURCE_SNAPSHOT_SYSTEM, + ATTR_SOURCE_SNAPSHOT_GLOBAL, + ATTR_SOURCE_SNAPSHOT_INFO, + ATTR_SOURCE_SNAPSHOT_NR, +}; + +struct attr_source_snapshot; + +int attr_fingerprint_sources( + const struct attr_fingerprint_source *sources, size_t nr, + const struct git_hash_algo *algo, struct attr_fingerprint *result); +int attr_fingerprint_repository(struct repository *repo, + struct attr_fingerprint *result); +int attr_source_snapshot_repository(struct repository *repo, + struct attr_source_snapshot **result); +int attr_source_snapshot_matches_repository( + struct repository *repo, + const struct attr_source_snapshot *snapshot); +const struct attr_fingerprint *attr_source_snapshot_fingerprint( + const struct attr_source_snapshot *snapshot); +int attr_source_snapshot_read( + const struct attr_source_snapshot *snapshot, + enum attr_source_snapshot_kind kind, + const char **path, const char **buf, size_t *len); +void attr_source_snapshot_free(struct attr_source_snapshot *snapshot); + +#endif /* ATTR_FINGERPRINT_H */ diff --git a/attr-manifest.c b/attr-manifest.c new file mode 100644 index 00000000000000..46aed49a430050 --- /dev/null +++ b/attr-manifest.c @@ -0,0 +1,232 @@ +#include "git-compat-util.h" +#include "attr-manifest.h" +#include "environment.h" +#include "read-cache-ll.h" +#include "strbuf.h" + +/* + * A manifest begins with a 32-bit entry count. Each entry contains a 32-bit + * path length, four bytes of source metadata, an object-format hash, and the + * unterminated path. Paths are strictly increasing. + */ +static int attr_manifest_path_valid(const unsigned char *path, size_t len) +{ + const char *base; + char *copy; + int valid; + + if (!len || path[0] == '/' || memchr(path, '\0', len)) + return 0; + copy = xmemdupz(path, len); + base = strrchr(copy, '/'); + base = base ? base + 1 : copy; + valid = !strcmp(base, GITATTRIBUTES_FILE) && + verify_path(copy, S_IFREG | 0644); + free(copy); + return valid; +} + +static int attr_manifest_entry_cmp(const struct attr_manifest_entry *a, + const struct attr_manifest_entry *b) +{ + size_t common = a->path_len < b->path_len ? a->path_len : b->path_len; + int cmp = memcmp(a->path, b->path, common); + + if (cmp) + return cmp; + return a->path_len < b->path_len ? -1 : a->path_len > b->path_len; +} + +static int attr_manifest_entry_equal(const struct attr_manifest_entry *a, + const struct attr_manifest_entry *b, + const struct git_hash_algo *algo) +{ + return !attr_manifest_entry_cmp(a, b) && a->source == b->source && + !memcmp(a->hash, b->hash, algo->rawsz); +} + +void attr_manifest_writer_init(struct attr_manifest_writer *writer, + struct strbuf *buf, + const struct git_hash_algo *algo) +{ + uint32_t count; + + if (!algo) + BUG("attribute manifest requires a hash algorithm"); + memset(writer, 0, sizeof(*writer)); + writer->buf = buf; + writer->algo = algo; + strbuf_reset(buf); + put_be32(&count, 0); + strbuf_add(buf, &count, sizeof(count)); +} + +int attr_manifest_writer_add(struct attr_manifest_writer *writer, + const char *path, + enum attr_manifest_source source, + const unsigned char *hash) +{ + struct attr_manifest_entry previous, current; + unsigned char metadata[4] = { source, 0, 0, 0 }; + uint32_t path_len_be; + size_t entry_offset, path_len = strlen(path); + + if (!writer->buf || !writer->algo || !hash || !path_len || + path_len > UINT32_MAX || writer->nr == UINT32_MAX || + (source != ATTR_MANIFEST_WORKTREE && + source != ATTR_MANIFEST_INDEX) || + !attr_manifest_path_valid((const unsigned char *)path, path_len)) + return -1; + + current.path = (const unsigned char *)path; + current.path_len = path_len; + if (writer->nr) { + previous.path = (const unsigned char *)writer->buf->buf + + writer->last_path_offset; + previous.path_len = writer->last_path_len; + if (attr_manifest_entry_cmp(&previous, ¤t) >= 0) + return -1; + } + + entry_offset = writer->buf->len; + put_be32(&path_len_be, path_len); + strbuf_add(writer->buf, &path_len_be, sizeof(path_len_be)); + strbuf_add(writer->buf, metadata, sizeof(metadata)); + strbuf_add(writer->buf, hash, writer->algo->rawsz); + strbuf_add(writer->buf, path, path_len); + writer->last_path_offset = entry_offset + sizeof(path_len_be) + + sizeof(metadata) + writer->algo->rawsz; + writer->last_path_len = path_len; + put_be32(writer->buf->buf, ++writer->nr); + return 0; +} + +int attr_manifest_cursor_init(struct attr_manifest_cursor *cursor, + const void *data, size_t len, + const struct git_hash_algo *algo) +{ + const unsigned char *bytes = data; + size_t minimum_entry_size; + + if (!algo) + BUG("attribute manifest requires a hash algorithm"); + if (len < sizeof(uint32_t)) + return -1; + minimum_entry_size = sizeof(uint32_t) + 4 + algo->rawsz + 1; + cursor->p = bytes + sizeof(uint32_t); + cursor->end = bytes + len; + cursor->last_path = NULL; + cursor->algo = algo; + cursor->last_path_len = 0; + cursor->remaining = get_be32(bytes); + if (cursor->remaining > + (len - sizeof(uint32_t)) / minimum_entry_size) + return -1; + return 0; +} + +int attr_manifest_cursor_next(struct attr_manifest_cursor *cursor, + struct attr_manifest_entry *entry) +{ + struct attr_manifest_entry previous; + uint32_t path_len; + size_t available; + + if (!cursor->remaining) + return cursor->p == cursor->end ? 0 : -1; + available = cursor->end - cursor->p; + if (available < sizeof(uint32_t) + 4 + cursor->algo->rawsz) + return -1; + path_len = get_be32(cursor->p); + cursor->p += sizeof(uint32_t); + entry->source = cursor->p[0]; + if ((entry->source != ATTR_MANIFEST_WORKTREE && + entry->source != ATTR_MANIFEST_INDEX) || + cursor->p[1] || cursor->p[2] || cursor->p[3]) + return -1; + cursor->p += 4; + entry->hash = cursor->p; + cursor->p += cursor->algo->rawsz; + available = cursor->end - cursor->p; + if (!path_len || available < path_len || + !attr_manifest_path_valid(cursor->p, path_len)) + return -1; + entry->path = cursor->p; + entry->path_len = path_len; + if (cursor->last_path) { + previous.path = cursor->last_path; + previous.path_len = cursor->last_path_len; + if (attr_manifest_entry_cmp(&previous, entry) >= 0) + return -1; + } + cursor->last_path = entry->path; + cursor->last_path_len = entry->path_len; + cursor->p += path_len; + cursor->remaining--; + return 1; +} + +int attr_manifest_valid(const void *data, size_t len, + const struct git_hash_algo *algo) +{ + struct attr_manifest_cursor cursor; + struct attr_manifest_entry entry; + int ret; + + if (attr_manifest_cursor_init(&cursor, data, len, algo)) + return 0; + while ((ret = attr_manifest_cursor_next(&cursor, &entry)) > 0) + ; + return !ret; +} + +int attr_manifest_for_each_changed(const void *old_data, size_t old_len, + const void *new_data, size_t new_len, + const struct git_hash_algo *algo, + attr_manifest_change_fn fn, void *data) +{ + struct attr_manifest_cursor old_cursor, new_cursor; + struct attr_manifest_entry old_entry, new_entry; + int old_ret, new_ret; + + /* + * Callers use this as a transactional change set. Validate both + * streams before allowing the callback to observe any entry. + */ + if (!attr_manifest_valid(old_data, old_len, algo) || + !attr_manifest_valid(new_data, new_len, algo)) + return -1; + if (attr_manifest_cursor_init(&old_cursor, old_data, old_len, algo) || + attr_manifest_cursor_init(&new_cursor, new_data, new_len, algo)) + return -1; + old_ret = attr_manifest_cursor_next(&old_cursor, &old_entry); + new_ret = attr_manifest_cursor_next(&new_cursor, &new_entry); + while (old_ret > 0 || new_ret > 0) { + struct attr_manifest_entry changed; + int has_changed = 1; + int cmp; + + if (old_ret <= 0) + cmp = 1; + else if (new_ret <= 0) + cmp = -1; + else + cmp = attr_manifest_entry_cmp(&old_entry, &new_entry); + if (cmp < 0) { + changed = old_entry; + old_ret = attr_manifest_cursor_next(&old_cursor, &old_entry); + } else if (cmp > 0) { + changed = new_entry; + new_ret = attr_manifest_cursor_next(&new_cursor, &new_entry); + } else { + changed = new_entry; + has_changed = !attr_manifest_entry_equal( + &old_entry, &new_entry, algo); + old_ret = attr_manifest_cursor_next(&old_cursor, &old_entry); + new_ret = attr_manifest_cursor_next(&new_cursor, &new_entry); + } + if (has_changed && fn(&changed, data)) + return -1; + } + return old_ret < 0 || new_ret < 0 ? -1 : 0; +} diff --git a/attr-manifest.h b/attr-manifest.h new file mode 100644 index 00000000000000..a38acccc224832 --- /dev/null +++ b/attr-manifest.h @@ -0,0 +1,59 @@ +#ifndef ATTR_MANIFEST_H +#define ATTR_MANIFEST_H + +#include "hash.h" + +struct strbuf; + +enum attr_manifest_source { + ATTR_MANIFEST_WORKTREE = 1, + ATTR_MANIFEST_INDEX = 2, +}; + +struct attr_manifest_entry { + const unsigned char *path; + uint32_t path_len; + enum attr_manifest_source source; + const unsigned char *hash; +}; + +struct attr_manifest_cursor { + const unsigned char *p; + const unsigned char *end; + const unsigned char *last_path; + const struct git_hash_algo *algo; + uint32_t last_path_len; + uint32_t remaining; +}; + +struct attr_manifest_writer { + struct strbuf *buf; + const struct git_hash_algo *algo; + size_t last_path_offset; + uint32_t last_path_len; + uint32_t nr; +}; + +typedef int (*attr_manifest_change_fn)(const struct attr_manifest_entry *entry, + void *data); + +void attr_manifest_writer_init(struct attr_manifest_writer *writer, + struct strbuf *buf, + const struct git_hash_algo *algo); +int attr_manifest_writer_add(struct attr_manifest_writer *writer, + const char *path, + enum attr_manifest_source source, + const unsigned char *hash); +int attr_manifest_cursor_init(struct attr_manifest_cursor *cursor, + const void *data, size_t len, + const struct git_hash_algo *algo); +int attr_manifest_cursor_next(struct attr_manifest_cursor *cursor, + struct attr_manifest_entry *entry); +int attr_manifest_valid(const void *data, size_t len, + const struct git_hash_algo *algo); +int attr_manifest_for_each_changed(const void *old_data, size_t old_len, + const void *new_data, size_t new_len, + const struct git_hash_algo *algo, + attr_manifest_change_fn fn, void *data); + +#endif /* ATTR_MANIFEST_H */ diff --git a/attr.c b/attr.c index 0e63f1b6de8f53..04f28e119f2361 100644 --- a/attr.c +++ b/attr.c @@ -14,6 +14,7 @@ #include "environment.h" #include "exec-cmd.h" #include "attr.h" +#include "attr-fingerprint.h" #include "dir.h" #include "gettext.h" #include "path.h" @@ -477,6 +478,8 @@ static struct check_vector { pthread_mutex_t mutex; } check_vector; +static const struct attr_source_snapshot *source_snapshot; + static inline void vector_lock(void) { pthread_mutex_lock(&check_vector.mutex); @@ -536,6 +539,31 @@ static void drop_all_attr_stacks(void) vector_unlock(); } +void git_attr_invalidate_all(void) +{ + drop_all_attr_stacks(); +} + +void git_attr_source_snapshot_begin( + const struct attr_source_snapshot *snapshot) +{ + if (!snapshot) + BUG("cannot begin a NULL attribute source snapshot"); + if (source_snapshot) + BUG("attribute source snapshots cannot be nested"); + drop_all_attr_stacks(); + source_snapshot = snapshot; +} + +void git_attr_source_snapshot_end( + const struct attr_source_snapshot *snapshot) +{ + if (!snapshot || source_snapshot != snapshot) + BUG("ending an inactive attribute source snapshot"); + drop_all_attr_stacks(); + source_snapshot = NULL; +} + struct attr_check *attr_check_alloc(void) { struct attr_check *c = xcalloc(1, sizeof(struct attr_check)); @@ -668,6 +696,15 @@ static struct attr_stack *read_attr_from_array(const char **list) return res; } +static void handle_attr_line_buf(struct attr_stack *res, + struct strbuf *line, const char *path, + int *lineno, unsigned flags) +{ + if (!*lineno && starts_with(line->buf, utf8_bom)) + strbuf_remove(line, 0, strlen(utf8_bom)); + handle_attr_line(res, line->buf, path, ++*lineno, flags); +} + /* * Callers into the attribute system assume there is a single, system-wide * global state where attributes are read from and when the state is flipped by @@ -721,17 +758,48 @@ static struct attr_stack *read_attr_from_file(const char *path, unsigned flags) } CALLOC_ARRAY(res, 1); - while (strbuf_getline(&buf, fp) != EOF) { - if (!lineno && starts_with(buf.buf, utf8_bom)) - strbuf_remove(&buf, 0, strlen(utf8_bom)); - handle_attr_line(res, buf.buf, path, ++lineno, flags); - } + while (strbuf_getline(&buf, fp) != EOF) + handle_attr_line_buf(res, &buf, path, &lineno, flags); fclose(fp); strbuf_release(&buf); return res; } +static struct attr_stack *read_attr_from_snapshot( + enum attr_source_snapshot_kind kind, unsigned flags) +{ + struct attr_stack *res; + struct strbuf line = STRBUF_INIT; + const char *path, *buf; + size_t length; + size_t offset = 0; + int lineno = 0; + + if (!source_snapshot) + BUG("attribute source snapshot is not set"); + if (!attr_source_snapshot_read(source_snapshot, kind, + &path, &buf, &length)) + return NULL; + + CALLOC_ARRAY(res, 1); + while (offset < length) { + const char *start = buf + offset; + const char *newline = memchr(start, '\n', length - offset); + size_t len = newline ? (size_t)(newline - start) : + length - offset; + + if (newline && len && start[len - 1] == '\r') + len--; + strbuf_reset(&line); + strbuf_add(&line, start, len); + handle_attr_line_buf(res, &line, path, &lineno, flags); + offset = newline ? (size_t)(newline - buf) + 1 : length; + } + strbuf_release(&line); + return res; +} + static struct attr_stack *read_attr_from_buf(char *buf, size_t length, const char *path, unsigned flags) { @@ -922,13 +990,21 @@ static void bootstrap_attr_stack(struct index_state *istate, push_stack(stack, e, NULL, 0); /* system-wide frame */ - if (git_attr_system_is_enabled()) { + if (source_snapshot) { + e = read_attr_from_snapshot( + ATTR_SOURCE_SNAPSHOT_SYSTEM, flags); + push_stack(stack, e, NULL, 0); + } else if (git_attr_system_is_enabled()) { e = read_attr_from_file(git_attr_system_file(), flags); push_stack(stack, e, NULL, 0); } /* home directory */ - if (git_attr_global_file()) { + if (source_snapshot) { + e = read_attr_from_snapshot( + ATTR_SOURCE_SNAPSHOT_GLOBAL, flags); + push_stack(stack, e, NULL, 0); + } else if (git_attr_global_file()) { e = read_attr_from_file(git_attr_global_file(), flags); push_stack(stack, e, NULL, 0); } @@ -938,7 +1014,9 @@ static void bootstrap_attr_stack(struct index_state *istate, push_stack(stack, e, xstrdup(""), 0); /* info frame */ - if (startup_info->have_repository) + if (source_snapshot) + e = read_attr_from_snapshot(ATTR_SOURCE_SNAPSHOT_INFO, flags); + else if (startup_info->have_repository) e = read_attr_from_file(git_path_info_attributes(), flags); else e = NULL; diff --git a/attr.h b/attr.h index a04a5210921e22..af2ae096d6642a 100644 --- a/attr.h +++ b/attr.h @@ -129,6 +129,7 @@ struct index_state; * `git_attr_name()`. */ struct git_attr; +struct attr_source_snapshot; /* opaque structures used internally for attribute collection */ struct all_attrs_item; @@ -227,6 +228,20 @@ enum git_attr_direction { }; void git_attr_set_direction(enum git_attr_direction new_direction); +/* Discard cached attributes after a provider-wide invalidation. */ +void git_attr_invalidate_all(void); + +/* + * Read system, global, and info attributes from an immutable snapshot. + * begin() and end() must be strictly paired, cannot nest, and the caller must + * keep the snapshot alive until end(). Readers may run concurrently while a + * snapshot is active, but begin() and end() require that there are no readers. + */ +void git_attr_source_snapshot_begin( + const struct attr_source_snapshot *snapshot); +void git_attr_source_snapshot_end( + const struct attr_source_snapshot *snapshot); + void attr_start(void); /* Return the system gitattributes file. */ diff --git a/builtin/add.c b/builtin/add.c index 60ffbede2be58a..b13ba9580cf896 100644 --- a/builtin/add.c +++ b/builtin/add.c @@ -6,6 +6,8 @@ #include "builtin.h" #include "advice.h" +#include "clean-status.h" +#include "clean-status-config.h" #include "config.h" #include "environment.h" #include "lockfile.h" @@ -285,18 +287,20 @@ static struct option builtin_add_options[] = { }; static int add_config(const char *var, const char *value, - const struct config_context *ctx, void *cb) + const struct config_context *ctx, void *data) { + clean_status_config_add(data, var, value, ctx); + if (!strcmp(var, "add.ignoreerrors") || !strcmp(var, "add.ignore-errors")) { ignore_add_errors = git_config_bool(var, value); return 0; } - if (git_color_config(var, value, cb) < 0) + if (git_color_config(var, value, NULL) < 0) return -1; - return git_default_config(var, value, ctx, cb); + return git_default_config(var, value, ctx, NULL); } static const char embedded_advice[] = N_( @@ -384,18 +388,25 @@ int cmd_add(int argc, const char *prefix, struct repository *repo) { + struct clean_status_config_digest clean_digest; int exit_status = 0; struct pathspec pathspec; struct dir_struct dir = DIR_INIT; int flags; int add_new_files; + int preserve_add_history = 0; int require_pathspec; char *seen = NULL; char *ps_matched = NULL; struct lock_file lock_file = LOCK_INIT; struct odb_transaction *transaction; - repo_config(repo, add_config, NULL); + show_usage_with_options_if_asked(argc, argv, + builtin_add_usage, builtin_add_options); + + clean_status_config_init(&clean_digest, repo->hash_algo); + repo_config(repo, add_config, &clean_digest); + clean_status_config_final(&clean_digest); argc = parse_options(argc, argv, prefix, builtin_add_options, builtin_add_usage, PARSE_OPT_KEEP_ARGV0); @@ -492,8 +503,27 @@ int cmd_add(int argc, (!(addremove || take_worktree_changes) ? ADD_CACHE_IGNORE_REMOVAL : 0)); + /* + * The refresh-only path below updates stat data and fsmonitor + * validity, but does not change the logical contents of the index. + * Ordinary add can do the same after an mtime-only change. Ask + * ADD_CACHE_TRACK_CLEAN_HISTORY to invalidate on any persistent + * add/remove decision below. + */ + if (refresh_only) { + clean_status_set_config_digest(repo, &clean_digest); + } else if (!show_only && !intent_to_add && !add_renormalize && + !chmod_arg && !include_sparse && !ignore_add_errors) { + preserve_add_history = 1; + flags |= ADD_CACHE_TRACK_CLEAN_HISTORY; + clean_status_set_config_digest(repo, &clean_digest); + } + if (repo_read_index_preload(repo, &pathspec, 0) < 0) die(_("index file corrupt")); + if (preserve_add_history && + (repo->index->split_index || repo->index->sparse_index)) + clean_status_invalidate_current_proof(repo->index); die_in_unpopulated_submodule(repo->index, prefix); die_path_inside_submodule(repo->index, &pathspec); @@ -603,6 +633,8 @@ int cmd_add(int argc, odb_transaction_commit(transaction); finish: + if (preserve_add_history && exit_status) + clean_status_invalidate_current_proof(repo->index); if (write_locked_index(repo->index, &lock_file, COMMIT_LOCK | SKIP_IF_UNCHANGED)) die(_("unable to write new index file")); diff --git a/builtin/checkout-index.c b/builtin/checkout-index.c index 311b94ff3174a6..1807696b1c92c8 100644 --- a/builtin/checkout-index.c +++ b/builtin/checkout-index.c @@ -8,6 +8,8 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "builtin.h" +#include "clean-status.h" +#include "clean-status-config.h" #include "config.h" #include "environment.h" #include "gettext.h" @@ -30,6 +32,13 @@ static char topath[4][TEMPORARY_FILENAME_LENGTH + 1]; static struct checkout state = CHECKOUT_INIT; +static int checkout_index_config(const char *key, const char *value, + const struct config_context *ctx, void *data) +{ + clean_status_config_add(data, key, value, ctx); + return git_default_config(key, value, ctx, NULL); +} + static void write_tempfile_record(const char *name, const char *prefix) { int i; @@ -215,6 +224,7 @@ int cmd_checkout_index(int argc, const char *prefix, struct repository *repo) { + struct clean_status_config_digest clean_digest; int i; struct lock_file lock_file = LOCK_INIT; int all = 0; @@ -253,7 +263,14 @@ int cmd_checkout_index(int argc, show_usage_with_options_if_asked(argc, argv, builtin_checkout_index_usage, builtin_checkout_index_options); - repo_config(repo, git_default_config, NULL); + clean_status_config_init(&clean_digest, repo->hash_algo); + repo_config(repo, checkout_index_config, &clean_digest); + clean_status_config_final(&clean_digest); + /* + * checkout-index never changes index contents. Keep closed semantic + * history attached when -u writes fresh stat data. + */ + clean_status_set_config_digest(repo, &clean_digest); prefix_length = prefix ? strlen(prefix) : 0; prepare_repo_settings(repo); diff --git a/builtin/checkout.c b/builtin/checkout.c index 55e3a89a852712..c18b8ce85f2a51 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -6,6 +6,8 @@ #include "branch.h" #include "cache-tree.h" #include "checkout.h" +#include "clean-status.h" +#include "clean-status-config.h" #include "commit.h" #include "config.h" #include "diff.h" @@ -46,6 +48,7 @@ #include "add-interactive.h" struct checkout_opts { + struct clean_status_config_digest clean_digest; int patch_mode; int patch_context; int patch_interhunk_context; @@ -142,6 +145,11 @@ static int post_checkout_hook(struct commit *old_commit, struct commit *new_comm return run_hooks_opt(the_repository, "post-checkout", &opt); } +struct tree_checkout_context { + int overlay_mode; + int *index_changed; +}; + /* * Handle a tree object and determine if we need to recurse into the * tree (READ_TREE_RECURSIVE) or skip it (0). @@ -149,11 +157,12 @@ static int post_checkout_hook(struct commit *old_commit, struct commit *new_comm static int try_update_sparse_directory(const struct object_id *oid, struct strbuf *base, const char *pathname, - int overlay_mode) + struct tree_checkout_context *context) { struct strbuf dirpath = STRBUF_INIT; struct cache_entry *old; int pos, result = READ_TREE_RECURSIVE; + int overlay_mode = context ? context->overlay_mode : 1; if (!the_repository->index->sparse_index) return result; @@ -180,6 +189,8 @@ static int try_update_sparse_directory(const struct object_id *oid, * sparse directory OID directly since files not present in * the source tree should be removed anyway. */ + if (context && context->index_changed) + *context->index_changed = 1; oidcpy(&old->oid, oid); old->ce_flags |= CE_UPDATE; result = 0; @@ -196,11 +207,11 @@ static int update_some(const struct object_id *oid, struct strbuf *base, int len; struct cache_entry *ce; int pos; - int overlay_mode = context ? *((int *)context) : 1; + struct tree_checkout_context *checkout_context = context; if (S_ISDIR(mode)) return try_update_sparse_directory(oid, base, pathname, - overlay_mode); + checkout_context); len = base->len + strlen(pathname); ce = make_empty_cache_entry(the_repository->index, len); @@ -228,16 +239,23 @@ static int update_some(const struct object_id *oid, struct strbuf *base, } } + if (checkout_context && checkout_context->index_changed) + *checkout_context->index_changed = 1; add_index_entry(the_repository->index, ce, ADD_CACHE_OK_TO_ADD | ADD_CACHE_OK_TO_REPLACE); return 0; } static int read_tree_some(struct tree *tree, const struct pathspec *pathspec, - int overlay_mode) + int overlay_mode, int *index_changed) { + struct tree_checkout_context context = { + .overlay_mode = overlay_mode, + .index_changed = index_changed, + }; + read_tree(the_repository, tree, - pathspec, update_some, &overlay_mode); + pathspec, update_some, &context); /* update the index with the given tree's info * for all args, expanding wildcards, and exit @@ -420,20 +438,24 @@ static void mark_ce_for_checkout_overlay(struct cache_entry *ce, static void mark_ce_for_checkout_no_overlay(struct cache_entry *ce, char *ps_matched, - const struct checkout_opts *opts) + const struct checkout_opts *opts, + int *index_changed) { ce->ce_flags &= ~CE_MATCHED; if (!opts->ignore_skipworktree && ce_skip_worktree(ce)) return; if (ce_path_match(the_repository->index, ce, &opts->pathspec, ps_matched)) { ce->ce_flags |= CE_MATCHED; - if (opts->source_tree && !(ce->ce_flags & CE_UPDATE)) + if (opts->source_tree && !(ce->ce_flags & CE_UPDATE)) { /* - * In overlay mode, but the path is not in + * In no-overlay mode, but the path is not in * tree-ish, which means we should remove it * from the index and the working tree. */ + if (index_changed) + *index_changed = 1; ce->ce_flags |= CE_REMOVE | CE_WT_REMOVE; + } } } @@ -524,6 +546,8 @@ static int checkout_paths(const struct checkout_opts *opts, int errs = 0; struct lock_file lock_file = LOCK_INIT; int checkout_index; + int preserve_source_tree_history = 0; + int source_tree_index_changed = 0; trace2_cmd_mode(opts->patch_mode ? "patch" : "path"); @@ -628,12 +652,34 @@ static int checkout_paths(const struct checkout_opts *opts, } repo_hold_locked_index(the_repository, &lock_file, LOCK_DIE_ON_ERROR); + /* + * A plain worktree checkout from the index only rewrites stat data. + * A source-tree checkout may do the same when every selected entry + * already matches the index; if not, invalidate its proof below. + * Keep written paths fsmonitor-invalid in either case. Do not do + * this for --merge, which may recreate unmerged index entries from + * resolve undo data. + */ + preserve_source_tree_history = + opts->source_tree && opts->checkout_index && + !opts->merge && !opts->writeout_stage; + if ((opts->checkout_worktree && !opts->source_tree && + !opts->merge && !opts->writeout_stage) || + preserve_source_tree_history) + clean_status_set_config_digest(the_repository, + &opts->clean_digest); if (repo_read_index_preload(the_repository, &opts->pathspec, 0) < 0) return error(_("index file corrupt")); + if (preserve_source_tree_history && + (the_repository->index->split_index || + the_repository->index->sparse_index)) + source_tree_index_changed = 1; if (opts->source_tree) read_tree_some(opts->source_tree, &opts->pathspec, - opts->overlay_mode); + opts->overlay_mode, + preserve_source_tree_history ? + &source_tree_index_changed : NULL); if (opts->merge) unmerge_index(the_repository->index, &opts->pathspec, CE_MATCHED); @@ -651,7 +697,10 @@ static int checkout_paths(const struct checkout_opts *opts, else mark_ce_for_checkout_no_overlay(the_repository->index->cache[pos], ps_matched, - opts); + opts, + preserve_source_tree_history ? + &source_tree_index_changed : + NULL); if (report_path_error(ps_matched, &opts->pathspec)) { free(ps_matched); @@ -698,6 +747,10 @@ static int checkout_paths(const struct checkout_opts *opts, checkout_index = opts->checkout_index; if (checkout_index) { + if (preserve_source_tree_history && + (source_tree_index_changed || errs)) + clean_status_invalidate_current_proof( + the_repository->index); if (write_locked_index(the_repository->index, &lock_file, COMMIT_LOCK)) die(_("unable to write new index file")); } else { @@ -846,6 +899,14 @@ static int merge_working_tree(const struct checkout_opts *opts, struct tree *new_tree; repo_hold_locked_index(the_repository, &lock_file, LOCK_DIE_ON_ERROR); + /* + * A discarding switch may rewrite only worktree/stat state when the + * target tree matches the index. Let unpack_trees() transfer the + * proof only after it proves that the rebuilt index is identical. + */ + if (opts->discard_changes) + clean_status_set_config_digest(the_repository, + &opts->clean_digest); if (repo_read_index_preload(the_repository, NULL, 0) < 0) { rollback_lock_file(&lock_file); return error(_("index file corrupt")); @@ -1282,6 +1343,8 @@ static int git_checkout_config(const char *var, const char *value, { struct checkout_opts *opts = cb; + clean_status_config_add(&opts->clean_digest, var, value, ctx); + if (!strcmp(var, "diff.ignoresubmodules")) { if (!value) return config_error_nonbool(var); @@ -1879,7 +1942,11 @@ static int checkout_main(int argc, const char **argv, const char *prefix, opts->prefix = prefix; opts->show_progress = -1; + show_usage_with_options_if_asked(argc, argv, usagestr, options); + + clean_status_config_init(&opts->clean_digest, the_repository->hash_algo); repo_config(the_repository, git_checkout_config, opts); + clean_status_config_final(&opts->clean_digest); if (the_repository->gitdir) { prepare_repo_settings(the_repository); the_repository->settings.command_requires_full_index = 0; diff --git a/builtin/commit.c b/builtin/commit.c index 28f61745034506..461c7e433b534c 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -13,12 +13,15 @@ #include "config.h" #include "lockfile.h" #include "cache-tree.h" +#include "clean-status.h" +#include "clean-status-index.h" #include "color.h" #include "dir.h" #include "editor.h" #include "environment.h" #include "diff.h" #include "commit.h" +#include "fsmonitor-settings.h" #include "add-interactive.h" #include "gettext.h" #include "revision.h" @@ -205,11 +208,34 @@ static void determine_whence(struct wt_status *s) s->whence = whence; } -static void status_init_config(struct wt_status *s, config_fn_t fn) +struct status_config_callback_data { + struct wt_status *status; + config_fn_t fn; + struct clean_status_config_digest *clean_digest; +}; + +static int status_config_callback(const char *key, const char *value, + const struct config_context *ctx, void *cb) +{ + struct status_config_callback_data *data = cb; + + clean_status_config_add(data->clean_digest, key, value, ctx); + return data->fn(key, value, ctx, data->status); +} + +static void status_init_config_with_clean_digest( + struct wt_status *s, config_fn_t fn, + struct clean_status_config_digest *clean_digest) { + struct status_config_callback_data data = { + .status = s, + .fn = fn, + .clean_digest = clean_digest, + }; + wt_status_prepare(the_repository, s); init_diff_ui_defaults(); - repo_config(the_repository, fn, s); + repo_config(the_repository, status_config_callback, &data); determine_whence(s); s->hints = advice_enabled(ADVICE_STATUS_HINTS); /* must come after repo_config() */ } @@ -349,7 +375,8 @@ static void refresh_cache_or_die(int refresh_flags) } static const char *prepare_index(const char **argv, const char *prefix, - const struct commit *current_head, int is_status) + const struct commit *current_head, int is_status, + struct wt_status *s) { struct string_list partial = STRING_LIST_INIT_DUP; struct pathspec pathspec; @@ -482,7 +509,14 @@ static const char *prepare_index(const char **argv, const char *prefix, if (!only && !pathspec.nr) { repo_hold_locked_index(the_repository, &index_lock, LOCK_DIE_ON_ERROR); - refresh_cache_or_die(refresh_flags); + if (!fstat_is_reliable() || + the_repository->index->split_index || + fsm_settings__get_mode(the_repository) != + FSMONITOR_MODE_IPC) + refresh_cache_or_die(refresh_flags); + else if (wt_status_refresh_index( + s, refresh_flags | REFRESH_IN_PORCELAIN, 0)) + die_resolve_conflict("commit"); if (the_repository->index->cache_changed || !cache_tree_fully_valid(the_repository->index->cache_tree)) cache_tree_update(the_repository->index, WRITE_TREE_SILENT); @@ -773,13 +807,29 @@ static int prepare_to_commit(const char *index_file, const char *prefix, int clean_message_contents = (cleanup_mode != COMMIT_MSG_CLEANUP_NONE); int old_display_comment_prefix; int invoked_hook; + int hook_index_matches = 0; + struct clean_status_index_snapshot hook_index = { .fd = -1 }; /* This checks and barfs if author is badly specified */ determine_author_info(author_ident); - if (!no_verify && run_commit_hook(use_editor, index_file, &invoked_hook, - "pre-commit", NULL)) - return 0; + if (!no_verify) { + int hook_failed = run_commit_hook( + use_editor, index_file, &invoked_hook, + "pre-commit", NULL); + + if (invoked_hook && fstat_is_reliable()) + wt_status_invalidate_refresh(s); + if (invoked_hook && fstat_is_reliable() && + commit_style == COMMIT_AS_IS) + hook_index_matches = + !clean_status_index_snapshot_pin( + &hook_index, the_repository->index); + if (hook_failed) { + clean_status_index_snapshot_release(&hook_index); + return 0; + } + } if (squash_message) { /* @@ -1095,10 +1145,29 @@ static int prepare_to_commit(const char *index_file, const char *prefix, else fputs(_(empty_rebase_pick_advice), stderr); } + clean_status_index_snapshot_release(&hook_index); return 0; } if (!no_verify && invoked_hook) { + struct lock_file refresh_lock = LOCK_INIT; + + /* + * Preserve any strong invalidation recorded while status + * closed the post-hook token. The pinned source prevents this + * write from replacing an index updated by the hook. + */ + if (hook_index_matches && + repo_hold_locked_index(the_repository, &refresh_lock, 0) >= 0) { + if (clean_status_index_snapshot_still_matches( + &hook_index, the_repository->index)) + repo_update_index_if_able( + the_repository, &refresh_lock); + else + rollback_lock_file(&refresh_lock); + } + clean_status_index_snapshot_release(&hook_index); + /* * Re-read the index as the pre-commit-commit hook was invoked * and could have updated it. We must do this before we invoke @@ -1431,7 +1500,7 @@ static int dry_run_commit(const char **argv, const char *prefix, int committable; const char *index_file; - index_file = prepare_index(argv, prefix, current_head, 1); + index_file = prepare_index(argv, prefix, current_head, 1, s); committable = run_status(stdout, index_file, prefix, 0, s); rollback_index_files(); @@ -1542,8 +1611,13 @@ struct repository *repo UNUSED) static int no_renames = -1; static const char *rename_score_arg = (const char *)-1; static struct wt_status s; + struct clean_status_config_digest clean_digest; unsigned int progress_flag = 0; int fd; + int default_status_command = argc == 1 && (!prefix || !*prefix); + int exact_clean_command = argc == 2 && + !strcmp(argv[1], "--porcelain=v2") && (!prefix || !*prefix); + int exact_clean_query; struct object_id oid; static struct option builtin_status_options[] = { OPT__VERBOSE(&verbose, N_("be verbose")), @@ -1605,7 +1679,11 @@ struct repository *repo UNUSED) prepare_repo_settings(the_repository); the_repository->settings.command_requires_full_index = 0; - status_init_config(&s, git_status_config); + clean_status_config_init(&clean_digest, the_repository->hash_algo); + status_init_config_with_clean_digest( + &s, git_status_config, &clean_digest); + clean_status_config_final(&clean_digest); + clean_status_set_config_digest(the_repository, &clean_digest); argc = parse_options(argc, argv, prefix, builtin_status_options, builtin_status_usage, 0); @@ -1622,19 +1700,39 @@ struct repository *repo UNUSED) parse_pathspec(&s.pathspec, 0, PATHSPEC_PREFER_FULL, prefix, argv); + s.allow_clean_status_shortcuts = + default_status_command && !s.pathspec.nr; + exact_clean_query = exact_clean_command && + status_format == STATUS_FORMAT_PORCELAIN_V2 && + !s.pathspec.nr && !s.show_branch && !s.show_stash && + !s.show_ignored_mode && !s.null_termination && !s.verbose && + s.show_untracked_files == SHOW_NORMAL_UNTRACKED_FILES; + s.certify_clean_status = exact_clean_query; + if (exact_clean_query && + clean_status_try_sidecar(the_repository, &clean_digest)) { + wt_status_collect_free_buffers(&s); + return 0; + } if (status_format != STATUS_FORMAT_PORCELAIN && status_format != STATUS_FORMAT_PORCELAIN_V2) progress_flag = REFRESH_PROGRESS; repo_read_index(the_repository); - refresh_index(the_repository->index, - REFRESH_QUIET|REFRESH_UNMERGED|progress_flag, - &s.pathspec, NULL, NULL); + wt_status_start_untracked_cache_preload(&s); + wt_status_refresh_index( + &s, REFRESH_QUIET | REFRESH_UNMERGED | progress_flag | + REFRESH_DEFER_BULK_DIRTY, + s.show_untracked_files != SHOW_NO_UNTRACKED_FILES && + !s.show_ignored_mode); if (use_optional_locks()) fd = repo_hold_locked_index(the_repository, &index_lock, 0); else fd = -1; + s.bulk_update_index_stat = + 0 <= fd && + the_repository->index->preload_bulk_tracked_nr == + the_repository->index->cache_nr; s.is_initial = repo_get_oid(the_repository, s.reference, &oid) ? 1 : 0; if (!s.is_initial) @@ -1654,6 +1752,9 @@ struct repository *repo UNUSED) wt_status_collect(&s); + if (exact_clean_command && 0 <= fd && + clean_status_issue_sidecar(&s, &clean_digest, &index_lock)) + fd = -1; if (0 <= fd) repo_update_index_if_able(the_repository, &index_lock); @@ -1701,6 +1802,7 @@ int cmd_commit(int argc, struct repository *repo UNUSED) { static struct wt_status s; + struct clean_status_config_digest clean_digest; static const char *cleanup_arg = NULL; static struct option builtin_commit_options[] = { OPT__QUIET(&quiet, N_("suppress summary after successful commit")), @@ -1805,7 +1907,11 @@ int cmd_commit(int argc, prepare_repo_settings(the_repository); the_repository->settings.command_requires_full_index = 0; - status_init_config(&s, git_commit_config); + clean_status_config_init(&clean_digest, the_repository->hash_algo); + status_init_config_with_clean_digest( + &s, git_commit_config, &clean_digest); + clean_status_config_final(&clean_digest); + clean_status_set_config_digest(the_repository, &clean_digest); s.commit_template = 1; status_format = STATUS_FORMAT_NONE; /* Ignore status.short */ s.colopts = 0; @@ -1835,7 +1941,7 @@ int cmd_commit(int argc, if (dry_run) return dry_run_commit(argv, prefix, current_head, &s); - index_file = prepare_index(argv, prefix, current_head, 0); + index_file = prepare_index(argv, prefix, current_head, 0, &s); /* Set up everything for writing the commit object. This includes running hooks, writing the trees, and interacting with the user. */ @@ -1845,6 +1951,7 @@ int cmd_commit(int argc, rollback_index_files(); goto cleanup; } + wt_status_collect_free_buffers(&s); /* Determine parents */ reflog_msg = getenv("GIT_REFLOG_ACTION"); @@ -1983,6 +2090,7 @@ int cmd_commit(int argc, NULL, NULL, NULL, NULL); cleanup: + wt_status_collect_free_buffers(&s); free_commit_extra_headers(extra); commit_list_free(parents); strbuf_release(&author_ident); diff --git a/builtin/describe.c b/builtin/describe.c index c0abc931a5948d..b39df0937ecd14 100644 --- a/builtin/describe.c +++ b/builtin/describe.c @@ -2,6 +2,8 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "builtin.h" +#include "clean-status.h" +#include "clean-status-config.h" #include "config.h" #include "environment.h" #include "gettext.h" @@ -598,11 +600,19 @@ static int option_parse_exact_match(const struct option *opt, const char *arg, return 0; } +static int describe_config(const char *key, const char *value, + const struct config_context *ctx, void *data) +{ + clean_status_config_add(data, key, value, ctx); + return git_default_config(key, value, ctx, NULL); +} + int cmd_describe(int argc, const char **argv, const char *prefix, struct repository *repo UNUSED ) { + struct clean_status_config_digest clean_digest; struct refs_for_each_ref_options for_each_ref_opts = { .flags = REFS_FOR_EACH_INCLUDE_BROKEN, }; @@ -647,7 +657,11 @@ int cmd_describe(int argc, OPT_END(), }; - repo_config(the_repository, git_default_config, NULL); + show_usage_with_options_if_asked(argc, argv, describe_usage, options); + + clean_status_config_init(&clean_digest, the_repository->hash_algo); + repo_config(the_repository, describe_config, &clean_digest); + clean_status_config_final(&clean_digest); argc = parse_options(argc, argv, prefix, options, describe_usage, 0); if (abbrev < 0) abbrev = DEFAULT_ABBREV; @@ -761,6 +775,12 @@ int cmd_describe(int argc, setup_work_tree(the_repository); prepare_repo_settings(the_repository); the_repository->settings.command_requires_full_index = 0; + /* + * The in-process dirty check only refreshes stat + * data before comparing the worktree with HEAD. + */ + clean_status_set_config_digest(the_repository, + &clean_digest); repo_read_index(the_repository); refresh_index(the_repository->index, REFRESH_QUIET|REFRESH_UNMERGED, NULL, NULL, NULL); diff --git a/builtin/fsmonitor--daemon.c b/builtin/fsmonitor--daemon.c index 4161dd82825b4c..953f68b4fc1185 100644 --- a/builtin/fsmonitor--daemon.c +++ b/builtin/fsmonitor--daemon.c @@ -42,9 +42,15 @@ static int fsmonitor__start_timeout_sec = 60; #define FSMONITOR__ANNOUNCE_STARTUP "fsmonitor.announcestartup" static int fsmonitor__announce_startup = 0; +struct fsmonitor_config_data { + unsigned int ignore_start_timeout : 1; +}; + static int fsmonitor_config(const char *var, const char *value, const struct config_context *ctx, void *cb) { + struct fsmonitor_config_data *data = cb; + if (!strcmp(var, FSMONITOR__IPC_THREADS)) { int i = git_config_int(var, value, ctx->kvi); if (i < 1) @@ -55,7 +61,12 @@ static int fsmonitor_config(const char *var, const char *value, } if (!strcmp(var, FSMONITOR__START_TIMEOUT)) { - int i = git_config_int(var, value, ctx->kvi); + int i; + + /* The run process does not consume this client-only setting. */ + if (data && data->ignore_start_timeout) + return 0; + i = git_config_int(var, value, ctx->kvi); if (i < 0) return error(_("value of '%s' out of range: %d"), FSMONITOR__START_TIMEOUT, i); @@ -73,7 +84,7 @@ static int fsmonitor_config(const char *var, const char *value, return 0; } - return git_default_config(var, value, ctx, cb); + return git_default_config(var, value, ctx, NULL); } /* @@ -694,18 +705,48 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, int do_trivial = 0; int do_flush = 0; int do_cookie = 0; + int invalid_binding = 0; enum fsmonitor_cookie_item_result cookie_result; + if (strcmp(command, "quit") && + strcmp(command, "flush") && + strcmp(command, FSMONITOR_IPC_CAPABILITY_COMMAND)) { + const char *identity; + const char *query; + + if (!skip_prefix(command, FSMONITOR_IPC_QUERY_PREFIX, + &identity) || + !(query = strchr(identity, '\n')) || + query - identity != FSMONITOR_IPC_WORKTREE_ID_HEX || + state->worktree_identity.len != FSMONITOR_IPC_WORKTREE_ID_HEX || + memcmp(identity, state->worktree_identity.buf, + FSMONITOR_IPC_WORKTREE_ID_HEX)) { + invalid_binding = 1; + trace2_data_intmax("fsmonitor", the_repository, + "query/worktree-mismatch", 1); + } else { + command = query + 1; + } + } + /* * We expect `command` to be of the form: * - * := quit NUL + * := get-capabilities NUL + * | quit NUL * | flush NUL * | NUL * | NUL */ - if (!strcmp(command, "quit")) { + if (!strcmp(command, FSMONITOR_IPC_CAPABILITY_COMMAND)) { + static const char capabilities[] = + FSMONITOR_IPC_QUERY_VERSION "\n"; + + return reply(reply_data, capabilities, + sizeof(capabilities) - 1); + + } else if (!strcmp(command, "quit")) { /* * A client has requested over the socket/pipe that the * daemon shutdown. @@ -728,6 +769,11 @@ static int do_handle_client(struct fsmonitor_daemon_state *state, do_flush = 1; do_trivial = 1; + } else if (invalid_binding) { + /* Never trust a token from an unbound or different worktree. */ + do_trivial = 1; + do_cookie = 1; + } else if (!skip_prefix(command, "builtin:", &p)) { /* assume V1 timestamp or garbage */ @@ -1311,6 +1357,12 @@ static int fsmonitor_run_daemon(void) strbuf_init(&state.path_worktree_watch, 0); strbuf_addstr(&state.path_worktree_watch, absolute_path(repo_get_work_tree(the_repository))); + strbuf_init(&state.worktree_identity, 0); + if (fsmonitor_ipc__get_worktree_identity(the_repository, + &state.worktree_identity)) { + err = error(_("could not identify worktree root")); + goto done; + } state.nr_paths_watching = 1; strbuf_init(&state.alias.alias, 0); @@ -1437,6 +1489,7 @@ static int fsmonitor_run_daemon(void) ipc_server_free(state.ipc_server_data); strbuf_release(&state.path_worktree_watch); + strbuf_release(&state.worktree_identity); strbuf_release(&state.path_gitdir_watch); strbuf_release(&state.path_cookie_prefix); strbuf_release(&state.path_ipc); @@ -1570,6 +1623,9 @@ int cmd_fsmonitor__daemon(int argc, const char *prefix, struct repository *repo UNUSED) { + struct fsmonitor_config_data config_data = { + .ignore_start_timeout = argc > 1 && !strcmp(argv[1], "run"), + }; const char *subcmd; enum fsmonitor_reason reason; int detach_console = 0; @@ -1586,7 +1642,7 @@ int cmd_fsmonitor__daemon(int argc, OPT_END() }; - repo_config(the_repository, fsmonitor_config, NULL); + repo_config(the_repository, fsmonitor_config, &config_data); argc = parse_options(argc, argv, prefix, options, builtin_fsmonitor__daemon_usage, 0); diff --git a/builtin/gc.c b/builtin/gc.c index 49c8474fade8ed..e9572940dcb22d 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -1593,7 +1593,8 @@ static int maintenance_task_geometric_repack(struct maintenance_run_opts *opts, child.odb_to_close = the_repository->objects; strvec_pushl(&child.args, "repack", "-d", "-l", NULL); - if (geometry.split < geometry.pack_nr) + if (geometry.split < geometry.pack_nr || + geometry.promisor_split < geometry.promisor_pack_nr) strvec_pushf(&child.args, "--geometric=%d", geometry.split_factor); else @@ -1648,7 +1649,7 @@ static int geometric_repack_auto_condition(struct gc_config *cfg UNUSED) * When we'd merge at least two packs with one another we always * perform the repack. */ - if (geometry.split) { + if (geometry.split || geometry.promisor_split) { ret = 1; goto out; } diff --git a/builtin/read-tree.c b/builtin/read-tree.c index 999a82ecdfd737..8e3b023271723c 100644 --- a/builtin/read-tree.c +++ b/builtin/read-tree.c @@ -5,6 +5,8 @@ */ #define USE_THE_REPOSITORY_VARIABLE #include "builtin.h" +#include "clean-status.h" +#include "clean-status-config.h" #include "config.h" #include "environment.h" #include "gettext.h" @@ -46,10 +48,11 @@ static const char * const read_tree_usage[] = { NULL }; -static int index_output_cb(const struct option *opt UNUSED, const char *arg, +static int index_output_cb(const struct option *opt, const char *arg, int unset) { BUG_ON_OPT_NEG(unset); + *(int *)opt->value = 1; set_alternate_index_output(arg); return 0; } @@ -100,12 +103,14 @@ static int debug_merge(const struct cache_entry * const *stages, } static int git_read_tree_config(const char *var, const char *value, - const struct config_context *ctx, void *cb) + const struct config_context *ctx, void *data) { + clean_status_config_add(data, var, value, ctx); + if (!strcmp(var, "submodule.recurse")) - return git_default_submodule_config(var, value, cb); + return git_default_submodule_config(var, value, NULL); - return git_default_config(var, value, ctx, cb); + return git_default_config(var, value, ctx, NULL); } int cmd_read_tree(int argc, @@ -113,7 +118,10 @@ int cmd_read_tree(int argc, const char *cmd_prefix, struct repository *repo UNUSED) { + struct clean_status_config_digest clean_digest; int i, stage = 0; + int index_output = 0; + int preserve_history = 0; struct object_id oid; struct tree_desc t[MAX_UNPACK_TREES]; struct unpack_trees_options opts; @@ -121,7 +129,7 @@ int cmd_read_tree(int argc, struct lock_file lock_file = LOCK_INIT; const struct option read_tree_options[] = { OPT__SUPER_PREFIX(&opts.super_prefix), - OPT_CALLBACK_F(0, "index-output", NULL, N_("file"), + OPT_CALLBACK_F(0, "index-output", &index_output, N_("file"), N_("write resulting index to "), PARSE_OPT_NONEG, index_output_cb), OPT_BOOL(0, "empty", &read_empty, @@ -169,7 +177,12 @@ int cmd_read_tree(int argc, opts.src_index = the_repository->index; opts.dst_index = the_repository->index; - repo_config(the_repository, git_read_tree_config, NULL); + show_usage_with_options_if_asked(argc, argv, + read_tree_usage, read_tree_options); + + clean_status_config_init(&clean_digest, the_repository->hash_algo); + repo_config(the_repository, git_read_tree_config, &clean_digest); + clean_status_config_final(&clean_digest); argc = parse_options(argc, argv, cmd_prefix, read_tree_options, read_tree_usage, 0); @@ -190,6 +203,23 @@ int cmd_read_tree(int argc, repo_hold_locked_index(the_repository, &lock_file, LOCK_DIE_ON_ERROR); + /* + * A one-tree merge or reset can rewrite only stat and fsmonitor + * state when its tree matches the existing index. Attach history + * before reading that index; unpack_trees() will transfer it only + * after proving that the result has the same logical entries. + */ + if (argc == 1 && !read_empty && !opts.prefix && + (opts.reset || opts.merge) && + !opts.dry_run && + !opts.skip_sparse_checkout && !opts.internal.debug_unpack && + !opts.trivial_merges_only && !opts.aggressive && + !opts.super_prefix && !index_output && + !should_update_submodules()) { + preserve_history = 1; + clean_status_set_config_digest(the_repository, &clean_digest); + } + /* * NEEDSWORK * @@ -200,10 +230,17 @@ int cmd_read_tree(int argc, */ if (opts.reset || opts.merge || opts.prefix) { - if (repo_read_index_unmerged(the_repository) && (opts.prefix || opts.merge)) + int unmerged = repo_read_index_unmerged(the_repository); + + if (preserve_history && unmerged) + clean_status_invalidate_current_proof( + the_repository->index); + if (unmerged && (opts.prefix || opts.merge)) die(_("You need to resolve your current index first")); stage = opts.merge = 1; } + if (preserve_history && the_repository->index->resolve_undo) + clean_status_invalidate_current_proof(the_repository->index); resolve_undo_clear_index(the_repository->index); for (i = 0; i < argc; i++) { diff --git a/builtin/reset.c b/builtin/reset.c index 78e69bd84ba2c3..dfa0b3624c792a 100644 --- a/builtin/reset.c +++ b/builtin/reset.c @@ -12,6 +12,8 @@ #include "builtin.h" #include "advice.h" +#include "clean-status.h" +#include "clean-status-config.h" #include "config.h" #include "environment.h" #include "gettext.h" @@ -325,12 +327,14 @@ static int reset_refs(const char *rev, const struct object_id *oid) } static int git_reset_config(const char *var, const char *value, - const struct config_context *ctx, void *cb) + const struct config_context *ctx, void *data) { + clean_status_config_add(data, var, value, ctx); + if (!strcmp(var, "submodule.recurse")) - return git_default_submodule_config(var, value, cb); + return git_default_submodule_config(var, value, NULL); - return git_default_config(var, value, ctx, cb); + return git_default_config(var, value, ctx, NULL); } int cmd_reset(int argc, @@ -338,9 +342,11 @@ int cmd_reset(int argc, const char *prefix, struct repository *repo UNUSED) { + struct clean_status_config_digest clean_digest; int reset_type = NONE, update_ref_status = 0, quiet = 0; int no_refresh = 0; int patch_mode = 0, pathspec_file_nul = 0, unborn; + int preserve_mixed_history = 0; const char *rev; char *pathspec_from_file = NULL; struct object_id oid; @@ -382,7 +388,11 @@ int cmd_reset(int argc, OPT_END() }; - repo_config(the_repository, git_reset_config, NULL); + show_usage_with_options_if_asked(argc, argv, git_reset_usage, options); + + clean_status_config_init(&clean_digest, the_repository->hash_algo); + repo_config(the_repository, git_reset_config, &clean_digest); + clean_status_config_final(&clean_digest); argc = parse_options(argc, argv, prefix, options, git_reset_usage, PARSE_OPT_KEEP_DASHDASH); @@ -477,6 +487,20 @@ int cmd_reset(int argc, if (intent_to_add && reset_type != MIXED) die(_("the option '%s' requires '%s'"), "-N", "--mixed"); + /* + * A no-path mixed or hard reset is a candidate for a stat-only + * rewrite even when its target commit differs from HEAD. Attach + * history early enough for the initial index read. Mixed reset + * checks its in-place result below; hard reset lets unpack_trees() + * transfer only an equal logical index. + */ + if ((reset_type == MIXED || reset_type == HARD) && + !pathspec.nr && !intent_to_add && + !unborn) { + preserve_mixed_history = reset_type == MIXED; + clean_status_set_config_digest(the_repository, &clean_digest); + } + if (repo_read_index(the_repository) < 0) die(_("index file corrupt")); @@ -496,6 +520,14 @@ int cmd_reset(int argc, update_ref_status = 1; goto cleanup; } + if (preserve_mixed_history && + (the_repository->index->split_index || + the_repository->index->sparse_index || + (the_repository->index->cache_changed & + (CE_ENTRY_CHANGED | CE_ENTRY_REMOVED | + CE_ENTRY_ADDED | RESOLVE_UNDO_CHANGED)))) + clean_status_invalidate_current_proof( + the_repository->index); the_repository->index->updated_skipworktree = 1; if (!no_refresh && repo_get_work_tree(the_repository)) { uint64_t t_begin, t_delta_in_ms; diff --git a/builtin/stash.c b/builtin/stash.c index 72c52571f8c06c..4fd7ec0c6258ad 100644 --- a/builtin/stash.c +++ b/builtin/stash.c @@ -2,6 +2,8 @@ #include "builtin.h" #include "abspath.h" +#include "clean-status.h" +#include "clean-status-config.h" #include "config.h" #include "environment.h" #include "gettext.h" @@ -150,6 +152,7 @@ static int show_stat = 1; static int show_patch; static int show_include_untracked; static int use_index; +static struct clean_status_config_digest stash_clean_digest; /* * w_commit is set to the commit containing the working tree @@ -975,6 +978,8 @@ static int list_stash(int argc, const char **argv, const char *prefix, static int git_stash_config(const char *var, const char *value, const struct config_context *ctx, void *cb) { + clean_status_config_add(cb, var, value, ctx); + if (!strcmp(var, "stash.showstat")) { show_stat = git_config_bool(var, value); return 0; @@ -991,7 +996,7 @@ static int git_stash_config(const char *var, const char *value, use_index = git_config_bool(var, value); return 0; } - return git_diff_basic_config(var, value, ctx, cb); + return git_diff_basic_config(var, value, ctx, NULL); } static void diff_include_untracked(const struct stash_info *info, struct diff_options *diff_opt) @@ -1671,6 +1676,7 @@ static int do_push_stash(const struct pathspec *ps, const char *stash_msg, int q int include_untracked, int only_staged) { int ret = 0; + int preserve_clean_history = !ps->nr && !include_untracked; struct stash_info info = STASH_INFO_INIT; struct strbuf patch = STRBUF_INIT; struct strbuf stash_msg_buf = STRBUF_INIT; @@ -1698,6 +1704,16 @@ static int do_push_stash(const struct pathspec *ps, const char *stash_msg, int q goto done; } + /* + * A clean stash push returns after its initial stat refresh. Keep + * that rewrite bound only for whole-worktree forms; paths and + * untracked discovery can change the index or its status inputs. + * If changes are found below, invalidate before the real stash + * machinery mutates the index or worktree. + */ + if (preserve_clean_history) + clean_status_set_config_digest(the_repository, + &stash_clean_digest); repo_read_index_preload(the_repository, NULL, 0); if (!include_untracked && ps->nr) { char *ps_matched = xcalloc(ps->nr, 1); @@ -1728,6 +1744,8 @@ static int do_push_stash(const struct pathspec *ps, const char *stash_msg, int q printf_ln(_("No local changes to save")); goto done; } + if (preserve_clean_history) + clean_status_invalidate_current_proof(the_repository->index); if (!refs_reflog_exists(get_main_ref_store(the_repository), ref_stash) && do_clear_stash()) { ret = -1; @@ -2478,7 +2496,12 @@ int cmd_stash(int argc, const char **args_copy; int ret; - repo_config(the_repository, git_stash_config, NULL); + show_usage_with_options_if_asked(argc, argv, git_stash_usage, options); + + clean_status_config_init(&stash_clean_digest, + the_repository->hash_algo); + repo_config(the_repository, git_stash_config, &stash_clean_digest); + clean_status_config_final(&stash_clean_digest); argc = parse_options(argc, argv, prefix, options, git_stash_usage, PARSE_OPT_SUBCOMMAND_OPTIONAL | diff --git a/builtin/update-index.c b/builtin/update-index.c index 241abd4332dcf9..b8b565f0d6f632 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -8,6 +8,8 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "builtin.h" +#include "clean-status.h" +#include "clean-status-config.h" #include "config.h" #include "environment.h" #include "gettext.h" @@ -54,6 +56,28 @@ static int ignore_skip_worktree_entries; #define UNMARK_FLAG 2 static struct strbuf mtime_dir = STRBUF_INIT; +static int update_index_config(const char *key, const char *value, + const struct config_context *ctx, void *data) +{ + clean_status_config_add(data, key, value, ctx); + return git_default_config(key, value, ctx, NULL); +} + +static int is_proof_preserving_rewrite(int argc, const char **argv) +{ + if (argc == 2) + return !strcmp(argv[1], "--refresh") || + !strcmp(argv[1], "--force-write-index"); + + if (argc != 3) + return 0; + + return (!strcmp(argv[1], "--refresh") && + !strcmp(argv[2], "--force-write-index")) || + (!strcmp(argv[1], "--force-write-index") && + !strcmp(argv[2], "--refresh")); +} + /* Untracked cache mode */ enum uc_mode { UC_UNSPECIFIED = -1, @@ -917,6 +941,7 @@ int cmd_update_index(int argc, const char *prefix, struct repository *repo UNUSED) { + struct clean_status_config_digest clean_digest; int newfd, entries, has_errors = 0, nul_term_line = 0; enum uc_mode untracked_cache = UC_UNSPECIFIED; int read_from_stdin = 0; @@ -932,6 +957,8 @@ int cmd_update_index(int argc, struct parse_opt_ctx_t ctx; strbuf_getline_fn getline_fn; int parseopt_state = PARSE_OPT_UNKNOWN; + int preserve_clean_history = + is_proof_preserving_rewrite(argc, argv); struct repository *r = the_repository; struct odb_transaction *transaction; struct option options[] = { @@ -1097,7 +1124,20 @@ int cmd_update_index(int argc, show_usage_with_options_if_asked(argc, argv, update_index_usage, options); - repo_config(the_repository, git_default_config, NULL); + if (preserve_clean_history) { + clean_status_config_init(&clean_digest, + the_repository->hash_algo); + repo_config(the_repository, update_index_config, + &clean_digest); + clean_status_config_final(&clean_digest); + /* + * These exact forms can refresh stat data, or no data at all, + * but cannot change the logical contents of the index. + */ + clean_status_set_config_digest(the_repository, &clean_digest); + } else { + repo_config(the_repository, git_default_config, NULL); + } prepare_repo_settings(r); the_repository->settings.command_requires_full_index = 0; diff --git a/cache-tree.c b/cache-tree.c index d92f5132865f13..c811e23b14705b 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -676,20 +676,34 @@ static struct cache_tree *read_one(const char **buffer, unsigned long *size_p) /* * Just a heuristic -- we do not add directories that often but * we do not want to have to extend it immediately when we do, - * hence +2. + * hence +2. Avoid a separate allocation for the common leaf case. */ - it->subtree_alloc = subtree_nr + 2; - CALLOC_ARRAY(it->down, it->subtree_alloc); + if (subtree_nr) { + it->subtree_alloc = subtree_nr + 2; + ALLOC_ARRAY(it->down, it->subtree_alloc); + } for (i = 0; i < subtree_nr; i++) { /* read each subtree */ struct cache_tree *sub; struct cache_tree_sub *subtree; const char *name = buf; + int namelen; sub = read_one(&buf, &size); if (!sub) goto free_return; - subtree = cache_tree_sub(it, name); + namelen = strlen(name); + if (!it->subtree_nr || + subtree_name_cmp(it->down[it->subtree_nr - 1]->name, + it->down[it->subtree_nr - 1]->namelen, + name, namelen) < 0) { + FLEX_ALLOC_MEM(subtree, name, name, namelen); + subtree->namelen = namelen; + it->down[it->subtree_nr++] = subtree; + } else { + /* Be liberal in what we accept from older writers. */ + subtree = cache_tree_sub(it, name); + } subtree->cache_tree = sub; } if (subtree_nr != it->subtree_nr) diff --git a/clean-status-config.c b/clean-status-config.c new file mode 100644 index 00000000000000..44481510bcc1c5 --- /dev/null +++ b/clean-status-config.c @@ -0,0 +1,106 @@ +#include "git-compat-util.h" +#include "clean-status-config.h" +#include "config.h" +#include "hash-framing.h" +#include "repository.h" +#include "strbuf.h" + +void clean_status_config_init(struct clean_status_config_digest *digest, + const struct git_hash_algo *algo) +{ + if (!algo) + BUG("clean-status config digest requires a hash algorithm"); + memset(digest, 0, sizeof(*digest)); + git_hash_init(&digest->ctx, algo); + git_hash_init(&digest->semantic_ctx, algo); + /* Invalidate proofs written before multiply-linked files stayed dirty. */ + hash_optional_cstring(&digest->ctx, + "clean-status-config-hardlink-v1"); + digest->initialized = 1; +} + +static void hash_config_entry(struct git_hash_ctx *ctx, + const char *key, const char *value, + const struct config_context *config_ctx) +{ + uint32_t metadata[2] = { 0 }; + + hash_optional_cstring(ctx, key); + hash_optional_cstring(ctx, value); + if (config_ctx && config_ctx->kvi) { + put_be32(&metadata[0], config_ctx->kvi->scope); + put_be32(&metadata[1], config_ctx->kvi->origin_type); + hash_length_delimited(ctx, metadata, sizeof(metadata)); + hash_optional_cstring(ctx, config_ctx->kvi->filename); + } else { + hash_length_delimited(ctx, metadata, sizeof(metadata)); + hash_optional_cstring(ctx, NULL); + } +} + +static void hash_effective_config_entry(struct git_hash_ctx *ctx, + const char *key, + const char *value) +{ + hash_optional_cstring(ctx, key); + hash_optional_cstring(ctx, value); +} + +void clean_status_config_add(struct clean_status_config_digest *digest, + const char *key, const char *value, + const struct config_context *ctx) +{ + const char *suffix; + int semantic; + + if (!digest->initialized || digest->finalized) + BUG("invalid clean-status config digest state"); + hash_config_entry(&digest->ctx, key, value, ctx); + semantic = !strcmp(key, "core.autocrlf") || + !strcmp(key, "core.eol") || + !strcmp(key, "core.checkroundtripencoding"); + if (skip_prefix(key, "filter.", &suffix) && + (ends_with(suffix, ".clean") || ends_with(suffix, ".process") || + ends_with(suffix, ".required"))) { + digest->unsafe_filter = 1; + semantic = 1; + } + if (semantic) { + hash_effective_config_entry(&digest->semantic_ctx, key, value); + digest->semantic_config_explicit = 1; + } +} + +void clean_status_config_final(struct clean_status_config_digest *digest) +{ + if (!digest->initialized || digest->finalized) + BUG("invalid clean-status config digest state"); + git_hash_final(digest->hash, &digest->ctx); + git_hash_final(digest->semantic_hash, &digest->semantic_ctx); + digest->finalized = 1; +} + +static int config_digest_callback(const char *key, const char *value, + const struct config_context *ctx, + void *data) +{ + clean_status_config_add(data, key, value, ctx); + return 0; +} + +int clean_status_config_read_repository( + struct repository *repo, + struct clean_status_config_digest *digest) +{ + struct config_options opts = { 0 }; + + clean_status_config_init(digest, repo->hash_algo); + opts.respect_includes = 1; + opts.commondir = repo->commondir; + opts.git_dir = repo->gitdir; + if (config_with_options(config_digest_callback, digest, NULL, + repo, &opts) < 0) + return -1; + clean_status_config_final(digest); + return 0; +} diff --git a/clean-status-config.h b/clean-status-config.h new file mode 100644 index 00000000000000..1ca173547a1154 --- /dev/null +++ b/clean-status-config.h @@ -0,0 +1,30 @@ +#ifndef CLEAN_STATUS_CONFIG_H +#define CLEAN_STATUS_CONFIG_H + +#include "hash.h" + +struct config_context; +struct repository; + +struct clean_status_config_digest { + struct git_hash_ctx ctx; + struct git_hash_ctx semantic_ctx; + unsigned char hash[GIT_MAX_RAWSZ]; + unsigned char semantic_hash[GIT_MAX_RAWSZ]; + unsigned initialized : 1; + unsigned finalized : 1; + unsigned unsafe_filter : 1; + unsigned semantic_config_explicit : 1; +}; + +void clean_status_config_init(struct clean_status_config_digest *digest, + const struct git_hash_algo *algo); +void clean_status_config_add(struct clean_status_config_digest *digest, + const char *key, const char *value, + const struct config_context *ctx); +void clean_status_config_final(struct clean_status_config_digest *digest); +int clean_status_config_read_repository( + struct repository *repo, + struct clean_status_config_digest *digest); + +#endif /* CLEAN_STATUS_CONFIG_H */ diff --git a/clean-status-epoch.c b/clean-status-epoch.c new file mode 100644 index 00000000000000..1257a455bf94c1 --- /dev/null +++ b/clean-status-epoch.c @@ -0,0 +1,210 @@ +#include "git-compat-util.h" +#include "attr-fingerprint.h" +#include "clean-status.h" +#include "clean-status-index.h" +#include "clean-status-internal.h" +#include "fsmonitor-clean-proof.h" +#include "fsmonitor-ll.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "trace2.h" + +/* + * State captured before a worktree scan. Every recorded input must still + * match after the closing provider query before scan results are accepted. + */ +struct clean_status_proof_epoch { + struct index_state *istate; + struct clean_status_index_snapshot index; + char *scan_start_token; + unsigned char config_hash[GIT_MAX_RAWSZ]; + unsigned char semantic_hash[GIT_MAX_RAWSZ]; + unsigned char attr_hash[GIT_MAX_RAWSZ]; + unsigned char attr_namespace_hash[GIT_MAX_RAWSZ]; + unsigned char manifest_hash[GIT_MAX_RAWSZ]; + uint32_t manifest_flags; + unsigned semantic_explicit : 1; + unsigned attr_sources_present : 1; + unsigned strong_mismatch : 1; + unsigned config_mismatch : 1; +}; + +static int config_matches_epoch( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch) +{ + struct clean_status_state *state = istate->clean_status; + struct clean_status_config_digest digest; + const struct git_hash_algo *algo = istate->repo->hash_algo; + + if (clean_status_config_read_repository(istate->repo, &digest)) + return 0; + return digest.finalized && !digest.unsafe_filter && + digest.semantic_config_explicit == epoch->semantic_explicit && + !memcmp(digest.hash, epoch->config_hash, algo->rawsz) && + !memcmp(digest.semantic_hash, epoch->semantic_hash, algo->rawsz) && + state && state->current_config_valid && + state->current_semantic_valid && + !memcmp(state->current_config_hash, epoch->config_hash, + algo->rawsz) && + !memcmp(state->current_semantic_hash, epoch->semantic_hash, + algo->rawsz); +} + +struct clean_status_proof_epoch *clean_status_capture_proof_epoch( + struct index_state *istate, + const struct attr_source_snapshot *attrs) +{ + struct clean_status_state *state = istate->clean_status; + struct clean_status_proof_epoch *epoch; + struct clean_status_config_digest digest; + struct clean_status_index_snapshot index; + const struct attr_fingerprint *fingerprint = + attr_source_snapshot_fingerprint(attrs); + uint32_t manifest_requirements = + FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | + FSMONITOR_CLEAN_PROOF_FULL_INDEX; + + if (istate->split_index || !state || !state->current_config_valid || + !state->config_enforced || + !state->current_semantic_valid || !state->current_attr_valid || + !state->manifest.current_valid || !state->manifest.checked || + state->manifest.global_fallback || state->unsafe_filter || + (state->manifest.current_flags & manifest_requirements) != + manifest_requirements || + !fsmonitor_pending_token_from_provider(istate) || + !istate->fsmonitor_last_update_pending || !fingerprint || + memcmp(fingerprint->content_hash, state->current_attr_hash, + istate->repo->hash_algo->rawsz) || + memcmp(fingerprint->namespace_hash, + state->current_attr_namespace_hash, + istate->repo->hash_algo->rawsz) || + fingerprint->sources_present != + state->current_attr_sources_present) + return NULL; + if (clean_status_config_read_repository(istate->repo, &digest) || + digest.unsafe_filter || !digest.finalized || + digest.semantic_config_explicit != + state->current_semantic_explicit || + memcmp(digest.hash, state->current_config_hash, + istate->repo->hash_algo->rawsz) || + memcmp(digest.semantic_hash, state->current_semantic_hash, + istate->repo->hash_algo->rawsz) || + clean_status_index_snapshot_pin(&index, istate)) + return NULL; + + CALLOC_ARRAY(epoch, 1); + epoch->istate = istate; + epoch->index = index; + epoch->scan_start_token = xstrdup(istate->fsmonitor_last_update_pending); + memcpy(epoch->config_hash, state->current_config_hash, + istate->repo->hash_algo->rawsz); + memcpy(epoch->semantic_hash, state->current_semantic_hash, + istate->repo->hash_algo->rawsz); + memcpy(epoch->attr_hash, state->current_attr_hash, + istate->repo->hash_algo->rawsz); + memcpy(epoch->attr_namespace_hash, + state->current_attr_namespace_hash, + istate->repo->hash_algo->rawsz); + memcpy(epoch->manifest_hash, state->manifest.current_hash, + istate->repo->hash_algo->rawsz); + epoch->manifest_flags = state->manifest.current_flags; + epoch->semantic_explicit = state->current_semantic_explicit; + epoch->attr_sources_present = state->current_attr_sources_present; + epoch->strong_mismatch = state->strong_mismatch; + epoch->config_mismatch = state->config_mismatch; + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/proof-epoch-captured", 1); + return epoch; +} + +int clean_status_proof_epoch_start_token_matches( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch) +{ + return epoch && epoch->istate == istate && epoch->scan_start_token && + fsmonitor_pending_token_from_provider(istate) && + istate->fsmonitor_last_update_pending && + !strcmp(epoch->scan_start_token, + istate->fsmonitor_last_update_pending); +} + +static int proof_epoch_matches( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch, + int check_attr_namespace) +{ + struct clean_status_state *state; + struct attr_fingerprint attrs; + const struct git_hash_algo *algo = istate->repo->hash_algo; + int matched = 0; + + if (!epoch || epoch->istate != istate || + !fsmonitor_pending_token_from_provider(istate) || + !istate->fsmonitor_last_update_pending) + goto done; + state = istate->clean_status; + if (!state || !state->current_config_valid || !state->config_enforced || + !state->current_semantic_valid || !state->current_attr_valid || + !state->manifest.current_valid || !state->manifest.checked || + state->manifest.global_fallback || state->unsafe_filter || + state->manifest.current_flags != epoch->manifest_flags || + state->current_semantic_explicit != epoch->semantic_explicit || + state->current_attr_sources_present != epoch->attr_sources_present || + state->strong_mismatch != epoch->strong_mismatch || + state->config_mismatch != epoch->config_mismatch) + goto done; + if (attr_fingerprint_repository(istate->repo, &attrs) || + memcmp(attrs.content_hash, epoch->attr_hash, algo->rawsz) || + (check_attr_namespace && + memcmp(attrs.namespace_hash, epoch->attr_namespace_hash, + algo->rawsz)) || + attrs.sources_present != epoch->attr_sources_present || + memcmp(state->manifest.current_hash, epoch->manifest_hash, + algo->rawsz) || + !config_matches_epoch(istate, epoch) || + !clean_status_index_snapshot_still_matches(&epoch->index, istate)) + goto done; + matched = 1; +done: + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/proof-epoch-matched", matched); + return matched; +} + +int clean_status_proof_epoch_prime_matches( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch) +{ + int matched = + clean_status_proof_epoch_start_token_matches(istate, epoch) && + proof_epoch_matches(istate, epoch, 1); + + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/proof-epoch-primed", matched); + return matched; +} + +int clean_status_proof_epoch_matches( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch) +{ + return proof_epoch_matches(istate, epoch, 1); +} + +int clean_status_proof_epoch_content_matches( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch) +{ + return proof_epoch_matches(istate, epoch, 0); +} + +void clean_status_release_proof_epoch( + struct clean_status_proof_epoch *epoch) +{ + if (!epoch) + return; + clean_status_index_snapshot_release(&epoch->index); + free(epoch->scan_start_token); + free(epoch); +} diff --git a/clean-status-fast.c b/clean-status-fast.c new file mode 100644 index 00000000000000..8b2ba3e36d0117 --- /dev/null +++ b/clean-status-fast.c @@ -0,0 +1,252 @@ +#include "git-compat-util.h" +#include "abspath.h" +#include "attr-fingerprint.h" +#include "clean-status.h" +#include "clean-status-index.h" +#include "clean-status-sidecar.h" +#include "dir.h" +#include "environment.h" +#include "exclude-source-proof.h" +#include "fsmonitor.h" +#include "fsmonitor-settings.h" +#include "object-name.h" +#include "repository.h" +#include "trace2.h" +#include "worktree.h" +#include "wrapper.h" + +#if !EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN + +int clean_status_try_sidecar( + struct repository *repo UNUSED, + const struct clean_status_config_digest *config UNUSED) +{ + return 0; +} + +#else + +struct fast_exclude_context { + int root_fd; +}; + +static void trace_miss(struct repository *repo, const char *reason) +{ + trace2_data_string("status", repo, "clean-proof/miss", reason); +} + +static int open_exclude_parent(void *data, const char *path) +{ + struct fast_exclude_context *context = data; + int flags = O_RDONLY | O_NONBLOCK | O_DIRECTORY | O_CLOEXEC; + +#ifdef O_NOFOLLOW + flags |= O_NOFOLLOW; +#endif + if (is_absolute_path(path)) + return open(path, flags); + return openat(context->root_fd, path, flags); +} + +static int capture_standard_excludes( + struct repository *repo, struct fast_exclude_context *context, + struct exclude_source_proof **proof, struct object_id *digest) +{ + struct dir_struct dir = DIR_INIT; + int ret; + + *proof = exclude_source_proof_create( + repo->index, context, open_exclude_parent, + EXCLUDE_SOURCE_PROOF_NONBLOCKING); + dir.internal.exclude_source_proof = *proof; + setup_standard_excludes(&dir); + ret = exclude_source_proof_digest(*proof, repo->hash_algo, digest); + dir_clear(&dir); + return ret; +} + +static int attr_snapshot_still_matches( + struct repository *repo, const struct attr_source_snapshot *snapshot) +{ + const struct attr_fingerprint *expected = + attr_source_snapshot_fingerprint(snapshot); + struct attr_fingerprint current; + + return expected && + !attr_fingerprint_repository(repo, ¤t) && + current.sources_present == expected->sources_present && + !memcmp(current.content_hash, expected->content_hash, + repo->hash_algo->rawsz) && + !memcmp(current.namespace_hash, expected->namespace_hash, + repo->hash_algo->rawsz); +} + +static int fast_path_test_barrier(void) +{ + const char *ready = + getenv("GIT_TEST_STATUS_CLEAN_SIDECAR_BARRIER_READY"); + const char *resume = + getenv("GIT_TEST_STATUS_CLEAN_SIDECAR_BARRIER_RESUME"); + struct strbuf buf = STRBUF_INIT; + int fd; + int ret; + + if (!ready && !resume) + return 0; + if (!ready || !resume) + return -1; + fd = open(resume, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + write_file(ready, "ready"); + ret = strbuf_read(&buf, fd, 1) > 0 ? 0 : -1; + close(fd); + strbuf_release(&buf); + return ret; +} + +static int current_worktree_is_main(struct repository *repo) +{ + struct worktree *worktree = get_current_worktree(repo); + int ret = worktree && is_main_worktree(worktree); + + free_worktree(worktree); + return ret; +} + +int clean_status_try_sidecar( + struct repository *repo, + const struct clean_status_config_digest *config) +{ + struct clean_status_sidecar_record record = + CLEAN_STATUS_SIDECAR_RECORD_INIT; + struct clean_status_index_snapshot index = { .fd = -1 }; + struct attr_source_snapshot *attrs = NULL; + struct exclude_source_proof *excludes = NULL; + struct fast_exclude_context exclude_context = { .root_fd = -1 }; + struct fsmonitor_query_result query = FSMONITOR_QUERY_RESULT_INIT; + struct clean_status_config_digest fresh_config; + struct object_id exclude_digest, head_tree; + struct stat scanned_worktree; + unsigned char repo_hash[GIT_MAX_RAWSZ]; + char *query_token = NULL; + int ret = 0; + + if (!config->finalized || config->unsafe_filter || + getenv(INDEX_ENVIRONMENT) || is_bare_repository(repo) || + !repo_get_work_tree(repo) || + !current_worktree_is_main(repo) || + fsm_settings__get_mode(repo) != FSMONITOR_MODE_IPC) { + trace_miss(repo, "fast-repository-shape"); + goto done; + } + if (clean_status_sidecar_load( + repo->index_file, repo->hash_algo, &record)) { + trace_miss(repo, "fast-sidecar-missing-or-corrupt"); + goto done; + } + if (clean_status_sidecar_pin_source( + repo->index_file, &record.sidecar, repo->hash_algo, + &index)) { + trace_miss(repo, "fast-index-mismatch"); + goto done; + } + if (memcmp(config->hash, record.sidecar.proof.config_hash, + repo->hash_algo->rawsz)) { + trace_miss(repo, "fast-config-changed"); + goto done; + } + if (attr_source_snapshot_repository(repo, &attrs)) { + trace_miss(repo, "fast-attributes"); + goto done; + } + exclude_context.root_fd = open_nofollow( + repo_get_work_tree(repo), + O_RDONLY | O_NONBLOCK | O_DIRECTORY | O_CLOEXEC); + if (exclude_context.root_fd < 0 || + fstat(exclude_context.root_fd, &scanned_worktree) || + capture_standard_excludes( + repo, &exclude_context, &excludes, &exclude_digest) || + !oideq(&exclude_digest, + &record.sidecar.proof.exclude_source_digest)) { + trace_miss(repo, "fast-excludes"); + goto done; + } + if (clean_status_repository_fingerprint( + repo, attrs, &index, &scanned_worktree, repo_hash)) { + trace_miss(repo, "fast-repository-unavailable"); + goto done; + } + if (memcmp(repo_hash, record.sidecar.proof.repo_hash, + repo->hash_algo->rawsz)) { + trace_miss(repo, "fast-repository-input"); + goto done; + } + if (repo_get_oid_tree(repo, "HEAD^{tree}", &head_tree) || + !oideq(&head_tree, &record.sidecar.proof.head_tree)) { + trace_miss(repo, "fast-head-changed"); + goto done; + } + + query_token = xmemdupz( + record.sidecar.token, record.sidecar.token_len); + if (query_builtin_fsmonitor(query_token, &query) != + FSMONITOR_QUERY_DELTA || + query.paths.len) { + trace_miss(repo, "fast-provider-changed"); + goto done; + } + if (fast_path_test_barrier()) { + trace_miss(repo, "fast-test-barrier"); + goto done; + } + + if (clean_status_config_read_repository(repo, &fresh_config) || + fresh_config.unsafe_filter || + memcmp(fresh_config.hash, config->hash, + repo->hash_algo->rawsz)) { + trace_miss(repo, "fast-config-raced"); + goto done; + } + if (repo_get_oid_tree(repo, "HEAD^{tree}", &head_tree) || + !oideq(&head_tree, &record.sidecar.proof.head_tree)) { + trace_miss(repo, "fast-head-raced"); + goto done; + } + if (clean_status_repository_fingerprint( + repo, attrs, &index, &scanned_worktree, repo_hash) || + memcmp(repo_hash, record.sidecar.proof.repo_hash, + repo->hash_algo->rawsz)) { + trace_miss(repo, "fast-repository-raced"); + goto done; + } + if (!attr_snapshot_still_matches(repo, attrs)) { + trace_miss(repo, "fast-attributes-raced"); + goto done; + } + if (!exclude_source_proof_validate(excludes)) { + trace_miss(repo, "fast-excludes-raced"); + goto done; + } + if (!clean_status_index_snapshot_still_matches_path( + &index, repo->index_file, repo->hash_algo)) { + trace_miss(repo, "fast-index-raced"); + goto done; + } + + trace2_data_intmax("status", repo, "clean-proof/hit", 1); + ret = 1; + +done: + free(query_token); + fsmonitor_query_result_release(&query); + if (exclude_context.root_fd >= 0) + close(exclude_context.root_fd); + exclude_source_proof_release(excludes); + attr_source_snapshot_free(attrs); + clean_status_index_snapshot_release(&index); + clean_status_sidecar_record_release(&record); + return ret; +} + +#endif /* EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN */ diff --git a/clean-status-history.c b/clean-status-history.c new file mode 100644 index 00000000000000..fb88e92ca1e9bb --- /dev/null +++ b/clean-status-history.c @@ -0,0 +1,357 @@ +#include "git-compat-util.h" +#include "clean-status.h" +#include "clean-status-internal.h" +#include "fsmonitor-clean-proof.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "strbuf.h" +#include "trace2.h" + +static void invalidate_disk_history(struct clean_status_state *state) +{ + state->disk_config_seen = 1; + state->disk_config_invalid = 1; + state->disk_config_valid = 0; + state->disk_semantic_valid = 0; + state->disk_attr_valid = 0; + FREE_AND_NULL(state->disk_config_token); + strbuf_reset(&state->disk_config_raw); + state->manifest.disk_valid = 0; + state->manifest.disk_flags = 0; + strbuf_reset(&state->manifest.disk); +} + +int clean_status_read_fsmonitor_config(struct index_state *istate, + const void *data, unsigned long size) +{ + struct clean_status_state *state = clean_status_get_state(istate); + struct fsmonitor_clean_proof proof; + + if (state->disk_config_seen || + fsmonitor_clean_proof_parse(&proof, data, size, + istate->repo->hash_algo) || + clean_status_manifest_load(&state->manifest, + proof.attr_manifest, + proof.attr_manifest_len, + proof.flags, + istate->repo->hash_algo)) { + invalidate_disk_history(state); + trace2_data_intmax("fsmonitor", istate->repo, + "config/invalid-extension", 1); + return 0; + } + + state->disk_config_seen = 1; + state->disk_config_token = xmemdupz(proof.token, proof.token_len); + memcpy(state->disk_config_hash, proof.config_hash, + istate->repo->hash_algo->rawsz); + memcpy(state->disk_semantic_hash, proof.semantic_hash, + istate->repo->hash_algo->rawsz); + memcpy(state->disk_attr_hash, proof.attr_hash, + istate->repo->hash_algo->rawsz); + strbuf_add(&state->disk_config_raw, data, size); + state->disk_config_valid = 1; + state->disk_semantic_valid = 1; + state->disk_attr_valid = 1; + return 0; +} + +void clean_status_prepare_fsmonitor_config(struct index_state *istate) +{ + struct clean_status_state *state = istate->clean_status; + const struct git_hash_algo *algo = istate->repo->hash_algo; + int token_coherent, config_coherent, semantic_changed, attr_changed; + int coherent; + + if (!state || !state->current_config_valid) + return; + token_coherent = state->disk_config_valid && + !state->disk_config_invalid && istate->fsmonitor_token_valid && + istate->fsmonitor_last_update && state->disk_config_token && + !strcmp(state->disk_config_token, istate->fsmonitor_last_update); + config_coherent = state->disk_config_valid && + !memcmp(state->disk_config_hash, state->current_config_hash, + algo->rawsz); + semantic_changed = state->disk_semantic_valid && + state->current_semantic_valid && + memcmp(state->disk_semantic_hash, state->current_semantic_hash, + algo->rawsz); + attr_changed = (state->disk_attr_valid && !state->current_attr_valid) || + (state->disk_attr_valid && state->current_attr_valid && + memcmp(state->disk_attr_hash, state->current_attr_hash, + algo->rawsz)); + coherent = token_coherent && config_coherent && + state->disk_semantic_valid && state->current_semantic_valid && + !semantic_changed && state->disk_attr_valid && + state->current_attr_valid && !attr_changed && + state->manifest.disk_valid && + (state->manifest.disk_flags & FSMONITOR_CLEAN_PROOF_ALL) == + FSMONITOR_CLEAN_PROOF_ALL && + !state->unsafe_filter; + state->config_revalidated = coherent; + state->initial_coherent = coherent; + FREE_AND_NULL(state->config_revalidated_token); + if (coherent) { + state->config_revalidated_token = + xstrdup(istate->fsmonitor_last_update); + clean_status_manifest_adopt_disk(&state->manifest); + } + state->config_mismatch = state->config_enforced && !coherent; + state->strong_mismatch = state->config_enforced && + (state->disk_config_invalid || state->unsafe_filter || + semantic_changed || attr_changed || + (state->disk_config_valid && !state->current_attr_valid) || + (!state->disk_semantic_valid && + state->current_semantic_explicit) || + (!state->disk_attr_valid && + state->current_attr_sources_present)); + trace2_data_intmax("fsmonitor", istate->repo, + "config/coherent", coherent); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/initial-mismatch", state->strong_mismatch); +} + +int clean_status_has_persistent_fsmonitor_semantic_history( + const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + + return state && state->disk_config_valid && + !state->disk_config_invalid && state->disk_semantic_valid && + state->disk_attr_valid && state->manifest.disk_valid && + (state->manifest.disk_flags & FSMONITOR_CLEAN_PROOF_ALL) == + FSMONITOR_CLEAN_PROOF_ALL && + state->disk_config_raw.len; +} + +int clean_status_has_worktree_manifest_history( + const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + uint32_t required = FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | + FSMONITOR_CLEAN_PROOF_FULL_INDEX; + + return state && state->disk_config_valid && + !state->disk_config_invalid && state->manifest.disk_valid && + (state->manifest.disk_flags & required) == required; +} + +int clean_status_fsmonitor_semantic_adoption_needed( + const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + int missing_history; + + if (!state || !state->current_config_valid || !state->config_enforced) + return 0; + if (state->semantic_baseline_pending) + return 0; + missing_history = + !clean_status_has_persistent_fsmonitor_semantic_history(istate) && + !clean_status_has_worktree_manifest_history(istate); + /* + * Keep missing history on the proof path until fsmonitor explicitly + * chooses the narrow forward-baseline lane for a valid legacy token. + */ + return state->strong_mismatch || missing_history; +} + +int clean_status_fsmonitor_semantic_baseline_needed( + const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + const struct repo_config_values *cfg; + + if (!state || !state->current_config_valid || !state->config_enforced || + state->strong_mismatch || state->disk_config_seen || + !istate->fsmonitor_token_valid || + !istate->fsmonitor_last_update || + !*istate->fsmonitor_last_update) + return 0; + /* + * This helper is exercised by isolated index-state unit fixtures, + * which are not the_repository. The config values are already + * initialized with the repository and need no lazy parsing here. + */ + cfg = &istate->repo->config_values_private_; + if (!cfg->trust_ctime || !cfg->check_stat) + return 0; + return !clean_status_has_persistent_fsmonitor_semantic_history(istate) && + !clean_status_has_worktree_manifest_history(istate); +} + +int clean_status_fsmonitor_semantic_baseline_pending( + const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + + return state && state->semantic_baseline_pending; +} + +void clean_status_begin_fsmonitor_semantic_baseline( + struct index_state *istate) +{ + struct clean_status_state *state = clean_status_get_state(istate); + + state->semantic_baseline_pending = 1; +} + +static int current_proof_is_writable(const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + + return state && istate->fsmonitor_token_valid && + istate->fsmonitor_last_update && + state->config_enforced && state->current_config_valid && + state->current_semantic_valid && state->current_attr_valid && + state->manifest.current_valid && + (state->manifest.current_flags & FSMONITOR_CLEAN_PROOF_ALL) == + FSMONITOR_CLEAN_PROOF_ALL && + clean_status_revalidated_token_matches(istate); +} + +void clean_status_advance_fsmonitor_config_token( + struct index_state *istate, const char *next_token) +{ + struct clean_status_state *state = istate->clean_status; + + if (!next_token || !current_proof_is_writable(istate)) + return; + FREE_AND_NULL(state->config_revalidated_token); + state->config_revalidated_token = xstrdup(next_token); + trace2_data_intmax("fsmonitor", istate->repo, + "config/token-advanced", 1); +} + +int clean_status_should_write_fsmonitor_config( + const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + + return current_proof_is_writable(istate) || + (state && state->disk_config_valid && + !state->disk_config_invalid && state->disk_config_raw.len); +} + +void clean_status_write_fsmonitor_config(struct strbuf *out, + const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + const struct git_hash_algo *algo = istate->repo->hash_algo; + + if (current_proof_is_writable(istate)) { + struct fsmonitor_clean_proof proof = { + .flags = state->manifest.current_flags, + .token = (const unsigned char *)istate->fsmonitor_last_update, + .token_len = strlen(istate->fsmonitor_last_update), + .config_hash = state->current_config_hash, + .semantic_hash = state->current_semantic_hash, + .attr_hash = state->current_attr_hash, + .attr_manifest = + (const unsigned char *)state->manifest.current.buf, + .attr_manifest_len = state->manifest.current.len, + }; + + if (fsmonitor_clean_proof_write(out, &proof, algo)) + BUG("cannot serialize validated fsmonitor clean proof"); + return; + } + if (fsmonitor_clean_proof_copy_without_bindings( + out, state->disk_config_raw.buf, state->disk_config_raw.len, algo)) + BUG("cannot preserve validated fsmonitor clean proof"); +} + +void clean_status_copy_fsmonitor_history(struct index_state *dst, + const struct index_state *src) +{ + const struct clean_status_state *src_state = src->clean_status; + struct clean_status_state *dst_state; + + if (!src_state || !src_state->disk_config_valid || + src_state->disk_config_invalid || !src_state->disk_config_raw.len) + return; + dst_state = clean_status_get_state(dst); + FREE_AND_NULL(dst_state->disk_config_token); + strbuf_reset(&dst_state->disk_config_raw); + dst_state->disk_config_token = + xstrdup_or_null(src_state->disk_config_token); + strbuf_addbuf(&dst_state->disk_config_raw, + &src_state->disk_config_raw); + memcpy(dst_state->disk_config_hash, src_state->disk_config_hash, + dst->repo->hash_algo->rawsz); + memcpy(dst_state->disk_semantic_hash, src_state->disk_semantic_hash, + dst->repo->hash_algo->rawsz); + memcpy(dst_state->disk_attr_hash, src_state->disk_attr_hash, + dst->repo->hash_algo->rawsz); + if (clean_status_manifest_load( + &dst_state->manifest, src_state->manifest.disk.buf, + src_state->manifest.disk.len, src_state->manifest.disk_flags, + dst->repo->hash_algo)) + BUG("cannot copy validated clean-status manifest"); + dst_state->disk_config_seen = 1; + dst_state->disk_config_valid = 1; + dst_state->disk_semantic_valid = src_state->disk_semantic_valid; + dst_state->disk_attr_valid = src_state->disk_attr_valid; + dst_state->disk_config_invalid = 0; +} + +static int same_persistent_index_contents(const struct index_state *a, + const struct index_state *b) +{ + const unsigned int semantic_flags = + CE_STAGEMASK | CE_VALID | CE_EXTENDED_FLAGS; + const unsigned int transient_flags = + CE_UPDATE | CE_REMOVE | CE_ADDED | CE_WT_REMOVE | + CE_CONFLICTED | CE_UNPACKED | CE_NEW_SKIP_WORKTREE | + CE_MATCHED | CE_STRIP_NAME; + unsigned int i; + + if (a->repo != b->repo || a->split_index || b->split_index || + a->sparse_index || b->sparse_index || + a->cache_nr != b->cache_nr) + return 0; + + for (i = 0; i < a->cache_nr; i++) { + const struct cache_entry *ce_a = a->cache[i]; + const struct cache_entry *ce_b = b->cache[i]; + + if (ce_namelen(ce_a) != ce_namelen(ce_b) || + memcmp(ce_a->name, ce_b->name, ce_namelen(ce_a) + 1) || + ce_a->ce_mode != ce_b->ce_mode || + !oideq(&ce_a->oid, &ce_b->oid) || + ((ce_a->ce_flags ^ ce_b->ce_flags) & semantic_flags) || + ((ce_a->ce_flags | ce_b->ce_flags) & transient_flags)) + return 0; + } + + return 1; +} + +int clean_status_transfer_current_proof_if_same_index( + struct index_state *dst, const struct index_state *src) +{ + struct strbuf proof = STRBUF_INIT; + int transferred; + + if (!current_proof_is_writable(src) || + !src->fsmonitor_last_update || + !dst->fsmonitor_last_update || + strcmp(src->fsmonitor_last_update, dst->fsmonitor_last_update) || + !same_persistent_index_contents(dst, src)) + return 0; + + /* + * Reparse the current proof as the destination's disk proof, then + * reattach the current command's digest. This copies only a proof + * which the destination's identical logical entries can support. + */ + clean_status_write_fsmonitor_config(&proof, src); + dst->fsmonitor_token_valid = src->fsmonitor_token_valid; + clean_status_read_fsmonitor_config(dst, proof.buf, proof.len); + clean_status_attach_config(dst); + clean_status_prepare_fsmonitor_config(dst); + transferred = current_proof_is_writable(dst); + strbuf_release(&proof); + + return transferred; +} diff --git a/clean-status-identity.c b/clean-status-identity.c new file mode 100644 index 00000000000000..98d5a7e51b9a73 --- /dev/null +++ b/clean-status-identity.c @@ -0,0 +1,56 @@ +#include "git-compat-util.h" +#include "clean-status-identity.h" +#include "strbuf.h" + +int clean_status_identity_from_stat(struct clean_status_identity *identity, + const struct stat *st) +{ + memset(identity, 0, sizeof(*identity)); + if (!S_ISREG(st->st_mode) || st->st_nlink != 1) + return -1; + path_stat_identity_init(&identity->stat, st); + return 0; +} + +int clean_status_identity_is_durable(void) +{ +#ifdef __APPLE__ + return 1; +#else + return 0; +#endif +} + +int clean_status_identity_equal(const struct clean_status_identity *a, + const struct clean_status_identity *b) +{ + return path_stat_identity_equal(&a->stat, &b->stat); +} + +void clean_status_identity_write(struct strbuf *out, + const struct clean_status_identity *identity) +{ + uint64_t value; + size_t i; + + for (i = 0; i < ARRAY_SIZE(identity->stat.fields); i++) { + put_be64(&value, identity->stat.fields[i]); + strbuf_add(out, &value, sizeof(value)); + } +} + +int clean_status_identity_read(const unsigned char **p, + const unsigned char *end, + struct clean_status_identity *identity) +{ + size_t i; + + memset(identity, 0, sizeof(*identity)); + for (i = 0; i < ARRAY_SIZE(identity->stat.fields); i++) { + if ((size_t)(end - *p) < sizeof(uint64_t)) + return -1; + identity->stat.fields[i] = get_be64(*p); + *p += sizeof(uint64_t); + } + return 0; +} diff --git a/clean-status-identity.h b/clean-status-identity.h new file mode 100644 index 00000000000000..a22f8438f50874 --- /dev/null +++ b/clean-status-identity.h @@ -0,0 +1,27 @@ +#ifndef CLEAN_STATUS_IDENTITY_H +#define CLEAN_STATUS_IDENTITY_H + +#include "path-namespace.h" + +struct strbuf; +struct stat; + +struct clean_status_identity { + struct path_stat_identity stat; +}; + +#define CLEAN_STATUS_IDENTITY_SIZE \ + (PATH_STAT_IDENTITY_FIELDS * sizeof(uint64_t)) + +int clean_status_identity_from_stat(struct clean_status_identity *identity, + const struct stat *st); +int clean_status_identity_is_durable(void); +int clean_status_identity_equal(const struct clean_status_identity *a, + const struct clean_status_identity *b); +void clean_status_identity_write(struct strbuf *out, + const struct clean_status_identity *identity); +int clean_status_identity_read(const unsigned char **p, + const unsigned char *end, + struct clean_status_identity *identity); + +#endif /* CLEAN_STATUS_IDENTITY_H */ diff --git a/clean-status-index.c b/clean-status-index.c new file mode 100644 index 00000000000000..48250acc533e79 --- /dev/null +++ b/clean-status-index.c @@ -0,0 +1,197 @@ +#include "git-compat-util.h" +#include "clean-status.h" +#include "clean-status-index.h" +#include "clean-status-internal.h" +#include "object.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "wrapper.h" + +static int snapshot_read( + int fd, const struct stat *st, const struct git_hash_algo *algo, + uint32_t *version, uint32_t *cache_nr, struct object_id *checksum) +{ + unsigned char header[12]; + unsigned char trailer[GIT_MAX_RAWSZ]; + + if (st->st_size < 0 || + (uintmax_t)st->st_size < sizeof(header) + algo->rawsz || + (size_t)pread_in_full(fd, header, sizeof(header), 0) != + sizeof(header) || + memcmp(header, "DIRC", 4) || + (size_t)pread_in_full(fd, trailer, algo->rawsz, + st->st_size - (off_t)algo->rawsz) != + algo->rawsz) + return -1; + *version = get_be32(header + 4); + *cache_nr = get_be32(header + 8); + if (*version < 2 || *version > 4) + return -1; + oidread(checksum, trailer, algo); + return 0; +} + +static int snapshot_matches( + int fd, const struct stat *st, uint32_t expected_version, + uint32_t expected_cache_nr, const struct object_id *expected_checksum, + const struct git_hash_algo *algo) +{ + struct object_id checksum; + uint32_t version, cache_nr; + + return !snapshot_read(fd, st, algo, &version, &cache_nr, &checksum) && + version == expected_version && cache_nr == expected_cache_nr && + oideq(&checksum, expected_checksum); +} + +static int snapshot_open( + struct clean_status_index_snapshot *snapshot, const char *path, + const struct git_hash_algo *algo, int allow_null_checksum) +{ + struct clean_status_identity named; + struct stat fd_st, named_st; + int fd; + + memset(snapshot, 0, sizeof(*snapshot)); + snapshot->fd = -1; + fd = open_nofollow(path, O_RDONLY); + if (fd < 0 || + fstat(fd, &fd_st) || + lstat(path, &named_st) || + clean_status_identity_from_stat(&snapshot->identity, &fd_st) || + clean_status_identity_from_stat(&named, &named_st) || + !clean_status_identity_equal(&snapshot->identity, &named) || + snapshot_read(fd, &fd_st, algo, &snapshot->version, + &snapshot->cache_nr, &snapshot->checksum) || + (!allow_null_checksum && is_null_oid(&snapshot->checksum))) + goto fail; + snapshot->fd = fd; + return 0; + +fail: + if (fd >= 0) + close(fd); + return -1; +} + +int clean_status_index_snapshot_open( + struct clean_status_index_snapshot *snapshot, const char *path, + const struct git_hash_algo *algo) +{ + return snapshot_open(snapshot, path, algo, 0); +} + +int clean_status_index_snapshot_still_matches_path( + const struct clean_status_index_snapshot *snapshot, const char *path, + const struct git_hash_algo *algo) +{ + struct clean_status_identity fd_identity, named_identity; + struct stat fd_st, named_st; + + return snapshot->fd >= 0 && + !fstat(snapshot->fd, &fd_st) && + !lstat(path, &named_st) && + !clean_status_identity_from_stat(&fd_identity, &fd_st) && + !clean_status_identity_from_stat(&named_identity, &named_st) && + clean_status_identity_equal(&fd_identity, &snapshot->identity) && + clean_status_identity_equal(&named_identity, + &snapshot->identity) && + snapshot_matches(snapshot->fd, &fd_st, snapshot->version, + snapshot->cache_nr, &snapshot->checksum, algo); +} + +static int snapshot_matches_index_state( + const struct clean_status_index_snapshot *snapshot, + const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + + return istate->version == snapshot->version && + istate->cache_nr == snapshot->cache_nr && + oideq(&istate->oid, &snapshot->checksum) && + (!is_null_oid(&snapshot->checksum) || + (clean_status_identity_is_durable() && state && + state->source_identity_valid && + clean_status_identity_equal(&snapshot->identity, + &state->source_identity))); +} + +int clean_status_index_snapshot_pin( + struct clean_status_index_snapshot *snapshot, + struct index_state *istate) +{ + if (snapshot_open(snapshot, istate->repo->index_file, + istate->repo->hash_algo, 1)) + return -1; + if (snapshot_matches_index_state(snapshot, istate)) + return 0; + clean_status_index_snapshot_release(snapshot); + return -1; +} + +int clean_status_index_snapshot_still_matches( + const struct clean_status_index_snapshot *snapshot, + const struct index_state *istate) +{ + return snapshot_matches_index_state(snapshot, istate) && + clean_status_index_snapshot_still_matches_path( + snapshot, istate->repo->index_file, + istate->repo->hash_algo); +} + +void clean_status_index_snapshot_release( + struct clean_status_index_snapshot *snapshot) +{ + if (snapshot->fd >= 0) + close(snapshot->fd); + snapshot->fd = -1; +} + +int clean_status_index_entries_are_certifiable( + const struct index_state *istate) +{ + for (size_t i = 0; i < istate->cache_nr; i++) { + const struct cache_entry *ce = istate->cache[i]; + + if (S_ISGITLINK(ce->ce_mode) || ce_stage(ce) || + ce_intent_to_add(ce) || ce_skip_worktree(ce) || + (ce->ce_flags & CE_VALID) || + !(ce->ce_flags & CE_FSMONITOR_VALID) || + (ce->ce_flags & ~(CE_UPTODATE | CE_HASHED | + CE_FSMONITOR_VALID | + CE_UPDATE_IN_BASE))) + return 0; + } + return 1; +} + +int clean_status_index_is_certifiable(const struct index_state *istate) +{ + return !is_null_oid(&istate->oid) && + clean_status_index_entries_are_certifiable(istate); +} + +void clean_status_record_source_identity(struct index_state *istate, + const struct stat *st) +{ + struct clean_status_state *state = istate->clean_status; + + if (!state || state->source_identity_valid || + !clean_status_identity_is_durable()) + return; + if (!clean_status_identity_from_stat(&state->source_identity, st)) + state->source_identity_valid = 1; +} + +int clean_status_verify_null_index(const struct index_state *istate, + const struct stat *st) +{ + const struct clean_status_state *state = istate->clean_status; + struct clean_status_identity identity; + + if (!state || !clean_status_identity_is_durable()) + return 1; + return state->source_identity_valid && + !clean_status_identity_from_stat(&identity, st) && + clean_status_identity_equal(&identity, &state->source_identity); +} diff --git a/clean-status-index.h b/clean-status-index.h new file mode 100644 index 00000000000000..def687ac534026 --- /dev/null +++ b/clean-status-index.h @@ -0,0 +1,35 @@ +#ifndef CLEAN_STATUS_INDEX_H +#define CLEAN_STATUS_INDEX_H + +#include "clean-status-identity.h" +#include "hash.h" + +struct index_state; + +struct clean_status_index_snapshot { + struct clean_status_identity identity; + uint32_t version; + uint32_t cache_nr; + struct object_id checksum; + int fd; +}; + +int clean_status_index_snapshot_open( + struct clean_status_index_snapshot *snapshot, const char *path, + const struct git_hash_algo *algo); +int clean_status_index_snapshot_still_matches_path( + const struct clean_status_index_snapshot *snapshot, const char *path, + const struct git_hash_algo *algo); +int clean_status_index_snapshot_pin( + struct clean_status_index_snapshot *snapshot, + struct index_state *istate); +int clean_status_index_snapshot_still_matches( + const struct clean_status_index_snapshot *snapshot, + const struct index_state *istate); +void clean_status_index_snapshot_release( + struct clean_status_index_snapshot *snapshot); +int clean_status_index_entries_are_certifiable( + const struct index_state *istate); +int clean_status_index_is_certifiable(const struct index_state *istate); + +#endif /* CLEAN_STATUS_INDEX_H */ diff --git a/clean-status-internal.h b/clean-status-internal.h new file mode 100644 index 00000000000000..000c7ce0077379 --- /dev/null +++ b/clean-status-internal.h @@ -0,0 +1,44 @@ +#ifndef CLEAN_STATUS_INTERNAL_H +#define CLEAN_STATUS_INTERNAL_H + +#include "clean-status-identity.h" +#include "clean-status-manifest.h" + +struct index_state; + +struct clean_status_state { + struct clean_status_identity source_identity; + struct clean_status_manifest_state manifest; + struct strbuf disk_config_raw; + char *disk_config_token; + char *config_revalidated_token; + unsigned char current_config_hash[GIT_MAX_RAWSZ]; + unsigned char disk_config_hash[GIT_MAX_RAWSZ]; + unsigned char current_semantic_hash[GIT_MAX_RAWSZ]; + unsigned char disk_semantic_hash[GIT_MAX_RAWSZ]; + unsigned char current_attr_hash[GIT_MAX_RAWSZ]; + unsigned char current_attr_namespace_hash[GIT_MAX_RAWSZ]; + unsigned char disk_attr_hash[GIT_MAX_RAWSZ]; + unsigned current_config_valid : 1; + unsigned current_semantic_valid : 1; + unsigned current_attr_valid : 1; + unsigned current_semantic_explicit : 1; + unsigned current_attr_sources_present : 1; + unsigned config_enforced : 1; + unsigned unsafe_filter : 1; + unsigned config_mismatch : 1; + unsigned strong_mismatch : 1; + unsigned config_revalidated : 1; + unsigned initial_coherent : 1; + unsigned source_identity_valid : 1; + unsigned disk_config_valid : 1; + unsigned disk_semantic_valid : 1; + unsigned disk_attr_valid : 1; + unsigned disk_config_seen : 1; + unsigned disk_config_invalid : 1; + unsigned semantic_baseline_pending : 1; +}; + +struct clean_status_state *clean_status_get_state(struct index_state *istate); + +#endif /* CLEAN_STATUS_INTERNAL_H */ diff --git a/clean-status-manifest.c b/clean-status-manifest.c new file mode 100644 index 00000000000000..2750ddeb7280a9 --- /dev/null +++ b/clean-status-manifest.c @@ -0,0 +1,189 @@ +#include "git-compat-util.h" +#include "attr-manifest.h" +#include "clean-status-index.h" +#include "clean-status-manifest.h" +#include "dir.h" +#include "fsmonitor-clean-proof.h" +#include "fsmonitor-ll.h" +#include "hash-framing.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "sparse-index.h" +#include "trace2.h" +#include "worktree-attr-manifest.h" + +struct invalidate_manifest_data { + struct index_state *istate; + int invalidated; +}; + +static int build_manifest(struct index_state *istate, + struct strbuf *manifest, + unsigned char *manifest_hash, + struct worktree_attr_manifest_stats *stats) +{ + struct clean_status_index_snapshot snapshot; + struct index_state scratch = INDEX_STATE_INIT(istate->repo); + int ret = -1; + + if (istate->sparse_index == INDEX_EXPANDED) + return worktree_attr_manifest_build( + istate, manifest, manifest_hash, stats); + if (clean_status_index_snapshot_pin(&snapshot, istate)) + return -1; + scratch.fsmonitor_has_run_once = 1; + if (read_index_from(&scratch, istate->repo->index_file, + istate->repo->gitdir) < 0 || + !clean_status_index_snapshot_still_matches(&snapshot, &scratch)) + goto done; + ensure_full_index(&scratch); + ret = worktree_attr_manifest_build( + &scratch, manifest, manifest_hash, stats); + if (ret || + !clean_status_index_snapshot_still_matches(&snapshot, istate)) { + strbuf_reset(manifest); + ret = -1; + } + +done: + release_index(&scratch); + clean_status_index_snapshot_release(&snapshot); + return ret; +} + +void clean_status_manifest_init(struct clean_status_manifest_state *state) +{ + memset(state, 0, sizeof(*state)); + strbuf_init(&state->disk, 0); + strbuf_init(&state->current, 0); +} + +void clean_status_manifest_release(struct clean_status_manifest_state *state) +{ + strbuf_release(&state->disk); + strbuf_release(&state->current); +} + +int clean_status_manifest_load(struct clean_status_manifest_state *state, + const void *data, size_t len, uint32_t flags, + const struct git_hash_algo *algo) +{ + state->disk_valid = 0; + state->disk_flags = 0; + strbuf_reset(&state->disk); + if (flags & ~FSMONITOR_CLEAN_PROOF_ALL || + !attr_manifest_valid(data, len, algo)) + return -1; + strbuf_add(&state->disk, data, len); + hash_buffer_digest(algo, data, len, state->disk_hash); + state->disk_flags = flags; + state->disk_valid = 1; + return 0; +} + +void clean_status_manifest_adopt_disk( + struct clean_status_manifest_state *state) +{ + if (!state->disk_valid) + BUG("cannot adopt an invalid clean-status manifest"); + strbuf_reset(&state->current); + strbuf_addbuf(&state->current, &state->disk); + memcpy(state->current_hash, state->disk_hash, + sizeof(state->current_hash)); + state->current_flags = state->disk_flags; + state->current_valid = 1; + state->checked = 1; + state->current_invalidated = 0; +} + +static int invalidate_manifest_path(const struct attr_manifest_entry *entry, + void *cb_data) +{ + struct invalidate_manifest_data *data = cb_data; + char *path = xmemdupz(entry->path, entry->path_len); + + untracked_cache_invalidate_trimmed_path(data->istate, path, 0); + data->invalidated += + fsmonitor_invalidate_attributes_path(data->istate, path); + free(path); + return 0; +} + +int clean_status_manifest_refresh(struct index_state *istate, + struct clean_status_manifest_state *state) +{ + struct worktree_attr_manifest_stats stats; + struct invalidate_manifest_data invalidation = { .istate = istate }; + const struct git_hash_algo *algo = istate->repo->hash_algo; + const struct strbuf *baseline = NULL; + struct strbuf next = STRBUF_INIT; + unsigned char next_hash[GIT_MAX_RAWSZ]; + + state->scan_count++; + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-scan-count", state->scan_count); + if (attr_manifest_valid(state->current.buf, state->current.len, algo)) + baseline = &state->current; + else if (state->disk_valid) + baseline = &state->disk; + state->checked = 1; + state->changed = 0; + state->global_fallback = 0; + state->current_valid = 0; + state->current_flags = 0; + if (build_manifest(istate, &next, next_hash, &stats)) { + state->global_fallback = !!baseline; + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-scan-failed", 1); + strbuf_release(&next); + return -1; + } + if (baseline) { + if (attr_manifest_for_each_changed( + baseline->buf, baseline->len, + next.buf, next.len, algo, + invalidate_manifest_path, &invalidation)) { + state->global_fallback = 1; + strbuf_release(&next); + return -1; + } + state->changed = baseline->len != next.len || + memcmp(baseline->buf, next.buf, next.len); + } + strbuf_swap(&state->current, &next); + strbuf_release(&next); + memcpy(state->current_hash, next_hash, algo->rawsz); + state->current_valid = 1; + state->current_flags = FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | + FSMONITOR_CLEAN_PROOF_FULL_INDEX; + state->current_invalidated = 0; + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-candidates", stats.candidates); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-threads", stats.threads); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-thread-failures", + stats.thread_failures); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-worktree-sources", + stats.worktree_sources); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-index-sources", stats.index_sources); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-bytes", state->current.len); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-changed", state->changed); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/manifest-invalidated", + invalidation.invalidated); + return invalidation.invalidated; +} + +void clean_status_manifest_invalidate( + struct clean_status_manifest_state *state) +{ + if (state->current_valid) + state->current_invalidated = 1; + state->current_valid = 0; + state->current_flags = 0; +} diff --git a/clean-status-manifest.h b/clean-status-manifest.h new file mode 100644 index 00000000000000..394fe25a888c0d --- /dev/null +++ b/clean-status-manifest.h @@ -0,0 +1,37 @@ +#ifndef CLEAN_STATUS_MANIFEST_H +#define CLEAN_STATUS_MANIFEST_H + +#include "hash.h" +#include "strbuf.h" + +struct index_state; + +struct clean_status_manifest_state { + struct strbuf disk; + struct strbuf current; + unsigned char disk_hash[GIT_MAX_RAWSZ]; + unsigned char current_hash[GIT_MAX_RAWSZ]; + uint32_t disk_flags; + uint32_t current_flags; + uint32_t scan_count; + unsigned disk_valid : 1; + unsigned current_valid : 1; + unsigned checked : 1; + unsigned changed : 1; + unsigned global_fallback : 1; + unsigned current_invalidated : 1; +}; + +void clean_status_manifest_init(struct clean_status_manifest_state *state); +void clean_status_manifest_release(struct clean_status_manifest_state *state); +int clean_status_manifest_load(struct clean_status_manifest_state *state, + const void *data, size_t len, uint32_t flags, + const struct git_hash_algo *algo); +void clean_status_manifest_adopt_disk( + struct clean_status_manifest_state *state); +int clean_status_manifest_refresh(struct index_state *istate, + struct clean_status_manifest_state *state); +void clean_status_manifest_invalidate( + struct clean_status_manifest_state *state); + +#endif /* CLEAN_STATUS_MANIFEST_H */ diff --git a/clean-status-sidecar-issue.c b/clean-status-sidecar-issue.c new file mode 100644 index 00000000000000..93fa79c2cbb540 --- /dev/null +++ b/clean-status-sidecar-issue.c @@ -0,0 +1,169 @@ +#include "git-compat-util.h" +#include "cache-tree.h" +#include "clean-status.h" +#include "clean-status-index.h" +#include "clean-status-internal.h" +#include "clean-status-sidecar.h" +#include "environment.h" +#include "fsmonitor-clean-proof.h" +#include "fsmonitor-ll.h" +#include "fsmonitor-settings.h" +#include "lockfile.h" +#include "object-name.h" +#include "preload-index.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "trace2.h" +#include "wt-status.h" + +static void trace_miss(struct repository *repo, const char *reason) +{ + trace2_data_string("status", repo, "clean-proof/miss", reason); +} + +static int issue_test_barrier(void) +{ + const char *ready = + getenv("GIT_TEST_STATUS_CLEAN_SIDECAR_ISSUE_BARRIER_READY"); + const char *resume = + getenv("GIT_TEST_STATUS_CLEAN_SIDECAR_ISSUE_BARRIER_RESUME"); + struct strbuf buf = STRBUF_INIT; + int ret; + + if (!ready && !resume) + return 0; + if (!ready || !resume) + return -1; + write_file(ready, "ready"); + ret = strbuf_read_file(&buf, resume, 1) > 0 ? 0 : -1; + strbuf_release(&buf); + return ret; +} + +static int output_is_certifiable(const struct wt_status *status) +{ + return status->status_format == STATUS_FORMAT_PORCELAIN_V2 && + !status->pathspec.nr && !status->show_branch && + !status->show_stash && !status->show_ignored_mode && + !status->null_termination && !status->verbose && + status->show_untracked_files == SHOW_NORMAL_UNTRACKED_FILES && + !status->change.nr && !status->untracked.nr && + !status->ignored.nr; +} + +static int history_is_certifiable(const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + + return state && + clean_status_has_persistent_fsmonitor_semantic_history(istate) && + clean_status_revalidated_token_matches(istate) && + state->manifest.current_valid && + (state->manifest.current_flags & FSMONITOR_CLEAN_PROOF_ALL) == + FSMONITOR_CLEAN_PROOF_ALL; +} + +static int fsmonitor_state_is_certifiable( + struct repository *repo, const struct index_state *istate) +{ + return !istate->split_index && + istate->sparse_index == INDEX_EXPANDED && + fsm_settings__get_mode(repo) == FSMONITOR_MODE_IPC && + !fsmonitor_has_pending_token(istate) && + istate->fsmonitor_token_valid && + istate->fsmonitor_last_update && + strlen(istate->fsmonitor_last_update) <= + FSMONITOR_CLEAN_PROOF_TOKEN_MAX && + clean_status_index_is_certifiable(istate); +} + +static int untracked_scan_is_certifiable( + struct wt_status *status, struct object_id *exclude_digest, + struct stat *scanned_worktree) +{ + if (status->untracked_from_preload) + return !preload_index_bulk_standard_excludes_digest( + status->repo->index, exclude_digest, + scanned_worktree); + if (status->untracked_from_token_closure) + return !wt_status_certified_excludes_digest( + status, exclude_digest, scanned_worktree); + return 0; +} + +int clean_status_issue_sidecar( + struct wt_status *status, + const struct clean_status_config_digest *config, + struct lock_file *index_lock) +{ + struct repository *repo = status->repo; + struct index_state *istate = repo->index; + struct clean_status_index_snapshot index = { .fd = -1 }; + struct clean_status_sidecar sidecar = { 0 }; + struct object_id exclude_digest, head_tree; + struct stat scanned_worktree; + unsigned char repo_hash[GIT_MAX_RAWSZ]; + int installed = 0; + + if (!is_lock_file_locked(index_lock) || + !config->finalized || config->unsafe_filter || + !output_is_certifiable(status)) { + trace_miss(repo, "issue-command-or-output"); + goto done; + } + if (!history_is_certifiable(istate)) { + trace_miss(repo, "issue-coherent-history"); + goto done; + } + if (getenv(INDEX_ENVIRONMENT) || + !fsmonitor_state_is_certifiable(repo, istate) || + !untracked_scan_is_certifiable( + status, &exclude_digest, &scanned_worktree)) { + trace_miss(repo, "issue-scan-or-index-shape"); + goto done; + } + if (issue_test_barrier()) { + trace_miss(repo, "issue-test-barrier"); + goto done; + } + if (!status->attr_source_snapshot || + clean_status_index_snapshot_pin(&index, istate) || + clean_status_repository_fingerprint( + repo, status->attr_source_snapshot, &index, + &scanned_worktree, repo_hash)) { + trace_miss(repo, "issue-pinned-inputs"); + goto done; + } + if (repo_get_oid_tree(repo, "HEAD^{tree}", &head_tree) || + !istate->cache_tree || istate->cache_tree->entry_count < 0 || + !oideq(&head_tree, &istate->cache_tree->oid)) { + trace_miss(repo, "issue-head-cache-tree"); + goto done; + } + + sidecar.identity = index.identity; + sidecar.proof.index_version = index.version; + sidecar.proof.cache_nr = index.cache_nr; + oidcpy(&sidecar.proof.index_checksum, &index.checksum); + oidcpy(&sidecar.proof.head_tree, &head_tree); + memcpy(sidecar.proof.config_hash, config->hash, + repo->hash_algo->rawsz); + memcpy(sidecar.proof.repo_hash, repo_hash, + repo->hash_algo->rawsz); + oidcpy(&sidecar.proof.exclude_source_digest, &exclude_digest); + sidecar.token = (const unsigned char *)istate->fsmonitor_last_update; + sidecar.token_len = strlen(istate->fsmonitor_last_update); + + if (clean_status_sidecar_install( + repo->index_file, &sidecar, &index, repo->hash_algo)) { + trace_miss(repo, "issue-sidecar-write"); + goto done; + } + rollback_lock_file(index_lock); + trace2_data_intmax("status", repo, "clean-proof/sidecar", 1); + installed = 1; + +done: + clean_status_index_snapshot_release(&index); + return installed; +} diff --git a/clean-status-sidecar.c b/clean-status-sidecar.c new file mode 100644 index 00000000000000..95db486a47b289 --- /dev/null +++ b/clean-status-sidecar.c @@ -0,0 +1,396 @@ +#include "git-compat-util.h" + +#ifdef __APPLE__ +#include +#endif + +#include "abspath.h" +#include "attr-fingerprint.h" +#include "clean-status-index.h" +#include "clean-status-sidecar.h" +#include "fsmonitor-clean-proof.h" +#include "hash-framing.h" +#include "lockfile.h" +#include "path.h" +#include "repository.h" +#include "replace-object.h" +#include "strbuf.h" +#include "worktree.h" +#include "wrapper.h" + +#define CLEAN_STATUS_SIDECAR_MAGIC "CSTS" +#define CLEAN_STATUS_SIDECAR_MAX_SIZE 8192 +#define CLEAN_STATUS_FILESYSTEM_ID_SIZE 16 + +struct clean_status_filesystem_id { + unsigned char value[CLEAN_STATUS_FILESYSTEM_ID_SIZE]; +}; + +static int checksum_valid(const void *data, size_t len, + const struct git_hash_algo *algo) +{ + const unsigned char *bytes = data; + unsigned char actual[GIT_MAX_RAWSZ]; + + if (len < algo->rawsz) + return 0; + hash_buffer_digest(algo, data, len - algo->rawsz, actual); + return !memcmp(actual, bytes + len - algo->rawsz, algo->rawsz); +} + +static int token_valid(const unsigned char *token, size_t token_len) +{ + static const char prefix[] = "builtin:"; + + return token && token_len && + token_len <= FSMONITOR_CLEAN_PROOF_TOKEN_MAX && + !memchr(token, '\0', token_len) && + token_len >= sizeof(prefix) - 1 && + !memcmp(token, prefix, sizeof(prefix) - 1); +} + +static int proof_valid(const struct clean_status_proof *proof, + const struct git_hash_algo *algo) +{ + return proof->index_version >= 2 && proof->index_version <= 4 && + !is_null_oid(&proof->index_checksum) && + !is_null_oid(&proof->head_tree) && + !is_null_oid(&proof->exclude_source_digest) && + proof->index_checksum.algo == hash_algo_by_ptr(algo) && + proof->head_tree.algo == hash_algo_by_ptr(algo) && + proof->exclude_source_digest.algo == hash_algo_by_ptr(algo); +} + +int clean_status_sidecar_parse(struct clean_status_sidecar *sidecar, + const void *data, size_t len, + const struct git_hash_algo *algo) +{ + const unsigned char *p = data; + const unsigned char *end; + size_t minimum = 4 + 2 * sizeof(uint32_t) + + CLEAN_STATUS_IDENTITY_SIZE + 3 * sizeof(uint32_t) + + 6 * algo->rawsz + 1; + uint32_t flags, token_len; + + memset(sidecar, 0, sizeof(*sidecar)); + if (len < minimum || memcmp(p, CLEAN_STATUS_SIDECAR_MAGIC, 4) || + !checksum_valid(data, len, algo)) + return -1; + end = p + len - algo->rawsz; + p += 4; + if (get_be32(p) != CLEAN_STATUS_SIDECAR_VERSION) + return -1; + p += sizeof(uint32_t); + flags = get_be32(p); + p += sizeof(uint32_t); + if (flags) + return -1; + if (clean_status_identity_read(&p, end, &sidecar->identity)) + return -1; + sidecar->proof.index_version = get_be32(p); + p += sizeof(uint32_t); + sidecar->proof.cache_nr = get_be32(p); + p += sizeof(uint32_t); + oidread(&sidecar->proof.index_checksum, p, algo); + p += algo->rawsz; + oidread(&sidecar->proof.head_tree, p, algo); + p += algo->rawsz; + memcpy(sidecar->proof.config_hash, p, algo->rawsz); + p += algo->rawsz; + memcpy(sidecar->proof.repo_hash, p, algo->rawsz); + p += algo->rawsz; + oidread(&sidecar->proof.exclude_source_digest, p, algo); + p += algo->rawsz; + token_len = get_be32(p); + p += sizeof(uint32_t); + if (!proof_valid(&sidecar->proof, algo) || + (size_t)(end - p) != token_len || + !token_valid(p, token_len)) + return -1; + sidecar->token = p; + sidecar->token_len = token_len; + return 0; +} + +int clean_status_sidecar_write(struct strbuf *out, + const struct clean_status_sidecar *sidecar, + const struct git_hash_algo *algo) +{ + uint32_t value; + + strbuf_reset(out); + if (!proof_valid(&sidecar->proof, algo) || + sidecar->token_len > UINT32_MAX || + !token_valid(sidecar->token, sidecar->token_len)) + return -1; + + strbuf_add(out, CLEAN_STATUS_SIDECAR_MAGIC, 4); + put_be32(&value, CLEAN_STATUS_SIDECAR_VERSION); + strbuf_add(out, &value, sizeof(value)); + put_be32(&value, 0); + strbuf_add(out, &value, sizeof(value)); + clean_status_identity_write(out, &sidecar->identity); + put_be32(&value, sidecar->proof.index_version); + strbuf_add(out, &value, sizeof(value)); + put_be32(&value, sidecar->proof.cache_nr); + strbuf_add(out, &value, sizeof(value)); + strbuf_add(out, sidecar->proof.index_checksum.hash, algo->rawsz); + strbuf_add(out, sidecar->proof.head_tree.hash, algo->rawsz); + strbuf_add(out, sidecar->proof.config_hash, algo->rawsz); + strbuf_add(out, sidecar->proof.repo_hash, algo->rawsz); + strbuf_add(out, sidecar->proof.exclude_source_digest.hash, + algo->rawsz); + put_be32(&value, sidecar->token_len); + strbuf_add(out, &value, sizeof(value)); + strbuf_add(out, sidecar->token, sidecar->token_len); + hash_append_checksum(out, algo); + return 0; +} + +static char *sidecar_path(const char *index_path) +{ + return xstrfmt("%s.csts", index_path); +} + +static int open_nofollow_nonblocking(const char *path, int flags) +{ +#ifdef O_NONBLOCK + return open_nofollow(path, flags | O_NONBLOCK); +#else + (void)path; + (void)flags; + errno = ENOSYS; + return -1; +#endif +} + +int clean_status_sidecar_load( + const char *index_path, const struct git_hash_algo *algo, + struct clean_status_sidecar_record *record) +{ + struct stat st; + char extra; + char *path = sidecar_path(index_path); + int fd = -1, ret = -1; + size_t size; + + memset(&record->sidecar, 0, sizeof(record->sidecar)); + strbuf_reset(&record->storage); + fd = open_nofollow_nonblocking(path, O_RDONLY | O_CLOEXEC); + if (fd < 0 || fstat(fd, &st) || !S_ISREG(st.st_mode) || + st.st_size < 0 || st.st_size > CLEAN_STATUS_SIDECAR_MAX_SIZE) + goto done; + size = xsize_t(st.st_size); + strbuf_grow(&record->storage, size); + strbuf_setlen(&record->storage, size); + if ((size_t)read_in_full(fd, record->storage.buf, size) != size || + read(fd, &extra, 1) != 0 || + clean_status_sidecar_parse(&record->sidecar, + record->storage.buf, + record->storage.len, algo)) + goto done; + ret = 0; + +done: + if (ret) + strbuf_reset(&record->storage); + if (fd >= 0) + close(fd); + free(path); + return ret; +} + +void clean_status_sidecar_record_release( + struct clean_status_sidecar_record *record) +{ + strbuf_release(&record->storage); + memset(&record->sidecar, 0, sizeof(record->sidecar)); +} + +static int local_apfs_id(int fd MAYBE_UNUSED, + struct clean_status_filesystem_id *id) +{ +#ifdef __APPLE__ + struct statfs fs; +#endif + + memset(id, 0, sizeof(*id)); +#ifdef __APPLE__ + if (fstatfs(fd, &fs) || !(fs.f_flags & MNT_LOCAL) || + strcmp(fs.f_fstypename, "apfs") || + sizeof(fs.f_fsid) > sizeof(id->value)) + return -1; + memcpy(id->value, &fs.f_fsid, sizeof(fs.f_fsid)); + return 0; +#else + return -1; +#endif +} + +static int sidecar_matches_snapshot( + const char *index_path, const struct clean_status_sidecar *sidecar, + const struct clean_status_index_snapshot *snapshot, + const struct git_hash_algo *algo) +{ + struct clean_status_filesystem_id fsid; + + return clean_status_identity_is_durable() && + snapshot && snapshot->fd >= 0 && + !local_apfs_id(snapshot->fd, &fsid) && + clean_status_identity_equal(&snapshot->identity, + &sidecar->identity) && + snapshot->version == sidecar->proof.index_version && + snapshot->cache_nr == sidecar->proof.cache_nr && + oideq(&snapshot->checksum, + &sidecar->proof.index_checksum) && + clean_status_index_snapshot_still_matches_path( + snapshot, index_path, algo); +} + +int clean_status_sidecar_pin_source( + const char *index_path, const struct clean_status_sidecar *sidecar, + const struct git_hash_algo *algo, + struct clean_status_index_snapshot *snapshot) +{ + if (clean_status_index_snapshot_open(snapshot, index_path, algo)) + return -1; + if (sidecar_matches_snapshot( + index_path, sidecar, snapshot, algo)) + return 0; + clean_status_index_snapshot_release(snapshot); + return -1; +} + +int clean_status_sidecar_install( + const char *index_path, const struct clean_status_sidecar *sidecar, + const struct clean_status_index_snapshot *snapshot, + const struct git_hash_algo *algo) +{ + struct strbuf encoded = STRBUF_INIT; + struct lock_file lock = LOCK_INIT; + char *path = sidecar_path(index_path); + int sidecar_fd = -1, ret = -1; + + if (!sidecar_matches_snapshot( + index_path, sidecar, snapshot, algo) || + clean_status_sidecar_write(&encoded, sidecar, algo)) + goto done; + sidecar_fd = hold_lock_file_for_update(&lock, path, 0); + if (sidecar_fd < 0 || + (size_t)write_in_full(sidecar_fd, encoded.buf, encoded.len) != + encoded.len || + !sidecar_matches_snapshot( + index_path, sidecar, snapshot, algo) || + commit_lock_file(&lock)) + goto done; + ret = 0; + +done: + if (ret) + rollback_lock_file(&lock); + free(path); + strbuf_release(&encoded); + return ret; +} + +static int current_worktree_is_main(struct repository *repo) +{ + struct worktree *worktree = get_current_worktree(repo); + int ret = worktree && is_main_worktree(worktree); + + free_worktree(worktree); + return ret; +} + +static int worktree_root_identity( + const struct stat *st, uint64_t *identity MAYBE_UNUSED) +{ + if (!S_ISDIR(st->st_mode)) + return -1; +#ifdef __APPLE__ + identity[0] = st->st_dev; + identity[1] = st->st_ino; + identity[2] = st->st_birthtimespec.tv_sec; + identity[3] = st->st_birthtimespec.tv_nsec; + identity[4] = st->st_gen; + return 0; +#else + return -1; +#endif +} + +int clean_status_repository_fingerprint( + struct repository *repo, + const struct attr_source_snapshot *attrs, + const struct clean_status_index_snapshot *index, + const struct stat *scanned_worktree, + unsigned char *out) +{ + static const char domain[] = "git-clean-status-repository-v1"; + const struct attr_fingerprint *attr_fingerprint = + attr_source_snapshot_fingerprint(attrs); + struct clean_status_filesystem_id index_fsid, worktree_fsid; + struct git_hash_ctx ctx; + struct stat st; + char *worktree = NULL, *gitdir = NULL, *commondir = NULL; + uint64_t root_identity[5]; + uint64_t scanned_root_identity[5]; + uint64_t value; + int worktree_fd = -1, ret = -1; + + if (!attr_fingerprint || attr_fingerprint->sources_present || + !index || index->fd < 0 || !scanned_worktree || + is_bare_repository(repo) || + !repo_get_work_tree(repo) || + !current_worktree_is_main(repo) || + repo_has_replace_refs_uncached(repo)) + goto done; + + worktree = real_pathdup(repo_get_work_tree(repo), 0); + gitdir = real_pathdup(repo_get_git_dir(repo), 0); + commondir = real_pathdup(repo_get_common_dir(repo), 0); + if (!worktree || !gitdir || !commondir) + goto done; + worktree_fd = open_nofollow_nonblocking( + worktree, O_RDONLY | O_CLOEXEC); + if (worktree_fd < 0 || + local_apfs_id(worktree_fd, &worktree_fsid) || + local_apfs_id(index->fd, &index_fsid) || + fstat(worktree_fd, &st) || + worktree_root_identity(&st, root_identity) || + worktree_root_identity( + scanned_worktree, scanned_root_identity) || + memcmp(root_identity, scanned_root_identity, + sizeof(root_identity))) + goto done; + + git_hash_init(&ctx, repo->hash_algo); + hash_length_delimited(&ctx, domain, sizeof(domain) - 1); + hash_length_delimited(&ctx, worktree, strlen(worktree)); + hash_length_delimited(&ctx, gitdir, strlen(gitdir)); + hash_length_delimited(&ctx, commondir, strlen(commondir)); + for (size_t i = 0; i < ARRAY_SIZE(root_identity); i++) { + put_be64(&value, root_identity[i]); + hash_length_delimited(&ctx, &value, sizeof(value)); + } + hash_length_delimited(&ctx, worktree_fsid.value, + sizeof(worktree_fsid.value)); + hash_length_delimited(&ctx, index_fsid.value, + sizeof(index_fsid.value)); + hash_length_delimited(&ctx, attr_fingerprint->content_hash, + repo->hash_algo->rawsz); + hash_optional_cstring(&ctx, setlocale(LC_CTYPE, NULL)); + hash_optional_cstring(&ctx, getenv("LC_ALL")); + hash_optional_cstring(&ctx, getenv("LC_CTYPE")); + hash_optional_cstring(&ctx, getenv("LANG")); + git_hash_final(out, &ctx); + ret = 0; + +done: + if (worktree_fd >= 0) + close(worktree_fd); + free(worktree); + free(gitdir); + free(commondir); + return ret; +} diff --git a/clean-status-sidecar.h b/clean-status-sidecar.h new file mode 100644 index 00000000000000..8149acbe5b86bb --- /dev/null +++ b/clean-status-sidecar.h @@ -0,0 +1,67 @@ +#ifndef CLEAN_STATUS_SIDECAR_H +#define CLEAN_STATUS_SIDECAR_H + +#include "clean-status-identity.h" +#include "hash.h" +#include "strbuf.h" + +struct clean_status_index_snapshot; +struct attr_source_snapshot; +struct repository; +struct stat; + +#define CLEAN_STATUS_SIDECAR_VERSION 1 + +struct clean_status_proof { + uint32_t index_version; + uint32_t cache_nr; + struct object_id index_checksum; + struct object_id head_tree; + unsigned char config_hash[GIT_MAX_RAWSZ]; + unsigned char repo_hash[GIT_MAX_RAWSZ]; + struct object_id exclude_source_digest; +}; + +struct clean_status_sidecar { + struct clean_status_identity identity; + struct clean_status_proof proof; + const unsigned char *token; + size_t token_len; +}; + +struct clean_status_sidecar_record { + struct clean_status_sidecar sidecar; + struct strbuf storage; +}; + +#define CLEAN_STATUS_SIDECAR_RECORD_INIT { \ + .storage = STRBUF_INIT, \ +} + +int clean_status_sidecar_parse(struct clean_status_sidecar *sidecar, + const void *data, size_t len, + const struct git_hash_algo *algo); +int clean_status_sidecar_write(struct strbuf *out, + const struct clean_status_sidecar *sidecar, + const struct git_hash_algo *algo); +int clean_status_sidecar_load( + const char *index_path, const struct git_hash_algo *algo, + struct clean_status_sidecar_record *record); +void clean_status_sidecar_record_release( + struct clean_status_sidecar_record *record); +int clean_status_sidecar_pin_source( + const char *index_path, const struct clean_status_sidecar *sidecar, + const struct git_hash_algo *algo, + struct clean_status_index_snapshot *snapshot); +int clean_status_sidecar_install( + const char *index_path, const struct clean_status_sidecar *sidecar, + const struct clean_status_index_snapshot *snapshot, + const struct git_hash_algo *algo); +int clean_status_repository_fingerprint( + struct repository *repo, + const struct attr_source_snapshot *attrs, + const struct clean_status_index_snapshot *index, + const struct stat *scanned_worktree, + unsigned char *out); + +#endif /* CLEAN_STATUS_SIDECAR_H */ diff --git a/clean-status.c b/clean-status.c new file mode 100644 index 00000000000000..2320bca5098cec --- /dev/null +++ b/clean-status.c @@ -0,0 +1,234 @@ +#include "git-compat-util.h" +#include "attr-fingerprint.h" +#include "clean-status.h" +#include "clean-status-internal.h" +#include "fsmonitor-clean-proof.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "trace2.h" + +static struct repository *configured_repo; +static unsigned char configured_hash[GIT_MAX_RAWSZ]; +static unsigned char configured_semantic_hash[GIT_MAX_RAWSZ]; +static int configured_hash_valid; +static int configured_unsafe_filter; +static int configured_semantic_explicit; + +struct clean_status_state *clean_status_get_state(struct index_state *istate) +{ + if (!istate->clean_status) { + CALLOC_ARRAY(istate->clean_status, 1); + clean_status_manifest_init(&istate->clean_status->manifest); + strbuf_init(&istate->clean_status->disk_config_raw, 0); + } + return istate->clean_status; +} + +void clean_status_set_config_digest( + struct repository *repo, + const struct clean_status_config_digest *digest) +{ + configured_repo = repo; + configured_hash_valid = digest && digest->finalized; + configured_unsafe_filter = configured_hash_valid && digest->unsafe_filter; + configured_semantic_explicit = configured_hash_valid && + digest->semantic_config_explicit; + if (!configured_hash_valid) + return; + memcpy(configured_hash, digest->hash, repo->hash_algo->rawsz); + memcpy(configured_semantic_hash, digest->semantic_hash, + repo->hash_algo->rawsz); +} + +void clean_status_attach_config(struct index_state *istate) +{ + struct clean_status_state *state = istate->clean_status; + struct attr_fingerprint attrs; + + if (state && state->current_config_valid) + return; + if (!configured_hash_valid || configured_repo != istate->repo) + return; + state = clean_status_get_state(istate); + memcpy(state->current_config_hash, configured_hash, + istate->repo->hash_algo->rawsz); + memcpy(state->current_semantic_hash, configured_semantic_hash, + istate->repo->hash_algo->rawsz); + state->current_config_valid = 1; + state->current_semantic_valid = 1; + state->current_semantic_explicit = configured_semantic_explicit; + state->config_enforced = 1; + state->unsafe_filter = configured_unsafe_filter; + if (!attr_fingerprint_repository(istate->repo, &attrs)) { + memcpy(state->current_attr_hash, attrs.content_hash, + istate->repo->hash_algo->rawsz); + memcpy(state->current_attr_namespace_hash, attrs.namespace_hash, + istate->repo->hash_algo->rawsz); + state->current_attr_valid = 1; + state->current_attr_sources_present = attrs.sources_present; + } +} + +int clean_status_revalidated_token_matches(const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + + return state && state->config_revalidated && + state->config_revalidated_token && + istate->fsmonitor_last_update && + !strcmp(state->config_revalidated_token, + istate->fsmonitor_last_update); +} + +void clean_status_invalidate_current_proof(struct index_state *istate) +{ + if (!istate->clean_status) + return; + istate->clean_status->config_revalidated = 0; + istate->clean_status->initial_coherent = 0; + istate->clean_status->semantic_baseline_pending = 0; +} + +int clean_status_capture_attr_snapshot( + struct index_state *istate, + struct attr_source_snapshot **snapshot) +{ + struct clean_status_state *state = istate->clean_status; + struct attr_source_snapshot *captured = NULL; + const struct attr_fingerprint *attrs; + int valid, changed = 0; + + if (!snapshot) + BUG("clean_status_capture_attr_snapshot requires an output"); + *snapshot = NULL; + if (!fstat_is_reliable() || !state || !state->current_config_valid || + !state->config_enforced) + return 0; + valid = !attr_source_snapshot_repository(istate->repo, &captured); + attrs = attr_source_snapshot_fingerprint(captured); + if (!valid || !state->current_attr_valid) { + changed = CLEAN_STATUS_ATTR_CONTENT_CHANGED | + CLEAN_STATUS_ATTR_NAMESPACE_CHANGED; + } else { + if (memcmp(attrs->content_hash, state->current_attr_hash, + istate->repo->hash_algo->rawsz)) + changed |= CLEAN_STATUS_ATTR_CONTENT_CHANGED; + if (memcmp(attrs->namespace_hash, + state->current_attr_namespace_hash, + istate->repo->hash_algo->rawsz)) + changed |= CLEAN_STATUS_ATTR_NAMESPACE_CHANGED; + } + if (valid) { + memcpy(state->current_attr_hash, attrs->content_hash, + istate->repo->hash_algo->rawsz); + memcpy(state->current_attr_namespace_hash, attrs->namespace_hash, + istate->repo->hash_algo->rawsz); + state->current_attr_valid = 1; + state->current_attr_sources_present = attrs->sources_present; + } else { + state->current_attr_valid = 0; + } + if (changed || state->unsafe_filter) { + state->config_mismatch = 1; + state->strong_mismatch = 1; + state->config_revalidated = 0; + } + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/rechecked", 1); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/mismatch", state->strong_mismatch); + if (!valid) + return -1; + *snapshot = captured; + return changed; +} + +int clean_status_fsmonitor_config_mismatch(const struct index_state *istate) +{ + return istate->clean_status && + istate->clean_status->current_config_valid && + (istate->clean_status->config_mismatch || + !istate->clean_status->config_revalidated); +} + +int clean_status_fsmonitor_strong_mismatch(const struct index_state *istate) +{ + return istate->clean_status && + istate->clean_status->current_config_valid && + istate->clean_status->strong_mismatch; +} + +int clean_status_refresh_worktree_manifest(struct index_state *istate) +{ + struct clean_status_state *state = clean_status_get_state(istate); + + return clean_status_manifest_refresh(istate, &state->manifest); +} + +int clean_status_manifest_global_fallback(const struct index_state *istate) +{ + return istate->clean_status && + istate->clean_status->manifest.global_fallback; +} + +int clean_status_worktree_manifest_needs_refresh( + const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + + return state && state->config_enforced && + state->manifest.current_invalidated; +} + +void clean_status_invalidate_current_manifest(struct index_state *istate) +{ + if (!istate->clean_status) + return; + clean_status_manifest_invalidate(&istate->clean_status->manifest); + clean_status_invalidate_current_proof(istate); +} + +void clean_status_mark_fsmonitor_config_valid(struct index_state *istate, + const char *closed_token) +{ + struct clean_status_state *state = istate->clean_status; + + if (!state || !state->current_config_valid) + return; + if (!closed_token || + !state->manifest.current_valid || !state->manifest.checked || + (state->manifest.current_flags & + (FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | + FSMONITOR_CLEAN_PROOF_FULL_INDEX)) != + (FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | + FSMONITOR_CLEAN_PROOF_FULL_INDEX)) { + state->config_revalidated = 0; + FREE_AND_NULL(state->config_revalidated_token); + trace2_data_intmax("fsmonitor", istate->repo, + "config/manifest-unbound", 1); + return; + } + state->config_mismatch = 0; + state->strong_mismatch = 0; + state->semantic_baseline_pending = 0; + state->manifest.current_flags = FSMONITOR_CLEAN_PROOF_ALL; + state->config_revalidated = state->current_semantic_valid && + state->current_attr_valid && state->manifest.current_valid; + state->initial_coherent = state->config_revalidated; + FREE_AND_NULL(state->config_revalidated_token); + if (state->config_revalidated) + state->config_revalidated_token = xstrdup(closed_token); + trace2_data_intmax("fsmonitor", istate->repo, + "config/revalidated", 1); +} + +void clean_status_release(struct index_state *istate) +{ + if (!istate->clean_status) + return; + clean_status_manifest_release(&istate->clean_status->manifest); + strbuf_release(&istate->clean_status->disk_config_raw); + free(istate->clean_status->disk_config_token); + free(istate->clean_status->config_revalidated_token); + FREE_AND_NULL(istate->clean_status); +} diff --git a/clean-status.h b/clean-status.h new file mode 100644 index 00000000000000..d5514af4363f06 --- /dev/null +++ b/clean-status.h @@ -0,0 +1,101 @@ +#ifndef CLEAN_STATUS_H +#define CLEAN_STATUS_H + +#include "clean-status-config.h" + +struct index_state; +struct attr_source_snapshot; +struct clean_status_proof_epoch; +struct lock_file; +struct repository; +struct stat; +struct strbuf; +struct wt_status; + +enum clean_status_attr_change { + CLEAN_STATUS_ATTR_CONTENT_CHANGED = 1 << 0, + CLEAN_STATUS_ATTR_NAMESPACE_CHANGED = 1 << 1, +}; + +void clean_status_set_config_digest( + struct repository *repo, + const struct clean_status_config_digest *digest); +void clean_status_attach_config(struct index_state *istate); +int clean_status_capture_attr_snapshot( + struct index_state *istate, + struct attr_source_snapshot **snapshot); +struct clean_status_proof_epoch *clean_status_capture_proof_epoch( + struct index_state *istate, + const struct attr_source_snapshot *attrs); +int clean_status_proof_epoch_start_token_matches( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch); +int clean_status_proof_epoch_prime_matches( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch); +int clean_status_proof_epoch_matches( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch); +int clean_status_proof_epoch_content_matches( + struct index_state *istate, + const struct clean_status_proof_epoch *epoch); +void clean_status_release_proof_epoch( + struct clean_status_proof_epoch *epoch); + +int clean_status_fsmonitor_config_mismatch(const struct index_state *istate); +int clean_status_fsmonitor_strong_mismatch(const struct index_state *istate); +int clean_status_revalidated_token_matches( + const struct index_state *istate); + +int clean_status_has_persistent_fsmonitor_semantic_history( + const struct index_state *istate); +int clean_status_has_worktree_manifest_history( + const struct index_state *istate); +int clean_status_fsmonitor_semantic_adoption_needed( + const struct index_state *istate); +int clean_status_fsmonitor_semantic_baseline_needed( + const struct index_state *istate); +int clean_status_fsmonitor_semantic_baseline_pending( + const struct index_state *istate); +void clean_status_begin_fsmonitor_semantic_baseline( + struct index_state *istate); + +int clean_status_refresh_worktree_manifest(struct index_state *istate); +int clean_status_manifest_global_fallback(const struct index_state *istate); +int clean_status_worktree_manifest_needs_refresh( + const struct index_state *istate); +void clean_status_invalidate_current_manifest(struct index_state *istate); +void clean_status_mark_fsmonitor_config_valid( + struct index_state *istate, const char *closed_token); + +void clean_status_record_source_identity(struct index_state *istate, + const struct stat *st); +int clean_status_verify_null_index(const struct index_state *istate, + const struct stat *st); + +int clean_status_issue_sidecar( + struct wt_status *status, + const struct clean_status_config_digest *config, + struct lock_file *index_lock); +int clean_status_try_sidecar( + struct repository *repo, + const struct clean_status_config_digest *config); + +int clean_status_read_fsmonitor_config(struct index_state *istate, + const void *data, unsigned long size); +void clean_status_prepare_fsmonitor_config(struct index_state *istate); +void clean_status_invalidate_current_proof(struct index_state *istate); +void clean_status_advance_fsmonitor_config_token( + struct index_state *istate, const char *next_token); +int clean_status_should_write_fsmonitor_config( + const struct index_state *istate); +void clean_status_write_fsmonitor_config(struct strbuf *out, + const struct index_state *istate); +void clean_status_copy_fsmonitor_history(struct index_state *dst, + const struct index_state *src); +int clean_status_transfer_current_proof_if_same_index( + struct index_state *dst, const struct index_state *src); + +void clean_status_release(struct index_state *istate); + +#endif /* CLEAN_STATUS_H */ diff --git a/compat/fsmonitor/fsm-listen-darwin.c b/compat/fsmonitor/fsm-listen-darwin.c index 43c3a915a0edfc..ffd8392262261b 100644 --- a/compat/fsmonitor/fsm-listen-darwin.c +++ b/compat/fsmonitor/fsm-listen-darwin.c @@ -138,6 +138,12 @@ static int ef_is_dropped(const FSEventStreamEventFlags ef) ef & kFSEventStreamEventFlagUserDropped); } +static int ef_is_hardlink(const FSEventStreamEventFlags ef) +{ + return ef & (kFSEventStreamEventFlagItemIsHardlink | + kFSEventStreamEventFlagItemIsLastHardlink); +} + /* * If an `xattr` change is the only reason we received this event, * then silently ignore it. Git doesn't care about xattr's. We @@ -208,6 +214,7 @@ static void fsevent_callback(ConstFSEventStreamRef streamRef UNUSED, const char *slash; char *resolved = NULL; struct strbuf tmp = STRBUF_INIT; + enum fsmonitor_path_type path_type; /* * Build a list of all filesystem changes into a private/local @@ -290,7 +297,23 @@ static void fsevent_callback(ConstFSEventStreamRef streamRef UNUSED, continue; } - switch (fsmonitor_classify_path_absolute(state, path_k)) { + path_type = fsmonitor_classify_path_absolute(state, path_k); + if (ef_is_hardlink(event_flags[k]) && + path_type == IS_WORKDIR_PATH) { + /* + * An event for one name does not prove that all names of the + * inode are in this watch cone. Make the client content-check + * the entire tracked set rather than trusting path-local stats. + */ + if (trace_pass_fl(&trace_fsmonitor)) + log_flags_set(path_k, event_flags[k]); + if (!batch) + batch = fsmonitor_batch__new(); + my_add_path(batch, FSMONITOR_PATH_GLOBAL_INVALIDATE); + continue; + } + + switch (path_type) { case IS_INSIDE_DOT_GIT_WITH_COOKIE_PREFIX: case IS_INSIDE_GITDIR_WITH_COOKIE_PREFIX: diff --git a/compat/precompose_utf8.c b/compat/precompose_utf8.c index 8077f6235b0cae..2be9f7577f9519 100644 --- a/compat/precompose_utf8.c +++ b/compat/precompose_utf8.c @@ -72,19 +72,32 @@ void probe_utf8_pathname_composition(void) strbuf_release(&path); } -const char *precompose_string_if_needed(const char *in) +void repo_precompose_utf8_prepare(struct repository *repo) +{ + struct repo_config_values *cfg = repo_config_values(repo); + + if (cfg->precomposed_unicode < 0 && + repo_config_get_bool(repo, "core.precomposeunicode", + &cfg->precomposed_unicode)) + cfg->precomposed_unicode = 0; +} + +const char *repo_precompose_string_if_needed(struct repository *repo, + const char *in) { size_t inlen; size_t outlen; - struct repo_config_values *cfg = repo_config_values(the_repository); + struct repo_config_values *cfg = repo_config_values(repo); if (!in) return NULL; if (has_non_ascii(in, (size_t)-1, &inlen)) { iconv_t ic_prec; char *out; + if (cfg->precomposed_unicode < 0) - repo_config_get_bool(the_repository, "core.precomposeunicode", &cfg->precomposed_unicode); + repo_config_get_bool(repo, "core.precomposeunicode", + &cfg->precomposed_unicode); if (cfg->precomposed_unicode != 1) return in; ic_prec = iconv_open(repo_encoding, path_encoding); @@ -104,6 +117,11 @@ const char *precompose_string_if_needed(const char *in) return in; } +const char *precompose_string_if_needed(const char *in) +{ + return repo_precompose_string_if_needed(the_repository, in); +} + const char *precompose_argv_prefix(int argc, const char **argv, const char *prefix) { int i = 0; diff --git a/compat/precompose_utf8.h b/compat/precompose_utf8.h index c7c3cc211e5031..6ec1fa973b7db9 100644 --- a/compat/precompose_utf8.h +++ b/compat/precompose_utf8.h @@ -29,8 +29,13 @@ typedef struct { struct dirent_prec_psx *dirent_nfc; } PREC_DIR; +struct repository; + const char *precompose_argv_prefix(int argc, const char **argv, const char *prefix); const char *precompose_string_if_needed(const char *in); +const char *repo_precompose_string_if_needed(struct repository *repo, + const char *in); +void repo_precompose_utf8_prepare(struct repository *repo); void probe_utf8_pathname_composition(void); PREC_DIR *precompose_utf8_opendir(const char *dirname); diff --git a/compat/preload-index/bulk-darwin-root.c b/compat/preload-index/bulk-darwin-root.c new file mode 100644 index 00000000000000..f12f730761c99c --- /dev/null +++ b/compat/preload-index/bulk-darwin-root.c @@ -0,0 +1,100 @@ +#include "git-compat-util.h" + +#include + +#include "compat/preload-index/bulk-darwin.h" +#include "path-namespace.h" +#include "repository.h" +#include "preload-index-bulk.h" + +static int same_fsid(const fsid_t *a, const fsid_t *b) +{ + return !memcmp(a, b, sizeof(*a)); +} + +static int stat_local_apfs(int fd, struct stat *st, struct statfs *fs) +{ + if (fstat(fd, st) || fstatfs(fd, fs)) + return -1; + if (!S_ISDIR(st->st_mode) || !(fs->f_flags & MNT_LOCAL) || + strcmp(fs->f_fstypename, "apfs")) { + errno = EXDEV; + return -1; + } + return 0; +} + +int preload_bulk_darwin_fd_on_root_mount(struct preload_bulk_scan *scan, + int fd, struct stat *st_out) +{ + struct preload_bulk_darwin_data *data = scan->platform_data; + struct statfs fs; + struct stat st; + + if (stat_local_apfs(fd, &st, &fs)) + return -1; + if (st.st_dev != data->root_stat.st_dev || + !same_fsid(&fs.f_fsid, &data->root_fsid)) { + errno = EXDEV; + return -1; + } + if (st_out) + *st_out = st; + return 0; +} + +const char *preload_bulk_darwin_open_root(struct preload_bulk_scan *scan) +{ + struct preload_bulk_darwin_data *data; + struct statfs fs; + struct stat st; + + CALLOC_ARRAY(data, 1); + scan->platform_data = data; + scan->root_fd = open(repo_get_work_tree(scan->repo), + O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (scan->root_fd < 0 || + stat_local_apfs(scan->root_fd, &st, &fs)) + return "unsupported-filesystem"; + return NULL; +} + +const char *preload_bulk_darwin_snapshot_root(struct preload_bulk_scan *scan) +{ + struct preload_bulk_darwin_data *data = scan->platform_data; + struct statfs fs; + struct stat st; + + if (stat_local_apfs(scan->root_fd, &st, &fs)) + return "unsupported-filesystem"; + data->root_stat = st; + data->root_fsid = fs.f_fsid; + return NULL; +} + +const char *preload_bulk_darwin_validate_root(struct preload_bulk_scan *scan) +{ + struct preload_bulk_darwin_data *data = scan->platform_data; + struct stat root_after; + int fd = open(repo_get_work_tree(scan->repo), + O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + + if (fd < 0 || + preload_bulk_darwin_fd_on_root_mount(scan, fd, &root_after) || + !path_namespace_stat_equal(&data->root_stat, &root_after)) { + if (fd >= 0) + close(fd); + return "namespace-race"; + } + close(fd); + return NULL; +} + +void preload_bulk_darwin_release(struct preload_bulk_scan *scan) +{ + if (scan->root_fd >= 0) { + close(scan->root_fd); + scan->root_fd = -1; + } + FREE_AND_NULL(scan->platform_data); +} diff --git a/compat/preload-index/bulk-darwin.c b/compat/preload-index/bulk-darwin.c new file mode 100644 index 00000000000000..8aef3a12a61413 --- /dev/null +++ b/compat/preload-index/bulk-darwin.c @@ -0,0 +1,620 @@ +#include "git-compat-util.h" + +#include +#include +#include + +#include "compat/precompose_utf8.h" +#include "compat/preload-index/bulk-darwin.h" +#include "dir.h" +#include "path-namespace.h" +#include "preload-index-bulk.h" + +#ifndef SF_FIRMLINK +#define SF_FIRMLINK 0x00800000 +#endif + +#define PRELOAD_INDEX_BULK_BUFFER_SIZE (1024 * 1024) + +static const attrgroup_t required_common = + ATTR_CMN_RETURNED_ATTRS | ATTR_CMN_ERROR | ATTR_CMN_NAME | + ATTR_CMN_DEVID | ATTR_CMN_OBJTYPE | + ATTR_CMN_CRTIME | ATTR_CMN_MODTIME | ATTR_CMN_CHGTIME | + ATTR_CMN_OWNERID | ATTR_CMN_GRPID | ATTR_CMN_ACCESSMASK | + ATTR_CMN_FLAGS | ATTR_CMN_FILEID; +static const attrgroup_t required_dir = ATTR_DIR_MOUNTSTATUS; +static const attrgroup_t required_file = + ATTR_FILE_LINKCOUNT | ATTR_FILE_DATALENGTH; + +static int valid_component(const char *component, size_t len) +{ + return len && + !(len == 1 && component[0] == '.') && + !(len == 2 && component[0] == '.' && component[1] == '.'); +} + +static int valid_relative_path(const char *path) +{ + const char *component = path; + + if (!strcmp(path, ".")) + return 1; + if (!*path || *path == '/') + return 0; + for (;;) { + const char *slash = strchr(component, '/'); + size_t len = slash ? (size_t)(slash - component) : + strlen(component); + + if (!valid_component(component, len)) + return 0; + if (!slash) + return 1; + component = slash + 1; + } +} + +static int preload_bulk_darwin_open_dir_at( + struct preload_bulk_worker *worker UNUSED, + int parent_fd, const char *name) +{ + if (!valid_component(name, strlen(name)) || strchr(name, '/')) { + errno = EINVAL; + return -1; + } + return openat(parent_fd, name, + O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); +} + +int preload_bulk_darwin_supports_nofollow_any(void) +{ +#ifdef O_NOFOLLOW_ANY + struct utsname uts; + char *end; + unsigned long major; + + /* + * O_NOFOLLOW_ANY arrived in Darwin 20. Older kernels accept the + * same bit as O_ALERT without enforcing no-follow semantics. + */ + if (uname(&uts) || !isdigit((unsigned char)uts.release[0])) + return 0; + errno = 0; + major = strtoul(uts.release, &end, 10); + return !errno && end != uts.release && *end == '.' && major >= 20; +#else + return 0; +#endif +} + +static int preload_bulk_darwin_open_relative(struct preload_bulk_scan *scan, + const char *path) +{ + if (!valid_relative_path(path)) { + errno = EINVAL; + return -1; + } + +#ifdef O_NOFOLLOW_ANY + return openat(scan->root_fd, path, + O_RDONLY | O_DIRECTORY | O_NOFOLLOW_ANY | O_CLOEXEC); +#else + errno = ENOTSUP; + return -1; +#endif +} + +static mode_t vnode_mode(fsobj_type_t type) +{ + switch (type) { + case VREG: + return S_IFREG; + case VLNK: + return S_IFLNK; + default: + return 0; + } +} + +static int fill_file_stat(struct stat *st, dev_t dev, uint64_t fileid, + fsobj_type_t type, struct timespec mtime, + struct timespec ctime, uid_t uid, gid_t gid, + uint32_t access, uint32_t linkcount, off_t size) +{ + mode_t mode = vnode_mode(type); + + if (!mode || size < 0 || + ((access & S_IFMT) && (access & S_IFMT) != mode) || + (access & ~(S_IFMT | 07777))) + return -1; + memset(st, 0, sizeof(*st)); + st->st_dev = dev; + st->st_ino = fileid; + st->st_mode = mode | (access & 07777); + st->st_uid = uid; + st->st_gid = gid; + st->st_nlink = linkcount; + st->st_size = size; + st->st_mtimespec = mtime; + st->st_ctimespec = ctime; + return 0; +} + +struct preload_bulk_darwin_entry { + const char *name; + uint32_t record_len; + dev_t dev; + fsobj_type_t type; + struct timespec birthtime; + struct timespec mtime; + struct timespec ctime; + uid_t uid; + gid_t gid; + uint32_t access; + uint32_t flags; + uint32_t linkcount; + uint32_t mountstatus; + uint64_t fileid; + off_t size; +}; + +static int decode_entry(const char *record, size_t remaining, + struct preload_bulk_darwin_entry *entry) +{ + uint32_t entry_error = 0; + attribute_set_t returned; + attrreference_t name_ref; + const char *p, *end, *name_ref_at; + size_t name_ref_offset, name_offset, name_remaining; + + if (remaining < sizeof(entry->record_len) + sizeof(returned)) + return -1; + memcpy(&entry->record_len, record, sizeof(entry->record_len)); + if ((entry->record_len % sizeof(uint64_t)) || + entry->record_len < sizeof(entry->record_len) + sizeof(returned) || + entry->record_len > remaining) + return -1; + + p = record + sizeof(entry->record_len); + end = record + entry->record_len; + memcpy(&returned, p, sizeof(returned)); + p += sizeof(returned); + if (returned.commonattr != required_common || + returned.volattr || returned.forkattr) + return -1; + +#define TAKE_ATTR(value) do { \ + if ((size_t)(end - p) < sizeof(value)) \ + return -1; \ + memcpy(&(value), p, sizeof(value)); \ + p += sizeof(value); \ +} while (0) + TAKE_ATTR(entry_error); + if (entry_error) + return -1; + name_ref_at = p; + TAKE_ATTR(name_ref); + TAKE_ATTR(entry->dev); + TAKE_ATTR(entry->type); + TAKE_ATTR(entry->birthtime); + TAKE_ATTR(entry->mtime); + TAKE_ATTR(entry->ctime); + TAKE_ATTR(entry->uid); + TAKE_ATTR(entry->gid); + TAKE_ATTR(entry->access); + TAKE_ATTR(entry->flags); + TAKE_ATTR(entry->fileid); + + if (entry->type == VDIR) { + if (returned.dirattr != required_dir || + returned.fileattr) + return -1; + TAKE_ATTR(entry->mountstatus); + } else { + if (returned.dirattr || + (returned.fileattr & ~required_file)) + return -1; + TAKE_ATTR(entry->linkcount); + TAKE_ATTR(entry->size); + if ((entry->type == VREG || entry->type == VLNK) && + returned.fileattr != required_file) + return -1; + } +#undef TAKE_ATTR + + if (name_ref.attr_dataoffset < 0 || + (name_ref.attr_dataoffset % (int32_t)sizeof(uint32_t))) + return -1; + name_ref_offset = name_ref_at - record; + if ((uint32_t)name_ref.attr_dataoffset > + entry->record_len - name_ref_offset) + return -1; + name_offset = name_ref_offset + name_ref.attr_dataoffset; + name_remaining = entry->record_len - name_offset; + entry->name = record + name_offset; + if (!name_ref.attr_length || + name_ref.attr_length > name_remaining || + entry->name < p) + return -1; + if (entry->name[name_ref.attr_length - 1] || + memchr(entry->name, '\0', name_ref.attr_length - 1) || + !valid_component(entry->name, name_ref.attr_length - 1) || + memchr(entry->name, '/', name_ref.attr_length - 1)) + return -1; + return 0; +} + +int preload_bulk_darwin_decode_record(const char *record, size_t len) +{ + struct preload_bulk_darwin_entry entry; + + return decode_entry(record, len, &entry); +} + +static struct preload_bulk_dir_identity directory_identity( + const struct stat *st) +{ + struct preload_bulk_dir_identity result = { + .stat = *st, + .complete = 1, + }; + + return result; +} + +static int directory_identity_matches( + const struct preload_bulk_dir_identity *before, + const struct stat *after) +{ + if (before->complete) + return path_namespace_stat_equal(&before->stat, after); + return S_ISDIR(after->st_mode) && + before->stat.st_dev == after->st_dev && + before->stat.st_ino == after->st_ino && + before->stat.st_birthtimespec.tv_sec == + after->st_birthtimespec.tv_sec && + before->stat.st_birthtimespec.tv_nsec == + after->st_birthtimespec.tv_nsec && + before->stat.st_mtimespec.tv_sec == after->st_mtimespec.tv_sec && + before->stat.st_mtimespec.tv_nsec == + after->st_mtimespec.tv_nsec && + before->stat.st_ctimespec.tv_sec == after->st_ctimespec.tv_sec && + before->stat.st_ctimespec.tv_nsec == + after->st_ctimespec.tv_nsec; +} + +static int enumerate_directory(struct preload_bulk_worker *worker, + const struct preload_bulk_task *task, int fd, + const struct preload_bulk_dir_identity *parent_identity) +{ + struct preload_bulk_scan *scan = worker->scan; + struct preload_bulk_darwin_data *data = scan->platform_data; + struct attrlist attrs = { 0 }; + char *buf = worker->buffer; + size_t path_prefix_len; + + if (!buf) { + buf = xmalloc(PRELOAD_INDEX_BULK_BUFFER_SIZE); + worker->buffer = buf; + } + + attrs.bitmapcount = ATTR_BIT_MAP_COUNT; + attrs.commonattr = required_common; + attrs.dirattr = required_dir; + attrs.fileattr = required_file; + worker->dirs++; + strbuf_reset(&worker->path); + if (strcmp(task->path, ".")) { + strbuf_addstr(&worker->path, task->path); + strbuf_addch(&worker->path, '/'); + } + path_prefix_len = worker->path.len; + + for (;;) { + int nr = getattrlistbulk(fd, &attrs, buf, + PRELOAD_INDEX_BULK_BUFFER_SIZE, + FSOPT_NOFOLLOW | + FSOPT_PACK_INVAL_ATTRS); + char *record = buf; + + worker->bulk_calls++; + if (nr < 0) + return -1; + if (!nr) + return 0; + + for (int i = 0; i < nr; i++) { + struct preload_bulk_darwin_entry entry; + struct stat st; + const char *path_name; + size_t remaining; + int pos; + + if (scan->collect_untracked && + preload_bulk_untracked_root_is_visible( + worker, task->untracked_root)) + return 0; + remaining = buf + PRELOAD_INDEX_BULK_BUFFER_SIZE - record; + if (decode_entry(record, remaining, &entry)) + goto malformed; + worker->entries++; + + /* + * The caller prepares the repository's Unicode policy + * before starting workers, so this is read-only here. + */ + path_name = repo_precompose_string_if_needed(scan->repo, + entry.name); + strbuf_setlen(&worker->path, path_prefix_len); + strbuf_addstr(&worker->path, path_name); + if (path_name != entry.name) + free((char *)path_name); + if (scan->collect_untracked) { + if (!strcmp(task->path, ".") && + !fspathcmp(worker->path.buf, ".git")) + goto next_record; + if (strcmp(task->path, ".") && + !fspathcmp(entry.name, ".git")) { + preload_bulk_invalidate_untracked( + worker); + goto next_record; + } + } + + pos = preload_bulk_index_position(scan, worker->path.buf, + worker->path.len); + if (entry.type == VDIR) { + struct preload_bulk_dir_identity child_identity = { + .stat = { + .st_dev = entry.dev, + .st_ino = entry.fileid, + .st_birthtimespec = + entry.birthtime, + .st_mtimespec = entry.mtime, + .st_ctimespec = entry.ctime, + }, + }; + struct preload_bulk_untracked_root *untracked_root = + task->untracked_root; + int has_tracked_descendants; + + if (pos >= 0) { + if (scan->collect_untracked && + preload_bulk_index_entry_is_gitlink( + scan, pos)) + goto next_record; + preload_bulk_record_tracked_fallback( + worker, pos); + goto next_record; + } + has_tracked_descendants = + preload_bulk_index_pos_has_tracked_descendants( + scan, worker->path.buf, + worker->path.len, pos); + if (!has_tracked_descendants && + preload_bulk_record_tracked_alias_fallback( + worker, worker->path.buf, + worker->path.len)) { + if (scan->collect_untracked) + preload_bulk_invalidate_untracked( + worker); + goto next_record; + } + if (!has_tracked_descendants) { + if (!scan->collect_untracked || + preload_bulk_untracked_is_invalid( + worker)) + goto next_record; + if (preload_bulk_untracked_root_is_visible( + worker, untracked_root)) + goto next_record; + if (preload_bulk_path_is_excluded( + worker, worker->path.buf, + DT_DIR)) + goto next_record; + if (!untracked_root) + untracked_root = + preload_bulk_untracked_root_new( + worker, + worker->path.buf, + worker->path.len); + } + if (((entry.access & S_IFMT) && + (entry.access & S_IFMT) != S_IFDIR) || + (entry.access & ~(S_IFMT | 07777))) + goto malformed_record; + if (entry.dev != data->root_stat.st_dev || + entry.mountstatus || + (entry.flags & SF_FIRMLINK)) { + preload_bulk_record_tracked_descendants_fallback( + worker, worker->path.buf, + worker->path.len); + if (scan->collect_untracked) + preload_bulk_invalidate_untracked( + worker); + goto next_record; + } + preload_bulk_schedule_directory( + worker, fd, parent_identity, + &child_identity, untracked_root, + entry.name, + worker->path.buf, + worker->path.len); + goto next_record; + } + + if (pos < 0) { + int found_alias = + preload_bulk_record_tracked_alias_fallback( + worker, worker->path.buf, + worker->path.len); + + if (scan->collect_untracked && + !preload_bulk_untracked_is_invalid( + worker)) { + int dtype; + + if (found_alias) { + preload_bulk_invalidate_untracked( + worker); + goto next_record; + } + if (entry.type == VREG) + dtype = DT_REG; + else if (entry.type == VLNK) + dtype = DT_LNK; + else + goto next_record; + if (!preload_bulk_path_is_excluded( + worker, worker->path.buf, + dtype)) + preload_bulk_record_untracked( + worker, + task->untracked_root, + worker->path.buf); + } + goto next_record; + } + if (entry.dev != data->root_stat.st_dev) { + preload_bulk_record_tracked_fallback( + worker, pos); + goto next_record; + } + if (entry.type != VREG && entry.type != VLNK) { + preload_bulk_record_tracked_fallback( + worker, pos); + goto next_record; + } + if (entry.linkcount != 1) { + preload_bulk_record_tracked_fallback( + worker, pos); + goto next_record; + } + if (fill_file_stat(&st, entry.dev, entry.fileid, + entry.type, entry.mtime, entry.ctime, + entry.uid, entry.gid, entry.access, + entry.linkcount, entry.size)) + goto malformed_record; + preload_bulk_record_tracked( + worker, pos, fd, entry.name, &st, 0); + +next_record: + record += entry.record_len; + continue; + +malformed_record: + worker->malformed++; + goto next_record; + } + } + +malformed: + worker->malformed++; + return -1; +} + +static int scan_directory(struct preload_bulk_worker *worker, + struct preload_bulk_task *task) +{ + struct preload_bulk_scan *scan = worker->scan; + struct preload_bulk_dir_identity before_identity; + struct stat before, after; + size_t path_len; + int fd = task->fd; + int ret = -1; + + if (fd < 0) + fd = preload_bulk_darwin_open_relative(scan, task->path); + if (fd < 0) + goto out; + if (preload_bulk_test_barrier(scan, task->path)) + goto out; + if (preload_bulk_darwin_fd_on_root_mount(scan, fd, &before)) { + if (errno != EXDEV) + goto out; + path_len = strlen(task->path); + preload_bulk_record_tracked_descendants_fallback( + worker, task->path, path_len); + if (scan->collect_untracked) + preload_bulk_invalidate_untracked(worker); + ret = 0; + goto out; + } + /* + * A child may have been replaced after its parent returned the bulk + * record, or while this task waited in the queue. + */ + if (task->has_child_identity && + !directory_identity_matches(&task->child_identity, &before)) { + worker->changed_dirs++; + ret = 0; + goto out; + } + before_identity = directory_identity(&before); + if ((!scan->collect_untracked || + !preload_bulk_untracked_root_is_visible( + worker, task->untracked_root)) && + enumerate_directory(worker, task, fd, &before_identity)) + goto out; + if (fstat(fd, &after)) + goto out; + if (!directory_identity_matches(&before_identity, &after)) + worker->changed_dirs++; + ret = 0; + +out: + if (task->has_parent_identity) { + struct stat parent_after; + int parent_changed = fd < 0; + + /* + * Resolve ".." through the child descriptor, not the worktree + * path, so a rename cannot redirect this parent check. + */ + if (!parent_changed) + parent_changed = fstatat(fd, "..", &parent_after, + AT_SYMLINK_NOFOLLOW); + if (parent_changed || + !directory_identity_matches(&task->parent_identity, + &parent_after)) + worker->changed_dirs++; + } + if (fd >= 0) + close(fd); + return ret; +} + +static const char *start_scan(struct preload_bulk_scan *scan) +{ + const char *error; + + repo_precompose_utf8_prepare(scan->repo); + error = preload_bulk_darwin_open_root(scan); + if (error) + return error; + return preload_bulk_darwin_snapshot_root(scan); +} + +static const char *finish_scan(struct preload_bulk_scan *scan) +{ + return preload_bulk_darwin_validate_root(scan); +} + +static const struct preload_bulk_backend darwin_backend = { + .collects_untracked = 1, + .start = start_scan, + .finish = finish_scan, + .release = preload_bulk_darwin_release, + .open_proof_parent = preload_bulk_darwin_open_relative, + .open_dir_at = preload_bulk_darwin_open_dir_at, + .scan_directory = scan_directory, +}; + +const struct preload_bulk_backend *preload_bulk_platform_backend(void) +{ +#ifdef O_NOFOLLOW_ANY + if (preload_bulk_darwin_supports_nofollow_any()) + return &darwin_backend; +#endif + return NULL; +} diff --git a/compat/preload-index/bulk-darwin.h b/compat/preload-index/bulk-darwin.h new file mode 100644 index 00000000000000..7c5c45ee947651 --- /dev/null +++ b/compat/preload-index/bulk-darwin.h @@ -0,0 +1,29 @@ +#ifndef PRELOAD_INDEX_BULK_DARWIN_H +#define PRELOAD_INDEX_BULK_DARWIN_H + +#ifdef __APPLE__ + +#include + +struct preload_bulk_scan; + +struct preload_bulk_darwin_data { + struct stat root_stat; + fsid_t root_fsid; +}; + +int preload_bulk_darwin_supports_nofollow_any(void); +/* + * Exposed so that tests can validate kernel-supplied records directly. + */ +int preload_bulk_darwin_decode_record(const char *record, size_t len); +int preload_bulk_darwin_fd_on_root_mount(struct preload_bulk_scan *scan, + int fd, struct stat *st_out); +const char *preload_bulk_darwin_open_root(struct preload_bulk_scan *scan); +const char *preload_bulk_darwin_snapshot_root(struct preload_bulk_scan *scan); +const char *preload_bulk_darwin_validate_root(struct preload_bulk_scan *scan); +void preload_bulk_darwin_release(struct preload_bulk_scan *scan); + +#endif /* __APPLE__ */ + +#endif /* PRELOAD_INDEX_BULK_DARWIN_H */ diff --git a/compat/preload-index/bulk-linux-entry.c b/compat/preload-index/bulk-linux-entry.c new file mode 100644 index 00000000000000..02cb27882bece3 --- /dev/null +++ b/compat/preload-index/bulk-linux-entry.c @@ -0,0 +1,268 @@ +#include "git-compat-util.h" + +#ifdef __linux__ + +#include +#include + +#include "compat/preload-index/bulk-linux.h" +#include "dir.h" +#include "preload-index-bulk.h" + +#define PRELOAD_INDEX_BULK_LINUX_BUFFER_SIZE (1024 * 1024) + +struct preload_linux_dirent64 { + uint64_t ino; + int64_t off; + uint16_t reclen; + uint8_t type; + char name[FLEX_ARRAY]; +}; + +#if defined(SYS_getdents64) && defined(SYS_statx) + +static void record_foreign_entry(struct preload_bulk_worker *worker, + const char *path, size_t path_len, + int pos, mode_t mode) +{ + if (pos >= 0) + preload_bulk_record_tracked_fallback(worker, pos); + if (S_ISDIR(mode)) + preload_bulk_record_tracked_descendants_fallback( + worker, path, path_len); + if (worker->scan->collect_untracked) + preload_bulk_invalidate_untracked(worker); +} + +static void handle_directory( + struct preload_bulk_worker *worker, + const struct preload_bulk_task *task, int fd, + const struct preload_bulk_dir_identity *parent_identity, + const char *name, int pos) +{ + struct preload_bulk_scan *scan = worker->scan; + struct preload_bulk_untracked_root *untracked_root = + task->untracked_root; + int has_tracked_descendants; + + if (pos >= 0) { + if (scan->collect_untracked && + preload_bulk_index_entry_is_gitlink(scan, pos)) + return; + preload_bulk_record_tracked_fallback(worker, pos); + return; + } + has_tracked_descendants = + preload_bulk_index_pos_has_tracked_descendants( + scan, worker->path.buf, worker->path.len, pos); + if (!has_tracked_descendants && + preload_bulk_record_tracked_alias_fallback( + worker, worker->path.buf, worker->path.len)) { + if (scan->collect_untracked) + preload_bulk_invalidate_untracked(worker); + return; + } + if (!has_tracked_descendants) { + if (!scan->collect_untracked || + preload_bulk_untracked_is_invalid(worker) || + preload_bulk_untracked_root_is_visible( + worker, untracked_root) || + preload_bulk_path_is_excluded( + worker, worker->path.buf, DT_DIR)) + return; + if (!untracked_root) + untracked_root = preload_bulk_untracked_root_new( + worker, worker->path.buf, worker->path.len); + } + preload_bulk_schedule_directory( + worker, fd, parent_identity, NULL, untracked_root, + name, worker->path.buf, worker->path.len); +} + +static void record_untracked(struct preload_bulk_worker *worker, + const struct preload_bulk_task *task, + int dtype) +{ + struct preload_bulk_scan *scan = worker->scan; + + if (!scan->collect_untracked || + preload_bulk_untracked_is_invalid(worker)) + return; + if (preload_bulk_record_tracked_alias_fallback( + worker, worker->path.buf, worker->path.len)) { + preload_bulk_invalidate_untracked(worker); + return; + } + if (!preload_bulk_path_is_excluded( + worker, worker->path.buf, dtype)) + preload_bulk_record_untracked( + worker, task->untracked_root, worker->path.buf); +} + +int preload_bulk_linux_enumerate( + struct preload_bulk_worker *worker, + struct preload_bulk_task *task, int fd, + const struct preload_bulk_dir_identity *parent_identity) +{ + struct preload_bulk_scan *scan = worker->scan; + struct preload_bulk_linux_data *data = scan->platform_data; + char *buf = worker->buffer; + size_t path_prefix_len; + + if (!buf) { + buf = xmalloc(PRELOAD_INDEX_BULK_LINUX_BUFFER_SIZE); + worker->buffer = buf; + } + worker->dirs++; + strbuf_reset(&worker->path); + if (strcmp(task->path, ".")) { + strbuf_addstr(&worker->path, task->path); + strbuf_addch(&worker->path, '/'); + } + path_prefix_len = worker->path.len; + + for (;;) { + long bytes = syscall(SYS_getdents64, fd, buf, + PRELOAD_INDEX_BULK_LINUX_BUFFER_SIZE); + size_t offset = 0; + + worker->bulk_calls++; + if (bytes < 0) + return -1; + if (!bytes) + return 0; + while (offset < (size_t)bytes) { + struct preload_linux_dirent64 *de = + (void *)(buf + offset); + size_t minimum = + offsetof(struct preload_linux_dirent64, name) + 1; + size_t name_space; + struct preload_linux_statx stx; + struct stat st; + char *nul; + unsigned char dtype; + int has_tracked_descendants = 0, pos; + + if (scan->collect_untracked && + preload_bulk_untracked_root_is_visible( + worker, task->untracked_root)) + return 0; + if ((size_t)bytes - offset < minimum || + de->reclen < minimum || + de->reclen > (size_t)bytes - offset) + goto malformed; + name_space = de->reclen - + offsetof(struct preload_linux_dirent64, name); + nul = memchr(de->name, '\0', name_space); + if (!nul || nul == de->name || + memchr(de->name, '/', nul - de->name)) + goto malformed; + offset += de->reclen; + if (is_dot_or_dotdot(de->name)) + continue; + worker->entries++; + if (!fspathcmp(de->name, ".git")) { + if (strcmp(task->path, ".") && + scan->collect_untracked) + preload_bulk_invalidate_untracked( + worker); + continue; + } + + strbuf_setlen(&worker->path, path_prefix_len); + strbuf_addstr(&worker->path, de->name); + if (worker->path.len > INT_MAX) + goto malformed; + pos = preload_bulk_index_position( + scan, worker->path.buf, worker->path.len); + dtype = de->type; + if (data->test_dirent_path && + !strcmp(data->test_dirent_path, worker->path.buf)) + dtype = data->test_dirent_type; + + /* + * Exact tracked paths always reach statx. A directory + * which may contain tracked descendants can be + * scheduled directly: the O_DIRECTORY open and + * descriptor statx remain authoritative. + * + * A hint cannot classify a wholly untracked entry: + * file-versus-directory changes exclude matching and + * result shape. Force those entries through statx before + * taking either shortcut. + */ + if (pos < 0 && dtype != DT_UNKNOWN) + has_tracked_descendants = + preload_bulk_index_pos_has_tracked_descendants( + scan, worker->path.buf, + worker->path.len, pos); + if (scan->collect_untracked && pos < 0 && + !has_tracked_descendants) + dtype = DT_UNKNOWN; + if (dtype == DT_DIR && pos < 0) { + handle_directory(worker, task, fd, + parent_identity, + de->name, pos); + continue; + } + if (pos < 0 && !has_tracked_descendants && + (dtype == DT_REG || dtype == DT_LNK)) { + record_untracked( + worker, task, + dtype == DT_LNK ? DT_LNK : DT_REG); + continue; + } + if (pos < 0 && !has_tracked_descendants && + dtype != DT_UNKNOWN) { + preload_bulk_record_tracked_alias_fallback( + worker, worker->path.buf, + worker->path.len); + continue; + } + if (preload_bulk_linux_entry_stat( + worker, fd, de->name, &stx, &st)) { + if (errno == EXDEV) { + record_foreign_entry( + worker, worker->path.buf, + worker->path.len, pos, + stx.mode); + continue; + } + goto malformed; + } + if (S_ISDIR(st.st_mode)) { + handle_directory(worker, task, fd, + parent_identity, + de->name, pos); + continue; + } + if (pos < 0) { + if (S_ISREG(st.st_mode)) + record_untracked(worker, task, DT_REG); + else if (S_ISLNK(st.st_mode)) + record_untracked(worker, task, DT_LNK); + else + preload_bulk_record_tracked_alias_fallback( + worker, worker->path.buf, + worker->path.len); + continue; + } + if ((!S_ISREG(st.st_mode) && !S_ISLNK(st.st_mode)) || + st.st_nlink != 1) { + preload_bulk_record_tracked_fallback( + worker, pos); + continue; + } + preload_bulk_record_tracked( + worker, pos, fd, de->name, &st, 0); + } + } + +malformed: + worker->malformed++; + return -1; +} + +#endif /* SYS_getdents64 && SYS_statx */ + +#endif /* __linux__ */ diff --git a/compat/preload-index/bulk-linux-open.c b/compat/preload-index/bulk-linux-open.c new file mode 100644 index 00000000000000..02cd5b55b87f43 --- /dev/null +++ b/compat/preload-index/bulk-linux-open.c @@ -0,0 +1,149 @@ +#include "git-compat-util.h" + +#ifdef __linux__ + +#include + +#include "compat/preload-index/bulk-linux.h" +#include "preload-index-bulk.h" + +#if defined(SYS_getdents64) && defined(SYS_statx) + +static int valid_component(const char *component, size_t len) +{ + return len && + !(len == 1 && component[0] == '.') && + !(len == 2 && component[0] == '.' && component[1] == '.'); +} + +static int verify_mount(struct preload_bulk_scan *scan, int fd) +{ + struct preload_bulk_linux_data *data = scan->platform_data; + struct preload_linux_statx stx; + + if (preload_bulk_linux_statx_raw(fd, "", PRELOAD_AT_EMPTY_PATH, + &stx)) + return -1; + if (!preload_bulk_linux_statx_complete(&stx)) { + errno = EOPNOTSUPP; + return -1; + } + if (stx.mnt_id != data->root_mnt_id) { + errno = EXDEV; + return -1; + } + return 0; +} + +#ifdef SYS_openat2 +int preload_bulk_linux_openat2_raw(int dirfd, const char *path) +{ + struct preload_linux_open_how how = { + .flags = O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC, + .resolve = PRELOAD_RESOLVE_BENEATH | + PRELOAD_RESOLVE_NO_SYMLINKS | + PRELOAD_RESOLVE_NO_MAGICLINKS | + PRELOAD_RESOLVE_NO_XDEV, + }; + + return syscall(SYS_openat2, dirfd, path, &how, sizeof(how)); +} +#endif + +static int open_one_fallback(struct preload_bulk_scan *scan, int parent_fd, + const char *name, int check_mount) +{ + int fd = openat(parent_fd, name, + O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + + if (fd < 0) + return -1; + if (check_mount && verify_mount(scan, fd)) { + int saved_errno = errno; + + close(fd); + errno = saved_errno; + return -1; + } + return fd; +} + +int preload_bulk_linux_open_dir_at( + struct preload_bulk_worker *worker, int parent_fd, + const char *name) +{ + struct preload_bulk_scan *scan = worker->scan; + struct preload_bulk_linux_data *data = scan->platform_data; + + if (!valid_component(name, strlen(name)) || strchr(name, '/')) { + errno = EINVAL; + return -1; + } +#ifdef SYS_openat2 + if (data->use_openat2) + return preload_bulk_linux_openat2_raw(parent_fd, name); +#else + (void)data; +#endif + /* scan_directory() verifies the opened descriptor's mount ID. */ + return open_one_fallback(scan, parent_fd, name, 0); +} + +int preload_bulk_linux_open_relative(struct preload_bulk_scan *scan, + const char *path) +{ + struct preload_bulk_linux_data *data = scan->platform_data; + const char *component = path; + int fd; + + if (!*path || *path == '/' || path[strlen(path) - 1] == '/') { + errno = EINVAL; + return -1; + } +#ifdef SYS_openat2 + if (data->use_openat2) + return preload_bulk_linux_openat2_raw(scan->root_fd, path); +#else + (void)data; +#endif + fd = fcntl(scan->root_fd, F_DUPFD_CLOEXEC, 0); + if (fd < 0) + return -1; + if (verify_mount(scan, fd)) { + int saved_errno = errno; + + close(fd); + errno = saved_errno; + return -1; + } + if (!strcmp(path, ".")) + return fd; + while (*component) { + const char *slash = strchr(component, '/'); + size_t len = slash ? (size_t)(slash - component) : + strlen(component); + char *name; + int next; + + if (!valid_component(component, len)) { + close(fd); + errno = EINVAL; + return -1; + } + name = xmemdupz(component, len); + next = open_one_fallback(scan, fd, name, 1); + free(name); + close(fd); + if (next < 0) + return -1; + fd = next; + if (!slash) + break; + component = slash + 1; + } + return fd; +} + +#endif /* SYS_getdents64 && SYS_statx */ + +#endif /* __linux__ */ diff --git a/compat/preload-index/bulk-linux-scan.c b/compat/preload-index/bulk-linux-scan.c new file mode 100644 index 00000000000000..c6da94aa41564b --- /dev/null +++ b/compat/preload-index/bulk-linux-scan.c @@ -0,0 +1,107 @@ +#include "git-compat-util.h" + +#ifdef __linux__ + +#include + +#include "compat/preload-index/bulk-linux.h" +#include "path-namespace.h" +#include "preload-index-bulk.h" + +#if defined(SYS_getdents64) && defined(SYS_statx) + +static struct preload_bulk_dir_identity directory_identity( + const struct preload_linux_statx *stx, const struct stat *st) +{ + struct preload_bulk_dir_identity result = { + .stat = *st, + .platform_id = stx->mnt_id, + .complete = 1, + }; + + return result; +} + +static int directory_identity_matches( + const struct preload_bulk_dir_identity *before, + const struct preload_linux_statx *stx, const struct stat *after) +{ + return S_ISDIR(after->st_mode) && + path_namespace_stat_equal(&before->stat, after) && + before->platform_id == stx->mnt_id; +} + +int preload_bulk_linux_scan_directory(struct preload_bulk_worker *worker, + struct preload_bulk_task *task) +{ + struct preload_bulk_scan *scan = worker->scan; + struct preload_linux_statx before_stx, after_stx; + struct preload_bulk_dir_identity before_identity; + struct stat before, after; + size_t path_len; + int fd = task->fd; + int ret = -1; + + if (fd < 0) + fd = preload_bulk_linux_open_relative(scan, task->path); + if (fd < 0) + goto out; + if (preload_bulk_test_barrier(scan, task->path)) + goto out; + if (preload_bulk_linux_fd_stat( + worker, fd, &before_stx, &before) || + !S_ISDIR(before.st_mode)) { + if (errno != EXDEV) + goto out; + path_len = strlen(task->path); + preload_bulk_record_tracked_descendants_fallback( + worker, task->path, path_len); + if (scan->collect_untracked) + preload_bulk_invalidate_untracked(worker); + ret = 0; + goto out; + } + before_identity = directory_identity(&before_stx, &before); + if ((!scan->collect_untracked || + !preload_bulk_untracked_root_is_visible( + worker, task->untracked_root)) && + preload_bulk_linux_enumerate( + worker, task, fd, &before_identity)) + goto out; + if (preload_bulk_linux_fd_stat( + worker, fd, &after_stx, &after)) + goto out; + if (!preload_bulk_linux_statx_same(&before_stx, &after_stx) || + !directory_identity_matches( + &before_identity, &after_stx, &after)) + worker->changed_dirs++; + ret = 0; + +out: + if (task->has_parent_identity) { + struct preload_linux_statx parent_stx; + struct stat parent_after; + int parent_changed = fd < 0; + + /* + * Resolve ".." through the held child descriptor so a move + * cannot redirect the parent check to the old path. + */ + if (!parent_changed) + parent_changed = preload_bulk_linux_entry_stat( + worker, fd, "..", &parent_stx, + &parent_after); + if (parent_changed || + !directory_identity_matches( + &task->parent_identity, &parent_stx, + &parent_after)) + worker->changed_dirs++; + } + if (fd >= 0) + close(fd); + return ret; +} + +#endif /* SYS_getdents64 && SYS_statx */ + +#endif /* __linux__ */ diff --git a/compat/preload-index/bulk-linux-stat.c b/compat/preload-index/bulk-linux-stat.c new file mode 100644 index 00000000000000..a8594d7230f9d0 --- /dev/null +++ b/compat/preload-index/bulk-linux-stat.c @@ -0,0 +1,140 @@ +#include "git-compat-util.h" + +#ifdef __linux__ + +#include +#include + +#include "compat/preload-index/bulk-linux.h" +#include "preload-index-bulk.h" + +#if defined(SYS_getdents64) && defined(SYS_statx) + +int preload_bulk_linux_statx_raw(int dirfd, const char *path, int flags, + struct preload_linux_statx *stx) +{ + memset(stx, 0, sizeof(*stx)); + return syscall(SYS_statx, dirfd, path, flags, + PRELOAD_STATX_BASIC_STATS | PRELOAD_STATX_MNT_ID, stx); +} + +int preload_bulk_linux_statx_complete( + const struct preload_linux_statx *stx) +{ + return (stx->mask & + (PRELOAD_STATX_BASIC_STATS | PRELOAD_STATX_MNT_ID)) == + (PRELOAD_STATX_BASIC_STATS | PRELOAD_STATX_MNT_ID) && + stx->mtime.tv_nsec < 1000000000 && + stx->ctime.tv_nsec < 1000000000; +} + +int preload_bulk_linux_statx_same(const struct preload_linux_statx *a, + const struct preload_linux_statx *b) +{ + return preload_bulk_linux_statx_complete(a) && + preload_bulk_linux_statx_complete(b) && + a->mnt_id == b->mnt_id && + a->dev_major == b->dev_major && + a->dev_minor == b->dev_minor && + a->ino == b->ino && a->mode == b->mode && + a->nlink == b->nlink && a->uid == b->uid && + a->gid == b->gid && a->size == b->size && + a->mtime.tv_sec == b->mtime.tv_sec && + a->mtime.tv_nsec == b->mtime.tv_nsec && + a->ctime.tv_sec == b->ctime.tv_sec && + a->ctime.tv_nsec == b->ctime.tv_nsec; +} + +static int statx_to_stat(const struct preload_linux_statx *stx, + struct stat *st) +{ + dev_t dev; + + if (!preload_bulk_linux_statx_complete(stx)) + return -1; + memset(st, 0, sizeof(*st)); + dev = makedev(stx->dev_major, stx->dev_minor); + if (major(dev) != stx->dev_major || minor(dev) != stx->dev_minor) + return -1; + st->st_dev = dev; + st->st_ino = stx->ino; + if ((uint64_t)st->st_ino != stx->ino) + return -1; + st->st_mode = stx->mode; + st->st_nlink = stx->nlink; + if ((uint64_t)st->st_nlink != stx->nlink) + return -1; + st->st_uid = stx->uid; + st->st_gid = stx->gid; + if ((uint64_t)st->st_uid != stx->uid || + (uint64_t)st->st_gid != stx->gid) + return -1; + st->st_size = stx->size; + if (st->st_size < 0 || (uint64_t)st->st_size != stx->size) + return -1; + st->st_mtim.tv_sec = stx->mtime.tv_sec; + st->st_mtim.tv_nsec = stx->mtime.tv_nsec; + st->st_ctim.tv_sec = stx->ctime.tv_sec; + st->st_ctim.tv_nsec = stx->ctime.tv_nsec; + if ((int64_t)st->st_mtim.tv_sec != stx->mtime.tv_sec || + (int64_t)st->st_ctim.tv_sec != stx->ctime.tv_sec) + return -1; + return 0; +} + +int preload_bulk_linux_entry_stat(struct preload_bulk_worker *worker, + int dirfd, const char *name, + struct preload_linux_statx *stx, + struct stat *st) +{ + struct preload_bulk_linux_data *data = + worker->scan->platform_data; + + if (preload_bulk_linux_statx_raw( + dirfd, name, + PRELOAD_AT_SYMLINK_NOFOLLOW | PRELOAD_AT_NO_AUTOMOUNT, + stx)) + return -1; + if (!preload_bulk_linux_statx_complete(stx)) { + errno = EOPNOTSUPP; + return -1; + } + if (stx->mnt_id != data->root_mnt_id) { + errno = EXDEV; + return -1; + } + if (statx_to_stat(stx, st)) { + errno = EOVERFLOW; + return -1; + } + return 0; +} + +int preload_bulk_linux_fd_stat(struct preload_bulk_worker *worker, int fd, + struct preload_linux_statx *stx, + struct stat *st) +{ + struct preload_bulk_linux_data *data = + worker->scan->platform_data; + + if (preload_bulk_linux_statx_raw(fd, "", PRELOAD_AT_EMPTY_PATH, + stx)) + return -1; + if (!preload_bulk_linux_statx_complete(stx)) { + errno = EOPNOTSUPP; + return -1; + } + if (stx->mnt_id != data->root_mnt_id) { + errno = EXDEV; + return -1; + } + if (statx_to_stat(stx, st)) { + errno = EOVERFLOW; + return -1; + } + return 0; +} + +#endif /* SYS_getdents64 && SYS_statx */ + +#endif /* __linux__ */ diff --git a/compat/preload-index/bulk-linux-topology.c b/compat/preload-index/bulk-linux-topology.c new file mode 100644 index 00000000000000..523e6c08b909b3 --- /dev/null +++ b/compat/preload-index/bulk-linux-topology.c @@ -0,0 +1,166 @@ +#include "git-compat-util.h" + +#ifdef __linux__ + +#include +#include +#include + +#include "compat/preload-index/bulk-linux.h" +#include "parse.h" +#include "preload-index-bulk.h" +#include "repository.h" +#include "trace2.h" + +#ifndef EXT_FAMILY_SUPER_MAGIC +#define EXT_FAMILY_SUPER_MAGIC 0xef53 +#endif +#ifndef XFS_SUPER_MAGIC +#define XFS_SUPER_MAGIC 0x58465342 +#endif + +#if defined(SYS_getdents64) && defined(SYS_statx) + +static void load_test_dirent_type(struct preload_bulk_linux_data *data) +{ + const char *path, *value; + + if (!git_env_bool("GIT_TEST_PRELOAD_INDEX_BULK", 0)) + return; + value = getenv("GIT_TEST_PRELOAD_INDEX_BULK_DIRENT_TYPE"); + if (!value) + return; + if (skip_prefix(value, "dir:", &path)) + data->test_dirent_type = DT_DIR; + else if (skip_prefix(value, "reg:", &path)) + data->test_dirent_type = DT_REG; + else + die("invalid GIT_TEST_PRELOAD_INDEX_BULK_DIRENT_TYPE"); + if (!*path) + die("GIT_TEST_PRELOAD_INDEX_BULK_DIRENT_TYPE needs a path"); + data->test_dirent_path = xstrdup(path); +} + +static int read_mountinfo(struct strbuf *out) +{ + int fd = open("/proc/self/mountinfo", O_RDONLY | O_CLOEXEC); + int ret = -1; + + if (fd < 0) + return -1; + strbuf_reset(out); + if (strbuf_read(out, fd, 0) >= 0) + ret = 0; + if (close(fd)) + ret = -1; + return ret; +} + +const char *preload_bulk_linux_start(struct preload_bulk_scan *scan) +{ + struct preload_bulk_linux_data *data; + struct statfs fs; + const char *fs_name; + + CALLOC_ARRAY(data, 1); + strbuf_init(&data->mountinfo, 0); + load_test_dirent_type(data); + scan->platform_data = data; + scan->root_fd = open(repo_get_work_tree(scan->repo), + O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (scan->root_fd < 0 || fstatfs(scan->root_fd, &fs)) + return "unsupported-filesystem"; + if ((unsigned long)fs.f_type == EXT_FAMILY_SUPER_MAGIC) + fs_name = "ext-family"; + else if ((unsigned long)fs.f_type == XFS_SUPER_MAGIC) + fs_name = "xfs"; + else + return "unsupported-filesystem"; + trace2_data_string("index", scan->repo, "preload/bulk_filesystem", + fs_name); + if (preload_bulk_linux_statx_raw( + scan->root_fd, "", PRELOAD_AT_EMPTY_PATH, + &data->root_statx) || + !preload_bulk_linux_statx_complete(&data->root_statx) || + !S_ISDIR(data->root_statx.mode)) + return "statx-unavailable"; + data->root_mnt_id = data->root_statx.mnt_id; + if (read_mountinfo(&data->mountinfo)) + return "namespace-check-unavailable"; +#ifdef SYS_openat2 + { + int fd = preload_bulk_linux_openat2_raw(scan->root_fd, "."); + + if (fd >= 0) { + struct preload_linux_statx probe; + + if (!preload_bulk_linux_statx_raw( + fd, "", PRELOAD_AT_EMPTY_PATH, &probe) && + preload_bulk_linux_statx_complete(&probe) && + probe.mnt_id == data->root_mnt_id) + data->use_openat2 = 1; + close(fd); + } + } +#endif + trace2_data_intmax("index", scan->repo, "preload/bulk_openat2", + data->use_openat2); + return NULL; +} + +const char *preload_bulk_linux_finish(struct preload_bulk_scan *scan) +{ + struct preload_bulk_linux_data *data = scan->platform_data; + struct strbuf after = STRBUF_INIT; + struct preload_linux_statx root_after; + const char *result = NULL; + int fd; + + if (read_mountinfo(&after)) { + result = "namespace-check-unavailable"; + goto out; + } + if (strbuf_cmp(&data->mountinfo, &after)) { + trace2_data_intmax( + "index", scan->repo, + "preload/bulk_namespace_churn", 1); + result = "namespace-churn"; + goto out; + } + fd = open(repo_get_work_tree(scan->repo), + O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (fd < 0) { + result = "namespace-race"; + goto out; + } + if (preload_bulk_linux_statx_raw( + fd, "", PRELOAD_AT_EMPTY_PATH, &root_after) || + !preload_bulk_linux_statx_same( + &data->root_statx, &root_after)) + result = "namespace-race"; + close(fd); + +out: + strbuf_release(&after); + return result; +} + +void preload_bulk_linux_release(struct preload_bulk_scan *scan) +{ + struct preload_bulk_linux_data *data = scan->platform_data; + + if (scan->root_fd >= 0) { + close(scan->root_fd); + scan->root_fd = -1; + } + if (!data) + return; + strbuf_release(&data->mountinfo); + free(data->test_dirent_path); + free(data); + scan->platform_data = NULL; +} + +#endif /* SYS_getdents64 && SYS_statx */ + +#endif /* __linux__ */ diff --git a/compat/preload-index/bulk-linux.c b/compat/preload-index/bulk-linux.c new file mode 100644 index 00000000000000..5c4c120a61840c --- /dev/null +++ b/compat/preload-index/bulk-linux.c @@ -0,0 +1,37 @@ +#include "git-compat-util.h" + +#ifdef __linux__ + +#include + +#include "compat/preload-index/bulk-linux.h" +#include "preload-index-bulk.h" + +#if defined(SYS_getdents64) && defined(SYS_statx) + +static const struct preload_bulk_backend linux_backend = { + .collects_untracked = 1, + .max_threads = 16, + .start = preload_bulk_linux_start, + .finish = preload_bulk_linux_finish, + .release = preload_bulk_linux_release, + .open_proof_parent = preload_bulk_linux_open_relative, + .open_dir_at = preload_bulk_linux_open_dir_at, + .scan_directory = preload_bulk_linux_scan_directory, +}; + +const struct preload_bulk_backend *preload_bulk_platform_backend(void) +{ + return &linux_backend; +} + +#else /* !SYS_getdents64 || !SYS_statx */ + +const struct preload_bulk_backend *preload_bulk_platform_backend(void) +{ + return NULL; +} + +#endif + +#endif /* __linux__ */ diff --git a/compat/preload-index/bulk-linux.h b/compat/preload-index/bulk-linux.h new file mode 100644 index 00000000000000..e25db006115584 --- /dev/null +++ b/compat/preload-index/bulk-linux.h @@ -0,0 +1,112 @@ +#ifndef PRELOAD_INDEX_BULK_LINUX_H +#define PRELOAD_INDEX_BULK_LINUX_H + +#ifdef __linux__ + +#include + +#include "strbuf.h" + +#define PRELOAD_AT_NO_AUTOMOUNT 0x800 +#define PRELOAD_AT_EMPTY_PATH 0x1000 +#define PRELOAD_AT_SYMLINK_NOFOLLOW 0x100 +#define PRELOAD_STATX_BASIC_STATS 0x000007ffU +#define PRELOAD_STATX_MNT_ID 0x00001000U +#define PRELOAD_RESOLVE_NO_XDEV 0x01 +#define PRELOAD_RESOLVE_NO_MAGICLINKS 0x02 +#define PRELOAD_RESOLVE_NO_SYMLINKS 0x04 +#define PRELOAD_RESOLVE_BENEATH 0x08 + +struct preload_linux_statx_timestamp { + int64_t tv_sec; + uint32_t tv_nsec; + int32_t reserved; +}; + +struct preload_linux_statx { + uint32_t mask; + uint32_t blksize; + uint64_t attributes; + uint32_t nlink; + uint32_t uid; + uint32_t gid; + uint16_t mode; + uint16_t spare0; + uint64_t ino; + uint64_t size; + uint64_t blocks; + uint64_t attributes_mask; + struct preload_linux_statx_timestamp atime; + struct preload_linux_statx_timestamp btime; + struct preload_linux_statx_timestamp ctime; + struct preload_linux_statx_timestamp mtime; + uint32_t rdev_major; + uint32_t rdev_minor; + uint32_t dev_major; + uint32_t dev_minor; + uint64_t mnt_id; + uint32_t dio_mem_align; + uint32_t dio_offset_align; + uint64_t spare3[12]; +}; + +struct preload_linux_open_how { + uint64_t flags; + uint64_t mode; + uint64_t resolve; +}; + +struct preload_bulk_linux_data { + struct preload_linux_statx root_statx; + struct strbuf mountinfo; + uint64_t root_mnt_id; + char *test_dirent_path; + unsigned char test_dirent_type; + int use_openat2; +}; + +struct preload_bulk_scan; +struct preload_bulk_task; +struct preload_bulk_worker; +struct preload_bulk_dir_identity; + +#if defined(SYS_getdents64) && defined(SYS_statx) + +int preload_bulk_linux_statx_raw(int dirfd, const char *path, int flags, + struct preload_linux_statx *stx); +int preload_bulk_linux_statx_complete( + const struct preload_linux_statx *stx); +int preload_bulk_linux_statx_same(const struct preload_linux_statx *a, + const struct preload_linux_statx *b); +int preload_bulk_linux_entry_stat(struct preload_bulk_worker *worker, + int dirfd, const char *name, + struct preload_linux_statx *stx, + struct stat *st); +int preload_bulk_linux_fd_stat(struct preload_bulk_worker *worker, int fd, + struct preload_linux_statx *stx, + struct stat *st); + +#ifdef SYS_openat2 +int preload_bulk_linux_openat2_raw(int dirfd, const char *path); +#endif +int preload_bulk_linux_open_dir_at(struct preload_bulk_worker *worker, + int parent_fd, const char *name); +int preload_bulk_linux_open_relative(struct preload_bulk_scan *scan, + const char *path); + +int preload_bulk_linux_enumerate( + struct preload_bulk_worker *worker, + struct preload_bulk_task *task, int fd, + const struct preload_bulk_dir_identity *parent_identity); +int preload_bulk_linux_scan_directory(struct preload_bulk_worker *worker, + struct preload_bulk_task *task); + +const char *preload_bulk_linux_start(struct preload_bulk_scan *scan); +const char *preload_bulk_linux_finish(struct preload_bulk_scan *scan); +void preload_bulk_linux_release(struct preload_bulk_scan *scan); + +#endif /* SYS_getdents64 && SYS_statx */ + +#endif /* __linux__ */ + +#endif /* PRELOAD_INDEX_BULK_LINUX_H */ diff --git a/config.mak.openai b/config.mak.openai new file mode 100644 index 00000000000000..0ae133c8bb2ecf --- /dev/null +++ b/config.mak.openai @@ -0,0 +1,74 @@ +# OpenAI release build settings. +# +# The Codex release workflow copies this file to config.mak before +# building. Keep release-only optimizations here instead of in the +# upstream Makefile. + +OPENAI_PROFILE_DIR := $(CURDIR) +OPENAI_PROFILE_RAW := $(OPENAI_PROFILE_DIR)/default_%m_%p.profraw +OPENAI_PROFILE_DATA := $(OPENAI_PROFILE_DIR)/default.profdata +OPENAI_PROFILE_TRAINING ?= .github/workflows/codex-pgo-training.sh +OPENAI_LLVM_PROFDATA ?= llvm-profdata + +ifdef OPENAI_LTO +CFLAGS_APPEND += -flto=$(OPENAI_LTO) +LDFLAGS_APPEND += -flto=$(OPENAI_LTO) +endif + +ifeq ("$(OPENAI_PROFILE_FORMAT)","LLVM") +ifeq ("$(OPENAI_PROFILE)","GEN") + BASIC_CFLAGS += -fprofile-instr-generate=$(OPENAI_PROFILE_RAW) + BASIC_CFLAGS += -DNO_NORETURN=1 + export CCACHE_DISABLE = t + V = 1 +else +ifneq ("$(OPENAI_PROFILE)","") + BASIC_CFLAGS += -fprofile-instr-use=$(OPENAI_PROFILE_DATA) + BASIC_CFLAGS += -Wno-profile-instr-unprofiled -DNO_NORETURN=1 + export CCACHE_DISABLE = t + V = 1 +endif +endif +else +ifeq ("$(OPENAI_PROFILE)","GEN") + BASIC_CFLAGS += -fprofile-generate=$(OPENAI_PROFILE_DIR) + BASIC_CFLAGS += -DNO_NORETURN=1 + EXTLIBS += -lgcov + export CCACHE_DISABLE = t + V = 1 +else +ifneq ("$(OPENAI_PROFILE)","") + BASIC_CFLAGS += -fprofile-use=$(OPENAI_PROFILE_DIR) + BASIC_CFLAGS += -fprofile-correction -DNO_NORETURN=1 + export CCACHE_DISABLE = t + V = 1 +endif +endif +endif + +# Every C object already waits for Git's forced GIT-CFLAGS target. Gate that +# target so "make strip install" finishes PGO before its normal prerequisites +# can start compiling with profile-use flags. +ifeq "$(OPENAI_PROFILE)" "BUILD" +GIT-CFLAGS: openai-profile +endif + +openai-profile: profile-clean openai-profile-clean + $(MAKE) OPENAI_PROFILE=GEN all + $(SHELL_PATH) $(OPENAI_PROFILE_TRAINING) + $(MAKE) OPENAI_PROFILE= openai-profile-merge + $(MAKE) OPENAI_PROFILE=USE all + +openai-profile-merge: +ifeq ("$(OPENAI_PROFILE_FORMAT)","LLVM") + $(OPENAI_LLVM_PROFDATA) merge \ + -output=$(OPENAI_PROFILE_DATA) \ + $(OPENAI_PROFILE_DIR)/*.profraw +endif + +clean: openai-profile-clean + +openai-profile-clean: + $(RM) $(OPENAI_PROFILE_DIR)/*.profraw $(OPENAI_PROFILE_DATA) + +.PHONY: openai-profile openai-profile-merge openai-profile-clean diff --git a/config.mak.uname b/config.mak.uname index 9ebd240378ca59..4d205c4c493939 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -63,6 +63,12 @@ ifeq ($(uname_S),Linux) PROCFS_EXECUTABLE_PATH = /proc/self/exe HAVE_PLATFORM_PROCINFO = YesPlease COMPAT_OBJS += compat/linux/procinfo.o + PRELOAD_INDEX_BULK_BACKEND = linux + PRELOAD_INDEX_BULK_PLATFORM_OBJS += compat/preload-index/bulk-linux-entry.o + PRELOAD_INDEX_BULK_PLATFORM_OBJS += compat/preload-index/bulk-linux-open.o + PRELOAD_INDEX_BULK_PLATFORM_OBJS += compat/preload-index/bulk-linux-scan.o + PRELOAD_INDEX_BULK_PLATFORM_OBJS += compat/preload-index/bulk-linux-stat.o + PRELOAD_INDEX_BULK_PLATFORM_OBJS += compat/preload-index/bulk-linux-topology.o EXTLIBS += -ldl # centos7/rhel7 provides gcc 4.8.5 and zlib 1.2.7. ifneq ($(findstring .el7.,$(uname_R)),) @@ -162,6 +168,8 @@ ifeq ($(uname_S),Darwin) USE_ENHANCED_BASIC_REGULAR_EXPRESSIONS = YesPlease HAVE_PLATFORM_PROCINFO = YesPlease COMPAT_OBJS += compat/darwin/procinfo.o + PRELOAD_INDEX_BULK_BACKEND = darwin + PRELOAD_INDEX_BULK_PLATFORM_OBJS += compat/preload-index/bulk-darwin-root.o ifeq ($(uname_M),arm64) HOMEBREW_PREFIX = /opt/homebrew diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index a57c4b464fa456..6440b31a4483a1 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -103,6 +103,7 @@ macro(parse_makefile_for_sources list_var makefile regex) file(STRINGS ${makefile} ${list_var} REGEX "^${regex} \\+=(.*)") string(REPLACE "${regex} +=" "" ${list_var} ${${list_var}}) string(REPLACE "$(COMPAT_OBJS)" "" ${list_var} ${${list_var}}) #remove "$(COMPAT_OBJS)" This is only for libgit. + string(REPLACE "$(PRELOAD_INDEX_BULK_OBJS)" "" ${list_var} ${${list_var}}) string(STRIP ${${list_var}} ${list_var}) #remove trailing/leading whitespaces string(REPLACE ".o" ".c;" ${list_var} ${${list_var}}) #change .o to .c, ; is for converting the string into a list list(TRANSFORM ${list_var} STRIP) #remove trailing/leading whitespaces for each element in list @@ -271,10 +272,26 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(NO_UNIX_SOCKETS 1) elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") - add_compile_definitions(PROCFS_EXECUTABLE_PATH="/proc/self/exe" HAVE_DEV_TTY ) - list(APPEND compat_SOURCES unix-socket.c unix-stream-server.c compat/linux/procinfo.c) + add_compile_definitions(HAVE_PRELOAD_INDEX_BULK + PROCFS_EXECUTABLE_PATH="/proc/self/exe" HAVE_DEV_TTY) + list(APPEND compat_SOURCES + unix-socket.c + unix-stream-server.c + compat/linux/procinfo.c + compat/preload-index/bulk-linux.c + compat/preload-index/bulk-linux-entry.c + compat/preload-index/bulk-linux-open.c + compat/preload-index/bulk-linux-scan.c + compat/preload-index/bulk-linux-stat.c + compat/preload-index/bulk-linux-topology.c) elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") - list(APPEND compat_SOURCES compat/darwin/procinfo.c) + add_compile_definitions(HAVE_PRELOAD_INDEX_BULK PRECOMPOSE_UNICODE + USE_ST_TIMESPEC) + list(APPEND compat_SOURCES + compat/darwin/procinfo.c + compat/precompose_utf8.c + compat/preload-index/bulk-darwin.c + compat/preload-index/bulk-darwin-root.c) endif() if(CMAKE_SYSTEM_NAME STREQUAL "Windows") @@ -664,6 +681,14 @@ include_directories(${CMAKE_BINARY_DIR}) #libgit parse_makefile_for_sources(libgit_SOURCES ${CMAKE_SOURCE_DIR}/Makefile "LIB_OBJS") +if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" OR + CMAKE_SYSTEM_NAME STREQUAL "Linux") + list(APPEND libgit_SOURCES + preload-index-bulk-index.c + preload-index-bulk-thread.c + preload-index-bulk.c) +endif() + list(TRANSFORM libgit_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") list(TRANSFORM compat_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") diff --git a/convert.c b/convert.c index 036506842c3d41..adacd107d84f31 100644 --- a/convert.c +++ b/convert.c @@ -1318,11 +1318,8 @@ static int git_path_check_ident(struct attr_check_item *check) static struct attr_check *check; -void convert_attrs(struct index_state *istate, - struct conv_attrs *ca, const char *path) +static void convert_attrs_init(void) { - struct attr_check_item *ccheck = NULL; - if (!check) { check = attr_check_initl("crlf", "ident", "filter", "eol", "text", "working-tree-encoding", @@ -1330,9 +1327,33 @@ void convert_attrs(struct index_state *istate, user_convert_tail = &user_convert; repo_config(the_repository, read_convert_config, NULL); } +} + +struct attr_check *convert_attrs_check_alloc(void) +{ + return attr_check_initl("crlf", "ident", "filter", + "eol", "text", "working-tree-encoding", + NULL); +} + +void convert_attrs_prepare(struct index_state *istate) +{ + convert_attrs_init(); + /* Prime default_attr_source() and the root attribute stack on main. */ + git_check_attr(istate, "", check); +} + +void convert_attrs_with_check(struct index_state *istate, + struct conv_attrs *ca, const char *path, + struct attr_check *attr_check) +{ + struct attr_check_item *ccheck; - git_check_attr(istate, path, check); - ccheck = check->items; + if (!attr_check || attr_check->nr != 6) + BUG("invalid per-thread conversion attribute check"); + + git_check_attr(istate, path, attr_check); + ccheck = attr_check->items; ca->crlf_action = git_path_check_crlf(ccheck + 4); if (ca->crlf_action == CRLF_UNDEFINED) ca->crlf_action = git_path_check_crlf(ccheck + 0); @@ -1363,6 +1384,23 @@ void convert_attrs(struct index_state *istate, ca->crlf_action = CRLF_AUTO_INPUT; } +void convert_attrs(struct index_state *istate, + struct conv_attrs *ca, const char *path) +{ + convert_attrs_init(); + convert_attrs_with_check(istate, ca, path, check); +} + +int convert_attrs_is_raw_safe(struct index_state *istate, const char *path, + struct attr_check *attr_check) +{ + struct conv_attrs ca; + + convert_attrs_with_check(istate, &ca, path, attr_check); + return !ca.drv && !ca.working_tree_encoding && !ca.ident && + ca.crlf_action == CRLF_BINARY; +} + void reset_parsed_attributes(void) { struct convert_driver *drv, *next; diff --git a/convert.h b/convert.h index 0a6e4086b8f932..97f399f2fe3846 100644 --- a/convert.h +++ b/convert.h @@ -8,6 +8,7 @@ #include "string-list.h" struct index_state; +struct attr_check; struct strbuf; #define CONV_EOL_RNDTRP_DIE (1<<0) /* Die if CRLF to LF to CRLF is different */ @@ -91,6 +92,28 @@ struct conv_attrs { void convert_attrs(struct index_state *istate, struct conv_attrs *ca, const char *path); +/* + * Prepare conversion configuration and the default attribute source on the + * main thread before using per-thread attribute checks below. + */ +void convert_attrs_prepare(struct index_state *istate); + +/* Allocate the exact six-attribute check used by convert_attrs(). */ +struct attr_check *convert_attrs_check_alloc(void); + +/* + * Thread-friendly variant. Each concurrent caller must supply a distinct + * check allocated by convert_attrs_check_alloc(), and conversion/attribute + * global state must remain immutable until all callers have finished. + */ +void convert_attrs_with_check(struct index_state *istate, + struct conv_attrs *ca, const char *path, + struct attr_check *check); + +/* True only when hashing the worktree bytes verbatim is exact. */ +int convert_attrs_is_raw_safe(struct index_state *istate, const char *path, + struct attr_check *check); + extern enum eol core_eol; extern char *check_roundtrip_encoding; const char *get_cached_convert_stats_ascii(struct index_state *istate, diff --git a/diff-lib.c b/diff-lib.c index 46cae637ecda83..8f8aba3038a8a6 100644 --- a/diff-lib.c +++ b/diff-lib.c @@ -90,7 +90,10 @@ static int match_stat_with_submodule(struct diff_options *diffopt, struct stat *st, unsigned ce_option, unsigned *dirty_submodule) { - int changed = ie_match_stat(diffopt->repo->index, ce, st, ce_option); + int changed; + + changed = ie_match_stat_with_content_check( + diffopt->repo->index, ce, st, ce_option); if (S_ISGITLINK(ce->ce_mode)) { struct diff_flags orig_flags = diffopt->flags; if (!diffopt->flags.override_submodule_config) @@ -127,7 +130,10 @@ void run_diff_files(struct rev_info *revs, unsigned int option) entries = istate->cache_nr; for (i = 0; i < entries; i++) { unsigned int oldmode, newmode; + int fsmonitor_valid = 0; struct cache_entry *ce = istate->cache[i]; + struct stat st; + int has_stat = 0; int changed; unsigned dirty_submodule = 0; const struct object_id *old_oid, *new_oid; @@ -246,9 +252,9 @@ void run_diff_files(struct rev_info *revs, unsigned int option) if (ce->ce_flags & (CE_VALID | CE_FSMONITOR_VALID)) { changed = 0; newmode = ce->ce_mode; + fsmonitor_valid = + !!(ce->ce_flags & CE_FSMONITOR_VALID); } else { - struct stat st; - changed = check_removed(ce, &st); if (changed) { if (changed < 0) { @@ -270,12 +276,20 @@ void run_diff_files(struct rev_info *revs, unsigned int option) changed = match_stat_with_submodule(&revs->diffopt, ce, &st, ce_option, &dirty_submodule); + has_stat = 1; newmode = ce_mode_from_stat(revs->repo, ce, st.st_mode); + fsmonitor_valid = fsmonitor_stat_can_be_valid(&st); } if (!changed && !dirty_submodule) { + if ((option & DIFF_UPDATE_INDEX_STAT) && has_stat && + (ce->ce_flags & CE_CONTENT_CHECK_REQUIRED)) { + refresh_index_entry_stat(istate, i, &st); + ce = istate->cache[i]; + } ce_mark_uptodate(ce); - mark_fsmonitor_valid(istate, ce); + if (fsmonitor_valid) + mark_fsmonitor_valid(istate, ce); if (revs->diffopt.flags.find_copies_harder) diff_same(&revs->diffopt, newmode, &ce->oid, ce->name); diff --git a/diff.h b/diff.h index bb5cddaf3499e9..eb81289415f8f3 100644 --- a/diff.h +++ b/diff.h @@ -698,6 +698,8 @@ void diff_get_merge_base(const struct rev_info *revs, struct object_id *mb); #define DIFF_SILENT_ON_REMOVED 01 /* report racily-clean paths as modified */ #define DIFF_RACY_IS_MODIFIED 02 +/* update index stat data for content-checked entries */ +#define DIFF_UPDATE_INDEX_STAT 04 void run_diff_files(struct rev_info *revs, unsigned int option); #define DIFF_INDEX_CACHED 01 diff --git a/dir.c b/dir.c index 95d8a1cce90f77..c1057362ea573f 100644 --- a/dir.c +++ b/dir.c @@ -15,10 +15,12 @@ #include "convert.h" #include "dir.h" #include "environment.h" +#include "exclude-source-proof.h" #include "gettext.h" #include "name-hash.h" #include "object-file.h" #include "path.h" +#include "path-namespace.h" #include "refs.h" #include "repository.h" #include "wildmatch.h" @@ -27,12 +29,15 @@ #include "varint.h" #include "ewah/ewok.h" #include "fsmonitor-ll.h" +#include "fsmonitor.h" #include "read-cache-ll.h" #include "setup.h" #include "sparse-index.h" #include "strbuf.h" #include "submodule-config.h" #include "symlinks.h" +#include "thread-utils.h" +#include "trace.h" #include "trace2.h" #include "tree.h" #include "hex.h" @@ -71,10 +76,865 @@ struct cached_dir { struct untracked_cache_dir *ucd; }; +struct untracked_cache_preload_task { + struct untracked_cache_dir *ucd; + char *path; + struct stat_data stat_data; + struct object_id exclude_oid; + unsigned int exclude_mode; + unsigned int was_valid : 1; + unsigned int stat_checked : 1; + unsigned int stat_matches : 1; + unsigned int exclude_matches : 1; + unsigned int exclude_index_present : 1; + unsigned int exclude_index_candidate : 1; + unsigned int exclude_index_matches : 1; + unsigned int exclude_index_content_matches : 1; + unsigned int normalize_exclude_oid : 1; + unsigned int update_stat_data : 1; +}; + +struct untracked_cache_preload; + +struct untracked_cache_preload_data { + pthread_t pthread; + struct untracked_cache_preload *preload; + size_t offset, nr; + int started; +}; + +struct untracked_cache_preload { + struct repository *repo; + struct index_state *istate; + struct untracked_cache *uc; + struct untracked_cache_dir *root; + struct untracked_cache_preload_task *tasks; + struct object_id *exclude_index_oids; + struct untracked_cache_preload_data *data; + struct cache_time index_timestamp; + char *exclude_per_dir; + size_t nr; + int threads; + unsigned int dir_flags; + uint64_t started_at; + unsigned int fsmonitor_excludes_only : 1; +}; + +#define UNTRACKED_CACHE_MAX_OVERLAP_THREADS 6 +#define UNTRACKED_CACHE_PRELOAD_COST 1000 +#define UNTRACKED_CACHE_EXCLUDE_PRELOAD_COST 256 +#define UNTRACKED_CACHE_MAX_EXCLUDE_SIZE (1024 * 1024) + +static void invalidate_gitignore(struct untracked_cache *uc, + struct untracked_cache_dir *dir); +static void invalidate_directory(struct untracked_cache *uc, + struct untracked_cache_dir *dir); + +static int match_untracked_dir_stat_racy(const struct cache_time *timestamp, + const struct stat_data *sd, + const struct stat *st); +static void *preload_untracked_cache_thread(void *data); + +static void collect_untracked_cache_preload_tasks( + struct untracked_cache_dir *ucd, + struct strbuf *path, + struct untracked_cache_preload_task **tasks, + size_t *nr, + size_t *alloc, + int fsmonitor_excludes_only) +{ + size_t i; + + if (!fsmonitor_excludes_only || + !is_null_oid(&ucd->exclude_oid)) { + ALLOC_GROW(*tasks, *nr + 1, *alloc); + memset(&(*tasks)[*nr], 0, sizeof(**tasks)); + (*tasks)[*nr].ucd = ucd; + (*tasks)[*nr].path = xstrdup(path->len ? path->buf : "."); + (*tasks)[*nr].stat_data = ucd->stat_data; + oidcpy(&(*tasks)[*nr].exclude_oid, &ucd->exclude_oid); + (*tasks)[*nr].was_valid = ucd->valid; + (*nr)++; + } + + for (i = 0; i < ucd->dirs_nr; i++) { + struct untracked_cache_dir *child = ucd->dirs[i]; + size_t old_len = path->len; + + if (path->len) + strbuf_addch(path, '/'); + strbuf_addstr(path, child->name); + collect_untracked_cache_preload_tasks(child, path, tasks, nr, + alloc, fsmonitor_excludes_only); + strbuf_setlen(path, old_len); + } +} + +static size_t count_untracked_cache_dirs_bounded( + const struct untracked_cache_dir *ucd, + size_t limit) +{ + size_t i, nr = 1; + + for (i = 0; i < ucd->dirs_nr && nr < limit; i++) + nr += count_untracked_cache_dirs_bounded(ucd->dirs[i], limit - nr); + return nr; +} + +#define UNTRACKED_CACHE_AUTO_PRELOAD_MIN_DIRS 2000 + +static int untracked_cache_auto_preload_worthwhile( + const struct untracked_cache *uc) +{ + if (!uc || !uc->root || uc->use_fsmonitor || !uc->root->valid) + return 0; + if (git_env_bool("GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD", 0)) + return 1; + return count_untracked_cache_dirs_bounded( + uc->root, UNTRACKED_CACHE_AUTO_PRELOAD_MIN_DIRS) >= + UNTRACKED_CACHE_AUTO_PRELOAD_MIN_DIRS; +} + +static int exclude_path_matches_fd(const char *path, + const struct stat *expected) +{ + struct stat st; + int fd = open_nofollow(path, O_RDONLY); + int ret = fd >= 0 && !fstat(fd, &st) && S_ISREG(st.st_mode) && + path_namespace_stat_equal(expected, &st); + + if (fd >= 0) + close(fd); + return ret; +} + +static int cached_exclude_file_matches( + const struct git_hash_algo *algo, + const char *path, const struct object_id *cached_oid, + struct object_id *raw_oid_out, unsigned int *mode_out) +{ + struct object_id raw_oid, normalized_oid; + struct stat st, st_after; + char *buf; + size_t size; + int fd, ret = 0; + + fd = open_nofollow(path, O_RDONLY); + if (fd < 0) + return 0; + if (fstat(fd, &st) < 0 || !S_ISREG(st.st_mode) || st.st_size < 0 || + st.st_size > UNTRACKED_CACHE_MAX_EXCLUDE_SIZE) + goto out_close; + + size = xsize_t(st.st_size); + buf = xmallocz(size + 1); + if (read_in_full(fd, buf, size) != size) + goto out; + /* Prove both the opened file and its pathname stayed unchanged. */ + if (fstat(fd, &st_after) || + !path_namespace_stat_equal(&st, &st_after) || + !exclude_path_matches_fd(path, &st_after)) + goto out; + if (mode_out) + *mode_out = st_after.st_mode; + + /* add_patterns() may record either the blob or its LF-normalized form. */ + hash_object_file(algo, buf, size, OBJ_BLOB, &raw_oid); + if (raw_oid_out) + oidcpy(raw_oid_out, &raw_oid); + if (oideq(&raw_oid, cached_oid)) { + ret = 1; + goto out; + } + buf[size] = '\n'; + hash_object_file(algo, buf, size + 1, OBJ_BLOB, + &normalized_oid); + ret = oideq(&normalized_oid, cached_oid); +out: + free(buf); +out_close: + close(fd); + return ret; +} + +static int cached_exclude_file_matches_index_stat( + const struct stat_data *sd, + const struct stat *st) +{ + struct stat_data current; + struct stat st_copy = *st; + + /* + * Compare every field saved in the index, independent of the user's + * ordinary stat-match settings. Unreliable object identities and + * multiply-linked files can conceal changes through paths outside + * the monitor's watch cone, so both retain the content-hash check. + */ + if (!fstat_is_reliable() || !S_ISREG(st->st_mode) || + st->st_nlink != 1) + return 0; + fill_stat_data(¤t, &st_copy); + return sd->sd_ctime.sec == current.sd_ctime.sec && + sd->sd_ctime.nsec == current.sd_ctime.nsec && + sd->sd_mtime.sec == current.sd_mtime.sec && + sd->sd_mtime.nsec == current.sd_mtime.nsec && + sd->sd_dev == current.sd_dev && + sd->sd_ino == current.sd_ino && + sd->sd_uid == current.sd_uid && + sd->sd_gid == current.sd_gid && + sd->sd_size == current.sd_size; +} + +static void preload_fsmonitor_excludes_from_index( + struct untracked_cache_preload *preload) +{ + struct repo_config_values *cfg = + repo_config_values(preload->istate->repo); + size_t i; + int stat_candidates = cfg->trust_ctime && cfg->check_stat; + + for (i = 0; i < preload->nr; i++) { + struct untracked_cache_preload_task *task = &preload->tasks[i]; + struct strbuf exclude_path = STRBUF_INIT; + struct cache_entry *ce; + int pos; + + if (!preload->exclude_per_dir) + continue; + if (strcmp(task->path, ".")) + strbuf_addstr(&exclude_path, task->path); + if (exclude_path.len) + strbuf_addch(&exclude_path, '/'); + strbuf_addstr(&exclude_path, preload->exclude_per_dir); + pos = index_name_pos_sparse( + preload->istate, exclude_path.buf, + exclude_path.len); + if (pos < 0) + goto next; + ce = preload->istate->cache[pos]; + if (!S_ISREG(ce->ce_mode) || + (!(ce->ce_flags & CE_FSMONITOR_VALID) && + fstat_is_reliable()) || + ce_skip_worktree(ce) || + (ce->ce_flags & CE_REMOVE) || + ce_intent_to_add(ce)) + goto next; + oidcpy(&preload->exclude_index_oids[i], &ce->oid); + task->exclude_index_present = 1; + if (!stat_candidates || + is_racy_timestamp(preload->istate, ce) || + (ce->ce_flags & CE_VALID)) + goto next; + /* + * Snapshot before launching workers. The main thread may + * refresh cache entries while exclude checks run. + */ + task->stat_data = ce->ce_stat_data; + task->exclude_index_candidate = 1; +next: + strbuf_release(&exclude_path); + } +} + +static struct untracked_cache_preload *untracked_cache_preload_start_1( + struct index_state *istate, unsigned int dir_flags, int automatic, + int fsmonitor_excludes_only) +{ + struct untracked_cache *uc = istate->untracked; + struct untracked_cache_preload *preload; + struct strbuf path = STRBUF_INIT; + size_t alloc = 0, offset = 0, work, i; + unsigned long test_threads; + int threads, online, create_threads = 1; + + if (!uc || !uc->root || uc->dir_flags != dir_flags || + (fsmonitor_excludes_only ? + !uc->use_fsmonitor : + uc->use_fsmonitor)) + return NULL; + + CALLOC_ARRAY(preload, 1); + preload->repo = istate->repo; + preload->istate = istate; + preload->uc = uc; + preload->root = uc->root; + preload->index_timestamp = istate->timestamp; + preload->exclude_per_dir = xstrdup_or_null(uc->exclude_per_dir); + preload->dir_flags = dir_flags; + preload->fsmonitor_excludes_only = fsmonitor_excludes_only; + collect_untracked_cache_preload_tasks( + uc->root, &path, &preload->tasks, &preload->nr, &alloc, + fsmonitor_excludes_only); + strbuf_release(&path); + if (fsmonitor_excludes_only) { + CALLOC_ARRAY(preload->exclude_index_oids, preload->nr); + preload_fsmonitor_excludes_from_index(preload); + } + + threads = HAVE_THREADS ? + preload->nr / (fsmonitor_excludes_only ? + UNTRACKED_CACHE_EXCLUDE_PRELOAD_COST : + UNTRACKED_CACHE_PRELOAD_COST) : + 1; + online = HAVE_THREADS ? online_cpus() : 1; + if (threads > online * 3) + threads = online * 3; + if (threads > UNTRACKED_CACHE_MAX_OVERLAP_THREADS) + threads = UNTRACKED_CACHE_MAX_OVERLAP_THREADS; + test_threads = git_env_ulong("GIT_TEST_UNTRACKED_CACHE_THREADS", 0); + if (test_threads && HAVE_THREADS) + threads = test_threads > UNTRACKED_CACHE_MAX_OVERLAP_THREADS ? + UNTRACKED_CACHE_MAX_OVERLAP_THREADS : test_threads; + if (threads < 1) + threads = 1; + if (preload->nr && (size_t)threads > preload->nr) + threads = preload->nr; + preload->threads = threads; + + preload->started_at = getnanotime(); + trace2_data_intmax("dir", istate->repo, + "preload_untracked_cache/threads", threads); + trace2_data_intmax("dir", istate->repo, + "preload_untracked_cache/automatic", automatic); + trace2_data_intmax("dir", istate->repo, + "preload_untracked_cache/fsmonitor-excludes-only", + fsmonitor_excludes_only); + CALLOC_ARRAY(preload->data, threads); + work = DIV_ROUND_UP(preload->nr, threads); + for (i = 0; i < threads; i++) { + struct untracked_cache_preload_data *data = &preload->data[i]; + int err; + + data->preload = preload; + data->offset = offset; + data->nr = offset < preload->nr ? + (preload->nr - offset < work ? + preload->nr - offset : work) : 0; + offset += data->nr; + if (threads == 1 || !create_threads) + continue; + err = pthread_create(&data->pthread, NULL, + preload_untracked_cache_thread, data); + if (err) { + create_threads = 0; + trace2_data_intmax( + "dir", istate->repo, + "preload_untracked_cache/thread_failure", err); + continue; + } + data->started = 1; + } + return preload; +} + +struct untracked_cache_preload * +untracked_cache_preload_start_fsmonitor_excludes( + struct index_state *istate, unsigned int dir_flags) +{ + return untracked_cache_preload_start_1( + istate, dir_flags, 0, 1); +} + +struct untracked_cache_preload *untracked_cache_preload_start_ordinary( + struct index_state *istate, unsigned int dir_flags) +{ + struct untracked_cache *uc = istate->untracked; + + if (!uc || uc->dir_flags != dir_flags || + !untracked_cache_auto_preload_worthwhile(uc)) + return NULL; + return untracked_cache_preload_start_1(istate, dir_flags, 1, 0); +} + +static void *preload_untracked_cache_thread(void *_data) +{ + struct untracked_cache_preload_data *data = _data; + struct untracked_cache_preload *preload = data->preload; + size_t i; + + for (i = data->offset; i < data->offset + data->nr; i++) { + struct untracked_cache_preload_task *task = &preload->tasks[i]; + struct strbuf exclude_path = STRBUF_INIT; + struct stat st; + + if (preload->fsmonitor_excludes_only) { + struct object_id raw_oid; + + if (!preload->exclude_per_dir) + continue; + if (strcmp(task->path, ".")) + strbuf_addstr(&exclude_path, task->path); + if (exclude_path.len) + strbuf_addch(&exclude_path, '/'); + strbuf_addstr(&exclude_path, + preload->exclude_per_dir); + if (task->exclude_index_candidate && + oideq(&preload->exclude_index_oids[i], + &task->exclude_oid) && + !lstat(exclude_path.buf, &st) && + cached_exclude_file_matches_index_stat( + &task->stat_data, &st)) { + task->exclude_mode = st.st_mode; + task->exclude_index_matches = 1; + task->exclude_index_content_matches = 1; + task->exclude_matches = 1; + strbuf_release(&exclude_path); + continue; + } + task->exclude_matches = cached_exclude_file_matches( + preload->repo->hash_algo, + exclude_path.buf, + &task->exclude_oid, &raw_oid, + &task->exclude_mode); + if (task->exclude_matches && + task->exclude_index_present && + oideq(&preload->exclude_index_oids[i], + &raw_oid)) { + task->exclude_index_content_matches = 1; + if (!oideq(&preload->exclude_index_oids[i], + &task->exclude_oid)) + task->normalize_exclude_oid = 1; + } + strbuf_release(&exclude_path); + continue; + } + if (!task->was_valid) + continue; + task->stat_checked = 1; + if (lstat(task->path, &st)) { + memset(&task->stat_data, 0, sizeof(task->stat_data)); + task->update_stat_data = 1; + continue; + } + if (!match_untracked_dir_stat_racy( + &preload->index_timestamp, &task->stat_data, &st)) { + task->stat_matches = 1; + } else { + fill_stat_data(&task->stat_data, &st); + task->update_stat_data = 1; + continue; + } + if (is_null_oid(&task->exclude_oid)) { + task->exclude_matches = 1; + continue; + } + if (!preload->exclude_per_dir) + continue; + if (strcmp(task->path, ".")) + strbuf_addstr(&exclude_path, task->path); + if (exclude_path.len) + strbuf_addch(&exclude_path, '/'); + strbuf_addstr(&exclude_path, preload->exclude_per_dir); + task->exclude_matches = cached_exclude_file_matches( + preload->repo->hash_algo, exclude_path.buf, + &task->exclude_oid, NULL, NULL); + strbuf_release(&exclude_path); + } + return NULL; +} + +static int untracked_cache_has_collapsed_child( + const struct untracked_cache_dir *parent, + const struct untracked_cache_dir *child) +{ + size_t i, len = strlen(child->name); + + for (i = 0; i < parent->untracked_nr; i++) { + const char *name = parent->untracked[i]; + + if (strlen(name) == len + 1 && name[len] == '/' && + !strncmp(name, child->name, len)) + return 1; + } + return 0; +} + +static int compute_untracked_cache_valid_recursive( + struct untracked_cache *uc, + struct untracked_cache_dir *ucd, + int invalidate_ancestors) +{ + size_t i; + int local_valid = ucd->valid && ucd->stat_checked && + ucd->stat_matches && ucd->exclude_matches; + int valid = local_valid; + int invalidate_self = !local_valid; + + for (i = 0; i < ucd->dirs_nr; i++) { + int child_valid = compute_untracked_cache_valid_recursive( + uc, ucd->dirs[i], invalidate_ancestors); + + if (!child_valid) { + valid = 0; + if (untracked_cache_has_collapsed_child(ucd, ucd->dirs[i])) + invalidate_self = 1; + } + } + if (invalidate_self && invalidate_ancestors) { + if (!local_valid && ucd->valid && ucd->stat_checked && + ucd->stat_matches && !ucd->exclude_matches) + invalidate_gitignore(uc, ucd); + else + invalidate_directory(uc, ucd); + } + ucd->valid_recursive = valid; + return valid; +} + +static int compute_untracked_cache_disk_valid_recursive( + struct untracked_cache_dir *ucd) +{ + size_t i; + int valid = ucd->valid && is_null_oid(&ucd->exclude_oid); + + for (i = 0; i < ucd->dirs_nr; i++) + if (!compute_untracked_cache_disk_valid_recursive(ucd->dirs[i])) + valid = 0; + ucd->valid_recursive = valid; + return valid; +} + +static int compute_untracked_cache_fsmonitor_valid_recursive( + struct untracked_cache_dir *ucd) +{ + size_t i; + int valid = ucd->valid; + + for (i = 0; i < ucd->dirs_nr; i++) + if (!compute_untracked_cache_fsmonitor_valid_recursive( + ucd->dirs[i])) + valid = 0; + ucd->valid_recursive = valid; + return valid; +} + +static void untracked_cache_preload_join( + struct untracked_cache_preload *preload) +{ + int i; + + if (preload->threads == 1) { + preload_untracked_cache_thread(&preload->data[0]); + return; + } + for (i = 0; i < preload->threads; i++) { + if (!preload->data[i].started) { + preload_untracked_cache_thread(&preload->data[i]); + continue; + } + if (pthread_join(preload->data[i].pthread, NULL)) + die(_("unable to join untracked-cache preload thread")); + } +} + +static void untracked_cache_preload_free( + struct untracked_cache_preload *preload) +{ + size_t i; + + trace2_data_intmax("dir", preload->repo, + "preload_untracked_cache/dirs", preload->nr); + trace2_data_intmax("dir", preload->repo, + "preload_untracked_cache/wall_us", + (getnanotime() - preload->started_at) / 1000); + free(preload->data); + for (i = 0; i < preload->nr; i++) + free(preload->tasks[i].path); + free(preload->tasks); + free(preload->exclude_index_oids); + free(preload->exclude_per_dir); + free(preload); +} + +static int converted_exclude_matches_cache_and_index( + struct untracked_cache_preload *preload, + struct untracked_cache_preload_task *task, + struct cache_entry *ce, + const char *path) +{ + struct object_id converted_oid, normalized_oid, raw_oid; + struct stat before, after; + char *buf = NULL; + size_t size; + int converted_fd, fd = -1; + int cached_matches, ret = 0; + + fd = open_nofollow(path, O_RDONLY); + if (fd < 0 || fstat(fd, &before) || !S_ISREG(before.st_mode) || + before.st_size < 0 || + before.st_size > UNTRACKED_CACHE_MAX_EXCLUDE_SIZE) + goto done; + size = xsize_t(before.st_size); + buf = xmallocz(size + 1); + if (read_in_full(fd, buf, size) != size) + goto done; + + hash_object_file(preload->repo->hash_algo, buf, size, OBJ_BLOB, + &raw_oid); + cached_matches = oideq(&raw_oid, &task->exclude_oid); + if (!cached_matches) { + buf[size] = '\n'; + hash_object_file(preload->repo->hash_algo, buf, size + 1, + OBJ_BLOB, &normalized_oid); + cached_matches = oideq(&normalized_oid, + &task->exclude_oid); + } + if (!cached_matches || lseek(fd, 0, SEEK_SET) < 0) + goto done; + + converted_fd = xdup(fd); + if (index_fd(preload->istate, &converted_oid, converted_fd, &before, + OBJ_BLOB, path, 0) || + !oideq(&converted_oid, &ce->oid)) + goto done; + + /* + * Keep the descriptor open while computing both identities, then + * prove that neither the opened file nor its pathname changed. + */ + if (fstat(fd, &after) || + !path_namespace_stat_equal(&before, &after) || + !exclude_path_matches_fd(path, &after)) + goto done; + fill_stat_data(&task->stat_data, &after); + task->exclude_mode = after.st_mode; + ret = 1; +done: + free(buf); + if (fd >= 0) + close(fd); + return ret; +} + +static int update_preloaded_exclude_index_uptodate( + struct untracked_cache_preload *preload, + struct untracked_cache_preload_task *task, + size_t task_nr, + size_t *normalized, + size_t *index_invalidated, + int *exclude_revalidated) +{ + struct strbuf path = STRBUF_INIT; + struct cache_entry *ce; + int content_matches, converts, pos, marked = 0; + + *exclude_revalidated = -1; + if (!task->exclude_index_present || !preload->exclude_per_dir) + return 0; + if (strcmp(task->path, ".")) + strbuf_addstr(&path, task->path); + if (path.len) + strbuf_addch(&path, '/'); + strbuf_addstr(&path, preload->exclude_per_dir); + pos = index_name_pos_sparse(preload->istate, path.buf, path.len); + if (pos < 0) + goto done; + ce = preload->istate->cache[pos]; + if (!ce_stage(ce) && S_ISREG(ce->ce_mode) && + oideq(&ce->oid, &preload->exclude_index_oids[task_nr])) { + if (task->exclude_index_matches) { + converts = 0; + content_matches = 1; + } else { + converts = would_convert_to_git( + preload->istate, path.buf); + content_matches = converts ? + converted_exclude_matches_cache_and_index( + preload, task, ce, path.buf) : + task->exclude_index_content_matches; + if (converts) + *exclude_revalidated = content_matches; + } + if (!converts && task->normalize_exclude_oid) { + oidcpy(&task->ucd->exclude_oid, + &preload->exclude_index_oids[task_nr]); + (*normalized)++; + } + if (content_matches && + (ce->ce_flags & CE_FSMONITOR_VALID) && + !ce_skip_worktree(ce) && + !(ce->ce_flags & CE_REMOVE) && + !ce_intent_to_add(ce) && + (!repo_trust_executable_bit(preload->istate->repo) || + !((ce->ce_mode ^ task->exclude_mode) & 0100))) { + if (converts && + memcmp(&ce->ce_stat_data, &task->stat_data, + sizeof(ce->ce_stat_data))) { + ce->ce_stat_data = task->stat_data; + ce->ce_flags |= CE_UPDATE_IN_BASE; + preload->istate->cache_changed |= + CE_ENTRY_CHANGED; + } + ce_mark_uptodate(ce); + marked = 1; + } else { + fsmonitor_invalidate_cache_entry(ce); + preload->istate->cache_changed |= FSMONITOR_CHANGED; + (*index_invalidated)++; + } + } +done: + strbuf_release(&path); + return marked; +} + +int untracked_cache_preload_finish(struct untracked_cache_preload *preload, + struct index_state *istate, + unsigned int dir_flags, + size_t *index_invalidated) +{ + struct untracked_cache *uc; + size_t i; + int applied = 0; + + if (index_invalidated) + *index_invalidated = 0; + if (!preload) + return 0; + untracked_cache_preload_join(preload); + uc = istate->untracked; + if (uc != preload->uc || !uc || uc->root != preload->root || + dir_flags != preload->dir_flags || + (preload->fsmonitor_excludes_only && !uc->use_fsmonitor)) + goto done; + + if (preload->fsmonitor_excludes_only) { + size_t index_matches = 0; + size_t invalidated = 0; + size_t index_uptodate = 0; + size_t normalized = 0; + + for (i = 0; i < preload->nr; i++) { + struct untracked_cache_preload_task *task = + &preload->tasks[i]; + int exclude_matches = + oideq(&task->exclude_oid, + &task->ucd->exclude_oid) && + task->exclude_matches; + int exclude_revalidated; + + if (!exclude_matches) + invalidate_gitignore(uc, task->ucd); + else { + if (task->exclude_index_matches) + index_matches++; + } + index_uptodate += + update_preloaded_exclude_index_uptodate( + preload, task, i, &normalized, + &invalidated, &exclude_revalidated); + if (exclude_matches && exclude_revalidated == 0) + invalidate_gitignore(uc, task->ucd); + } + if (normalized) + istate->cache_changed |= UNTRACKED_CHANGED; + trace2_data_intmax( + "dir", istate->repo, + "preload_untracked_cache/index-excludes", + index_matches); + trace2_data_intmax( + "dir", istate->repo, + "preload_untracked_cache/index-uptodate", + index_uptodate); + trace2_data_intmax( + "dir", istate->repo, + "preload_untracked_cache/index-invalidated", + invalidated); + trace2_data_intmax( + "dir", istate->repo, + "preload_untracked_cache/normalized-excludes", + normalized); + trace2_data_intmax( + "dir", istate->repo, + "preload_untracked_cache/valid", + compute_untracked_cache_fsmonitor_valid_recursive( + preload->root)); + if (index_invalidated) + *index_invalidated = invalidated; + applied = 1; + goto done; + } + + for (i = 0; i < preload->nr; i++) { + struct untracked_cache_preload_task *task = &preload->tasks[i]; + struct untracked_cache_dir *ucd = task->ucd; + + ucd->stat_checked = 0; + ucd->stat_matches = 0; + ucd->exclude_matches = 0; + ucd->valid_recursive = 0; + /* Invalidation performed after the snapshot always wins. */ + if (!task->was_valid || !ucd->valid) + continue; + ucd->stat_checked = task->stat_checked; + ucd->stat_matches = task->stat_matches; + ucd->exclude_matches = task->exclude_matches; + if (task->update_stat_data) + ucd->stat_data = task->stat_data; + } + trace2_data_intmax("dir", istate->repo, + "preload_untracked_cache/valid", + compute_untracked_cache_valid_recursive( + uc, preload->root, + dir_flags & DIR_SHOW_OTHER_DIRECTORIES)); + applied = 1; +done: + trace2_data_intmax("dir", istate->repo, + "preload_untracked_cache/applied", applied); + untracked_cache_preload_free(preload); + return applied; +} + +void untracked_cache_preload_release(struct untracked_cache_preload *preload) +{ + if (!preload) + return; + untracked_cache_preload_join(preload); + untracked_cache_preload_free(preload); +} + +/* + * Unlike cache entries, an untracked-cache directory has no later content + * check to correct a false stat match. Compare every field saved in UNTR, + * regardless of core.checkStat or core.trustCtime. + */ +static int match_untracked_dir_stat(const struct stat_data *sd, + const struct stat *st) +{ + return !S_ISDIR(st->st_mode) || + sd->sd_ctime.sec != (unsigned int)st->st_ctime || + sd->sd_ctime.nsec != ST_CTIME_NSEC(*st) || + sd->sd_mtime.sec != (unsigned int)st->st_mtime || + sd->sd_mtime.nsec != ST_MTIME_NSEC(*st) || + sd->sd_dev != (unsigned int)st->st_dev || + sd->sd_ino != (unsigned int)st->st_ino || + sd->sd_uid != (unsigned int)st->st_uid || + sd->sd_gid != (unsigned int)st->st_gid || + sd->sd_size != (unsigned int)st->st_size; +} + +static int match_untracked_dir_stat_racy(const struct cache_time *timestamp, + const struct stat_data *sd, + const struct stat *st) +{ + if (timestamp->sec && +#ifdef USE_NSEC + (timestamp->sec < sd->sd_mtime.sec || + (timestamp->sec == sd->sd_mtime.sec && + timestamp->nsec <= sd->sd_mtime.nsec))) +#else + timestamp->sec <= sd->sd_mtime.sec) +#endif + return MTIME_CHANGED; + return match_untracked_dir_stat(sd, st); +} + static enum path_treatment read_directory_recursive(struct dir_struct *dir, struct index_state *istate, const char *path, int len, struct untracked_cache_dir *untracked, int check_only, int stop_at_first_file, const struct pathspec *pathspec); +static int resolve_dtype_with_error(int dtype, struct index_state *istate, + const char *path, int len, int *failed); static int resolve_dtype(int dtype, struct index_state *istate, const char *path, int len); struct dirent *readdir_skip_dot_and_dotdot(DIR *dirp) @@ -1098,6 +1958,8 @@ static void do_invalidate_gitignore(struct untracked_cache_dir *dir) { int i; dir->valid = 0; + dir->valid_recursive = 0; + dir->has_untracked = 0; for (size_t i = 0; i < dir->untracked_nr; i++) free(dir->untracked[i]); dir->untracked_nr = 0; @@ -1112,6 +1974,15 @@ static void invalidate_gitignore(struct untracked_cache *uc, do_invalidate_gitignore(dir); } +void untracked_cache_invalidate_all(struct index_state *istate) +{ + if (!istate->untracked || !istate->untracked->root) + return; + invalidate_gitignore(istate->untracked, istate->untracked->root); + istate->untracked->use_fsmonitor = 0; + istate->cache_changed |= UNTRACKED_CHANGED; +} + static void invalidate_directory(struct untracked_cache *uc, struct untracked_cache_dir *dir) { @@ -1127,6 +1998,7 @@ static void invalidate_directory(struct untracked_cache *uc, uc->dir_invalidated++; dir->valid = 0; + dir->valid_recursive = 0; for (size_t i = 0; i < dir->untracked_nr; i++) free(dir->untracked[i]); dir->untracked_nr = 0; @@ -1148,34 +2020,63 @@ static void invalidate_directory(struct untracked_cache *uc, */ static int add_patterns(const char *fname, const char *base, int baselen, struct pattern_list *pl, struct index_state *istate, - unsigned flags, struct oid_stat *oid_stat) + unsigned flags, struct oid_stat *oid_stat, + struct exclude_source_proof *source_proof) { + struct exclude_source_capture *capture = + exclude_source_capture_begin(source_proof, fname, + !!(flags & PATTERN_NOFOLLOW)); struct stat st; int r; int fd; size_t size = 0; char *buf; - if (flags & PATTERN_NOFOLLOW) + if (capture) + fd = exclude_source_capture_open(capture); + else if (flags & PATTERN_NOFOLLOW) fd = open_nofollow(fname, O_RDONLY); else fd = open(fname, O_RDONLY); if (fd < 0 || fstat(fd, &st) < 0) { - if (fd < 0) + if (fd < 0) { warn_on_fopen_errors(fname); - else + if (capture && exclude_source_capture_absent(capture)) + exclude_source_capture_record(capture, -1, NULL, + NULL, 0); + else + exclude_source_capture_error(capture); + } else { + exclude_source_capture_error(capture); close(fd); - if (!istate) + } + if (!istate) { + exclude_source_capture_release(capture); return -1; + } r = read_skip_worktree_file_from_index(istate, fname, &size, &buf, oid_stat); + if (r == 1) + exclude_source_capture_error(capture); + exclude_source_capture_release(capture); + capture = NULL; if (r != 1) return r; } else { size = xsize_t(st.st_size); + if (size > PATTERN_MAX_FILE_SIZE) { + exclude_source_capture_error(capture); + exclude_source_capture_release(capture); + warning("ignoring excessively large pattern file: %s", + fname); + close(fd); + return -1; + } if (size == 0) { + exclude_source_capture_record(capture, fd, &st, NULL, 0); + exclude_source_capture_release(capture); if (oid_stat) { fill_stat_data(&oid_stat->stat, &st); oidcpy(&oid_stat->oid, the_hash_algo->empty_blob); @@ -1186,10 +2087,15 @@ static int add_patterns(const char *fname, const char *base, int baselen, } buf = xmallocz(size); if (read_in_full(fd, buf, size) != size) { + exclude_source_capture_error(capture); + exclude_source_capture_release(capture); free(buf); close(fd); return -1; } + exclude_source_capture_record(capture, fd, &st, buf, size); + exclude_source_capture_release(capture); + capture = NULL; buf[size++] = '\n'; close(fd); if (oid_stat) { @@ -1201,6 +2107,10 @@ static int add_patterns(const char *fname, const char *base, int baselen, (pos = index_name_pos(istate, fname, strlen(fname))) >= 0 && !ce_stage(istate->cache[pos]) && ce_uptodate(istate->cache[pos]) && + !(istate->cache[pos]->ce_flags & + (CE_VALID | CE_REMOVE)) && + !ce_skip_worktree(istate->cache[pos]) && + !ce_intent_to_add(istate->cache[pos]) && !would_convert_to_git(istate, fname)) oidcpy(&oid_stat->oid, &istate->cache[pos]->oid); @@ -1258,7 +2168,8 @@ int add_patterns_from_file_to_list(const char *fname, const char *base, struct index_state *istate, unsigned flags) { - return add_patterns(fname, base, baselen, pl, istate, flags, NULL); + return add_patterns(fname, base, baselen, pl, istate, flags, NULL, + NULL); } int add_patterns_from_blob_to_list( @@ -1303,10 +2214,11 @@ struct pattern_list *add_pattern_list(struct dir_struct *dir, /* * Used to set up core.excludesfile and .git/info/exclude lists. */ -static void add_patterns_from_file_1(struct dir_struct *dir, const char *fname, - struct oid_stat *oid_stat) +static int add_patterns_from_file_1(struct dir_struct *dir, const char *fname, + struct oid_stat *oid_stat, int gentle) { struct pattern_list *pl; + int ret; /* * catch setup_standard_excludes() that's called before * dir->untracked is assigned. That function behaves @@ -1315,14 +2227,17 @@ static void add_patterns_from_file_1(struct dir_struct *dir, const char *fname, if (!dir->untracked) dir->internal.unmanaged_exclude_files++; pl = add_pattern_list(dir, EXC_FILE, fname); - if (add_patterns(fname, "", 0, pl, NULL, 0, oid_stat) < 0) + ret = add_patterns(fname, "", 0, pl, NULL, 0, oid_stat, + dir->internal.exclude_source_proof); + if (ret < 0 && !gentle) die(_("cannot use %s as an exclude file"), fname); + return ret; } void add_patterns_from_file(struct dir_struct *dir, const char *fname) { dir->internal.unmanaged_exclude_files++; /* see validate_untracked_cache() */ - add_patterns_from_file_1(dir, fname, NULL); + add_patterns_from_file_1(dir, fname, NULL, 0); } int match_basename(const char *basename, int basenamelen, @@ -1770,9 +2685,25 @@ static void prep_exclude(struct dir_struct *dir, strbuf_addbuf(&sb, &dir->internal.basebuf); strbuf_addstr(&sb, dir->exclude_per_dir); pl->src = strbuf_detach(&sb, NULL); - add_patterns(pl->src, pl->src, stk->baselen, pl, istate, - PATTERN_NOFOLLOW, - untracked ? &oid_stat : NULL); + if (add_patterns(pl->src, pl->src, stk->baselen, pl, + istate, PATTERN_NOFOLLOW, + untracked ? &oid_stat : NULL, + dir->internal.exclude_source_proof) < 0 && + untracked && is_null_oid(&oid_stat.oid)) { + struct stat st; + + /* + * Keep a non-blob sentinel for a source that is + * present but unreadable. Otherwise a valid + * untracked-cache directory cannot distinguish + * that state from an absent per-directory + * exclude file. + */ + if (!lstat(pl->src, &st) || + !is_missing_file_error(errno)) + oidcpy(&oid_stat.oid, + the_hash_algo->empty_tree); + } } /* * NEEDSWORK: when untracked cache is enabled, prep_exclude() @@ -2367,6 +3298,12 @@ unsigned char get_dtype(struct dirent *e, struct strbuf *path, static int resolve_dtype(int dtype, struct index_state *istate, const char *path, int len) +{ + return resolve_dtype_with_error(dtype, istate, path, len, NULL); +} + +static int resolve_dtype_with_error(int dtype, struct index_state *istate, + const char *path, int len, int *failed) { struct stat st; @@ -2375,8 +3312,11 @@ static int resolve_dtype(int dtype, struct index_state *istate, dtype = get_index_dtype(istate, path, len); if (dtype != DT_UNKNOWN) return dtype; - if (lstat(path, &st)) + if (lstat(path, &st)) { + if (failed && !is_missing_file_error(errno)) + *failed = 1; return dtype; + } if (S_ISREG(st.st_mode)) return DT_REG; if (S_ISDIR(st.st_mode)) @@ -2432,6 +3372,7 @@ static enum path_treatment treat_path(struct dir_struct *dir, const struct pathspec *pathspec) { int has_path_in_index, dtype, excluded; + int dtype_failed = 0; if (!cdir->d_name) return treat_path_fast(dir, cdir, istate, path, @@ -2443,7 +3384,11 @@ static enum path_treatment treat_path(struct dir_struct *dir, if (simplify_away(path->buf, path->len, pathspec)) return path_none; - dtype = resolve_dtype(cdir->d_type, istate, path->buf, path->len); + dtype = resolve_dtype_with_error( + cdir->d_type, istate, path->buf, path->len, + &dtype_failed); + if (dtype_failed) + dir->internal.traversal_failed = 1; /* Always exclude indexed files */ has_path_in_index = !!index_file_exists(istate, path->buf, path->len, @@ -2518,6 +3463,7 @@ static void add_untracked(struct untracked_cache_dir *dir, const char *name) ALLOC_GROW(dir->untracked, dir->untracked_nr + 1, dir->untracked_alloc); dir->untracked[dir->untracked_nr++] = xstrdup(name); + dir->has_untracked = 1; } static int valid_cached_dir(struct dir_struct *dir, @@ -2536,14 +3482,24 @@ static int valid_cached_dir(struct dir_struct *dir, */ refresh_fsmonitor(istate); if (!(dir->untracked->use_fsmonitor && untracked->valid)) { - if (lstat(path->len ? path->buf : ".", &st)) { - memset(&untracked->stat_data, 0, sizeof(untracked->stat_data)); - return 0; - } - if (!untracked->valid || - match_stat_data_racy(istate, &untracked->stat_data, &st)) { - fill_stat_data(&untracked->stat_data, &st); - return 0; + if (dir->internal.untracked_cache_preloaded && + untracked->stat_checked) { + if (!untracked->valid || !untracked->stat_matches || + !untracked->exclude_matches) + return 0; + } else { + if (lstat(path->len ? path->buf : ".", &st)) { + memset(&untracked->stat_data, 0, + sizeof(untracked->stat_data)); + return 0; + } + if (!untracked->valid || + match_untracked_dir_stat_racy( + &istate->timestamp, + &untracked->stat_data, &st)) { + fill_stat_data(&untracked->stat_data, &st); + return 0; + } } } @@ -2583,8 +3539,10 @@ static int open_cached_dir(struct cached_dir *cdir, return 0; c_path = path->len ? path->buf : "."; cdir->fdir = opendir(c_path); - if (!cdir->fdir) + if (!cdir->fdir) { + dir->internal.traversal_failed = 1; warning_errno(_("could not open directory '%s'"), c_path); + } if (dir->untracked) { invalidate_directory(dir->untracked, untracked); dir->untracked->dir_opened++; @@ -2594,13 +3552,16 @@ static int open_cached_dir(struct cached_dir *cdir, return 0; } -static int read_cached_dir(struct cached_dir *cdir) +static int read_cached_dir(struct cached_dir *cdir, struct dir_struct *dir) { struct dirent *de; if (cdir->fdir) { + errno = 0; de = readdir_skip_dot_and_dotdot(cdir->fdir); if (!de) { + if (errno) + dir->internal.traversal_failed = 1; cdir->d_name = NULL; cdir->d_type = DT_UNKNOWN; return -1; @@ -2628,8 +3589,32 @@ static int read_cached_dir(struct cached_dir *cdir) return -1; } +static void remove_collapsed_untracked_child( + struct untracked_cache *uc, + struct untracked_cache_dir *parent, + const struct untracked_cache_dir *child) +{ + size_t i, len = strlen(child->name); + + for (i = 0; i < parent->untracked_nr; i++) { + char *name = parent->untracked[i]; + + if (strlen(name) != len + 1 || name[len] != '/' || + strncmp(name, child->name, len)) + continue; + free(name); + MOVE_ARRAY(parent->untracked + i, parent->untracked + i + 1, + parent->untracked_nr - i - 1); + parent->untracked_nr--; + uc->dir_invalidated++; + return; + } +} + static void close_cached_dir(struct cached_dir *cdir) { + int i; + if (cdir->fdir) closedir(cdir->fdir); /* @@ -2639,6 +3624,12 @@ static void close_cached_dir(struct cached_dir *cdir) if (cdir->untracked) { cdir->untracked->valid = 1; cdir->untracked->recurse = 1; + cdir->untracked->has_untracked = !!cdir->untracked->untracked_nr; + for (i = 0; !cdir->untracked->has_untracked && + i < cdir->untracked->dirs_nr; i++) + cdir->untracked->has_untracked = + cdir->untracked->dirs[i]->recurse && + cdir->untracked->dirs[i]->has_untracked; } } @@ -2710,6 +3701,16 @@ static enum path_treatment read_directory_recursive(struct dir_struct *dir, struct strbuf path = STRBUF_INIT; strbuf_add(&path, base, baselen); + if (untracked && + (dir->internal.untracked_cache_preloaded || + dir->untracked->use_fsmonitor) && + untracked->valid && untracked->valid_recursive && + untracked->check_only == !!check_only && + !untracked->has_untracked && + (dir->flags & DIR_SHOW_OTHER_DIRECTORIES)) { + untracked->recurse = 1; + goto out; + } if (open_cached_dir(&cdir, dir, untracked, istate, &path, check_only)) goto out; @@ -2718,10 +3719,27 @@ static enum path_treatment read_directory_recursive(struct dir_struct *dir, if (untracked) untracked->check_only = !!check_only; - while (!read_cached_dir(&cdir)) { + while (!read_cached_dir(&cdir, dir)) { /* check how the file or directory should be treated */ state = treat_path(dir, untracked, &cdir, istate, &path, baselen, pathspec); + if (!cdir.d_name && cdir.ucd && cdir.ucd->check_only && + state < path_untracked && untracked && + untracked_cache_has_collapsed_child(untracked, cdir.ucd) && + (dir->flags & DIR_SHOW_OTHER_DIRECTORIES)) { + /* + * A collapsed directory retains one descendant as its + * untracked witness. Rescan before dropping a stale witness; + * an unvisited sibling may still make the directory untracked. + */ + invalidate_directory(dir->untracked, cdir.ucd); + state = read_directory_recursive( + dir, istate, path.buf, path.len, cdir.ucd, + 1, 0, pathspec); + if (state < path_untracked) + remove_collapsed_untracked_child( + dir->untracked, untracked, cdir.ucd); + } dir->internal.visited_paths++; if (state > dir_state) @@ -2879,6 +3897,8 @@ static int treat_leading_path(struct dir_struct *dir, return state == path_recurse; } +#define UNTRACKED_CACHE_IDENT_VERSION 2 + static const char *get_ident_string(void) { static struct strbuf sb = STRBUF_INIT; @@ -2888,8 +3908,9 @@ static const char *get_ident_string(void) return sb.buf; if (uname(&uts) < 0) die_errno(_("failed to get kernel name and information")); - strbuf_addf(&sb, "Location %s, system %s", repo_get_work_tree(the_repository), - uts.sysname); + strbuf_addf(&sb, "Location %s, system %s, cache version %d", + repo_get_work_tree(the_repository), uts.sysname, + UNTRACKED_CACHE_IDENT_VERSION); return sb.buf; } @@ -3142,6 +4163,7 @@ int read_directory(struct dir_struct *dir, struct index_state *istate, dir->internal.visited_directories = 0; if (has_symlink_leading_path(path, len)) { + dir->internal.untracked_cache_preloaded = 0; trace2_region_leave("dir", "read_directory", istate->repo); return dir->nr; } @@ -3180,6 +4202,7 @@ int read_directory(struct dir_struct *dir, struct index_state *istate, } } + dir->internal.untracked_cache_preloaded = 0; return dir->nr; } @@ -3486,16 +4509,23 @@ void setup_standard_excludes(struct dir_struct *dir) dir->exclude_per_dir = ".gitignore"; /* core.excludesfile defaulting to $XDG_CONFIG_HOME/git/ignore */ - if (excludes_file && !access_or_warn(excludes_file, R_OK, 0)) + if (excludes_file && + (dir->internal.exclude_source_proof || + !access_or_warn(excludes_file, R_OK, 0))) add_patterns_from_file_1(dir, excludes_file, - dir->untracked ? &dir->internal.ss_excludes_file : NULL); + dir->untracked ? + &dir->internal.ss_excludes_file : NULL, + !!dir->internal.exclude_source_proof); /* per repository user preference */ if (startup_info->have_repository) { const char *path = git_path_info_exclude(); - if (!access_or_warn(path, R_OK, 0)) + if (dir->internal.exclude_source_proof || + !access_or_warn(path, R_OK, 0)) add_patterns_from_file_1(dir, path, - dir->untracked ? &dir->internal.ss_info_exclude : NULL); + dir->untracked ? + &dir->internal.ss_info_exclude : NULL, + !!dir->internal.exclude_source_proof); } } @@ -3824,7 +4854,11 @@ static int read_one_dir(struct untracked_cache_dir **untracked_, for (i = 0; i < untracked->dirs_nr; i++) { if (read_one_dir(untracked->dirs + i, rd) < 0) return -1; + if (untracked->dirs[i]->has_untracked) + untracked->has_untracked = 1; } + if (untracked->untracked_nr) + untracked->has_untracked = 1; return 0; } @@ -3947,6 +4981,8 @@ struct untracked_cache *read_untracked_extension(const void *data, unsigned long ewah_each_bit(rd.valid, read_stat, &rd); ewah_each_bit(rd.sha1_valid, read_oid, &rd); next = rd.data; + if (next == end) + compute_untracked_cache_disk_valid_recursive(uc->root); done: free(rd.ucd); @@ -3966,6 +5002,7 @@ static void invalidate_one_directory(struct untracked_cache *uc, { uc->dir_invalidated++; ucd->valid = 0; + ucd->valid_recursive = 0; for (size_t i = 0; i < ucd->untracked_nr; i++) free(ucd->untracked[i]); ucd->untracked_nr = 0; diff --git a/dir.h b/dir.h index 83e0f648a81f36..088e06c1ada4d8 100644 --- a/dir.h +++ b/dir.h @@ -7,6 +7,7 @@ #include "statinfo.h" #include "strbuf.h" +struct exclude_source_proof; struct repository; /** @@ -182,7 +183,17 @@ struct untracked_cache_dir { /* all data except 'dirs' in this struct are good */ unsigned int valid : 1; unsigned int recurse : 1; - /* null object ID means this directory does not have .gitignore */ + /* this subtree contains at least one cached untracked entry */ + unsigned int has_untracked : 1; + /* transient results from directory-stat preloading */ + unsigned int stat_checked : 1; + unsigned int stat_matches : 1; + unsigned int exclude_matches : 1; + unsigned int valid_recursive : 1; + /* + * A null object ID means this directory does not have .gitignore. + * The empty-tree ID records a present source that could not be read. + */ struct object_id exclude_oid; char name[FLEX_ARRAY]; }; @@ -353,6 +364,14 @@ struct dir_struct { /* Stats about the traversal */ unsigned visited_paths; unsigned visited_directories; + unsigned untracked_cache_preloaded : 1; + unsigned traversal_failed : 1; + + /* + * Optional borrowed proof that covers every exclusion source + * consulted by this traversal. + */ + struct exclude_source_proof *exclude_source_proof; } internal; }; @@ -597,6 +616,7 @@ int cmp_dir_entry(const void *p1, const void *p2); int check_dir_entry_contains(const struct dir_entry *out, const struct dir_entry *in); void untracked_cache_invalidate_path(struct index_state *, const char *, int safe_path); +void untracked_cache_invalidate_all(struct index_state *); /* * Invalidate the untracked-cache for this path, but first strip * off a trailing slash, if present. @@ -607,6 +627,17 @@ void untracked_cache_invalidate_trimmed_path(struct index_state *, void untracked_cache_remove_from_index(struct index_state *, const char *); void untracked_cache_add_to_index(struct index_state *, const char *); +struct untracked_cache_preload; +struct untracked_cache_preload * +untracked_cache_preload_start_fsmonitor_excludes( + struct index_state *, unsigned int dir_flags); +struct untracked_cache_preload *untracked_cache_preload_start_ordinary( + struct index_state *, unsigned int dir_flags); +int untracked_cache_preload_finish(struct untracked_cache_preload *, + struct index_state *, unsigned int dir_flags, + size_t *index_invalidated); +void untracked_cache_preload_release(struct untracked_cache_preload *); + void free_untracked_cache(struct untracked_cache *); struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz); void write_untracked_extension(struct strbuf *out, struct untracked_cache *untracked); diff --git a/entry.c b/entry.c index 1c4f0f44070ea3..9284e0d0d49123 100644 --- a/entry.c +++ b/entry.c @@ -512,8 +512,9 @@ int checkout_entry_ca(struct cache_entry *ce, struct conv_attrs *ca, if (!check_path(path.buf, path.len, &st, state->base_dir_len)) { const struct submodule *sub; - unsigned changed = ie_match_stat(state->istate, ce, &st, - CE_MATCH_IGNORE_VALID | CE_MATCH_IGNORE_SKIP_WORKTREE); + unsigned changed = ie_match_stat_with_content_check( + state->istate, ce, &st, + CE_MATCH_IGNORE_VALID | CE_MATCH_IGNORE_SKIP_WORKTREE); /* * Needs to be checked before !changed returns early, * as the possibly empty directory was not changed diff --git a/exclude-source-proof.c b/exclude-source-proof.c new file mode 100644 index 00000000000000..83ee1626d93746 --- /dev/null +++ b/exclude-source-proof.c @@ -0,0 +1,473 @@ +#include "git-compat-util.h" +#include "exclude-source-proof.h" +#include "hash-framing.h" +#include "object-file.h" +#include "path-namespace.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "strmap.h" +#include "trace2.h" + +/* + * Each entry describes one path/policy observation for replay or a stable + * digest. Filesystem identities make capture and validation coherent; the + * durable observation is the source's existence and bytes. + */ +struct exclude_source_proof_entry { + char *path; + size_t size; + struct object_id oid; + unsigned exists : 1; + unsigned nofollow : 1; +}; + +struct exclude_source_proof { + struct index_state *istate; + void *open_data; + exclude_source_open_parent_fn open_parent; + struct exclude_source_proof_entry *entries; + struct strintmap entries_by_path[2]; + size_t nr; + size_t alloc; + unsigned nonblocking : 1; + unsigned invalid : 1; +}; + +struct exclude_source_capture { + struct exclude_source_proof *proof; + char *path; + char *parent; + char *relative; + int parent_fd; + struct stat parent_stat; + unsigned nofollow : 1; +}; + +static char *source_parent(const char *path) +{ + const char *slash = strrchr(path, '/'); + + if (!slash) + return xstrdup("."); + if (slash == path) + return xstrdup("/"); + return xmemdupz(path, slash - path); +} + +static char *source_relative(const char *path, const char *parent) +{ + const char *relative; + size_t len; + + if (!strcmp(parent, ".")) + return xstrdup(path); + if (!strcmp(parent, "/")) { + relative = path + 1; + } else { + len = strlen(parent); + if (strncmp(path, parent, len) || path[len] != '/') + BUG("exclude source is not below its parent"); + relative = path + len + 1; + } + return xstrdup(*relative ? relative : "."); +} + +static int parent_up(char *parent) +{ + char *slash; + + if (!strcmp(parent, ".") || !strcmp(parent, "/")) + return 0; + slash = strrchr(parent, '/'); + if (!slash) { + parent[0] = '.'; + parent[1] = '\0'; + } else if (slash == parent) { + parent[1] = '\0'; + } else { + *slash = '\0'; + } + return 1; +} + +static int open_source_at(int parent_fd, const char *relative, int nofollow, + int nonblocking) +{ +#if EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN + int flags = O_RDONLY | O_CLOEXEC; + + if (nofollow) + flags |= O_NOFOLLOW; + if (nonblocking) + flags |= O_NONBLOCK; + return openat(parent_fd, relative, flags); +#else + (void)parent_fd; + (void)relative; + (void)nofollow; + (void)nonblocking; + errno = ENOSYS; + return -1; +#endif +} + +static int parent_identity_stable( + struct exclude_source_proof *proof, const char *parent, + int held_fd, const struct stat *expected) +{ + struct stat held, reopened; + int fd = proof->open_parent(proof->open_data, parent); + int stable = !fstat(held_fd, &held) && + fd >= 0 && !fstat(fd, &reopened) && + path_namespace_stat_equal(expected, &held) && + path_namespace_stat_equal(expected, &reopened); + + if (fd >= 0) + close(fd); + return stable; +} + +static int parent_stable(struct exclude_source_capture *capture) +{ + return parent_identity_stable( + capture->proof, capture->parent, capture->parent_fd, + &capture->parent_stat); +} + +static void capture_free(struct exclude_source_capture *capture) +{ + if (!capture) + return; + if (capture->parent_fd >= 0) + close(capture->parent_fd); + free(capture->path); + free(capture->parent); + free(capture->relative); + free(capture); +} + +static struct exclude_source_capture *capture_begin( + struct exclude_source_proof *proof, const char *path, + int nofollow, int invalidate) +{ + struct exclude_source_capture *capture; + + if (!proof || proof->invalid) + return NULL; + if (!path) { + if (invalidate) + proof->invalid = 1; + return NULL; + } + CALLOC_ARRAY(capture, 1); + capture->proof = proof; + capture->nofollow = nofollow; + capture->parent_fd = -1; + capture->path = xstrdup(path); + capture->parent = source_parent(path); + for (;;) { + capture->parent_fd = proof->open_parent(proof->open_data, + capture->parent); + if (capture->parent_fd >= 0) + break; + if (!is_missing_file_error(errno) || + !parent_up(capture->parent)) + break; + } + if (capture->parent_fd < 0 || + fstat(capture->parent_fd, &capture->parent_stat) || + !S_ISDIR(capture->parent_stat.st_mode)) { + if (invalidate) + proof->invalid = 1; + capture_free(capture); + return NULL; + } + capture->relative = source_relative(path, capture->parent); + return capture; +} + +struct exclude_source_proof *exclude_source_proof_create( + struct index_state *istate, void *open_data, + exclude_source_open_parent_fn open_parent, unsigned flags) +{ + struct exclude_source_proof *proof; + + CALLOC_ARRAY(proof, 1); + proof->istate = istate; + proof->open_data = open_data; + proof->open_parent = open_parent; + proof->nonblocking = + !!(flags & EXCLUDE_SOURCE_PROOF_NONBLOCKING); + strintmap_init_with_options(&proof->entries_by_path[0], -1, + NULL, 0); + strintmap_init_with_options(&proof->entries_by_path[1], -1, + NULL, 0); + if (!EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN || + !istate || !istate->repo || !istate->repo->hash_algo || + !open_parent || + (flags & ~EXCLUDE_SOURCE_PROOF_NONBLOCKING)) + proof->invalid = 1; + return proof; +} + +struct exclude_source_capture *exclude_source_capture_begin( + struct exclude_source_proof *proof, const char *path, + int nofollow) +{ + return capture_begin(proof, path, nofollow, 1); +} + +int exclude_source_capture_open(struct exclude_source_capture *capture) +{ + if (!capture) { + errno = EINVAL; + return -1; + } + return open_source_at(capture->parent_fd, capture->relative, + capture->nofollow, + capture->proof->nonblocking); +} + +int exclude_source_capture_absent(struct exclude_source_capture *capture) +{ +#if EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN + struct stat st; + + if (!capture) + return 0; + if (!fstatat(capture->parent_fd, capture->relative, &st, + AT_SYMLINK_NOFOLLOW)) + return 0; + return is_missing_file_error(errno); +#else + (void)capture; + return 0; +#endif +} + +static int source_matches(struct exclude_source_capture *capture, + const struct stat *expected) +{ + struct stat st; + int fd = open_source_at(capture->parent_fd, capture->relative, + capture->nofollow, 1); + int ret = fd >= 0 && !fstat(fd, &st) && + path_namespace_stat_equal(expected, &st); + + if (fd >= 0) + close(fd); + return ret; +} + +static int same_observation( + const struct exclude_source_proof_entry *entry, + int exists, size_t size, const struct object_id *oid) +{ + return entry->exists == exists && + (!exists || + (entry->size == size && oideq(&entry->oid, oid))); +} + +static void record_observation( + struct exclude_source_capture *capture, int exists, + size_t size, const struct object_id *oid) +{ + struct exclude_source_proof *proof = capture->proof; + struct strintmap *map = + &proof->entries_by_path[!!capture->nofollow]; + struct exclude_source_proof_entry *entry; + int index = strintmap_get(map, capture->path); + + if (index >= 0) { + if (!same_observation(&proof->entries[index], + exists, size, oid)) + proof->invalid = 1; + return; + } + + ALLOC_GROW(proof->entries, proof->nr + 1, proof->alloc); + entry = &proof->entries[proof->nr]; + memset(entry, 0, sizeof(*entry)); + entry->path = xstrdup(capture->path); + entry->nofollow = capture->nofollow; + entry->exists = exists; + if (exists) { + entry->size = size; + oidcpy(&entry->oid, oid); + } + strintmap_set(map, entry->path, proof->nr); + proof->nr++; +} + +void exclude_source_capture_record( + struct exclude_source_capture *capture, + int source_fd, + const struct stat *source_stat, + const void *buf, size_t size) +{ + struct exclude_source_proof *proof; + struct object_id oid; + struct stat final; + + if (!capture) + return; + proof = capture->proof; + if (proof->invalid) + return; + + if (!source_stat) { + if (!exclude_source_capture_absent(capture) || + !parent_stable(capture) || + !exclude_source_capture_absent(capture)) { + proof->invalid = 1; + return; + } + record_observation(capture, 0, 0, NULL); + return; + } + + if (source_fd < 0 || source_stat->st_size < 0 || + (!buf && size) || + xsize_t(source_stat->st_size) != size || + fstat(source_fd, &final) || + !path_namespace_stat_equal(source_stat, &final) || + !source_matches(capture, &final) || + !parent_stable(capture)) { + proof->invalid = 1; + return; + } + hash_object_file(proof->istate->repo->hash_algo, buf, size, + OBJ_BLOB, &oid); + record_observation(capture, 1, size, &oid); +} + +void exclude_source_capture_error(struct exclude_source_capture *capture) +{ + if (capture) + capture->proof->invalid = 1; +} + +void exclude_source_capture_release(struct exclude_source_capture *capture) +{ + capture_free(capture); +} + +static int proof_entry_matches( + struct exclude_source_proof *proof, + const struct exclude_source_proof_entry *entry) +{ + struct exclude_source_capture *capture = + capture_begin(proof, entry->path, entry->nofollow, 0); + struct object_id oid; + struct stat before, after, final; + char *buf = NULL; + size_t size; + int fd = -1; + int ret = 0; + + if (!capture) + goto done; + if (!entry->exists) { + ret = exclude_source_capture_absent(capture) && + parent_stable(capture) && + exclude_source_capture_absent(capture); + goto done; + } + + fd = open_source_at(capture->parent_fd, capture->relative, + entry->nofollow, 1); + if (fd < 0 || fstat(fd, &before) || before.st_size < 0 || + xsize_t(before.st_size) != entry->size) + goto done; + size = entry->size; + buf = xmalloc(size ? size : 1); + if ((size_t)read_in_full(fd, buf, size) != size || + fstat(fd, &after) || + !path_namespace_stat_equal(&before, &after) || + !source_matches(capture, &after)) + goto done; + hash_object_file(proof->istate->repo->hash_algo, buf, size, + OBJ_BLOB, &oid); + if (!oideq(&oid, &entry->oid) || + !parent_stable(capture) || + fstat(fd, &final) || + !path_namespace_stat_equal(&after, &final) || + !source_matches(capture, &final)) + goto done; + ret = 1; +done: + free(buf); + if (fd >= 0) + close(fd); + capture_free(capture); + return ret; +} + +int exclude_source_proof_validate(struct exclude_source_proof *proof) +{ + int valid; + + if (!proof) + return 0; + valid = !proof->invalid; + for (size_t i = 0; valid && i < proof->nr; i++) + valid = proof_entry_matches(proof, &proof->entries[i]); + if (proof->istate && proof->istate->repo) { + trace2_data_intmax("exclude", proof->istate->repo, + "proof_entries", proof->nr); + trace2_data_intmax("exclude", proof->istate->repo, + "proof_valid", valid); + } + return valid; +} + +int exclude_source_proof_digest( + struct exclude_source_proof *proof, + const struct git_hash_algo *algo, + struct object_id *oid) +{ + static const char domain[] = "git-exclude-source-proof-digest-v1"; + const struct git_hash_algo *source_algo; + struct git_hash_ctx ctx; + unsigned char count[sizeof(uint64_t)]; + unsigned char format[sizeof(uint32_t)]; + + if (!proof || !algo || !oid || + !exclude_source_proof_validate(proof)) + return -1; + source_algo = proof->istate->repo->hash_algo; + git_hash_init(&ctx, algo); + hash_length_delimited(&ctx, domain, sizeof(domain) - 1); + put_be32(format, source_algo->format_id); + hash_length_delimited(&ctx, format, sizeof(format)); + put_be64(count, proof->nr); + hash_length_delimited(&ctx, count, sizeof(count)); + for (size_t i = 0; i < proof->nr; i++) { + const struct exclude_source_proof_entry *entry = + &proof->entries[i]; + unsigned char policy[] = { + entry->nofollow, + entry->exists, + }; + + hash_length_delimited(&ctx, entry->path, + strlen(entry->path)); + hash_length_delimited(&ctx, policy, sizeof(policy)); + hash_length_delimited(&ctx, entry->oid.hash, + entry->exists ? source_algo->rawsz : 0); + } + git_hash_final_oid(oid, &ctx); + return 0; +} + +void exclude_source_proof_release(struct exclude_source_proof *proof) +{ + if (!proof) + return; + strintmap_clear(&proof->entries_by_path[0]); + strintmap_clear(&proof->entries_by_path[1]); + for (size_t i = 0; i < proof->nr; i++) + free(proof->entries[i].path); + free(proof->entries); + free(proof); +} diff --git a/exclude-source-proof.h b/exclude-source-proof.h new file mode 100644 index 00000000000000..ab9b626c664380 --- /dev/null +++ b/exclude-source-proof.h @@ -0,0 +1,52 @@ +#ifndef EXCLUDE_SOURCE_PROOF_H +#define EXCLUDE_SOURCE_PROOF_H + +#if (defined(__APPLE__) || defined(__linux__)) && \ + defined(O_CLOEXEC) && defined(O_NONBLOCK) && \ + defined(O_NOFOLLOW) && defined(O_DIRECTORY) && \ + defined(AT_SYMLINK_NOFOLLOW) +#define EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN 1 +#else +#define EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN 0 +#endif + +struct exclude_source_capture; +struct exclude_source_proof; +struct git_hash_algo; +struct index_state; +struct object_id; +struct stat; + +typedef int (*exclude_source_open_parent_fn)(void *data, const char *path); + +enum exclude_source_proof_flags { + EXCLUDE_SOURCE_PROOF_NONBLOCKING = (1 << 0), +}; + +struct exclude_source_proof *exclude_source_proof_create( + struct index_state *istate, void *open_data, + exclude_source_open_parent_fn open_parent, unsigned flags); +struct exclude_source_capture *exclude_source_capture_begin( + struct exclude_source_proof *proof, const char *path, + int nofollow); +int exclude_source_capture_open(struct exclude_source_capture *capture); +int exclude_source_capture_absent(struct exclude_source_capture *capture); +void exclude_source_capture_record( + struct exclude_source_capture *capture, + int source_fd, + const struct stat *source_stat, + const void *buf, size_t size); +void exclude_source_capture_error(struct exclude_source_capture *capture); +void exclude_source_capture_release(struct exclude_source_capture *capture); +int exclude_source_proof_validate(struct exclude_source_proof *proof); +/* + * Hash unique observations in first-observation order. The caller must + * capture them in a deterministic order when comparing across processes. + */ +int exclude_source_proof_digest( + struct exclude_source_proof *proof, + const struct git_hash_algo *algo, + struct object_id *oid); +void exclude_source_proof_release(struct exclude_source_proof *proof); + +#endif /* EXCLUDE_SOURCE_PROOF_H */ diff --git a/exec-cmd.c b/exec-cmd.c index 507e67d528b0dd..dc801d1a6d35d4 100644 --- a/exec-cmd.c +++ b/exec-cmd.c @@ -27,6 +27,14 @@ static const char *system_prefix(void); +/* + * Absolute path to the current executable, when it can be determined. Keep + * this separately from executable_dirname because some callers need to + * re-execute this exact Git rather than resolve "git" through PATH. + */ +static const char *executable_path; +static int executable_is_dispatcher; + #ifdef RUNTIME_PREFIX /** @@ -257,7 +265,8 @@ void git_resolve_executable_dir(const char *argv0) return; } - resolved = strbuf_detach(&buf, NULL); + executable_path = strbuf_detach(&buf, NULL); + resolved = xstrdup(executable_path); slash = find_last_dir_sep(resolved); if (slash) resolved[slash - resolved] = '\0'; @@ -278,15 +287,42 @@ static const char *system_prefix(void) } /* - * This is called during initialization, but No work needs to be done here when - * runtime prefix is not being used. + * A non-runtime-prefix build does not need the executable directory for path + * discovery, but an explicit argv[0] is still useful for exact re-execution. */ -void git_resolve_executable_dir(const char *argv0 UNUSED) +void git_resolve_executable_dir(const char *argv0) { + struct strbuf buf = STRBUF_INIT; + + /* A bare argv[0] would require a PATH lookup and is not authoritative. */ + if (!argv0 || !*argv0 || !find_last_dir_sep(argv0)) + return; + strbuf_add_absolute_path(&buf, argv0); + if (strbuf_normalize_path(&buf)) { + trace_printf("trace: could not normalize executable path: %s\n", + buf.buf); + strbuf_release(&buf); + return; + } + executable_path = strbuf_detach(&buf, NULL); + trace2_cmd_path(executable_path); } #endif /* RUNTIME_PREFIX */ +const char *git_executable_path(void) +{ + /* Helpers have an exact path too, but cannot dispatch Git builtins. */ + if (!executable_path || !executable_is_dispatcher) + return NULL; + return executable_path; +} + +void git_mark_executable_as_dispatcher(void) +{ + executable_is_dispatcher = 1; +} + char *system_path(const char *path) { struct strbuf d = STRBUF_INIT; diff --git a/exec-cmd.h b/exec-cmd.h index 330b41d54dec52..0613765ef7e4ee 100644 --- a/exec-cmd.h +++ b/exec-cmd.h @@ -5,6 +5,8 @@ struct strvec; void git_set_exec_path(const char *exec_path); void git_resolve_executable_dir(const char *path); +const char *git_executable_path(void); +void git_mark_executable_as_dispatcher(void); const char *git_exec_path(void); void setup_path(void); const char **prepare_git_cmd(struct strvec *out, const char **argv); diff --git a/fsmonitor--daemon.h b/fsmonitor--daemon.h index 5cbbec8d940ba7..850188f872b783 100644 --- a/fsmonitor--daemon.h +++ b/fsmonitor--daemon.h @@ -40,6 +40,7 @@ struct fsmonitor_daemon_state { pthread_mutex_t main_lock; struct strbuf path_worktree_watch; + struct strbuf worktree_identity; struct strbuf path_gitdir_watch; struct alias_info alias; int nr_paths_watching; diff --git a/fsmonitor-clean-proof.c b/fsmonitor-clean-proof.c new file mode 100644 index 00000000000000..3c45c014469737 --- /dev/null +++ b/fsmonitor-clean-proof.c @@ -0,0 +1,116 @@ +#include "git-compat-util.h" +#include "attr-manifest.h" +#include "fsmonitor-clean-proof.h" +#include "hash-framing.h" +#include "strbuf.h" + +#define FSMONITOR_CLEAN_PROOF_MAGIC 0x46534331 /* "FSC1" */ +#define FSMONITOR_CLEAN_PROOF_HEADER_WORDS 5 + +int fsmonitor_clean_proof_parse(struct fsmonitor_clean_proof *proof, + const void *data, size_t len, + const struct git_hash_algo *algo) +{ + struct fsmonitor_clean_proof parsed = { 0 }; + const unsigned char *p = data; + const unsigned char *end = p + len; + unsigned char checksum[GIT_MAX_RAWSZ]; + size_t hashes_len = 4 * algo->rawsz; + uint32_t token_len, manifest_len; + + memset(proof, 0, sizeof(*proof)); + if (len < FSMONITOR_CLEAN_PROOF_HEADER_WORDS * sizeof(uint32_t) + + hashes_len + 1) + return -1; + if (get_be32(p) != FSMONITOR_CLEAN_PROOF_VERSION) + return -1; + p += sizeof(uint32_t); + if (get_be32(p) != FSMONITOR_CLEAN_PROOF_MAGIC) + return -1; + p += sizeof(uint32_t); + parsed.flags = get_be32(p); + p += sizeof(uint32_t); + token_len = get_be32(p); + p += sizeof(uint32_t); + manifest_len = get_be32(p); + p += sizeof(uint32_t); + if (parsed.flags & ~FSMONITOR_CLEAN_PROOF_ALL || !token_len || + token_len > FSMONITOR_CLEAN_PROOF_TOKEN_MAX || + manifest_len < sizeof(uint32_t) || + (size_t)(end - p) < token_len || memchr(p, '\0', token_len)) + return -1; + parsed.token = p; + parsed.token_len = token_len; + p += token_len; + if ((size_t)(end - p) < hashes_len || + (size_t)(end - p) - hashes_len != manifest_len) + return -1; + parsed.config_hash = p; + p += algo->rawsz; + parsed.semantic_hash = p; + p += algo->rawsz; + parsed.attr_hash = p; + p += algo->rawsz; + parsed.attr_manifest = p; + parsed.attr_manifest_len = manifest_len; + p += manifest_len; + if (!attr_manifest_valid(parsed.attr_manifest, + parsed.attr_manifest_len, algo)) + return -1; + hash_buffer_digest(algo, data, len - algo->rawsz, checksum); + if (memcmp(checksum, p, algo->rawsz)) + return -1; + *proof = parsed; + return 0; +} + +int fsmonitor_clean_proof_write(struct strbuf *out, + const struct fsmonitor_clean_proof *proof, + const struct git_hash_algo *algo) +{ + uint32_t value; + + strbuf_reset(out); + if (!proof->token || !proof->token_len || + proof->token_len > FSMONITOR_CLEAN_PROOF_TOKEN_MAX || + proof->token_len > UINT32_MAX || + memchr(proof->token, '\0', proof->token_len) || + proof->flags & ~FSMONITOR_CLEAN_PROOF_ALL || + !proof->config_hash || !proof->semantic_hash || !proof->attr_hash || + !proof->attr_manifest || proof->attr_manifest_len > UINT32_MAX || + !attr_manifest_valid(proof->attr_manifest, + proof->attr_manifest_len, algo)) + return -1; + + put_be32(&value, FSMONITOR_CLEAN_PROOF_VERSION); + strbuf_add(out, &value, sizeof(value)); + put_be32(&value, FSMONITOR_CLEAN_PROOF_MAGIC); + strbuf_add(out, &value, sizeof(value)); + put_be32(&value, proof->flags); + strbuf_add(out, &value, sizeof(value)); + put_be32(&value, proof->token_len); + strbuf_add(out, &value, sizeof(value)); + put_be32(&value, proof->attr_manifest_len); + strbuf_add(out, &value, sizeof(value)); + strbuf_add(out, proof->token, proof->token_len); + strbuf_add(out, proof->config_hash, algo->rawsz); + strbuf_add(out, proof->semantic_hash, algo->rawsz); + strbuf_add(out, proof->attr_hash, algo->rawsz); + strbuf_add(out, proof->attr_manifest, proof->attr_manifest_len); + hash_append_checksum(out, algo); + return 0; +} + +int fsmonitor_clean_proof_copy_without_bindings( + struct strbuf *out, const void *data, size_t len, + const struct git_hash_algo *algo) +{ + struct fsmonitor_clean_proof proof; + + strbuf_reset(out); + if (fsmonitor_clean_proof_parse(&proof, data, len, algo)) + return -1; + proof.flags &= ~(FSMONITOR_CLEAN_PROOF_TOKEN_BOUND | + FSMONITOR_CLEAN_PROOF_STAT_BOUND); + return fsmonitor_clean_proof_write(out, &proof, algo); +} diff --git a/fsmonitor-clean-proof.h b/fsmonitor-clean-proof.h new file mode 100644 index 00000000000000..0d4da4cd725803 --- /dev/null +++ b/fsmonitor-clean-proof.h @@ -0,0 +1,42 @@ +#ifndef FSMONITOR_CLEAN_PROOF_H +#define FSMONITOR_CLEAN_PROOF_H + +#include "hash.h" + +struct strbuf; + +#define FSMONITOR_CLEAN_PROOF_VERSION 1 +#define FSMONITOR_CLEAN_PROOF_TOKEN_MAX 4096 + +#define FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE (1u << 0) +#define FSMONITOR_CLEAN_PROOF_TOKEN_BOUND (1u << 1) +#define FSMONITOR_CLEAN_PROOF_STAT_BOUND (1u << 2) +#define FSMONITOR_CLEAN_PROOF_FULL_INDEX (1u << 3) +#define FSMONITOR_CLEAN_PROOF_ALL \ + (FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | \ + FSMONITOR_CLEAN_PROOF_TOKEN_BOUND | \ + FSMONITOR_CLEAN_PROOF_STAT_BOUND | \ + FSMONITOR_CLEAN_PROOF_FULL_INDEX) + +struct fsmonitor_clean_proof { + uint32_t flags; + const unsigned char *token; + size_t token_len; + const unsigned char *config_hash; + const unsigned char *semantic_hash; + const unsigned char *attr_hash; + const unsigned char *attr_manifest; + size_t attr_manifest_len; +}; + +int fsmonitor_clean_proof_parse(struct fsmonitor_clean_proof *proof, + const void *data, size_t len, + const struct git_hash_algo *algo); +int fsmonitor_clean_proof_write(struct strbuf *out, + const struct fsmonitor_clean_proof *proof, + const struct git_hash_algo *algo); +int fsmonitor_clean_proof_copy_without_bindings( + struct strbuf *out, const void *data, size_t len, + const struct git_hash_algo *algo); + +#endif /* FSMONITOR_CLEAN_PROOF_H */ diff --git a/fsmonitor-ipc.c b/fsmonitor-ipc.c index 6112d130644f04..f6eb03cfd9442f 100644 --- a/fsmonitor-ipc.c +++ b/fsmonitor-ipc.c @@ -1,7 +1,14 @@ #define USE_THE_REPOSITORY_VARIABLE #include "git-compat-util.h" +#include "abspath.h" +#include "config.h" +#include "environment.h" +#include "exec-cmd.h" #include "gettext.h" +#include "hash.h" +#include "lockfile.h" +#include "parse.h" #include "simple-ipc.h" #include "fsmonitor-ipc.h" #include "repository.h" @@ -9,6 +16,46 @@ #include "strbuf.h" #include "trace2.h" +int fsmonitor_ipc__get_worktree_identity(struct repository *r, + struct strbuf *identity) +{ + static const char hex[] = "0123456789abcdef"; + struct strbuf canonical = STRBUF_INIT; + struct strbuf stable = STRBUF_INIT; + git_SHA256_CTX ctx; + unsigned char hash[GIT_SHA256_RAWSZ]; + struct stat st; + const char *worktree = repo_get_work_tree(r); + int ret = -1; + + if (!worktree || + !strbuf_realpath(&canonical, worktree, 0) || + stat(canonical.buf, &st)) + goto done; + strbuf_addf(&stable, "v1\n%"PRIuMAX":", (uintmax_t)canonical.len); + strbuf_addbuf(&stable, &canonical); + strbuf_addf(&stable, "\n%"PRIuMAX"\n%"PRIuMAX, + (uintmax_t)st.st_dev, (uintmax_t)st.st_ino); +#ifdef __APPLE__ + strbuf_addf(&stable, "\n%"PRIdMAX"\n%ld\n%"PRIu32, + (intmax_t)st.st_birthtimespec.tv_sec, + st.st_birthtimespec.tv_nsec, st.st_gen); +#endif + git_SHA256_Init(&ctx); + git_SHA256_Update(&ctx, stable.buf, stable.len); + git_SHA256_Final(hash, &ctx); + strbuf_reset(identity); + for (size_t i = 0; i < ARRAY_SIZE(hash); i++) { + strbuf_addch(identity, hex[hash[i] >> 4]); + strbuf_addch(identity, hex[hash[i] & 0xf]); + } + ret = 0; +done: + strbuf_release(&stable); + strbuf_release(&canonical); + return ret; +} + #ifndef HAVE_FSMONITOR_DAEMON_BACKEND /* @@ -45,6 +92,17 @@ int fsmonitor_ipc__send_command(const char *command UNUSED, #else +static void prepare_spawn_env(struct strvec *env) +{ + /* Let the child rediscover this repository from the worktree. */ + strvec_push(env, GIT_DIR_ENVIRONMENT); + strvec_push(env, GIT_WORK_TREE_ENVIRONMENT); + strvec_push(env, GIT_COMMON_DIR_ENVIRONMENT); + strvec_push(env, GIT_PREFIX_ENVIRONMENT); + strvec_push(env, GIT_IMPLICIT_WORK_TREE_ENVIRONMENT); + strvec_push(env, INDEX_ENVIRONMENT); +} + int fsmonitor_ipc__is_supported(void) { return 1; @@ -55,59 +113,291 @@ enum ipc_active_state fsmonitor_ipc__get_state(void) return ipc_get_active_state(fsmonitor_ipc__get_path(the_repository)); } +#define FSMONITOR_START_TIMEOUT_KEY "fsmonitor.starttimeout" +#define FSMONITOR_START_TIMEOUT_DEFAULT 60 +#define FSMONITOR_RESTART_ATTEMPTS 3 + +static unsigned int get_start_timeout(void) +{ + const char *value; + int timeout; + + if (!repo_config_get_value(the_repository, + FSMONITOR_START_TIMEOUT_KEY, &value) && + value && git_parse_int(value, &timeout) && timeout >= 0) + return timeout; + return FSMONITOR_START_TIMEOUT_DEFAULT; +} + +static int spawn_wait_cb(const struct child_process *cmd UNUSED, + void *cb_data UNUSED) +{ + switch (fsmonitor_ipc__get_state()) { + case IPC_STATE__LISTENING: + return 0; + case IPC_STATE__NOT_LISTENING: + case IPC_STATE__PATH_NOT_FOUND: + return 1; + default: + case IPC_STATE__INVALID_PATH: + case IPC_STATE__OTHER_ERROR: + return -1; + } +} + static int spawn_daemon(void) { struct child_process cmd = CHILD_PROCESS_INIT; + struct strbuf canonical_worktree = STRBUF_INIT; + enum start_bg_result result; + unsigned int timeout = get_start_timeout(); + const char *git = git_executable_path(); + const char *worktree = repo_get_work_tree(the_repository); + int ret = -1; - cmd.git_cmd = 1; + if (!worktree || + !strbuf_realpath(&canonical_worktree, worktree, 0)) { + error(_("cannot start fsmonitor daemon without a work tree")); + goto done; + } + + prepare_spawn_env(&cmd.env); + cmd.dir = canonical_worktree.buf; + if (git) + strvec_push(&cmd.args, git); + else + cmd.git_cmd = 1; cmd.no_stdin = 1; cmd.no_stdout = 1; cmd.no_stderr = 1; cmd.close_fd_above_stderr = 1; cmd.trace2_child_class = "fsmonitor"; - strvec_pushl(&cmd.args, "fsmonitor--daemon", "start", NULL); + strvec_pushl(&cmd.args, "fsmonitor--daemon", "run", "--detach", NULL); + + result = start_bg_command(&cmd, spawn_wait_cb, NULL, timeout); + if (result == SBGR_READY || + fsmonitor_ipc__get_state() == IPC_STATE__LISTENING) + ret = 0; +done: + strbuf_release(&canonical_worktree); + return ret; +} + +static int try_send_command(const char *command, struct strbuf *answer, + enum ipc_active_state *state_out) +{ + struct ipc_client_connection *connection = NULL; + struct ipc_client_connect_options options + = IPC_CLIENT_CONNECT_OPTIONS_INIT; + enum ipc_active_state state; + int ret = -1; + + strbuf_reset(answer); + options.wait_if_busy = 1; + options.wait_if_not_found = 0; + + state = ipc_client_try_connect(fsmonitor_ipc__get_path(the_repository), + &options, &connection); + if (state == IPC_STATE__LISTENING) { + ret = ipc_client_send_command_to_connection( + connection, command, strlen(command), answer); + ipc_client_close_connection(connection); + } + + if (state_out) + *state_out = state; + return ret; +} + +static int is_trivial_response(const struct strbuf *answer) +{ + const char *nul = memchr(answer->buf, '\0', answer->len); - return run_command(&cmd); + return nul && nul != answer->buf && + answer->len == (size_t)(nul - answer->buf) + 3 && + nul[1] == '/' && nul[2] == '\0'; +} + +static int has_capability(const struct strbuf *answer, + const char *capability) +{ + const char *p = answer->buf; + const char *end = answer->buf + answer->len; + size_t capability_len = strlen(capability); + + while (p < end) { + const char *eol = memchr(p, '\n', end - p); + const char *line_end = eol ? eol : end; + + if ((size_t)(line_end - p) == capability_len && + !memcmp(p, capability, capability_len)) + return 1; + if (!eol) + break; + p = eol + 1; + } + return 0; +} + +static int server_supports_bound_queries(void) +{ + struct strbuf answer = STRBUF_INIT; + int ret; + + ret = !try_send_command(FSMONITOR_IPC_CAPABILITY_COMMAND, + &answer, NULL) && + has_capability(&answer, FSMONITOR_IPC_QUERY_VERSION); + strbuf_release(&answer); + return ret; +} + +static int wait_for_daemon_exit(void) +{ + uintmax_t elapsed_ms = 0; + uintmax_t timeout_ms = (uintmax_t)get_start_timeout() * 1000; + + while (fsmonitor_ipc__get_state() == IPC_STATE__LISTENING) { + if (elapsed_ms >= timeout_ms) + return -1; + sleep_millisec(50); + elapsed_ms += 50; + } + return 0; +} + +static int restart_incompatible_daemon(void) +{ + struct strbuf answer = STRBUF_INIT; + struct strbuf lock_path = STRBUF_INIT; + struct lock_file restart_lock = LOCK_INIT; + uintmax_t timeout_ms = (uintmax_t)get_start_timeout() * 1000; + long lock_timeout_ms = timeout_ms > LONG_MAX ? + LONG_MAX : (long)timeout_ms; + int have_lock = 0; + int ret = -1; + + /* + * Serialize the re-probe, quit, wait, and spawn sequence. This uses a + * different lock from the one used briefly while binding the socket. + */ + strbuf_addf(&lock_path, "%s.restart", + fsmonitor_ipc__get_path(the_repository)); + if (hold_lock_file_for_update_timeout(&restart_lock, lock_path.buf, + LOCK_NO_DEREF, + lock_timeout_ms) < 0) { + if (server_supports_bound_queries()) + ret = 0; + goto done; + } + have_lock = 1; + + /* Another client may have replaced the daemon while we waited. */ + if (server_supports_bound_queries()) + goto success; + + trace2_data_intmax("fsm_client", NULL, + "query/incompatible-daemon", 1); + if (try_send_command("quit", &answer, NULL)) { + /* + * The connection state describes the failed attempt, not + * necessarily the state after the failure. Re-read it before + * deciding whether there is still a daemon to replace. + */ + if (fsmonitor_ipc__get_state() == IPC_STATE__LISTENING) { + if (server_supports_bound_queries()) + ret = 0; + goto done; + } + } + + if (wait_for_daemon_exit()) + goto done; + + /* + * A concurrent client may already have started a replacement. + * The retried bound query will verify its capability if needed. + */ + if (fsmonitor_ipc__get_state() != IPC_STATE__LISTENING && + spawn_daemon()) + goto done; + +success: + ret = 0; + +done: + if (have_lock) + rollback_lock_file(&restart_lock); + strbuf_release(&lock_path); + strbuf_release(&answer); + return ret; } int fsmonitor_ipc__send_query(const char *since_token, struct strbuf *answer) { + struct strbuf command = STRBUF_INIT; + struct strbuf identity = STRBUF_INIT; int ret = -1; - int tried_to_spawn = 0; + int lifecycle_attempts = 0; enum ipc_active_state state = IPC_STATE__OTHER_ERROR; struct ipc_client_connection *connection = NULL; struct ipc_client_connect_options options = IPC_CLIENT_CONNECT_OPTIONS_INIT; const char *tok = since_token ? since_token : ""; - size_t tok_len = since_token ? strlen(since_token) : 0; + + trace2_region_enter("fsm_client", "query", NULL); + if (fsmonitor_ipc__get_worktree_identity(the_repository, &identity)) { + trace2_data_intmax("fsm_client", NULL, + "query/worktree-identity-error", 1); + goto done; + } + strbuf_addstr(&command, FSMONITOR_IPC_QUERY_PREFIX); + strbuf_addbuf(&command, &identity); + strbuf_addch(&command, '\n'); + strbuf_addstr(&command, tok); options.wait_if_busy = 1; options.wait_if_not_found = 0; - trace2_region_enter("fsm_client", "query", NULL); trace2_data_string("fsm_client", NULL, "query/command", tok); try_again: + strbuf_reset(answer); state = ipc_client_try_connect(fsmonitor_ipc__get_path(the_repository), &options, &connection); switch (state) { case IPC_STATE__LISTENING: ret = ipc_client_send_command_to_connection( - connection, tok, tok_len, answer); + connection, command.buf, command.len, answer); ipc_client_close_connection(connection); + connection = NULL; trace2_data_intmax("fsm_client", NULL, "query/response-length", answer->len); + if (!ret && is_trivial_response(answer) && + !server_supports_bound_queries()) { + /* + * A daemon predating bound queries treats query-v1 as + * garbage and returns a valid trivial response. Never + * accept that unbound result. Replace the daemon with + * the invoking Git executable and retry instead. + */ + strbuf_reset(answer); + ret = -1; + if (lifecycle_attempts++ >= FSMONITOR_RESTART_ATTEMPTS || + restart_incompatible_daemon()) + goto done; + options.wait_if_not_found = 1; + goto try_again; + } goto done; case IPC_STATE__NOT_LISTENING: case IPC_STATE__PATH_NOT_FOUND: - if (tried_to_spawn) + if (lifecycle_attempts++ >= FSMONITOR_RESTART_ATTEMPTS) goto done; - tried_to_spawn++; if (spawn_daemon()) goto done; @@ -137,6 +427,8 @@ int fsmonitor_ipc__send_query(const char *since_token, done: trace2_region_leave("fsm_client", "query", NULL); + strbuf_release(&identity); + strbuf_release(&command); return ret; } @@ -144,30 +436,15 @@ int fsmonitor_ipc__send_query(const char *since_token, int fsmonitor_ipc__send_command(const char *command, struct strbuf *answer) { - struct ipc_client_connection *connection = NULL; - struct ipc_client_connect_options options - = IPC_CLIENT_CONNECT_OPTIONS_INIT; - int ret; enum ipc_active_state state; const char *c = command ? command : ""; - size_t c_len = command ? strlen(command) : 0; - - strbuf_reset(answer); - - options.wait_if_busy = 1; - options.wait_if_not_found = 0; + int ret = try_send_command(c, answer, &state); - state = ipc_client_try_connect(fsmonitor_ipc__get_path(the_repository), - &options, &connection); if (state != IPC_STATE__LISTENING) { die(_("fsmonitor--daemon is not running")); return -1; } - ret = ipc_client_send_command_to_connection(connection, c, c_len, - answer); - ipc_client_close_connection(connection); - if (ret == -1) { die(_("could not send '%s' command to fsmonitor--daemon"), c); return -1; diff --git a/fsmonitor-ipc.h b/fsmonitor-ipc.h index 8b489da762b047..006ee0750cf134 100644 --- a/fsmonitor-ipc.h +++ b/fsmonitor-ipc.h @@ -5,6 +5,15 @@ struct repository; +#define FSMONITOR_IPC_QUERY_VERSION "query-v1" +#define FSMONITOR_IPC_QUERY_PREFIX FSMONITOR_IPC_QUERY_VERSION " " +#define FSMONITOR_IPC_CAPABILITY_COMMAND "get-capabilities" +#define FSMONITOR_IPC_WORKTREE_ID_HEX 64 + +/* Hash the canonical worktree root and its stable filesystem identity. */ +int fsmonitor_ipc__get_worktree_identity(struct repository *r, + struct strbuf *identity); + /* * Returns true if built-in file system monitor daemon is defined * for this platform. diff --git a/fsmonitor-ll.h b/fsmonitor-ll.h index 0504ca07d62fa1..759775506fc01a 100644 --- a/fsmonitor-ll.h +++ b/fsmonitor-ll.h @@ -4,6 +4,17 @@ struct index_state; struct strbuf; +/* A provider-only marker; worktree-relative paths cannot begin with '/'. */ +#define FSMONITOR_PATH_GLOBAL_INVALIDATE "//" + +enum fsmonitor_token_result { + FSMONITOR_TOKEN_NOT_PENDING = 0, + FSMONITOR_TOKEN_CLEAN, + FSMONITOR_TOKEN_CHANGED, + FSMONITOR_TOKEN_TRIVIAL, + FSMONITOR_TOKEN_ERROR, +}; + extern struct trace_key trace_fsmonitor; /* @@ -12,6 +23,12 @@ extern struct trace_key trace_fsmonitor; */ int read_fsmonitor_extension(struct index_state *istate, const void *data, unsigned long sz); +int read_fsmonitor_untracked_extension(struct index_state *istate, + const void *data, unsigned long sz); +void write_fsmonitor_untracked_extension(struct strbuf *sb, + struct index_state *istate); +void prepare_fsmonitor_untracked(struct index_state *istate); + /* * Fill the fsmonitor_dirty ewah bits with their state from the index, * before it is split during writing. @@ -44,6 +61,22 @@ void tweak_fsmonitor(struct index_state *istate); */ void refresh_fsmonitor(struct index_state *istate); +int fsmonitor_invalidate_attributes_path(struct index_state *istate, + const char *name); + +/* Close a provider token which was obtained before a required scan. */ +int fsmonitor_has_pending_token(const struct index_state *istate); +int fsmonitor_pending_token_from_provider(const struct index_state *istate); +/* Reopen the last accepted IPC token after an in-process operation. */ +int fsmonitor_reopen_token(struct index_state *istate); +enum fsmonitor_token_result fsmonitor_query_pending_token( + struct index_state *istate, int untracked_ready); +void fsmonitor_accept_pending_token(struct index_state *istate, + int untracked_proof_complete, + int untracked_cache_valid); +void fsmonitor_reject_pending_token(struct index_state *istate); +void fsmonitor_mark_untracked_cache_valid(struct index_state *istate); + /* * Does the received result contain the "trivial" response? */ diff --git a/fsmonitor.c b/fsmonitor.c index 107767527ebec7..66db89c6c07447 100644 --- a/fsmonitor.c +++ b/fsmonitor.c @@ -2,10 +2,13 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "git-compat-util.h" +#include "attr.h" +#include "clean-status.h" #include "config.h" #include "dir.h" #include "environment.h" #include "ewah/ewok.h" +#include "ewah/ewok_rlw.h" #include "fsmonitor.h" #include "fsmonitor-ipc.h" #include "name-hash.h" @@ -16,6 +19,7 @@ #define INDEX_EXTENSION_VERSION1 (1) #define INDEX_EXTENSION_VERSION2 (2) +#define FSMONITOR_TOKEN_MAX (4096) #define HOOK_INTERFACE_VERSION1 (1) #define HOOK_INTERFACE_VERSION2 (2) @@ -39,6 +43,46 @@ static void fsmonitor_ewah_callback(size_t pos, void *is) ce->ce_flags &= ~CE_FSMONITOR_VALID; } +static int fsmonitor_ewah_is_valid(struct ewah_bitmap *bitmap) +{ + size_t pointer = 0, expanded_words = 0; + size_t logical_words = bitmap->bit_size / BITS_IN_EWORD + + !!(bitmap->bit_size % BITS_IN_EWORD); + size_t padding = bitmap->bit_size % BITS_IN_EWORD; + eword_t *last_rlw = NULL; + + while (pointer < bitmap->buffer_size) { + eword_t *rlw = &bitmap->buffer[pointer]; + size_t running_words = rlw_get_running_len(rlw); + size_t literal_words = rlw_get_literal_words(rlw); + size_t i; + + last_rlw = rlw; + if (literal_words > bitmap->buffer_size - pointer - 1) + return 0; + if (running_words > logical_words - expanded_words) + return 0; + expanded_words += running_words; + if (rlw_get_run_bit(rlw) && running_words && padding && + expanded_words == logical_words) + return 0; + if (literal_words > logical_words - expanded_words) + return 0; + for (i = 0; i < literal_words; i++) { + eword_t literal = bitmap->buffer[pointer + 1 + i]; + + if (padding && + expanded_words + i + 1 == logical_words && + literal >> padding) + return 0; + } + expanded_words += literal_words; + pointer += 1 + literal_words; + } + + return expanded_words == logical_words && bitmap->rlw == last_rlw; +} + static int fsmonitor_hook_version(void) { int hook_version; @@ -59,44 +103,82 @@ int read_fsmonitor_extension(struct index_state *istate, const void *data, unsigned long sz) { const char *index = data; + const char *end = index + sz; + const char *nul; uint32_t hdr_version; uint32_t ewah_size; + uint32_t ewah_words; + uint32_t ewah_rlw; struct ewah_bitmap *fsmonitor_dirty; int ret; uint64_t timestamp; struct strbuf last_update = STRBUF_INIT; - if (sz < sizeof(uint32_t) + 1 + sizeof(uint32_t)) - return error("corrupt fsmonitor extension (too short)"); + if (istate->fsmonitor_extension_seen) + goto invalid; + istate->fsmonitor_extension_seen = 1; + if (end - index < sizeof(uint32_t)) + goto invalid; hdr_version = get_be32(index); index += sizeof(uint32_t); if (hdr_version == INDEX_EXTENSION_VERSION1) { + if (end - index < sizeof(uint64_t)) + goto invalid; timestamp = get_be64(index); strbuf_addf(&last_update, "%"PRIu64"", timestamp); index += sizeof(uint64_t); } else if (hdr_version == INDEX_EXTENSION_VERSION2) { - strbuf_addstr(&last_update, index); - index += last_update.len + 1; + nul = memchr(index, '\0', end - index); + if (!nul || nul == index || nul - index > FSMONITOR_TOKEN_MAX) + goto invalid; + strbuf_add(&last_update, index, nul - index); + index = nul + 1; } else { - return error("bad fsmonitor version %d", hdr_version); + goto invalid; } - istate->fsmonitor_last_update = strbuf_detach(&last_update, NULL); - + if (end - index < sizeof(uint32_t)) + goto invalid; ewah_size = get_be32(index); index += sizeof(uint32_t); + if (ewah_size != end - index || ewah_size < 3 * sizeof(uint32_t)) + goto invalid; + + /* Reject impossible EWAH lengths before its parser allocates memory. */ + ewah_words = get_be32(index + sizeof(uint32_t)); + if (ewah_words > (ewah_size - 3 * sizeof(uint32_t)) / + sizeof(eword_t) || + 3 * sizeof(uint32_t) + (size_t)ewah_words * sizeof(eword_t) != + ewah_size) + goto invalid; + ewah_rlw = get_be32(index + ewah_size - sizeof(uint32_t)); + if (ewah_rlw >= ewah_words) + goto invalid; fsmonitor_dirty = ewah_new(); ret = ewah_read_mmap(fsmonitor_dirty, index, ewah_size); if (ret != ewah_size) { ewah_free(fsmonitor_dirty); - return error("failed to parse ewah bitmap reading fsmonitor index extension"); + goto invalid; + } + if (!fsmonitor_ewah_is_valid(fsmonitor_dirty)) { + ewah_free(fsmonitor_dirty); + goto invalid; + } + if (!istate->split_index && + fsmonitor_dirty->bit_size > istate->cache_nr) { + ewah_free(fsmonitor_dirty); + goto invalid; } - istate->fsmonitor_dirty = fsmonitor_dirty; - if (!istate->split_index) - assert_index_minimum(istate, istate->fsmonitor_dirty->bit_size); + /* Publish only after the complete optional extension is validated. */ + FREE_AND_NULL(istate->fsmonitor_last_update); + if (istate->fsmonitor_dirty) + ewah_free(istate->fsmonitor_dirty); + istate->fsmonitor_last_update = strbuf_detach(&last_update, NULL); + istate->fsmonitor_dirty = fsmonitor_dirty; + istate->fsmonitor_token_valid = 1; trace2_data_string("index", NULL, "extension/fsmn/read/token", istate->fsmonitor_last_update); @@ -104,6 +186,81 @@ int read_fsmonitor_extension(struct index_state *istate, const void *data, "read fsmonitor extension successful '%s'", istate->fsmonitor_last_update); return 0; + +invalid: + istate->fsmonitor_extension_seen = 1; + istate->fsmonitor_token_valid = 0; + istate->fsmonitor_untracked_valid = 0; + FREE_AND_NULL(istate->fsmonitor_last_update); + if (istate->fsmonitor_dirty) { + ewah_free(istate->fsmonitor_dirty); + istate->fsmonitor_dirty = NULL; + } + strbuf_release(&last_update); + trace2_data_intmax("fsmonitor", istate->repo, + "extension/invalid", 1); + return 0; +} + +#define FSMONITOR_UNTRACKED_EXTENSION_VERSION 1 + +int read_fsmonitor_untracked_extension(struct index_state *istate, + const void *data, unsigned long sz) +{ + const char *p = data; + const char *nul; + uint32_t version; + + if (istate->fsmonitor_untracked_extension_seen) + goto invalid; + istate->fsmonitor_untracked_extension_seen = 1; + if (sz < sizeof(version) + 2) + goto invalid; + version = get_be32(p); + p += sizeof(version); + sz -= sizeof(version); + if (version != FSMONITOR_UNTRACKED_EXTENSION_VERSION) + goto invalid; + nul = memchr(p, '\0', sz); + if (!nul || nul == p || (size_t)(nul - p + 1) != sz || + nul - p > FSMONITOR_TOKEN_MAX) + goto invalid; + + FREE_AND_NULL(istate->fsmonitor_untracked_token); + istate->fsmonitor_untracked_token = xstrdup(p); + return 0; + +invalid: + istate->fsmonitor_untracked_extension_seen = 1; + istate->fsmonitor_untracked_extension_invalid = 1; + istate->fsmonitor_untracked_valid = 0; + FREE_AND_NULL(istate->fsmonitor_untracked_token); + trace2_data_intmax("fsmonitor", istate->repo, + "untracked/invalid-extension", 1); + return 0; +} + +void write_fsmonitor_untracked_extension(struct strbuf *sb, + struct index_state *istate) +{ + uint32_t version; + + put_be32(&version, FSMONITOR_UNTRACKED_EXTENSION_VERSION); + strbuf_add(sb, &version, sizeof(version)); + strbuf_addstr(sb, istate->fsmonitor_last_update); + strbuf_addch(sb, '\0'); +} + +void prepare_fsmonitor_untracked(struct index_state *istate) +{ + istate->fsmonitor_untracked_valid = + !istate->fsmonitor_untracked_extension_invalid && + (!istate->untracked || !istate->untracked->root || + (istate->fsmonitor_token_valid && + istate->fsmonitor_last_update && + istate->fsmonitor_untracked_token && + !strcmp(istate->fsmonitor_last_update, + istate->fsmonitor_untracked_token))); } void fill_fsmonitor_bitmap(struct index_state *istate) @@ -189,13 +346,14 @@ static int query_fsmonitor_hook(struct repository *r, } /* - * Invalidate the FSM bit on this CE. This is like mark_fsmonitor_invalid() - * but we've already handled the untracked-cache, so let's not repeat that - * work. This also lets us have a different trace message so that we can - * see everything that was done as part of the refresh-callback. + * Strongly invalidate one cache entry without touching attributes or the + * untracked cache. Callers choose those wider invalidation scopes explicitly. */ -static void invalidate_ce_fsm(struct cache_entry *ce) +void fsmonitor_invalidate_cache_entry(struct cache_entry *ce) { + ce->ce_flags &= ~CE_UPTODATE; + memset(&ce->ce_stat_data, 0, sizeof(ce->ce_stat_data)); + ce->ce_flags |= CE_CONTENT_CHECK_REQUIRED; if (ce->ce_flags & CE_FSMONITOR_VALID) { trace_printf_key(&trace_fsmonitor, "fsmonitor_refresh_callback INV: '%s'", @@ -207,6 +365,39 @@ static void invalidate_ce_fsm(struct cache_entry *ce) static size_t handle_path_with_trailing_slash( struct index_state *istate, const char *name, int pos); +int fsmonitor_invalidate_attributes_path(struct index_state *istate, + const char *name) +{ + size_t len = strlen(name), base, attr_len = strlen(GITATTRIBUTES_FILE); + size_t invalidated = 0; + unsigned int i; + + while (len && is_dir_sep(name[len - 1])) + len--; + base = len; + while (base && !is_dir_sep(name[base - 1])) + base--; + if (len - base != attr_len || + fspathncmp(name + base, GITATTRIBUTES_FILE, attr_len)) + return 0; + + git_attr_invalidate_all(); + for (i = 0; i < istate->cache_nr; i++) { + struct cache_entry *ce = istate->cache[i]; + + if (base && (ce->ce_namelen < base || + fspathncmp(ce->name, name, base))) + continue; + fsmonitor_invalidate_cache_entry(ce); + invalidated++; + } + if (invalidated) + istate->cache_changed |= FSMONITOR_CHANGED; + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/attributes-scope", base); + return invalidated > 0; +} + /* * Use the name-hash to do a case-insensitive cache-entry lookup with * the pathname and invalidate the cache-entry. @@ -254,7 +445,7 @@ static size_t handle_using_name_hash_icase( */ untracked_cache_invalidate_trimmed_path(istate, ce->name, 0); - invalidate_ce_fsm(ce); + fsmonitor_invalidate_cache_entry(ce); return 1; } @@ -347,7 +538,7 @@ static size_t handle_path_without_trailing_slash( * cache-entry with the same pathname, nor for a cone * at that directory. (That is, assume no D/F conflicts.) */ - invalidate_ce_fsm(istate->cache[pos]); + fsmonitor_invalidate_cache_entry(istate->cache[pos]); return 1; } else { size_t nr_in_cone; @@ -425,27 +616,57 @@ static size_t handle_path_with_trailing_slash( for (i = pos; i < istate->cache_nr; i++) { if (!starts_with(istate->cache[i]->name, name)) break; - invalidate_ce_fsm(istate->cache[i]); + fsmonitor_invalidate_cache_entry(istate->cache[i]); nr_in_cone++; } + if (nr_in_cone) { + /* + * A matched directory event may stand in for a nested + * attribute-file change. + */ + git_attr_invalidate_all(); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/attributes-cone", nr_in_cone); + } + return nr_in_cone; } static void fsmonitor_refresh_callback(struct index_state *istate, char *name) { int len = strlen(name); - int pos = index_name_pos(istate, name, len); + int pos; + int attributes_may_have_changed; size_t nr_in_cone; trace_printf_key(&trace_fsmonitor, "fsmonitor_refresh_callback '%s' (pos %d)", - name, pos); + name, !strcmp(name, FSMONITOR_PATH_GLOBAL_INVALIDATE) ? + -1 : index_name_pos(istate, name, len)); + if (!strcmp(name, FSMONITOR_PATH_GLOBAL_INVALIDATE)) { + unsigned int i; + + clean_status_invalidate_current_manifest(istate); + git_attr_invalidate_all(); + untracked_cache_invalidate_all(istate); + for (i = 0; i < istate->cache_nr; i++) + fsmonitor_invalidate_cache_entry(istate->cache[i]); + istate->cache_changed |= FSMONITOR_CHANGED; + trace2_data_intmax("fsmonitor", istate->repo, + "apply/global-invalidation", 1); + return; + } + pos = index_name_pos(istate, name, len); + attributes_may_have_changed = + fsmonitor_invalidate_attributes_path(istate, name); if (name[len - 1] == '/') nr_in_cone = handle_path_with_trailing_slash(istate, name, pos); else nr_in_cone = handle_path_without_trailing_slash(istate, name, pos); + if (pos < 0 && nr_in_cone) + attributes_may_have_changed = 1; /* * If we did not find an exact match for this pathname or any @@ -455,10 +676,15 @@ static void fsmonitor_refresh_callback(struct index_state *istate, char *name) */ if (!nr_in_cone && repo_ignore_case(the_repository)) { nr_in_cone = handle_using_name_hash_icase(istate, name); - if (!nr_in_cone) + if (!nr_in_cone) { nr_in_cone = handle_using_dir_name_hash_icase( istate, name); + if (nr_in_cone) + attributes_may_have_changed = 1; + } } + if (attributes_may_have_changed) + clean_status_invalidate_current_manifest(istate); if (nr_in_cone) trace_printf_key(&trace_fsmonitor, @@ -503,6 +729,253 @@ static void fsmonitor_refresh_callback(struct index_state *istate, char *name) */ static int fsmonitor_force_update_threshold = 100; +static int is_trivial_response_at(const struct strbuf *result, size_t offset) +{ + size_t i; + + if (offset >= result->len || result->buf[offset] != '/') + return 0; + for (i = offset + 1; i < result->len; i++) + if (result->buf[i] != '\0' && result->buf[i] != '\n' && + result->buf[i] != '\r') + return 0; + return 1; +} + +void fsmonitor_query_result_release(struct fsmonitor_query_result *result) +{ + strbuf_release(&result->token); + strbuf_release(&result->paths); +} + +static int fsmonitor_valid_worktree_path(const char *path, size_t len) +{ + struct strbuf copy = STRBUF_INIT; + int valid = 0; + + if (!len || is_dir_sep(path[0]) || has_dos_drive_prefix(path)) + return 0; + strbuf_add(©, path, len); + if (is_dir_sep(copy.buf[copy.len - 1])) + strbuf_setlen(©, copy.len - 1); + if (!copy.len || is_dir_sep(copy.buf[copy.len - 1])) + goto done; + valid = verify_path(copy.buf, 0); + +done: + strbuf_release(©); + return valid; +} + +enum fsmonitor_query_outcome fsmonitor_parse_builtin_response( + const struct strbuf *raw, struct fsmonitor_query_result *result) +{ + const char *nul, *p, *end; + + if (!raw->len) + goto malformed; + nul = memchr(raw->buf, '\0', raw->len); + if (!nul || nul == raw->buf || nul - raw->buf > FSMONITOR_TOKEN_MAX) + goto malformed; + strbuf_add(&result->token, raw->buf, nul - raw->buf); + if (!starts_with(result->token.buf, "builtin:")) + goto malformed; + + p = nul + 1; + end = raw->buf + raw->len; + if (p == end) { + result->outcome = FSMONITOR_QUERY_DELTA; + return result->outcome; + } + if (end[-1] != '\0') + goto malformed; + if (end - p == 2 && p[0] == '/' && p[1] == '\0') { + result->outcome = FSMONITOR_QUERY_TRIVIAL; + return result->outcome; + } + + while (p < end) { + nul = memchr(p, '\0', end - p); + if (!nul || nul == p) + goto malformed; + if (strcmp(p, FSMONITOR_PATH_GLOBAL_INVALIDATE) && + !fsmonitor_valid_worktree_path(p, nul - p)) + goto malformed; + p = nul + 1; + } + strbuf_add(&result->paths, raw->buf + result->token.len + 1, + end - (raw->buf + result->token.len + 1)); + result->outcome = FSMONITOR_QUERY_DELTA; + return result->outcome; + +malformed: + strbuf_reset(&result->token); + strbuf_reset(&result->paths); + trace2_data_intmax("fsm_client", NULL, "query/invalid-response", 1); + return FSMONITOR_QUERY_ERROR; +} + +enum fsmonitor_query_outcome query_builtin_fsmonitor( + const char *since_token, struct fsmonitor_query_result *result) +{ + const char *test_sequence = + getenv("GIT_TEST_FSMONITOR_QUERY_SEQUENCE"); + struct strbuf raw = STRBUF_INIT; + + /* + * Tests may script clean, delta, trivial, and error responses with + * C, D, T, and E. A delta uses GIT_TEST_FSMONITOR_QUERY_PATH. + */ + if (test_sequence && *test_sequence) { + static size_t query_nr; + const char *path; + char outcome; + + if (query_nr >= strlen(test_sequence)) + return FSMONITOR_QUERY_ERROR; + outcome = test_sequence[query_nr++]; + if (outcome == 'E') + return FSMONITOR_QUERY_ERROR; + + strbuf_addf(&result->token, "builtin:test:%"PRIuMAX, + (uintmax_t)query_nr); + if (outcome == 'T') { + result->outcome = FSMONITOR_QUERY_TRIVIAL; + return result->outcome; + } + if (outcome == 'D') { + path = getenv("GIT_TEST_FSMONITOR_QUERY_PATH"); + if (!path || !*path) + return FSMONITOR_QUERY_ERROR; + strbuf_addstr(&result->paths, path); + strbuf_addch(&result->paths, '\0'); + } else if (outcome != 'C') { + return FSMONITOR_QUERY_ERROR; + } + result->outcome = FSMONITOR_QUERY_DELTA; + return result->outcome; + } + + if (!fsmonitor_ipc__send_query(since_token, &raw)) + fsmonitor_parse_builtin_response(&raw, result); + strbuf_release(&raw); + return result->outcome; +} + +static int apply_fsmonitor_paths(struct index_state *istate, + const struct strbuf *paths) +{ + const char *p = paths->buf; + const char *end = paths->buf + paths->len; + int count = 0; + + while (p < end) { + size_t len = strlen(p); + + fsmonitor_refresh_callback(istate, (char *)p); + count++; + p += len + 1; + } + return count; +} + +static void invalidate_all_fsmonitor(struct index_state *istate) +{ + unsigned int i; + int changed = 0; + + for (i = 0; i < istate->cache_nr; i++) { + if (istate->cache[i]->ce_flags & CE_FSMONITOR_VALID) + changed = 1; + istate->cache[i]->ce_flags &= ~CE_FSMONITOR_VALID; + } + istate->fsmonitor_untracked_valid = 0; + if (istate->untracked) + istate->untracked->use_fsmonitor = 0; + if (changed) + istate->cache_changed |= FSMONITOR_CHANGED; +} + +/* + * A forward baseline still needs one ordinary stat refresh before its + * provider token can certify the index. Clear only process-local + * uptodate state so that refresh_index() performs those stats without + * escalating to content checks. + */ +static void invalidate_all_fsmonitor_for_baseline( + struct index_state *istate) +{ + unsigned int i; + + invalidate_all_fsmonitor(istate); + for (i = 0; i < istate->cache_nr; i++) + istate->cache[i]->ce_flags &= ~CE_UPTODATE; +} + +static void invalidate_all_fsmonitor_strong(struct index_state *istate) +{ + unsigned int i; + int provider_disabled = + fsm_settings__get_mode(istate->repo) == FSMONITOR_MODE_DISABLED; + + invalidate_all_fsmonitor(istate); + for (i = 0; i < istate->cache_nr; i++) { + struct cache_entry *ce = istate->cache[i]; + + if (provider_disabled && ce_skip_worktree(ce)) + continue; + fsmonitor_invalidate_cache_entry(ce); + } +} + +void fsmonitor_invalidate_semantics(struct index_state *istate) +{ + clean_status_invalidate_current_proof(istate); + git_attr_invalidate_all(); + invalidate_all_fsmonitor_strong(istate); + istate->cache_changed |= FSMONITOR_CHANGED; + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/strong-invalidation", 1); +} + +static void invalidate_fsmonitor_for_bootstrap( + struct index_state *istate, enum fsmonitor_mode mode, + int semantic_adoption_needed, int semantic_baseline_needed, + int physical_history_unavailable) +{ + int manifest_refresh_failed; + + if (!fstat_is_reliable() || mode != FSMONITOR_MODE_IPC || + istate->split_index) { + invalidate_all_fsmonitor(istate); + return; + } + + if (physical_history_unavailable) { + clean_status_refresh_worktree_manifest(istate); + fsmonitor_invalidate_semantics(istate); + untracked_cache_invalidate_all(istate); + return; + } + + manifest_refresh_failed = + clean_status_refresh_worktree_manifest(istate) < 0; + if (manifest_refresh_failed || + clean_status_manifest_global_fallback(istate) || + (semantic_adoption_needed && !semantic_baseline_needed)) { + fsmonitor_invalidate_semantics(istate); + } else { + if (semantic_baseline_needed) { + clean_status_begin_fsmonitor_semantic_baseline(istate); + invalidate_all_fsmonitor_for_baseline(istate); + trace2_data_intmax("fsmonitor", istate->repo, + "semantic/adoption-baseline", 1); + } else { + invalidate_all_fsmonitor(istate); + } + } +} + void refresh_fsmonitor(struct index_state *istate) { static int warn_once = 0; @@ -514,6 +987,10 @@ void refresh_fsmonitor(struct index_state *istate) char *buf; unsigned int i; int is_trivial = 0; + int tracked_requires_bootstrap; + int untracked_requires_bootstrap; + int semantic_adoption_needed; + int semantic_baseline_needed; struct repository *r = istate->repo; enum fsmonitor_mode fsm_mode = fsm_settings__get_mode(r); enum fsmonitor_reason reason = fsm_settings__get_reason(r); @@ -530,28 +1007,31 @@ void refresh_fsmonitor(struct index_state *istate) return; istate->fsmonitor_has_run_once = 1; + semantic_adoption_needed = fstat_is_reliable() && + !istate->split_index && + fsm_mode == FSMONITOR_MODE_IPC && + clean_status_fsmonitor_semantic_adoption_needed(istate); + semantic_baseline_needed = fstat_is_reliable() && + !istate->split_index && + fsm_mode == FSMONITOR_MODE_IPC && + clean_status_fsmonitor_semantic_baseline_needed(istate); trace_printf_key(&trace_fsmonitor, "refresh fsmonitor"); if (fsm_mode == FSMONITOR_MODE_IPC) { - query_success = !fsmonitor_ipc__send_query( + struct fsmonitor_query_result result = + FSMONITOR_QUERY_RESULT_INIT; + + query_builtin_fsmonitor( istate->fsmonitor_last_update ? istate->fsmonitor_last_update : "builtin:fake", - &query_result); - if (query_success) { - /* - * The response contains a series of nul terminated - * strings. The first is the new token. - * - * Use `char *buf` as an interlude to trick the CI - * static analysis to let us use `strbuf_addstr()` - * here (and only copy the token) rather than - * `strbuf_addbuf()`. - */ - buf = query_result.buf; - strbuf_addstr(&last_update_token, buf); - bol = last_update_token.len + 1; - is_trivial = query_result.buf[bol] == '/'; + &result); + if (result.outcome != FSMONITOR_QUERY_ERROR) { + query_success = 1; + strbuf_addbuf(&last_update_token, &result.token); + is_trivial = result.outcome == FSMONITOR_QUERY_TRIVIAL; + if (!is_trivial) + strbuf_addbuf(&query_result, &result.paths); if (is_trivial) trace2_data_intmax("fsm_client", NULL, "query/trivial-response", 1); @@ -567,6 +1047,7 @@ void refresh_fsmonitor(struct index_state *istate) */ strbuf_addstr(&last_update_token, "builtin:fake"); } + fsmonitor_query_result_release(&result); goto apply_results; } @@ -612,7 +1093,8 @@ void refresh_fsmonitor(struct index_state *istate) query_success = 0; } else { bol = last_update_token.len + 1; - is_trivial = query_result.buf[bol] == '/'; + is_trivial = is_trivial_response_at( + &query_result, bol); } } else if (hook_version < 0) { hook_version = HOOK_INTERFACE_VERSION1; @@ -626,7 +1108,7 @@ void refresh_fsmonitor(struct index_state *istate) r, HOOK_INTERFACE_VERSION1, istate->fsmonitor_last_update, &query_result); if (query_success) - is_trivial = query_result.buf[0] == '/'; + is_trivial = is_trivial_response_at(&query_result, 0); } if (is_trivial) @@ -657,6 +1139,13 @@ void refresh_fsmonitor(struct index_state *istate) */ trace2_region_enter("fsmonitor", "apply_results", istate->repo); + tracked_requires_bootstrap = !query_success || is_trivial || + !istate->fsmonitor_token_valid || + (fstat_is_reliable() && !istate->split_index && + fsm_mode == FSMONITOR_MODE_IPC && + clean_status_fsmonitor_config_mismatch(istate)); + untracked_requires_bootstrap = !istate->fsmonitor_untracked_valid; + if (query_success && !is_trivial) { /* * Mark all pathnames returned by the monitor as dirty. @@ -665,22 +1154,62 @@ void refresh_fsmonitor(struct index_state *istate) */ int count = 0; - buf = query_result.buf; - for (i = bol; i < query_result.len; i++) { - if (buf[i] != '\0') - continue; - fsmonitor_refresh_callback(istate, buf + bol); - bol = i + 1; - count++; + if (fsm_mode == FSMONITOR_MODE_IPC) { + count = apply_fsmonitor_paths(istate, &query_result); + } else { + buf = query_result.buf; + for (i = bol; i < query_result.len; i++) { + if (buf[i] != '\0') + continue; + if (i > bol) { + fsmonitor_refresh_callback(istate, buf + bol); + count++; + } + bol = i + 1; + } + if (bol < query_result.len) { + fsmonitor_refresh_callback(istate, buf + bol); + count++; + } } - if (bol < query_result.len) { - fsmonitor_refresh_callback(istate, buf + bol); - count++; + + /* + * Applying a provider event may expire semantic history after + * the initial bootstrap decision. Keep the new token pending + * until status has rescanned against rebuilt inputs. + */ + if (fstat_is_reliable() && !istate->split_index && + fsm_mode == FSMONITOR_MODE_IPC && + clean_status_fsmonitor_config_mismatch(istate)) + tracked_requires_bootstrap = 1; + + if (tracked_requires_bootstrap) { + /* + * Provider paths can invalidate the manifest or + * semantic inputs after our pre-query snapshot. + * Recheck before choosing the narrow baseline lane. + */ + semantic_adoption_needed = fstat_is_reliable() && + !istate->split_index && + fsm_mode == FSMONITOR_MODE_IPC && + clean_status_fsmonitor_semantic_adoption_needed( + istate); + semantic_baseline_needed = fstat_is_reliable() && + !istate->split_index && + fsm_mode == FSMONITOR_MODE_IPC && + clean_status_fsmonitor_semantic_baseline_needed( + istate); + invalidate_fsmonitor_for_bootstrap( + istate, fsm_mode, semantic_adoption_needed, + semantic_baseline_needed, + !istate->fsmonitor_token_valid); } /* Now mark the untracked cache for fsmonitor usage */ if (istate->untracked) - istate->untracked->use_fsmonitor = 1; + istate->untracked->use_fsmonitor = + !tracked_requires_bootstrap && + !untracked_requires_bootstrap; if (count > fsmonitor_force_update_threshold) istate->cache_changed |= FSMONITOR_CHANGED; @@ -697,32 +1226,198 @@ void refresh_fsmonitor(struct index_state *istate) * we've actually changed entries, so keep track if we * actually changed entries or not. */ - int is_cache_changed = 0; + invalidate_fsmonitor_for_bootstrap( + istate, fsm_mode, semantic_adoption_needed, + semantic_baseline_needed, 1); + } + trace2_region_leave("fsmonitor", "apply_results", istate->repo); - for (i = 0; i < istate->cache_nr; i++) { - if (istate->cache[i]->ce_flags & CE_FSMONITOR_VALID) { - is_cache_changed = 1; - istate->cache[i]->ce_flags &= ~CE_FSMONITOR_VALID; - } - } + strbuf_release(&query_result); + /* + * A token obtained before a full scan cannot describe changes which + * race with that scan. Keep it in memory until the caller closes the + * race with a second query. The last valid token remains safe because + * a query relative to it will return a superset of changes. + */ + if (tracked_requires_bootstrap) { + if (!last_update_token.len) { + if (istate->fsmonitor_last_update) + strbuf_addstr(&last_update_token, + istate->fsmonitor_last_update); + else + strbuf_addstr(&last_update_token, "builtin:fake"); + } + FREE_AND_NULL(istate->fsmonitor_last_update_pending); + istate->fsmonitor_last_update_pending = + strbuf_detach(&last_update_token, NULL); /* - * If we're going to check every file, ensure we save - * the results. + * A trivial response cannot validate prior state, but its + * returned token is still a provider-owned boundary. Use it + * to anchor the complete scan which the caller will close with + * another query. Hook providers cannot perform that closing + * query, so do not publish their trivial-response tokens. */ - if (is_cache_changed) - istate->cache_changed |= FSMONITOR_CHANGED; + istate->fsmonitor_pending_token_from_provider = + query_success && + (fsm_mode == FSMONITOR_MODE_IPC || !is_trivial); + istate->fsmonitor_untracked_valid = 0; + } else { + /* + * The applied delta carries an existing proof forward: + * tracked paths are now invalid in FSMN, while semantic + * events have already expired the proof itself. + */ + if (fsm_mode == FSMONITOR_MODE_IPC) + clean_status_advance_fsmonitor_config_token( + istate, last_update_token.buf); + FREE_AND_NULL(istate->fsmonitor_last_update); + istate->fsmonitor_last_update = + strbuf_detach(&last_update_token, NULL); + if (untracked_requires_bootstrap) { + FREE_AND_NULL(istate->fsmonitor_last_update_pending); + istate->fsmonitor_last_update_pending = + xstrdup(istate->fsmonitor_last_update); + istate->fsmonitor_pending_token_from_provider = 1; + } else { + FREE_AND_NULL(istate->fsmonitor_last_update_pending); + istate->fsmonitor_pending_token_from_provider = 0; + } + if (istate->fsmonitor_untracked_valid && istate->untracked) { + FREE_AND_NULL(istate->fsmonitor_untracked_token); + istate->fsmonitor_untracked_token = + xstrdup(istate->fsmonitor_last_update); + } + } +} - if (istate->untracked) - istate->untracked->use_fsmonitor = 0; +int fsmonitor_has_pending_token(const struct index_state *istate) +{ + return !!istate->fsmonitor_last_update_pending; +} + +int fsmonitor_pending_token_from_provider(const struct index_state *istate) +{ + return istate->fsmonitor_last_update_pending && + istate->fsmonitor_pending_token_from_provider; +} + +int fsmonitor_reopen_token(struct index_state *istate) +{ + if (!fstat_is_reliable() || istate->split_index || + fsm_settings__get_mode(istate->repo) != FSMONITOR_MODE_IPC) + return 0; + if (istate->fsmonitor_last_update_pending) + return istate->fsmonitor_pending_token_from_provider; + if (!istate->fsmonitor_token_valid || !istate->fsmonitor_last_update) + return 0; + istate->fsmonitor_last_update_pending = + xstrdup(istate->fsmonitor_last_update); + istate->fsmonitor_pending_token_from_provider = 1; + return 1; +} + +enum fsmonitor_token_result fsmonitor_query_pending_token( + struct index_state *istate, int untracked_ready) +{ + struct fsmonitor_query_result result = FSMONITOR_QUERY_RESULT_INIT; + enum fsmonitor_token_result ret; + int count; + + if (!istate->fsmonitor_last_update_pending) + return FSMONITOR_TOKEN_NOT_PENDING; + if (fsm_settings__get_mode(istate->repo) != FSMONITOR_MODE_IPC) + return FSMONITOR_TOKEN_ERROR; + + query_builtin_fsmonitor(istate->fsmonitor_last_update_pending, &result); + if (result.outcome == FSMONITOR_QUERY_ERROR) { + istate->fsmonitor_pending_token_from_provider = 0; + ret = FSMONITOR_TOKEN_ERROR; + goto done; } - trace2_region_leave("fsmonitor", "apply_results", istate->repo); - strbuf_release(&query_result); + FREE_AND_NULL(istate->fsmonitor_last_update_pending); + istate->fsmonitor_last_update_pending = + strbuf_detach(&result.token, NULL); + istate->fsmonitor_pending_token_from_provider = 1; + if (result.outcome == FSMONITOR_QUERY_TRIVIAL) { + invalidate_all_fsmonitor_strong(istate); + trace2_data_intmax("fsmonitor", istate->repo, + "token_closure/trivial", 1); + ret = FSMONITOR_TOKEN_TRIVIAL; + goto done; + } + + count = apply_fsmonitor_paths(istate, &result.paths); + if (istate->untracked) + istate->untracked->use_fsmonitor = !!untracked_ready; + trace2_data_intmax("fsmonitor", istate->repo, + "token_closure/apply_count", count); + ret = count ? FSMONITOR_TOKEN_CHANGED : FSMONITOR_TOKEN_CLEAN; - /* Now that we've updated istate, save the last_update_token */ +done: + fsmonitor_query_result_release(&result); + return ret; +} + +void fsmonitor_accept_pending_token(struct index_state *istate, + int untracked_proof_complete, + int untracked_cache_valid) +{ + if (untracked_cache_valid && !untracked_proof_complete) + BUG("valid untracked cache without a complete proof"); + if (!fsmonitor_pending_token_from_provider(istate)) + return; FREE_AND_NULL(istate->fsmonitor_last_update); - istate->fsmonitor_last_update = strbuf_detach(&last_update_token, NULL); + istate->fsmonitor_last_update = istate->fsmonitor_last_update_pending; + istate->fsmonitor_last_update_pending = NULL; + istate->fsmonitor_pending_token_from_provider = 0; + istate->fsmonitor_token_valid = 1; + istate->fsmonitor_untracked_valid = !!untracked_cache_valid; + if (istate->untracked) + istate->untracked->use_fsmonitor = !!untracked_cache_valid; + istate->cache_changed |= FSMONITOR_CHANGED; + FREE_AND_NULL(istate->fsmonitor_untracked_token); + if (untracked_cache_valid) + istate->fsmonitor_untracked_token = + xstrdup(istate->fsmonitor_last_update); + else if (!untracked_proof_complete) { + /* + * Keep a query anchored at the accepted tracked token. A + * later in-process status may need to close work done after + * this point before validating its untracked cache. + */ + istate->fsmonitor_last_update_pending = + xstrdup(istate->fsmonitor_last_update); + istate->fsmonitor_pending_token_from_provider = 1; + } + trace2_data_intmax("fsmonitor", istate->repo, + "token_closure/accepted", 1); +} + +void fsmonitor_reject_pending_token(struct index_state *istate) +{ + FREE_AND_NULL(istate->fsmonitor_last_update_pending); + istate->fsmonitor_pending_token_from_provider = 0; + if (!istate->fsmonitor_token_valid) + FREE_AND_NULL(istate->fsmonitor_last_update); + invalidate_all_fsmonitor_strong(istate); + trace2_data_intmax("fsmonitor", istate->repo, + "token_closure/rejected", 1); +} + +void fsmonitor_mark_untracked_cache_valid(struct index_state *istate) +{ + if (istate->fsmonitor_last_update_pending || + !istate->fsmonitor_token_valid || + !istate->fsmonitor_last_update || !istate->untracked || + istate->fsmonitor_untracked_valid) + return; + istate->fsmonitor_untracked_valid = 1; + FREE_AND_NULL(istate->fsmonitor_untracked_token); + istate->fsmonitor_untracked_token = + xstrdup(istate->fsmonitor_last_update); + istate->cache_changed |= FSMONITOR_CHANGED; } /* @@ -754,6 +1449,7 @@ static void initialize_fsmonitor_last_update(struct index_state *istate) strbuf_addf(&last_update, "%"PRIu64"", getnanotime()); istate->fsmonitor_last_update = strbuf_detach(&last_update, NULL); + istate->fsmonitor_token_valid = 0; } void add_fsmonitor(struct index_state *istate) @@ -772,7 +1468,7 @@ void add_fsmonitor(struct index_state *istate) /* reset the untracked cache */ if (istate->untracked) { add_untracked_cache(istate); - istate->untracked->use_fsmonitor = 1; + istate->untracked->use_fsmonitor = 0; } /* Update the fsmonitor state */ @@ -782,6 +1478,13 @@ void add_fsmonitor(struct index_state *istate) void remove_fsmonitor(struct index_state *istate) { + istate->fsmonitor_token_valid = 0; + istate->fsmonitor_untracked_valid = 0; + FREE_AND_NULL(istate->fsmonitor_last_update_pending); + istate->fsmonitor_pending_token_from_provider = 0; + FREE_AND_NULL(istate->fsmonitor_untracked_token); + if (istate->untracked) + istate->untracked->use_fsmonitor = 0; if (istate->fsmonitor_last_update) { trace_printf_key(&trace_fsmonitor, "remove fsmonitor"); istate->cache_changed |= FSMONITOR_CHANGED; diff --git a/fsmonitor.h b/fsmonitor.h index 5195a8624db82b..136f4769c36fc2 100644 --- a/fsmonitor.h +++ b/fsmonitor.h @@ -6,8 +6,55 @@ #include "fsmonitor-settings.h" #include "object.h" #include "read-cache-ll.h" +#include "strbuf.h" #include "trace.h" +/* + * Force the next stat-aware caller to verify this entry's content. Wider + * invalidation, such as attributes or untracked-cache state, is the caller's + * responsibility. + */ +void fsmonitor_invalidate_cache_entry(struct cache_entry *ce); + +enum fsmonitor_query_outcome { + FSMONITOR_QUERY_ERROR = 0, + FSMONITOR_QUERY_DELTA, + FSMONITOR_QUERY_TRIVIAL, +}; + +struct fsmonitor_query_result { + enum fsmonitor_query_outcome outcome; + struct strbuf token; + struct strbuf paths; +}; + +#define FSMONITOR_QUERY_RESULT_INIT { \ + .outcome = FSMONITOR_QUERY_ERROR, \ + .token = STRBUF_INIT, \ + .paths = STRBUF_INIT, \ +} + +void fsmonitor_query_result_release(struct fsmonitor_query_result *result); +enum fsmonitor_query_outcome fsmonitor_parse_builtin_response( + const struct strbuf *raw, struct fsmonitor_query_result *result); +enum fsmonitor_query_outcome query_builtin_fsmonitor( + const char *since_token, struct fsmonitor_query_result *result); + +/* + * A pathname monitor cannot prove that every name for a multiply-linked + * inode is inside its watch cone. When the platform reports real link + * counts, keep such regular files out of the persistent valid bitmap so + * that every new process checks their stat data. Platforms that synthesize + * link counts retain their existing fsmonitor behavior. The in-process + * CE_UPTODATE bit is still safe after the caller's lstat(). + */ +static inline int fsmonitor_stat_can_be_valid(const struct stat *st) +{ + return !S_ISREG(st->st_mode) || st->st_nlink <= 1; +} + +void fsmonitor_invalidate_semantics(struct index_state *istate); + /* * Check if refresh_fsmonitor has been called at least once. * refresh_fsmonitor is idempotent. Returns true if fsmonitor is diff --git a/git.c b/git.c index e5f1811b6bb762..956a0c2e4d8f54 100644 --- a/git.c +++ b/git.c @@ -929,6 +929,9 @@ int cmd_main(int argc, const char **argv) if (slash) cmd = slash + 1; } + /* A renamed dispatcher is still safer to re-exec than a Git from PATH. */ + if (!starts_with(cmd, "git-")) + git_mark_executable_as_dispatcher(); trace_command_performance(argv); diff --git a/hash-framing.h b/hash-framing.h new file mode 100644 index 00000000000000..6808cc288faa04 --- /dev/null +++ b/hash-framing.h @@ -0,0 +1,51 @@ +#ifndef HASH_FRAMING_H +#define HASH_FRAMING_H + +#include "hash.h" +#include "strbuf.h" + +static inline void hash_length_delimited(struct git_hash_ctx *ctx, + const void *data, size_t len) +{ + uint32_t size; + + if (len > UINT32_MAX) + BUG("length-delimited hash input too long"); + put_be32(&size, len); + git_hash_update(ctx, &size, sizeof(size)); + if (len) + git_hash_update(ctx, data, len); +} + +static inline void hash_optional_cstring(struct git_hash_ctx *ctx, + const char *value) +{ + static const unsigned char missing = 0; + + if (value) + hash_length_delimited(ctx, value, strlen(value)); + else + hash_length_delimited(ctx, &missing, sizeof(missing)); +} + +static inline void hash_buffer_digest(const struct git_hash_algo *algo, + const void *data, size_t len, + unsigned char *hash) +{ + struct git_hash_ctx ctx; + + git_hash_init(&ctx, algo); + git_hash_update(&ctx, data, len); + git_hash_final(hash, &ctx); +} + +static inline void hash_append_checksum(struct strbuf *out, + const struct git_hash_algo *algo) +{ + unsigned char hash[GIT_MAX_RAWSZ]; + + hash_buffer_digest(algo, out->buf, out->len, hash); + strbuf_add(out, hash, algo->rawsz); +} + +#endif /* HASH_FRAMING_H */ diff --git a/meson.build b/meson.build index 7073d5844d2531..f510267b8270c4 100644 --- a/meson.build +++ b/meson.build @@ -317,6 +317,8 @@ libgit_sources = [ 'archive-tar.c', 'archive-zip.c', 'archive.c', + 'attr-fingerprint.c', + 'attr-manifest.c', 'attr.c', 'base85.c', 'bisect.c', @@ -331,6 +333,16 @@ libgit_sources = [ 'chdir-notify.c', 'checkout.c', 'chunk-format.c', + 'clean-status.c', + 'clean-status-config.c', + 'clean-status-epoch.c', + 'clean-status-history.c', + 'clean-status-identity.c', + 'clean-status-index.c', + 'clean-status-manifest.c', + 'clean-status-sidecar.c', + 'clean-status-fast.c', + 'clean-status-sidecar-issue.c', 'color.c', 'column.c', 'combine-diff.c', @@ -368,6 +380,7 @@ libgit_sources = [ 'editor.c', 'entry.c', 'environment.c', + 'exclude-source-proof.c', 'ewah/bitmap.c', 'ewah/ewah_bitmap.c', 'ewah/ewah_io.c', @@ -378,6 +391,7 @@ libgit_sources = [ 'fetch-pack.c', 'fmt-merge-msg.c', 'fsck.c', + 'fsmonitor-clean-proof.c', 'fsmonitor.c', 'fsmonitor-ipc.c', 'fsmonitor-settings.c', @@ -460,6 +474,7 @@ libgit_sources = [ 'patch-delta.c', 'patch-ids.c', 'path.c', + 'path-namespace.c', 'path-walk.c', 'pathspec.c', 'pkt-line.c', @@ -522,6 +537,11 @@ libgit_sources = [ 'run-command.c', 'send-pack.c', 'sequencer.c', + 'semantic-verify-file.c', + 'semantic-verify-path.c', + 'semantic-verify-root.c', + 'semantic-verify-worker.c', + 'semantic-verify.c', 'serve.c', 'server-info.c', 'setup.c', @@ -576,6 +596,8 @@ libgit_sources = [ 'walker.c', 'wildmatch.c', 'worktree.c', + 'worktree-attr-manifest.c', + 'worktree-attr-source.c', 'wrapper.c', 'write-or-die.c', 'ws.c', @@ -1296,8 +1318,11 @@ endif if host_machine.system() == 'darwin' compat_sources += 'compat/precompose_utf8.c' + libgit_c_args += '-DHAVE_PRELOAD_INDEX_BULK' libgit_c_args += '-DPRECOMPOSE_UNICODE' libgit_c_args += '-DPROTECT_HFS_DEFAULT' +elif host_machine.system() == 'linux' + libgit_c_args += '-DHAVE_PRELOAD_INDEX_BULK' endif # Configure general compatibility wrappers. @@ -1339,11 +1364,33 @@ elif host_machine.system() == 'windows' endif if host_machine.system() == 'linux' - compat_sources += 'compat/linux/procinfo.c' + compat_sources += [ + 'compat/linux/procinfo.c', + 'compat/preload-index/bulk-linux.c', + 'compat/preload-index/bulk-linux-entry.c', + 'compat/preload-index/bulk-linux-open.c', + 'compat/preload-index/bulk-linux-scan.c', + 'compat/preload-index/bulk-linux-stat.c', + 'compat/preload-index/bulk-linux-topology.c', + ] + libgit_sources += [ + 'preload-index-bulk-index.c', + 'preload-index-bulk-thread.c', + 'preload-index-bulk.c', + ] elif host_machine.system() == 'windows' compat_sources += 'compat/win32/trace2_win32_process_info.c' elif host_machine.system() == 'darwin' - compat_sources += 'compat/darwin/procinfo.c' + compat_sources += [ + 'compat/darwin/procinfo.c', + 'compat/preload-index/bulk-darwin.c', + 'compat/preload-index/bulk-darwin-root.c', + ] + libgit_sources += [ + 'preload-index-bulk-index.c', + 'preload-index-bulk-thread.c', + 'preload-index-bulk.c', + ] else compat_sources += 'compat/stub/procinfo.c' endif diff --git a/name-hash.c b/name-hash.c index 83757db8746230..47c659d6c75374 100644 --- a/name-hash.c +++ b/name-hash.c @@ -619,6 +619,14 @@ static void lazy_init_name_hash(struct index_state *istate) trace_performance_leave("initialize name hash"); } +int prepare_index_casefolding(struct index_state *istate) +{ + if (!repo_ignore_case(istate->repo)) + return 0; + lazy_init_name_hash(istate); + return 1; +} + /* * A test routine for t/helper/ sources. * diff --git a/name-hash.h b/name-hash.h index 0cbfc4286316b2..cc7e752ab1e27d 100644 --- a/name-hash.h +++ b/name-hash.h @@ -10,6 +10,8 @@ int index_dir_find(struct index_state *istate, const char *name, int namelen, #define index_dir_exists(i, n, l) index_dir_find((i), (n), (l), NULL) +/* Prepare the name and directory hashes for concurrent case-folded lookups. */ +int prepare_index_casefolding(struct index_state *istate); void adjust_dirname_case(struct index_state *istate, char *name); struct cache_entry *index_file_exists(struct index_state *istate, const char *name, int namelen, int igncase); diff --git a/path-namespace.c b/path-namespace.c new file mode 100644 index 00000000000000..533c8b53262899 --- /dev/null +++ b/path-namespace.c @@ -0,0 +1,256 @@ +#include "git-compat-util.h" +#include "abspath.h" +#include "hash.h" +#include "hash-framing.h" +#include "path-namespace.h" +#include "strbuf.h" + +enum namespace_entry_state { + NAMESPACE_ENTRY_MISSING = 0, + NAMESPACE_ENTRY_PRESENT = 1, +}; + +struct stat_fingerprint { + struct path_stat_identity identity; + unsigned int state; +}; + +struct path_namespace_snapshot { + struct stat_fingerprint *entries; + size_t nr; + size_t alloc; +}; + +void path_stat_identity_init(struct path_stat_identity *identity, + const struct stat *st) +{ + memset(identity, 0, sizeof(*identity)); + identity->fields[0] = st->st_dev; + identity->fields[1] = st->st_ino; + identity->fields[2] = st->st_mode; + identity->fields[3] = st->st_nlink; + identity->fields[4] = st->st_uid; + identity->fields[5] = st->st_gid; + identity->fields[6] = st->st_size; + identity->fields[7] = st->st_mtime; +#ifdef __APPLE__ + identity->fields[8] = st->st_mtimespec.tv_nsec; +#else + identity->fields[8] = ST_MTIME_NSEC(*st); +#endif + identity->fields[9] = st->st_ctime; +#ifdef __APPLE__ + identity->fields[10] = st->st_ctimespec.tv_nsec; +#else + identity->fields[10] = ST_CTIME_NSEC(*st); +#endif +#ifdef __APPLE__ + identity->fields[11] = st->st_birthtimespec.tv_sec; + identity->fields[12] = st->st_birthtimespec.tv_nsec; + identity->fields[13] = st->st_gen; +#endif +} + +int path_stat_identity_equal(const struct path_stat_identity *a, + const struct path_stat_identity *b) +{ + return !memcmp(a, b, sizeof(*a)); +} + +static void stat_fingerprint_init(struct stat_fingerprint *fingerprint, + const struct stat *st) +{ + memset(fingerprint, 0, sizeof(*fingerprint)); + fingerprint->state = NAMESPACE_ENTRY_PRESENT; + path_stat_identity_init(&fingerprint->identity, st); + if (S_ISDIR(st->st_mode)) { + /* Unrelated children do not change which object a path names. */ + fingerprint->identity.fields[3] = 0; + fingerprint->identity.fields[6] = 0; + for (size_t i = 7; i <= 10; i++) + fingerprint->identity.fields[i] = 0; + } +} + +static int stat_fingerprint_equal(const struct stat_fingerprint *a, + const struct stat_fingerprint *b) +{ + return a->state == b->state && + path_stat_identity_equal(&a->identity, &b->identity); +} + +static int capture_entry(const char *path, + struct path_namespace_snapshot *snapshot) +{ + struct stat st; + struct stat_fingerprint *entry; + + ALLOC_GROW(snapshot->entries, snapshot->nr + 1, snapshot->alloc); + entry = &snapshot->entries[snapshot->nr++]; + memset(entry, 0, sizeof(*entry)); + if (!lstat(path, &st)) { + stat_fingerprint_init(entry, &st); + return 0; + } + if (errno == ENOENT || errno == ENOTDIR) { + entry->state = NAMESPACE_ENTRY_MISSING; + return 0; + } + return -1; +} + +int path_namespace_capture(const char *path, + struct path_namespace_snapshot **snapshot_out) +{ + struct path_namespace_snapshot *snapshot; + struct strbuf prefix = STRBUF_INIT; + size_t root_len, pos; + int ret = -1; + + if (!fstat_is_reliable()) { + errno = EAGAIN; + return -1; + } + + CALLOC_ARRAY(snapshot, 1); + root_len = offset_1st_component(path); + if (!root_len) + goto done; + strbuf_add(&prefix, path, root_len); + if (capture_entry(prefix.buf, snapshot)) + goto done; + pos = root_len; + while (path[pos]) { + size_t start, end; + + while (path[pos] && is_dir_sep(path[pos])) + pos++; + if (!path[pos]) + break; + start = pos; + while (path[pos] && !is_dir_sep(path[pos])) + pos++; + end = pos; + strbuf_complete(&prefix, '/'); + strbuf_add(&prefix, path + start, end - start); + if (capture_entry(prefix.buf, snapshot)) + goto done; + } + *snapshot_out = snapshot; + snapshot = NULL; + ret = 0; +done: + path_namespace_clear(snapshot); + strbuf_release(&prefix); + return ret; +} + +int path_namespace_equal(const struct path_namespace_snapshot *a, + const struct path_namespace_snapshot *b) +{ + if (a->nr != b->nr) + return 0; + for (size_t i = 0; i < a->nr; i++) + if (!stat_fingerprint_equal(&a->entries[i], &b->entries[i])) + return 0; + return 1; +} + +int path_namespace_target_present( + const struct path_namespace_snapshot *snapshot) +{ + return snapshot->nr && + snapshot->entries[snapshot->nr - 1].state == + NAMESPACE_ENTRY_PRESENT; +} + +void path_namespace_hash(struct git_hash_ctx *ctx, + const struct path_namespace_snapshot *snapshot) +{ + uint32_t value; + uint64_t field; + + put_be32(&value, snapshot->nr); + hash_length_delimited(ctx, &value, sizeof(value)); + for (size_t i = 0; i < snapshot->nr; i++) { + put_be32(&value, snapshot->entries[i].state); + hash_length_delimited(ctx, &value, sizeof(value)); + for (size_t j = 0; + j < ARRAY_SIZE(snapshot->entries[i].identity.fields); j++) { + put_be64(&field, + snapshot->entries[i].identity.fields[j]); + hash_length_delimited(ctx, &field, sizeof(field)); + } + } +} + +void path_namespace_hash_stat(struct git_hash_ctx *ctx, const struct stat *st) +{ + struct path_stat_identity identity; + uint64_t field; + + path_stat_identity_init(&identity, st); + for (size_t i = 0; i < ARRAY_SIZE(identity.fields); i++) { + put_be64(&field, identity.fields[i]); + hash_length_delimited(ctx, &field, sizeof(field)); + } +} + +int path_namespace_stat_equal(const struct stat *a, const struct stat *b) +{ + struct path_stat_identity first, second; + + path_stat_identity_init(&first, a); + path_stat_identity_init(&second, b); + return path_stat_identity_equal(&first, &second); +} + +int path_namespace_reopen_component( + int parent_fd, const char *component, int flags, + path_namespace_open_fn open_fn, const struct stat *expected) +{ + struct stat reopened; + int fd, saved_errno; + + if (!open_fn || !component || !*component || + !strcmp(component, ".") || !strcmp(component, "..")) { + errno = EINVAL; + return -1; + } + for (const char *p = component; *p; p++) { + if (is_dir_sep(*p)) { + errno = EINVAL; + return -1; + } + } + if (!fstat_is_reliable()) { + errno = EAGAIN; + return -1; + } + + fd = open_fn(parent_fd, component, flags); + if (fd < 0) + return -1; + if (fstat(fd, &reopened)) { + saved_errno = errno; + goto error; + } + if (!path_namespace_stat_equal(expected, &reopened)) { + saved_errno = EAGAIN; + goto error; + } + return close(fd); + +error: + close(fd); + errno = saved_errno; + return -1; +} + +void path_namespace_clear(struct path_namespace_snapshot *snapshot) +{ + if (!snapshot) + return; + free(snapshot->entries); + free(snapshot); +} diff --git a/path-namespace.h b/path-namespace.h new file mode 100644 index 00000000000000..d702b2570aae73 --- /dev/null +++ b/path-namespace.h @@ -0,0 +1,36 @@ +#ifndef PATH_NAMESPACE_H +#define PATH_NAMESPACE_H + +struct git_hash_ctx; +struct path_namespace_snapshot; +struct stat; + +typedef int (*path_namespace_open_fn)(int dirfd, const char *path, int flags); + +#define PATH_STAT_IDENTITY_FIELDS 14 + +struct path_stat_identity { + uint64_t fields[PATH_STAT_IDENTITY_FIELDS]; +}; + +void path_stat_identity_init(struct path_stat_identity *identity, + const struct stat *st); +int path_stat_identity_equal(const struct path_stat_identity *a, + const struct path_stat_identity *b); +int path_namespace_capture(const char *path, + struct path_namespace_snapshot **snapshot_out); +int path_namespace_equal(const struct path_namespace_snapshot *a, + const struct path_namespace_snapshot *b); +int path_namespace_target_present( + const struct path_namespace_snapshot *snapshot); +void path_namespace_hash(struct git_hash_ctx *ctx, + const struct path_namespace_snapshot *snapshot); +void path_namespace_hash_stat(struct git_hash_ctx *ctx, + const struct stat *st); +int path_namespace_stat_equal(const struct stat *a, const struct stat *b); +int path_namespace_reopen_component( + int parent_fd, const char *component, int flags, + path_namespace_open_fn open_fn, const struct stat *expected); +void path_namespace_clear(struct path_namespace_snapshot *snapshot); + +#endif /* PATH_NAMESPACE_H */ diff --git a/preload-index-bulk-index.c b/preload-index-bulk-index.c new file mode 100644 index 00000000000000..a4edb54230a3d7 --- /dev/null +++ b/preload-index-bulk-index.c @@ -0,0 +1,251 @@ +#include "git-compat-util.h" +#include "name-hash.h" +#include "object.h" +#include "preload-index-bulk.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "semantic-verify.h" +#include "semantic-verify-internal.h" + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +int preload_bulk_index_position(struct preload_bulk_scan *scan, + const char *path, size_t path_len) +{ + if (path_len > INT_MAX) + return -1; + return index_name_pos_sparse(scan->istate, path, path_len); +} + +static int first_tracked_descendant(struct index_state *istate, + const char *path, size_t path_len, + int pos) +{ + while ((unsigned int)pos < istate->cache_nr) { + const struct cache_entry *ce = istate->cache[pos]; + + if (ce_namelen(ce) <= path_len || + memcmp(ce->name, path, path_len)) + return -1; + if (ce->name[path_len] == '/') + return pos; + if ((unsigned char)ce->name[path_len] > '/') + return -1; + pos++; + } + return -1; +} + +int preload_bulk_index_pos_has_tracked_descendants( + struct preload_bulk_scan *scan, const char *path, size_t path_len, + int pos) +{ + struct index_state *istate = scan->istate; + + if (pos >= 0) + return 0; + pos = -pos - 1; + return first_tracked_descendant(istate, path, path_len, pos) >= 0; +} + +static int record_tracked_state(struct preload_bulk_worker *worker, int pos, + unsigned char state) +{ + struct preload_bulk_scan *scan = worker->scan; + int recorded = 1; + +#if GIT_GNUC_PREREQ(4, 7) || \ + (__has_builtin(__atomic_compare_exchange_n) && \ + __has_builtin(__atomic_store_n)) + unsigned char expected = PRELOAD_BULK_TRACKED_UNSEEN; + + if (!__atomic_compare_exchange_n(&scan->tracked_state[pos], &expected, + state, 0, __ATOMIC_RELAXED, + __ATOMIC_RELAXED)) { + __atomic_store_n(&scan->tracked_state[pos], + PRELOAD_BULK_TRACKED_FALLBACK, + __ATOMIC_RELAXED); + recorded = 0; + } +#else + pthread_mutex_lock(&scan->queue.mutex); + if (scan->tracked_state[pos] != PRELOAD_BULK_TRACKED_UNSEEN) { + state = PRELOAD_BULK_TRACKED_FALLBACK; + recorded = 0; + } + scan->tracked_state[pos] = state; + pthread_mutex_unlock(&scan->queue.mutex); +#endif + return recorded; +} + +static void record_stat_update(struct preload_bulk_worker *worker, int pos, + const struct stat_data *stat_data) +{ + struct preload_bulk_stat_update *update; + + ALLOC_GROW(worker->stat_updates, worker->stat_updates_nr + 1, + worker->stat_updates_alloc); + update = &worker->stat_updates[worker->stat_updates_nr++]; + update->cache_pos = pos; + memcpy(&update->stat_data, stat_data, sizeof(update->stat_data)); +} + +static int tracked_entry_is_eligible(const struct cache_entry *ce) +{ + return !ce_stage(ce) && + !ce_intent_to_add(ce) && + !ce_skip_worktree(ce) && + !(ce->ce_flags & (CE_VALID | CE_REMOVE)) && + (S_ISREG(ce->ce_mode) || S_ISLNK(ce->ce_mode)); +} + +/* + * Match ie_modified(): a nonzero cached size mismatch is a conclusive + * content change. Zero sizes and the historical Windows symlink sentinel + * still require an ordinary content check. Recompute the stat-data match + * because CE_MATCH_RACY_IS_DIRTY may make ie_match_stat() report a data + * change without a size mismatch. + */ +static int size_change_is_definitive(const struct cache_entry *ce, + const struct stat *st, + unsigned int changed) +{ + if (changed & (MODE_CHANGED | TYPE_CHANGED)) + return 0; +#ifdef GIT_WINDOWS_NATIVE + if (S_ISLNK(st->st_mode) && ce->ce_stat_data.sd_size == MAX_PATH) + return 0; +#endif + return ce->ce_stat_data.sd_size && + (match_stat_data(&ce->ce_stat_data, (struct stat *)st) & + DATA_CHANGED); +} + +static unsigned char verify_content_at( + struct preload_bulk_worker *worker, int pos, int parent_fd, + const char *basename, const struct stat *st, + int observed_has_platform_identity, struct stat_data *stat_data, + int *has_stat_update) +{ + struct preload_bulk_scan *scan = worker->scan; + struct cache_entry *ce = scan->istate->cache[pos]; + struct semantic_verify_file_result file; + + *has_stat_update = 0; + if (!scan->verify_content || + !semantic_verify_classify_entry( + scan->istate, ce, worker->attr_check, &file)) + return PRELOAD_BULK_TRACKED_CONTENT_CHECK; + semantic_verify_file_at( + parent_fd, basename, st, observed_has_platform_identity, + scan->root_dev, ce, scan->istate->repo, + worker->hash_buffer, &file); + worker->bytes_hashed += file.bytes_hashed; + if (file.kind == SEMANTIC_VERIFY_RAW_MODIFIED) + return PRELOAD_BULK_TRACKED_DEFINITIVE_MODIFIED; + if (file.kind != SEMANTIC_VERIFY_RAW_CLEAN || !file.persistable) + return PRELOAD_BULK_TRACKED_CONTENT_CHECK; + if (memcmp(&file.stat_data, &ce->ce_stat_data, + sizeof(file.stat_data))) { + memcpy(stat_data, &file.stat_data, sizeof(*stat_data)); + *has_stat_update = 1; + } + return PRELOAD_BULK_TRACKED_CLEAN; +} + +int preload_bulk_index_entry_is_gitlink(struct preload_bulk_scan *scan, + int pos) +{ + return pos >= 0 && S_ISGITLINK(scan->istate->cache[pos]->ce_mode); +} + +void preload_bulk_record_tracked( + struct preload_bulk_worker *worker, int pos, int parent_fd, + const char *basename, const struct stat *st, + int observed_has_platform_identity) +{ + struct preload_bulk_scan *scan = worker->scan; + struct cache_entry *ce = scan->istate->cache[pos]; + struct stat_data stat_data; + unsigned int changed; + unsigned char state; + int has_stat_update = 0; + + if (!tracked_entry_is_eligible(ce)) + return; + changed = ie_match_stat( + scan->istate, ce, (struct stat *)st, + CE_MATCH_RACY_IS_DIRTY | CE_MATCH_IGNORE_FSMONITOR); + if (!changed) + state = PRELOAD_BULK_TRACKED_CLEAN; + else if (size_change_is_definitive(ce, st, changed)) + state = PRELOAD_BULK_TRACKED_DEFINITIVE_MODIFIED; + else + state = verify_content_at( + worker, pos, parent_fd, basename, st, + observed_has_platform_identity, &stat_data, + &has_stat_update); + if (record_tracked_state(worker, pos, state) && has_stat_update) + record_stat_update(worker, pos, &stat_data); +} + +void preload_bulk_record_tracked_fallback( + struct preload_bulk_worker *worker, int pos) +{ + if (!tracked_entry_is_eligible(worker->scan->istate->cache[pos])) + return; + record_tracked_state(worker, pos, + PRELOAD_BULK_TRACKED_FALLBACK); +} + +void preload_bulk_record_tracked_descendants_fallback( + struct preload_bulk_worker *worker, const char *path, + size_t path_len) +{ + struct index_state *istate = worker->scan->istate; + int pos = preload_bulk_index_position(worker->scan, path, path_len); + + if (pos < 0) + pos = -pos - 1; + else + pos++; + while ((pos = first_tracked_descendant(istate, path, path_len, pos)) >= 0) { + preload_bulk_record_tracked_fallback(worker, pos); + pos++; + } +} + +int preload_bulk_record_tracked_alias_fallback( + struct preload_bulk_worker *worker, const char *path, + size_t path_len) +{ + struct preload_bulk_scan *scan = worker->scan; + struct index_state *istate = scan->istate; + struct cache_entry *ce; + struct strbuf canonical = STRBUF_INIT; + int found = 0; + int pos; + + if (!scan->case_insensitive || path_len > INT_MAX) + return 0; + if (index_dir_find(istate, path, path_len, &canonical)) { + found = 1; + preload_bulk_record_tracked_descendants_fallback( + worker, canonical.buf, canonical.len); + goto out; + } + ce = index_file_exists(istate, path, path_len, 1); + if (!ce) + goto out; + found = 1; + pos = index_name_pos_sparse(istate, ce->name, ce_namelen(ce)); + if (pos >= 0) + preload_bulk_record_tracked_fallback(worker, pos); + +out: + strbuf_release(&canonical); + return found; +} diff --git a/preload-index-bulk-thread.c b/preload-index-bulk-thread.c new file mode 100644 index 00000000000000..6cfc4f8409c45a --- /dev/null +++ b/preload-index-bulk-thread.c @@ -0,0 +1,333 @@ +#include "git-compat-util.h" + +#include + +#include "attr.h" +#include "clean-status.h" +#include "convert.h" +#include "preload-index-bulk.h" +#include "read-cache-ll.h" +#include "semantic-verify-internal.h" +#include "trace2.h" + +#define PRELOAD_INDEX_BULK_OPEN_FD_CAP 128 +#define PRELOAD_INDEX_BULK_OPEN_FD_RESERVE 16 + +static void queue_set_failed(struct preload_bulk_queue *queue) +{ + pthread_mutex_lock(&queue->mutex); + queue->failed = 1; + pthread_mutex_unlock(&queue->mutex); +} + +static void enqueue_task(struct preload_bulk_scan *scan, + struct preload_bulk_task *task) +{ + struct preload_bulk_queue *queue = &scan->queue; + + pthread_mutex_lock(&queue->mutex); + task->next = queue->head; + queue->head = task; + queue->pending++; + pthread_cond_signal(&queue->cond); + pthread_mutex_unlock(&queue->mutex); +} + +static int reserve_open_fd(struct preload_bulk_queue *queue) +{ + int reserved = 0; + + pthread_mutex_lock(&queue->mutex); + if (queue->open_fds < queue->open_fd_limit) { + queue->open_fds++; + reserved = 1; + } + pthread_mutex_unlock(&queue->mutex); + return reserved; +} + +static void release_open_fd(struct preload_bulk_queue *queue) +{ + pthread_mutex_lock(&queue->mutex); + if (!queue->open_fds) + BUG("bulk preload open-fd count underflow"); + queue->open_fds--; + pthread_mutex_unlock(&queue->mutex); +} + +void preload_bulk_schedule_directory( + struct preload_bulk_worker *worker, int parent_fd, + const struct preload_bulk_dir_identity *parent_identity, + const struct preload_bulk_dir_identity *child_identity, + struct preload_bulk_untracked_root *untracked_root, + const char *name, const char *path, size_t path_len) +{ + struct preload_bulk_scan *scan = worker->scan; + struct preload_bulk_task *task; + + FLEX_ALLOC_MEM(task, path, path, path_len); + if (parent_identity) { + task->parent_identity = *parent_identity; + task->has_parent_identity = 1; + } + if (child_identity) { + task->child_identity = *child_identity; + task->has_child_identity = 1; + } + task->untracked_root = untracked_root; + task->fd = -1; + if (reserve_open_fd(&scan->queue)) { + task->reserved_fd = 1; + task->fd = scan->backend->open_dir_at(worker, parent_fd, name); + if (task->fd < 0) { + int saved_errno = errno; + + task->reserved_fd = 0; + release_open_fd(&scan->queue); + if (saved_errno == EXDEV) { + preload_bulk_record_tracked_descendants_fallback( + worker, path, path_len); + if (scan->collect_untracked) + preload_bulk_invalidate_untracked(worker); + free(task); + return; + } + if (saved_errno != EMFILE && saved_errno != ENFILE) { + free(task); + queue_set_failed(&scan->queue); + return; + } + } + } + enqueue_task(scan, task); +} + +static size_t preload_bulk_open_fd_limit(void) +{ + struct rlimit limit; + rlim_t value; + + if (getrlimit(RLIMIT_NOFILE, &limit)) + return 1; + if (limit.rlim_cur == RLIM_INFINITY) + return PRELOAD_INDEX_BULK_OPEN_FD_CAP; + if (limit.rlim_cur <= PRELOAD_INDEX_BULK_OPEN_FD_RESERVE) + return 1; + value = limit.rlim_cur - PRELOAD_INDEX_BULK_OPEN_FD_RESERVE; + if (value > PRELOAD_INDEX_BULK_OPEN_FD_CAP) + value = PRELOAD_INDEX_BULK_OPEN_FD_CAP; + return value; +} + +static int queue_init(struct preload_bulk_queue *queue) +{ + memset(queue, 0, sizeof(*queue)); +#if HAVE_THREADS + if (pthread_mutex_init(&queue->mutex, NULL)) + return -1; + if (pthread_cond_init(&queue->cond, NULL)) { + pthread_mutex_destroy(&queue->mutex); + return -1; + } +#endif + queue->open_fd_limit = preload_bulk_open_fd_limit(); + return 0; +} + +static void queue_release(struct preload_bulk_queue *queue) +{ + if (queue->head || queue->pending || queue->open_fds) + BUG("releasing non-empty bulk preload queue"); +#if HAVE_THREADS + pthread_cond_destroy(&queue->cond); + pthread_mutex_destroy(&queue->mutex); +#endif + memset(queue, 0, sizeof(*queue)); +} + +static void *preload_bulk_worker_main(void *data) +{ + struct preload_bulk_worker *worker = data; + struct preload_bulk_queue *queue = &worker->scan->queue; + + for (;;) { + struct preload_bulk_task *task; + int failed, reserved_fd; + + pthread_mutex_lock(&queue->mutex); + while (!queue->head && queue->pending) + pthread_cond_wait(&queue->cond, &queue->mutex); + if (!queue->pending) { + pthread_mutex_unlock(&queue->mutex); + break; + } + task = queue->head; + queue->head = task->next; + pthread_mutex_unlock(&queue->mutex); + + failed = + worker->scan->backend->scan_directory(worker, task); + reserved_fd = task->reserved_fd; + free(task); + + pthread_mutex_lock(&queue->mutex); + if (failed) + queue->failed = 1; + if (reserved_fd) { + if (!queue->open_fds) + BUG("bulk preload open-fd count underflow"); + queue->open_fds--; + } + if (!queue->pending) + BUG("bulk preload task count underflow"); + queue->pending--; + if (!queue->pending) + pthread_cond_broadcast(&queue->cond); + pthread_mutex_unlock(&queue->mutex); + } + return NULL; +} + +static void release_workers(struct preload_bulk_scan *scan) +{ + for (int i = 0; i < scan->threads; i++) { + attr_check_free(scan->workers[i].attr_check); + free(scan->workers[i].buffer); + free(scan->workers[i].hash_buffer); + free(scan->workers[i].stat_updates); + strbuf_release(&scan->workers[i].path); + } + FREE_AND_NULL(scan->workers); +} + +static void prepare_content_verification(struct preload_bulk_scan *scan) +{ + if (!scan->proof_epoch) + return; + + convert_attrs_prepare(scan->istate); + for (int i = 0; i < scan->threads; i++) { + scan->workers[i].attr_check = convert_attrs_check_alloc(); + git_check_attr( + scan->istate, "", scan->workers[i].attr_check); + } + if (!clean_status_proof_epoch_prime_matches( + scan->istate, scan->proof_epoch)) { + for (int i = 0; i < scan->threads; i++) { + attr_check_free(scan->workers[i].attr_check); + scan->workers[i].attr_check = NULL; + } + git_attr_invalidate_all(); + return; + } + for (int i = 0; i < scan->threads; i++) + scan->workers[i].hash_buffer = + xmalloc(SEMANTIC_VERIFY_HASH_BUFFER_SIZE); + scan->verify_content = 1; + trace2_data_intmax("index", scan->repo, + "preload/bulk_content_verify", 1); +} + +static void collect_stat_updates(struct preload_bulk_scan *scan) +{ + size_t nr = 0; + + for (int i = 0; i < scan->threads; i++) { + struct preload_bulk_worker *worker = &scan->workers[i]; + + for (size_t j = 0; j < worker->stat_updates_nr; j++) { + struct preload_bulk_stat_update *update = + &worker->stat_updates[j]; + + if (update->cache_pos >= scan->istate->cache_nr) + BUG("bulk stat update position out of range"); + if (scan->tracked_state[update->cache_pos] == + PRELOAD_BULK_TRACKED_CLEAN) + nr++; + } + } + ALLOC_ARRAY(scan->stat_updates, nr); + for (int i = 0; i < scan->threads; i++) { + struct preload_bulk_worker *worker = &scan->workers[i]; + + for (size_t j = 0; j < worker->stat_updates_nr; j++) { + struct preload_bulk_stat_update *update = + &worker->stat_updates[j]; + + if (scan->tracked_state[update->cache_pos] != + PRELOAD_BULK_TRACKED_CLEAN) + continue; + scan->stat_updates[scan->stat_updates_nr++] = *update; + } + } +} + +int preload_bulk_run_scan(struct preload_bulk_scan *scan, + struct preload_bulk_run_result *result) +{ + struct preload_bulk_task *root_task; + int failed, started_threads = 1; + + if (scan->threads < 1) + BUG("bulk preload scan requires at least one worker"); + memset(result, 0, sizeof(*result)); + if (queue_init(&scan->queue)) + return -1; + CALLOC_ARRAY(scan->workers, scan->threads); + for (int i = 0; i < scan->threads; i++) { + scan->workers[i].scan = scan; + strbuf_init(&scan->workers[i].path, 0); + } + prepare_content_verification(scan); + + FLEX_ALLOC_STR(root_task, path, "."); + if (!reserve_open_fd(&scan->queue)) + BUG("bulk preload queue cannot reserve its root descriptor"); + root_task->reserved_fd = 1; + root_task->fd = fcntl(scan->root_fd, F_DUPFD_CLOEXEC, 0); + if (root_task->fd < 0) { + release_open_fd(&scan->queue); + free(root_task); + release_workers(scan); + queue_release(&scan->queue); + return -1; + } + enqueue_task(scan, root_task); + + for (int i = 1; i < scan->threads; i++) { + int err = pthread_create(&scan->workers[i].thread, NULL, + preload_bulk_worker_main, + &scan->workers[i]); + + if (err) + break; + scan->workers[i].started = 1; + started_threads++; + } + preload_bulk_worker_main(&scan->workers[0]); + for (int i = 1; i < scan->threads; i++) + if (scan->workers[i].started && + pthread_join(scan->workers[i].thread, NULL)) + BUG("unable to join bulk preload worker"); + + for (int i = 0; i < scan->threads; i++) { + struct preload_bulk_worker *worker = &scan->workers[i]; + + result->dirs += worker->dirs; + result->entries += worker->entries; + result->bulk_calls += worker->bulk_calls; + result->bytes_hashed += worker->bytes_hashed; + result->changed_dirs += worker->changed_dirs; + result->malformed += worker->malformed; + } + collect_stat_updates(scan); + result->threads = started_threads; + result->untracked_complete = + scan->collect_untracked && !scan->queue.untracked_invalid; + failed = scan->queue.failed || result->malformed || + result->changed_dirs; + + release_workers(scan); + queue_release(&scan->queue); + return failed ? -1 : 0; +} diff --git a/preload-index-bulk.c b/preload-index-bulk.c new file mode 100644 index 00000000000000..6dd2d1e552ec56 --- /dev/null +++ b/preload-index-bulk.c @@ -0,0 +1,363 @@ +#include "git-compat-util.h" +#include "abspath.h" +#include "dir.h" +#include "exclude-source-proof.h" +#include "name-hash.h" +#include "parse.h" +#include "preload-index-bulk.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "trace2.h" + +struct preload_bulk_untracked_root { + struct preload_bulk_untracked_root *next; + /* + * Normal-mode status reports an untracked directory after finding + * one visible descendant. Share that decision among workers below + * the directory. + */ + unsigned visible : 1; + char path[FLEX_ARRAY]; +}; + +static int backend_available(const struct preload_bulk_backend *backend) +{ + return backend && backend->start && backend->finish && + backend->release && backend->open_dir_at && + backend->scan_directory; +} + +static int open_exclude_parent(void *data, const char *path) +{ + struct preload_bulk_scan *scan = data; + + if (is_absolute_path(path)) + return open(path, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + return scan->backend->open_proof_parent(scan, path); +} + +int preload_bulk_available(void) +{ + return backend_available(preload_bulk_platform_backend()); +} + +int preload_bulk_test_barrier(struct preload_bulk_scan *scan, + const char *path) +{ + struct strbuf buf = STRBUF_INIT; + int fd; + int result; + + if (!scan->test_barrier_path || + strcmp(scan->test_barrier_path, path)) + return 0; + if (!scan->test_barrier_ready || !scan->test_barrier_resume) + return -1; + + fd = open(scan->test_barrier_resume, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + write_file(scan->test_barrier_ready, "ready"); + result = strbuf_read(&buf, fd, 1) > 0 ? 0 : -1; + close(fd); + strbuf_release(&buf); + return result; +} + +int preload_bulk_path_is_excluded(struct preload_bulk_worker *worker, + const char *path, int dtype) +{ + struct preload_bulk_scan *scan = worker->scan; + int result; + + if (!scan->exclude_dir) + BUG("bulk preload has no exclude state"); + pthread_mutex_lock(&scan->exclude_mutex); + result = is_excluded(scan->exclude_dir, scan->istate, path, &dtype); + pthread_mutex_unlock(&scan->exclude_mutex); + return result; +} + +void preload_bulk_invalidate_untracked( + struct preload_bulk_worker *worker) +{ + struct preload_bulk_queue *queue = &worker->scan->queue; + + pthread_mutex_lock(&queue->mutex); + queue->untracked_invalid = 1; + pthread_mutex_unlock(&queue->mutex); +} + +int preload_bulk_untracked_is_invalid( + struct preload_bulk_worker *worker) +{ + struct preload_bulk_queue *queue = &worker->scan->queue; + int invalid; + + pthread_mutex_lock(&queue->mutex); + invalid = queue->untracked_invalid; + pthread_mutex_unlock(&queue->mutex); + return invalid; +} + +struct preload_bulk_untracked_root *preload_bulk_untracked_root_new( + struct preload_bulk_worker *worker, const char *path, + size_t path_len) +{ + struct preload_bulk_scan *scan = worker->scan; + struct preload_bulk_untracked_root *root; + + FLEX_ALLOC_MEM(root, path, path, path_len + 1); + root->path[path_len] = '/'; + root->path[path_len + 1] = '\0'; + + pthread_mutex_lock(&scan->queue.mutex); + root->next = scan->untracked_roots; + scan->untracked_roots = root; + pthread_mutex_unlock(&scan->queue.mutex); + return root; +} + +int preload_bulk_untracked_root_is_visible( + struct preload_bulk_worker *worker MAYBE_UNUSED, + const struct preload_bulk_untracked_root *root) +{ + int visible; + + if (!root) + return 0; + pthread_mutex_lock(&worker->scan->queue.mutex); + visible = root->visible; + pthread_mutex_unlock(&worker->scan->queue.mutex); + return visible; +} + +void preload_bulk_record_untracked( + struct preload_bulk_worker *worker, + struct preload_bulk_untracked_root *root, + const char *path) +{ + struct preload_bulk_scan *scan = worker->scan; + struct preload_bulk_queue *queue = &scan->queue; + int record = 1; + + pthread_mutex_lock(&queue->mutex); + if (queue->untracked_invalid) + record = 0; + else if (root) { + if (root->visible) + record = 0; + else + root->visible = 1; + } + if (record) + string_list_append(&scan->untracked, + root ? root->path : path); + pthread_mutex_unlock(&queue->mutex); +} + +static int collect_untracked_paths(struct preload_bulk_scan *scan, + struct preload_bulk_result *result) +{ + /* + * Do not publish provisional output until all closing validations + * have succeeded. + */ + string_list_sort(&scan->untracked); + for (size_t i = 1; i < scan->untracked.nr; i++) + if (!strcmp(scan->untracked.items[i - 1].string, + scan->untracked.items[i].string)) + return -1; + result->untracked = scan->untracked; + scan->untracked = (struct string_list)STRING_LIST_INIT_DUP; + return 0; +} + +int preload_bulk_collect(struct index_state *istate, int threads, + struct preload_bulk_result *result) +{ + struct dir_struct exclude_dir = DIR_INIT; + struct exclude_source_proof *exclude_proof = NULL; + const struct preload_bulk_backend *backend = + preload_bulk_platform_backend(); + struct preload_bulk_scan scan = { + .repo = istate->repo, + .istate = istate, + .backend = backend, + .proof_epoch = istate->preload_bulk_proof_epoch, + .root_fd = -1, + .threads = threads, + .untracked = STRING_LIST_INIT_DUP, + }; + struct preload_bulk_run_result run_result = { 0 }; + struct object_id standard_excludes_digest; + struct stat root_stat; + const char *start_error, *finish_error = NULL; + const char *untracked_reason = NULL; + int standard_excludes_digest_valid = 0; + int scan_error = -1; + int clean; + + memset(result, 0, sizeof(*result)); + result->untracked.strdup_strings = 1; + result->outcome = "start-fallback"; + result->reason = "backend-unavailable"; + if (!backend_available(backend)) + return -1; + if (istate->sparse_index == INDEX_EXPANDED) { + /* + * Workers may need case-folding lookups for names returned by + * the filesystem. Build the lazy hash before they start. + * + * A collapsed sparse index cannot expand itself concurrently + * from the worker threads. Leave its unseen entries to the + * existing preload path, which expands them on the main thread. + */ + scan.case_insensitive = prepare_index_casefolding(istate); + scan.can_skip_unseen_preload = 1; + } + scan.collect_untracked = + !!istate->preload_untracked && + backend->collects_untracked && + backend->open_proof_parent; + if (istate->preload_untracked && !scan.collect_untracked) + untracked_reason = "backend-unsupported"; + if (backend->max_threads > 0 && + scan.threads > backend->max_threads) + scan.threads = backend->max_threads; + if (scan.collect_untracked) { + scan.exclude_dir = &exclude_dir; +#if HAVE_THREADS + if (pthread_mutex_init(&scan.exclude_mutex, NULL)) { + return -1; + } +#endif + } + + if (git_env_bool("GIT_TEST_PRELOAD_INDEX_BULK", 0)) { + scan.test_barrier_path = getenv( + "GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_PATH"); + scan.test_barrier_ready = getenv( + "GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_READY"); + scan.test_barrier_resume = getenv( + "GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_RESUME"); + } + + CALLOC_ARRAY(scan.tracked_state, istate->cache_nr); + start_error = backend->start(&scan); + if (!start_error && (scan.proof_epoch || scan.collect_untracked) && + (scan.root_fd < 0 || fstat(scan.root_fd, &root_stat))) + start_error = "root-stat"; + if (!start_error && scan.proof_epoch) + scan.root_dev = root_stat.st_dev; + if (!start_error) { + if (scan.collect_untracked) { + exclude_proof = exclude_source_proof_create( + istate, &scan, open_exclude_parent, 0); + exclude_dir.internal.exclude_source_proof = + exclude_proof; + setup_standard_excludes(&exclude_dir); + standard_excludes_digest_valid = + !exclude_source_proof_digest( + exclude_proof, + istate->repo->hash_algo, + &standard_excludes_digest); + } + scan_error = preload_bulk_run_scan(&scan, &run_result); + if (!scan_error) + scan_error = preload_bulk_test_barrier(&scan, ""); + finish_error = backend->finish(&scan); + if (!scan_error && !finish_error && + run_result.untracked_complete) { + int exclude_proof_valid; + + trace2_region_enter( + "index", "preload/bulk_excludes", istate->repo); + exclude_proof_valid = + exclude_source_proof_validate(exclude_proof); + if (!standard_excludes_digest_valid || + !exclude_proof_valid) { + run_result.untracked_complete = 0; + untracked_reason = "exclude-race"; + } + trace2_region_leave( + "index", "preload/bulk_excludes", istate->repo); + } + } + + clean = !start_error && !scan_error && !finish_error && + !run_result.changed_dirs && + !run_result.malformed; + if (clean && run_result.untracked_complete && + collect_untracked_paths(&scan, result)) { + run_result.untracked_complete = 0; + untracked_reason = "duplicate-path"; + } + result->run = run_result; + result->untracked_reason = untracked_reason; + if (start_error) { + result->outcome = "start-fallback"; + result->reason = start_error; + } else if (run_result.changed_dirs) { + result->outcome = "scan-fallback"; + result->reason = "filesystem-race"; + } else if (run_result.malformed) { + result->outcome = "scan-fallback"; + result->reason = "malformed-record"; + } else if (scan_error) { + result->outcome = "scan-fallback"; + result->reason = "scan-error"; + } else if (finish_error) { + result->outcome = "finish-fallback"; + result->reason = finish_error; + } else { + result->outcome = "complete"; + result->reason = NULL; + } + if (clean) { + result->tracked_state = scan.tracked_state; + result->stat_updates = scan.stat_updates; + result->stat_updates_nr = scan.stat_updates_nr; + result->nr = istate->cache_nr; + result->can_skip_unseen_preload = + scan.can_skip_unseen_preload; + result->untracked_complete = run_result.untracked_complete; + if (result->untracked_complete) { + result->standard_excludes_digest_valid = 1; + oidcpy(&result->standard_excludes_digest, + &standard_excludes_digest); + result->scanned_worktree = root_stat; + } + scan.tracked_state = NULL; + scan.stat_updates = NULL; + scan.stat_updates_nr = 0; + } + + backend->release(&scan); + while (scan.untracked_roots) { + struct preload_bulk_untracked_root *next = + scan.untracked_roots->next; + + free(scan.untracked_roots); + scan.untracked_roots = next; + } + string_list_clear(&scan.untracked, 0); + if (scan.exclude_dir) { +#if HAVE_THREADS + pthread_mutex_destroy(&scan.exclude_mutex); +#endif + dir_clear(&exclude_dir); + exclude_source_proof_release(exclude_proof); + } + free(scan.tracked_state); + free(scan.stat_updates); + return clean ? 0 : -1; +} + +void preload_bulk_result_release(struct preload_bulk_result *result) +{ + FREE_AND_NULL(result->tracked_state); + FREE_AND_NULL(result->stat_updates); + string_list_clear(&result->untracked, 0); + memset(result, 0, sizeof(*result)); +} diff --git a/preload-index-bulk.h b/preload-index-bulk.h new file mode 100644 index 00000000000000..ff5583438fd07a --- /dev/null +++ b/preload-index-bulk.h @@ -0,0 +1,202 @@ +#ifndef PRELOAD_INDEX_BULK_H +#define PRELOAD_INDEX_BULK_H + +#include "git-compat-util.h" +#include "hash.h" +#include "preload-index.h" +#include "statinfo.h" +#include "strbuf.h" +#include "string-list.h" +#include "thread-utils.h" + +struct dir_struct; +struct attr_check; +struct clean_status_proof_epoch; +struct index_state; +struct repository; +struct preload_bulk_untracked_root; + +struct preload_bulk_dir_identity { + struct stat stat; + uint64_t platform_id; + unsigned complete : 1; +}; + +struct preload_bulk_task { + struct preload_bulk_task *next; + struct preload_bulk_dir_identity parent_identity; + struct preload_bulk_dir_identity child_identity; + struct preload_bulk_untracked_root *untracked_root; + int fd; + unsigned reserved_fd : 1; + unsigned has_parent_identity : 1; + unsigned has_child_identity : 1; + char path[FLEX_ARRAY]; +}; + +struct preload_bulk_queue { + pthread_mutex_t mutex; + pthread_cond_t cond; + struct preload_bulk_task *head; + /* + * pending includes queued and in-flight tasks. open_fds counts only + * descriptor reservations held by tasks. + */ + size_t pending; + size_t open_fds; + size_t open_fd_limit; + int failed; + unsigned untracked_invalid : 1; +}; + +struct preload_bulk_scan; + +struct preload_bulk_worker { + struct preload_bulk_scan *scan; + pthread_t thread; + void *buffer; + void *hash_buffer; + struct attr_check *attr_check; + struct preload_bulk_stat_update *stat_updates; + size_t stat_updates_nr; + size_t stat_updates_alloc; + struct strbuf path; + uint64_t dirs; + uint64_t entries; + uint64_t bulk_calls; + uint64_t bytes_hashed; + uint64_t changed_dirs; + uint64_t malformed; + unsigned started : 1; +}; + +struct preload_bulk_backend { + unsigned collects_untracked : 1; + int max_threads; + const char *(*start)(struct preload_bulk_scan *scan); + const char *(*finish)(struct preload_bulk_scan *scan); + void (*release)(struct preload_bulk_scan *scan); + int (*open_proof_parent)(struct preload_bulk_scan *scan, + const char *path); + int (*open_dir_at)(struct preload_bulk_worker *worker, int parent_fd, + const char *name); + /* + * Consume task->fd when it is non-negative, and close it before + * returning. + */ + int (*scan_directory)(struct preload_bulk_worker *worker, + struct preload_bulk_task *task); +}; + +struct preload_bulk_scan { + struct repository *repo; + struct index_state *istate; + const struct preload_bulk_backend *backend; + void *platform_data; + const char *test_barrier_path; + const char *test_barrier_ready; + const char *test_barrier_resume; + struct preload_bulk_queue queue; + struct preload_bulk_worker *workers; + unsigned char *tracked_state; + struct preload_bulk_stat_update *stat_updates; + size_t stat_updates_nr; + struct clean_status_proof_epoch *proof_epoch; + struct dir_struct *exclude_dir; + pthread_mutex_t exclude_mutex; + struct preload_bulk_untracked_root *untracked_roots; + struct string_list untracked; + int root_fd; + int threads; + dev_t root_dev; + unsigned collect_untracked : 1; + unsigned verify_content : 1; + unsigned case_insensitive : 1; + unsigned can_skip_unseen_preload : 1; +}; + +struct preload_bulk_run_result { + uint64_t dirs; + uint64_t entries; + uint64_t bulk_calls; + uint64_t changed_dirs; + uint64_t malformed; + uint64_t bytes_hashed; + int threads; + unsigned untracked_complete : 1; +}; + +struct preload_bulk_result { + unsigned char *tracked_state; + struct preload_bulk_stat_update *stat_updates; + size_t stat_updates_nr; + size_t nr; + const char *outcome; + const char *reason; + const char *untracked_reason; + struct preload_bulk_run_result run; + unsigned can_skip_unseen_preload : 1; + struct string_list untracked; + unsigned untracked_complete : 1; + unsigned standard_excludes_digest_valid : 1; + struct object_id standard_excludes_digest; + struct stat scanned_worktree; +}; + +struct preload_bulk_stat_update { + uint32_t cache_pos; + struct stat_data stat_data; +}; + +void preload_bulk_schedule_directory( + struct preload_bulk_worker *worker, int parent_fd, + const struct preload_bulk_dir_identity *parent_identity, + const struct preload_bulk_dir_identity *child_identity, + struct preload_bulk_untracked_root *untracked_root, + const char *name, const char *path, size_t path_len); +int preload_bulk_index_position(struct preload_bulk_scan *scan, + const char *path, size_t path_len); +int preload_bulk_index_pos_has_tracked_descendants( + struct preload_bulk_scan *scan, const char *path, size_t path_len, + int pos); +int preload_bulk_index_entry_is_gitlink(struct preload_bulk_scan *scan, + int pos); +void preload_bulk_record_tracked( + struct preload_bulk_worker *worker, int pos, int parent_fd, + const char *basename, const struct stat *st, + int observed_has_platform_identity); +void preload_bulk_record_tracked_fallback( + struct preload_bulk_worker *worker, int pos); +void preload_bulk_record_tracked_descendants_fallback( + struct preload_bulk_worker *worker, const char *path, + size_t path_len); +int preload_bulk_record_tracked_alias_fallback( + struct preload_bulk_worker *worker, const char *path, + size_t path_len); +int preload_bulk_path_is_excluded(struct preload_bulk_worker *worker, + const char *path, int dtype); +void preload_bulk_invalidate_untracked( + struct preload_bulk_worker *worker); +int preload_bulk_untracked_is_invalid( + struct preload_bulk_worker *worker); +struct preload_bulk_untracked_root *preload_bulk_untracked_root_new( + struct preload_bulk_worker *worker, const char *path, + size_t path_len); +int preload_bulk_untracked_root_is_visible( + struct preload_bulk_worker *worker, + const struct preload_bulk_untracked_root *root); +void preload_bulk_record_untracked( + struct preload_bulk_worker *worker, + struct preload_bulk_untracked_root *root, + const char *path); +int preload_bulk_run_scan(struct preload_bulk_scan *scan, + struct preload_bulk_run_result *result); +const struct preload_bulk_backend *preload_bulk_platform_backend(void); +int preload_bulk_collect(struct index_state *istate, int threads, + struct preload_bulk_result *result); +int preload_bulk_available(void); +int preload_bulk_test_barrier(struct preload_bulk_scan *scan, + const char *path); +void preload_bulk_result_release(struct preload_bulk_result *result); + +#endif /* PRELOAD_INDEX_BULK_H */ diff --git a/preload-index.c b/preload-index.c index b222821b448526..892f1204676829 100644 --- a/preload-index.c +++ b/preload-index.c @@ -12,6 +12,9 @@ #include "gettext.h" #include "parse.h" #include "preload-index.h" +#ifdef HAVE_PRELOAD_INDEX_BULK +#include "preload-index-bulk.h" +#endif #include "progress.h" #include "read-cache.h" #include "thread-utils.h" @@ -28,6 +31,8 @@ */ #define MAX_PARALLEL (20) #define THREAD_COST (500) +#define BULK_MAX_PARALLEL (32) +#define BULK_ENTRIES_PER_THREAD (5000) struct progress_data { unsigned long n; @@ -40,10 +45,23 @@ struct thread_data { struct index_state *index; struct pathspec pathspec; struct progress_data *progress; +#ifdef HAVE_PRELOAD_INDEX_BULK + const unsigned char *bulk_state; + unsigned bulk_provider_pending : 1; +#endif int offset, nr; int t2_nr_lstat; }; +static int preload_entry_needs_stat(const struct cache_entry *ce) +{ + return !ce_stage(ce) && + !S_ISGITLINK(ce->ce_mode) && + !ce_uptodate(ce) && + !ce_skip_worktree(ce) && + !(ce->ce_flags & CE_FSMONITOR_VALID); +} + static void *preload_thread(void *_data) { int nr, last_nr; @@ -51,6 +69,9 @@ static void *preload_thread(void *_data) struct index_state *index = p->index; struct cache_entry **cep = index->cache + p->offset; struct cache_def cache = CACHE_DEF_INIT; +#ifdef HAVE_PRELOAD_INDEX_BULK + const unsigned char *bulk_state = p->bulk_state; +#endif nr = p->nr; if (nr + p->offset > index->cache_nr) @@ -60,17 +81,21 @@ static void *preload_thread(void *_data) do { struct cache_entry *ce = *cep++; struct stat st; +#ifdef HAVE_PRELOAD_INDEX_BULK + unsigned char state = bulk_state ? + *bulk_state++ : PRELOAD_BULK_TRACKED_UNSEEN; +#endif - if (ce_stage(ce)) - continue; - if (S_ISGITLINK(ce->ce_mode)) + if (!preload_entry_needs_stat(ce)) continue; - if (ce_uptodate(ce)) - continue; - if (ce_skip_worktree(ce)) - continue; - if (ce->ce_flags & CE_FSMONITOR_VALID) +#ifdef HAVE_PRELOAD_INDEX_BULK + if (state == PRELOAD_BULK_TRACKED_CONTENT_CHECK || + state == PRELOAD_BULK_TRACKED_DEFINITIVE_MODIFIED || + state == PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED || + (p->bulk_provider_pending && + state == PRELOAD_BULK_TRACKED_CLEAN)) continue; +#endif if (p->progress && !(nr & 31)) { struct progress_data *pd = p->progress; @@ -90,7 +115,8 @@ static void *preload_thread(void *_data) if (ie_match_stat(index, ce, &st, CE_MATCH_RACY_IS_DIRTY|CE_MATCH_IGNORE_FSMONITOR)) continue; ce_mark_uptodate(ce); - mark_fsmonitor_valid(index, ce); + if (fsmonitor_stat_can_be_valid(&st)) + mark_fsmonitor_valid(index, ce); } while (--nr > 0); if (p->progress) { struct progress_data *pd = p->progress; @@ -103,6 +129,429 @@ static void *preload_thread(void *_data) return NULL; } +#ifdef HAVE_PRELOAD_INDEX_BULK +struct preload_bulk_pending { + unsigned char *tracked_state; + struct preload_bulk_stat_update *stat_updates; + size_t stat_updates_nr; + struct object_id standard_excludes_digest; + struct stat scanned_worktree; + unsigned provider : 1; + unsigned standard_excludes_digest_valid : 1; +}; + +static int stat_data_is_zero(const struct stat_data *sd) +{ + return !sd->sd_ctime.sec && + !sd->sd_ctime.nsec && + !sd->sd_mtime.sec && + !sd->sd_mtime.nsec && + !sd->sd_dev && + !sd->sd_ino && + !sd->sd_uid && + !sd->sd_gid && + !sd->sd_size; +} + +static int preload_bulk_entry_is_useful(const struct cache_entry *ce, + int allow_zero_stat) +{ + return preload_entry_needs_stat(ce) && + !ce_intent_to_add(ce) && + !(ce->ce_flags & (CE_VALID | CE_REMOVE)) && + (S_ISREG(ce->ce_mode) || S_ISLNK(ce->ce_mode)) && + (allow_zero_stat || !stat_data_is_zero(&ce->ce_stat_data)); +} + +static size_t preload_bulk_useful_candidates(struct index_state *index, + int allow_zero_stat) +{ + size_t useful = 0; + + for (size_t i = 0; i < index->cache_nr; i++) + if (preload_bulk_entry_is_useful( + index->cache[i], allow_zero_stat)) + useful++; + return useful; +} + +static size_t preload_bulk_apply_result( + struct index_state *index, + struct preload_bulk_result *result, + int defer_all, + int *has_deferred) +{ + size_t applied = 0; + + if (result->nr != index->cache_nr) + BUG("bulk preload result does not match the index"); + + for (size_t i = 0; i < result->nr; i++) { + struct cache_entry *ce = index->cache[i]; + unsigned char state = result->tracked_state[i]; + + /* + * A complete scan which did not observe a useful entry proves + * that the entry is absent. Avoid repeating the same lookup in + * speculative preload. A status consumer may use this result + * directly; other callers retain the authoritative refresh. + */ + if (result->can_skip_unseen_preload && + state == PRELOAD_BULK_TRACKED_UNSEEN && + preload_bulk_entry_is_useful(ce, defer_all)) { + state = PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED; + result->tracked_state[i] = state; + } + if ((state == PRELOAD_BULK_TRACKED_CONTENT_CHECK || + state == PRELOAD_BULK_TRACKED_DEFINITIVE_MODIFIED || + state == PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED) && + preload_entry_needs_stat(ce)) + *has_deferred = 1; + if (defer_all && state == PRELOAD_BULK_TRACKED_CLEAN && + preload_entry_needs_stat(ce)) + *has_deferred = 1; + if (state != PRELOAD_BULK_TRACKED_CLEAN) + continue; + if (!preload_bulk_entry_is_useful(ce, defer_all)) + continue; + if (defer_all) + continue; + ce_mark_uptodate(ce); + mark_fsmonitor_valid(index, ce); + applied++; + } + return applied; +} + +static int preload_bulk_threads(size_t useful) +{ + int cpus = online_cpus(); + int threads = DIV_ROUND_UP(useful, BULK_ENTRIES_PER_THREAD); + + if (threads < 1) + threads = 1; + if (cpus > 0) { + int cpu_limit = cpus > BULK_MAX_PARALLEL / 2 ? + BULK_MAX_PARALLEL : cpus * 2; + + if (threads > cpu_limit) + threads = cpu_limit; + } + if (threads > BULK_MAX_PARALLEL) + threads = BULK_MAX_PARALLEL; + return threads; +} + +static void preload_bulk_trace_result( + struct index_state *index, + const struct preload_bulk_result *result, + size_t applied) +{ + uint64_t content_check = 0, definitive_modified = 0; + uint64_t definitive_deleted = 0, fallback = 0; + + for (size_t i = 0; i < result->nr; i++) { + switch (result->tracked_state[i]) { + case PRELOAD_BULK_TRACKED_DEFINITIVE_MODIFIED: + definitive_modified++; + break; + case PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED: + definitive_deleted++; + break; + case PRELOAD_BULK_TRACKED_CONTENT_CHECK: + content_check++; + break; + case PRELOAD_BULK_TRACKED_FALLBACK: + fallback++; + break; + default: + break; + } + } + trace2_data_string("index", index->repo, "preload/bulk_result", + result->outcome); + if (result->reason) + trace2_data_string("index", index->repo, + "preload/bulk_reason", result->reason); + if (result->untracked_reason) + trace2_data_string("index", index->repo, + "preload/bulk_untracked_reason", + result->untracked_reason); + trace2_data_intmax("index", index->repo, "preload/bulk_applied", + applied); + trace2_data_intmax("index", index->repo, "preload/bulk_dirs", + result->run.dirs); + trace2_data_intmax("index", index->repo, "preload/bulk_entries", + result->run.entries); + trace2_data_intmax("index", index->repo, "preload/bulk_calls", + result->run.bulk_calls); + trace2_data_intmax("index", index->repo, + "preload/bulk_bytes_hashed", + result->run.bytes_hashed); + trace2_data_intmax("index", index->repo, "preload/bulk_workers", + result->run.threads); + trace2_data_intmax("index", index->repo, + "preload/bulk_definitive_modified", + definitive_modified); + trace2_data_intmax("index", index->repo, + "preload/bulk_definitive_deleted", + definitive_deleted); + trace2_data_intmax("index", index->repo, + "preload/bulk_content_check", content_check); + trace2_data_intmax("index", index->repo, + "preload/bulk_fallback", fallback); + trace2_data_intmax("index", index->repo, + "preload/bulk_untracked_complete", + result->untracked_complete); + trace2_data_intmax("index", index->repo, + "preload/bulk_untracked_count", + result->untracked.nr); +} + +static int preload_bulk_config_enabled(struct index_state *index) +{ + int enabled = 0; + int control; + + control = git_env_bool("GIT_TEST_PRELOAD_INDEX_BULK", -1); + if (control < 0) + repo_config_get_bool(index->repo, "core.preloadindexbulk", + &enabled); + else + enabled = control; + return enabled; +} + +static void preload_bulk_try(struct index_state *index, + unsigned int refresh_flags, + struct preload_bulk_pending *pending) +{ + struct preload_bulk_result result = { 0 }; + size_t useful; + size_t applied = 0; + int provider = !!index->preload_bulk_proof_epoch; + int has_deferred = 0; + int threads; + + /* + * Let the test variable override configuration without bypassing + * any of the proof checks. + */ + if (!preload_bulk_config_enabled(index) || + !preload_bulk_available()) + return; + if (provider) { + if (!(refresh_flags & REFRESH_DEFER_BULK_DIRTY) || + fsm_settings__get_mode(index->repo) != FSMONITOR_MODE_IPC || + !fsmonitor_pending_token_from_provider(index)) + return; + } else if (fsm_settings__get_mode(index->repo) != + FSMONITOR_MODE_DISABLED) { + return; + } + useful = preload_bulk_useful_candidates(index, provider); + trace2_data_intmax("index", index->repo, "preload/bulk_useful", + useful); + trace2_data_intmax("index", index->repo, "preload/bulk_cache_nr", + index->cache_nr); + if (!useful && !index->preload_untracked) + return; + threads = preload_bulk_threads(useful); + trace2_region_enter("index", "preload/bulk", index->repo); + if (!preload_bulk_collect(index, threads, &result)) { + applied = preload_bulk_apply_result(index, &result, + provider, + &has_deferred); + } + preload_bulk_trace_result(index, &result, applied); + if (has_deferred) { + pending->tracked_state = result.tracked_state; + result.tracked_state = NULL; + pending->provider = provider; + if (provider) { + pending->stat_updates = result.stat_updates; + pending->stat_updates_nr = result.stat_updates_nr; + result.stat_updates = NULL; + result.stat_updates_nr = 0; + } + } + if (result.standard_excludes_digest_valid) { + pending->provider = provider; + pending->standard_excludes_digest_valid = 1; + oidcpy(&pending->standard_excludes_digest, + &result.standard_excludes_digest); + pending->scanned_worktree = result.scanned_worktree; + } + if (result.untracked_complete && index->preload_untracked) { + *index->preload_untracked = result.untracked; + index->preload_untracked_complete = 1; + result.untracked = + (struct string_list)STRING_LIST_INIT_DUP; + } + trace2_region_leave("index", "preload/bulk", index->repo); + preload_bulk_result_release(&result); +} + +static void preload_bulk_finish_state(struct index_state *index, + struct preload_bulk_pending *pending, + unsigned int refresh_flags) +{ + if ((refresh_flags & REFRESH_DEFER_BULK_DIRTY) && + pending->tracked_state) { + index->preload_bulk_tracked_state = pending->tracked_state; + index->preload_bulk_tracked_nr = index->cache_nr; + index->preload_bulk_stat_updates = pending->stat_updates; + index->preload_bulk_stat_updates_nr = + pending->stat_updates_nr; + index->preload_bulk_provider_pending = pending->provider; + pending->tracked_state = NULL; + pending->stat_updates = NULL; + pending->stat_updates_nr = 0; + } + if (pending->standard_excludes_digest_valid) { + oidcpy(&index->preload_bulk_standard_excludes_digest, + &pending->standard_excludes_digest); + index->preload_bulk_scanned_worktree = + pending->scanned_worktree; + if (pending->provider) + index->preload_bulk_excludes_digest_pending = 1; + else + index->preload_bulk_excludes_digest_valid = 1; + } + free(pending->tracked_state); + free(pending->stat_updates); +} + +static int compare_stat_update(const void *va, const void *vb) +{ + const struct preload_bulk_stat_update *a = va; + const struct preload_bulk_stat_update *b = vb; + + return a->cache_pos < b->cache_pos ? -1 : + a->cache_pos > b->cache_pos ? 1 : 0; +} +#endif + +void preload_index_bulk_result_consume(struct index_state *index) +{ + FREE_AND_NULL(index->preload_bulk_tracked_state); + FREE_AND_NULL(index->preload_bulk_stat_updates); + index->preload_bulk_tracked_nr = 0; + index->preload_bulk_stat_updates_nr = 0; + index->preload_bulk_provider_pending = 0; + index->preload_bulk_excludes_digest_pending = 0; +} + +void preload_index_bulk_result_clear(struct index_state *index) +{ + preload_index_bulk_result_consume(index); + index->preload_bulk_excludes_digest_valid = 0; + oidclr(&index->preload_bulk_standard_excludes_digest, + index->repo->hash_algo); +} + +int preload_index_bulk_can_close_provider(struct index_state *index) +{ +#ifdef HAVE_PRELOAD_INDEX_BULK + int core_preload_index = 1; + + repo_config_get_bool(index->repo, "core.preloadindex", + &core_preload_index); + return core_preload_index && + preload_bulk_config_enabled(index) && + preload_bulk_available() && + index->sparse_index == INDEX_EXPANDED && + fsm_settings__get_mode(index->repo) == FSMONITOR_MODE_IPC && + fsmonitor_pending_token_from_provider(index) && + (preload_bulk_useful_candidates(index, 1) || + index->preload_untracked); +#else + (void)index; + return 0; +#endif +} + +int preload_index_bulk_result_accept(struct index_state *index) +{ +#ifdef HAVE_PRELOAD_INDEX_BULK + size_t update_nr = 0; + int applied = 0; + + if (!index->preload_bulk_provider_pending && + !index->preload_bulk_excludes_digest_pending) + return 0; + if (!index->preload_bulk_provider_pending) + goto accept_digest; + if (!index->preload_bulk_tracked_state || + index->preload_bulk_tracked_nr != index->cache_nr) + return -1; + + QSORT(index->preload_bulk_stat_updates, + index->preload_bulk_stat_updates_nr, compare_stat_update); + for (size_t i = 0; i < index->preload_bulk_stat_updates_nr; i++) { + struct preload_bulk_stat_update *update = + &index->preload_bulk_stat_updates[i]; + + if (update->cache_pos >= index->cache_nr || + index->preload_bulk_tracked_state[update->cache_pos] != + PRELOAD_BULK_TRACKED_CLEAN || + (i && update[-1].cache_pos == update->cache_pos)) + return -1; + } + + for (size_t i = 0; i < index->cache_nr; i++) { + struct cache_entry *ce = index->cache[i]; + struct preload_bulk_stat_update *update = NULL; + + if (index->preload_bulk_tracked_state[i] != + PRELOAD_BULK_TRACKED_CLEAN) + continue; + if (update_nr < index->preload_bulk_stat_updates_nr && + index->preload_bulk_stat_updates[update_nr].cache_pos == i) + update = + &index->preload_bulk_stat_updates[update_nr++]; + if (update && + memcmp(&ce->ce_stat_data, &update->stat_data, + sizeof(ce->ce_stat_data))) { + memcpy(&ce->ce_stat_data, &update->stat_data, + sizeof(ce->ce_stat_data)); + ce->ce_flags |= CE_UPDATE_IN_BASE; + index->cache_changed |= CE_ENTRY_CHANGED; + } + ce_mark_uptodate(ce); + mark_fsmonitor_valid(index, ce); + applied++; + } + if (update_nr != index->preload_bulk_stat_updates_nr) + BUG("validated bulk stat update was not applied"); + + FREE_AND_NULL(index->preload_bulk_stat_updates); + index->preload_bulk_stat_updates_nr = 0; + index->preload_bulk_provider_pending = 0; +accept_digest: + if (index->preload_bulk_excludes_digest_pending) { + index->preload_bulk_excludes_digest_pending = 0; + index->preload_bulk_excludes_digest_valid = 1; + } + trace2_data_intmax("index", index->repo, + "preload/bulk_provider_applied", applied); +#else + (void)index; +#endif + return 0; +} + +int preload_index_bulk_standard_excludes_digest( + const struct index_state *index, struct object_id *digest, + struct stat *scanned_worktree) +{ + if (!index->preload_bulk_excludes_digest_valid) + return -1; + oidcpy(digest, &index->preload_bulk_standard_excludes_digest); + *scanned_worktree = index->preload_bulk_scanned_worktree; + return 0; +} + void preload_index(struct index_state *index, const struct pathspec *pathspec, unsigned int refresh_flags) @@ -110,19 +559,41 @@ void preload_index(struct index_state *index, int threads, i, work, offset; struct thread_data data[MAX_PARALLEL]; struct progress_data pd; +#ifdef HAVE_PRELOAD_INDEX_BULK + struct preload_bulk_pending bulk = { 0 }; +#endif int t2_sum_lstat = 0; int core_preload_index = 1; + preload_index_bulk_result_clear(index); + index->preload_untracked_complete = 0; + if (index->preload_untracked) + string_list_clear(index->preload_untracked, 0); repo_config_get_bool(index->repo, "core.preloadindex", &core_preload_index); - if (!HAVE_THREADS || !core_preload_index) + if (!core_preload_index) + return; + +#ifdef HAVE_PRELOAD_INDEX_BULK + if (!pathspec || !pathspec->nr) + preload_bulk_try(index, refresh_flags, &bulk); +#endif + if (!HAVE_THREADS) { +#ifdef HAVE_PRELOAD_INDEX_BULK + preload_bulk_finish_state(index, &bulk, refresh_flags); +#endif return; + } threads = index->cache_nr / THREAD_COST; if ((index->cache_nr > 1) && (threads < 2) && git_env_bool("GIT_TEST_PRELOAD_INDEX", 0)) threads = 2; - if (threads < 2) + if (threads < 2) { +#ifdef HAVE_PRELOAD_INDEX_BULK + preload_bulk_finish_state(index, &bulk, refresh_flags); +#endif return; + } trace2_region_enter("index", "preload", NULL); @@ -146,6 +617,11 @@ void preload_index(struct index_state *index, int err; p->index = index; +#ifdef HAVE_PRELOAD_INDEX_BULK + p->bulk_state = bulk.tracked_state ? + bulk.tracked_state + offset : NULL; + p->bulk_provider_pending = bulk.provider; +#endif if (pathspec) copy_pathspec(&p->pathspec, pathspec); p->offset = offset; @@ -165,6 +641,9 @@ void preload_index(struct index_state *index, t2_sum_lstat += p->t2_nr_lstat; } stop_progress(&pd.progress); +#ifdef HAVE_PRELOAD_INDEX_BULK + preload_bulk_finish_state(index, &bulk, refresh_flags); +#endif if (pathspec) { /* earlier we made deep copies for each thread to work with */ diff --git a/preload-index.h b/preload-index.h index 251b1ed88e9820..87e6d0b89276a7 100644 --- a/preload-index.h +++ b/preload-index.h @@ -2,8 +2,19 @@ #define PRELOAD_INDEX_H struct index_state; +struct object_id; struct pathspec; struct repository; +struct stat; + +enum preload_bulk_tracked_state { + PRELOAD_BULK_TRACKED_UNSEEN = 0, + PRELOAD_BULK_TRACKED_CLEAN, + PRELOAD_BULK_TRACKED_DEFINITIVE_MODIFIED, + PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED, + PRELOAD_BULK_TRACKED_CONTENT_CHECK, + PRELOAD_BULK_TRACKED_FALLBACK, +}; void preload_index(struct index_state *index, const struct pathspec *pathspec, @@ -11,5 +22,12 @@ void preload_index(struct index_state *index, int repo_read_index_preload(struct repository *, const struct pathspec *pathspec, unsigned refresh_flags); +void preload_index_bulk_result_clear(struct index_state *index); +void preload_index_bulk_result_consume(struct index_state *index); +int preload_index_bulk_can_close_provider(struct index_state *index); +int preload_index_bulk_result_accept(struct index_state *index); +int preload_index_bulk_standard_excludes_digest( + const struct index_state *index, struct object_id *digest, + struct stat *scanned_worktree); #endif /* PRELOAD_INDEX_H */ diff --git a/read-cache-ll.h b/read-cache-ll.h index 71b87615ebc6d3..9c44302486794d 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -31,6 +31,9 @@ struct cache_entry { char name[FLEX_ARRAY]; /* more */ }; +struct clean_status_proof_epoch; +struct preload_bulk_stat_update; + #define CE_STAGEMASK (0x3000) #define CE_EXTENDED (0x4000) #define CE_VALID (0x8000) @@ -69,14 +72,18 @@ struct cache_entry { */ #define CE_INTENT_TO_ADD (1 << 29) #define CE_SKIP_WORKTREE (1 << 30) -/* CE_EXTENDED2 is for future extension */ -#define CE_EXTENDED2 (1U << 31) +/* + * In-memory only. The cached stat data cannot be trusted, and callers which + * normally trust stat differences must verify content. This occupies the + * former never-persisted extension slot. + */ +#define CE_CONTENT_CHECK_REQUIRED (1U << 31) #define CE_EXTENDED_FLAGS (CE_INTENT_TO_ADD | CE_SKIP_WORKTREE) /* * Safeguard to avoid saving wrong flags: - * - CE_EXTENDED2 won't get saved until its semantic is known + * - CE_CONTENT_CHECK_REQUIRED is transient and must not be saved * - Bits in 0x0000FFFF have been saved in ce_flags already * - Bits in 0x003F0000 are currently in-memory flags */ @@ -120,7 +127,9 @@ static inline unsigned create_ce_flags(unsigned stage) #define ce_stage(ce) ((CE_STAGEMASK & (ce)->ce_flags) >> CE_STAGESHIFT) #define ce_uptodate(ce) ((ce)->ce_flags & CE_UPTODATE) #define ce_skip_worktree(ce) ((ce)->ce_flags & CE_SKIP_WORKTREE) -#define ce_mark_uptodate(ce) ((ce)->ce_flags |= CE_UPTODATE) +#define ce_mark_uptodate(ce) \ + ((ce)->ce_flags = ((ce)->ce_flags | CE_UPTODATE) & \ + ~CE_CONTENT_CHECK_REQUIRED) #define ce_intent_to_add(ce) ((ce)->ce_flags & CE_INTENT_TO_ADD) #define cache_entry_size(len) (offsetof(struct cache_entry,name) + (len) + 1) @@ -136,7 +145,9 @@ static inline unsigned create_ce_flags(unsigned stage) #define FSMONITOR_CHANGED (1 << 8) struct split_index; +struct clean_status_state; struct untracked_cache; +struct string_list; struct progress; struct pattern_list; @@ -176,18 +187,41 @@ struct index_state { drop_cache_tree : 1, updated_workdir : 1, updated_skipworktree : 1, - fsmonitor_has_run_once : 1; + fsmonitor_has_run_once : 1, + fsmonitor_token_valid : 1, + fsmonitor_extension_seen : 1, + fsmonitor_untracked_valid : 1, + fsmonitor_untracked_extension_seen : 1, + fsmonitor_untracked_extension_invalid : 1, + fsmonitor_pending_token_from_provider : 1, + preload_untracked_complete : 1, + preload_bulk_provider_pending : 1, + preload_bulk_excludes_digest_pending : 1, + preload_bulk_excludes_digest_valid : 1; enum sparse_index_mode sparse_index; struct hashmap name_hash; struct hashmap dir_hash; struct object_id oid; struct untracked_cache *untracked; + unsigned char *preload_bulk_tracked_state; + size_t preload_bulk_tracked_nr; + struct preload_bulk_stat_update *preload_bulk_stat_updates; + size_t preload_bulk_stat_updates_nr; + struct object_id preload_bulk_standard_excludes_digest; + struct stat preload_bulk_scanned_worktree; + /* Borrowed only while refresh_index() performs a provider scan. */ + struct clean_status_proof_epoch *preload_bulk_proof_epoch; + /* Borrowed for the duration of preload_index(). */ + struct string_list *preload_untracked; char *fsmonitor_last_update; + char *fsmonitor_last_update_pending; + char *fsmonitor_untracked_token; struct ewah_bitmap *fsmonitor_dirty; struct mem_pool *ce_mem_pool; struct progress *progress; struct repository *repo; struct pattern_list *sparse_checkout_patterns; + struct clean_status_state *clean_status; }; /** @@ -396,6 +430,7 @@ int remove_file_from_index(struct index_state *, const char *path); #define ADD_CACHE_IGNORE_ERRORS 4 #define ADD_CACHE_IGNORE_REMOVAL 8 #define ADD_CACHE_INTENT 16 +#define ADD_CACHE_TRACK_CLEAN_HISTORY 32 /* * These two are used to add the contents of the file at path * to the index, marking the working tree up-to-date by storing @@ -431,6 +466,13 @@ int is_racy_timestamp(const struct index_state *istate, int has_racy_timestamp(struct index_state *istate); int ie_match_stat(struct index_state *, const struct cache_entry *, struct stat *, unsigned int); int ie_modified(struct index_state *, const struct cache_entry *, struct stat *, unsigned int); +/* + * Unlike ie_match_stat(), verify content for marked non-gitlinks. Ordinary + * entries, including unmarked zero-stat entries, retain stat-only matching. + */ +int ie_match_stat_with_content_check(struct index_state *, + const struct cache_entry *, + struct stat *, unsigned int); int match_stat_data_racy(const struct index_state *istate, const struct stat_data *sd, struct stat *st); @@ -453,6 +495,7 @@ int fake_lstat(const struct cache_entry *ce, struct stat *st); #define REFRESH_IN_PORCELAIN (1 << 5) /* user friendly output, not "needs update" */ #define REFRESH_PROGRESS (1 << 6) /* show progress bar if stderr is tty */ #define REFRESH_IGNORE_SKIP_WORKTREE (1 << 7) /* ignore skip_worktree entries */ +#define REFRESH_DEFER_BULK_DIRTY (1 << 8) /* leave bulk results to diff */ int refresh_index(struct index_state *, unsigned int flags, const struct pathspec *pathspec, char *seen, const char *header_msg); /* * Refresh the index and write it to disk. @@ -473,6 +516,9 @@ int repo_refresh_and_write_index(struct repository*, unsigned int refresh_flags, struct cache_entry *refresh_cache_entry(struct index_state *, struct cache_entry *, unsigned int); +/* The caller must first verify the entry's content and mode against st. */ +void refresh_index_entry_stat(struct index_state *, int, struct stat *); + void set_alternate_index_output(const char *); extern int verify_index_checksum; diff --git a/read-cache.c b/read-cache.c index 6c449f393d8d4d..067ac8403155fd 100644 --- a/read-cache.c +++ b/read-cache.c @@ -16,6 +16,7 @@ #include "tempfile.h" #include "lockfile.h" #include "cache-tree.h" +#include "clean-status.h" #include "refs.h" #include "dir.h" #include "object-file.h" @@ -71,6 +72,8 @@ #define CACHE_EXT_LINK 0x6c696e6b /* "link" */ #define CACHE_EXT_UNTRACKED 0x554E5452 /* "UNTR" */ #define CACHE_EXT_FSMONITOR 0x46534D4E /* "FSMN" */ +#define CACHE_EXT_FSMONITOR_CONFIG 0x46534346 /* "FSCF" */ +#define CACHE_EXT_FSMONITOR_UNTRACKED 0x46535543 /* "FSUC" */ #define CACHE_EXT_ENDOFINDEXENTRIES 0x454F4945 /* "EOIE" */ #define CACHE_EXT_INDEXENTRYOFFSETTABLE 0x49454F54 /* "IEOT" */ #define CACHE_EXT_SPARSE_DIRECTORIES 0x73646972 /* "sdir" */ @@ -199,10 +202,35 @@ void fill_stat_cache_info(struct index_state *istate, struct cache_entry *ce, st if (S_ISREG(st->st_mode)) { ce_mark_uptodate(ce); - mark_fsmonitor_valid(istate, ce); + if (fsmonitor_stat_can_be_valid(st)) + mark_fsmonitor_valid(istate, ce); } } +static struct cache_entry *make_refreshed_cache_entry( + struct index_state *istate, const struct cache_entry *ce, + struct stat *st, int preserve_valid) +{ + struct cache_entry *updated = + make_empty_cache_entry(istate, ce_namelen(ce)); + + copy_cache_entry(updated, ce); + memcpy(updated->name, ce->name, ce->ce_namelen + 1); + fill_stat_cache_info(istate, updated, st); + /* Do not let assume-unchanged reacquire a caller-cleared CE_VALID. */ + if (preserve_valid && assume_unchanged && + !(ce->ce_flags & CE_VALID)) + updated->ce_flags &= ~CE_VALID; + return updated; +} + +void refresh_index_entry_stat(struct index_state *istate, int nr, + struct stat *st) +{ + replace_index_entry(istate, nr, make_refreshed_cache_entry( + istate, istate->cache[nr], st, 1)); +} + static unsigned int st_mode_from_ce(const struct cache_entry *ce) { switch (ce->ce_mode & S_IFMT) { @@ -492,6 +520,32 @@ int ie_modified(struct index_state *istate, return 0; } +int ie_match_stat_with_content_check(struct index_state *istate, + const struct cache_entry *ce, + struct stat *st, unsigned int options) +{ + struct cache_entry *current; + int changed, pos; + + if (S_ISGITLINK(ce->ce_mode) || + !(ce->ce_flags & CE_CONTENT_CHECK_REQUIRED)) + return ie_match_stat(istate, ce, st, options); + + changed = ie_modified(istate, ce, st, options); + if (changed) + return changed; + + pos = index_name_pos(istate, ce->name, ce_namelen(ce)); + if (pos < 0 || istate->cache[pos] != ce) + return 0; + + current = istate->cache[pos]; + fill_stat_data(¤t->ce_stat_data, st); + current->ce_flags |= CE_UPDATE_IN_BASE; + istate->cache_changed |= CE_ENTRY_CHANGED; + return 0; +} + static int cache_name_stage_compare(const char *name1, int len1, int stage1, const char *name2, int len2, int stage2) { @@ -699,6 +753,20 @@ static struct cache_entry *create_alias_ce(struct index_state *istate, return new_entry; } +static int same_persistent_add_entry(const struct cache_entry *a, + const struct cache_entry *b) +{ + const unsigned int flags = + CE_STAGEMASK | CE_VALID | CE_EXTENDED_FLAGS; + + return a && b && + ce_namelen(a) == ce_namelen(b) && + !memcmp(a->name, b->name, ce_namelen(a)) && + a->ce_mode == b->ce_mode && + oideq(&a->oid, &b->oid) && + ((a->ce_flags ^ b->ce_flags) & flags) == 0; +} + void set_object_name_for_intent_to_add_entry(struct cache_entry *ce) { struct object_id oid; @@ -709,7 +777,8 @@ void set_object_name_for_intent_to_add_entry(struct cache_entry *ce) int add_to_index(struct index_state *istate, const char *path, struct stat *st, int flags) { - int namelen, was_same; + int namelen, was_same, logical_same; + int cache_nr = istate->cache_nr; mode_t st_mode = st->st_mode; struct cache_entry *ce, *alias = NULL; unsigned ce_option = CE_MATCH_IGNORE_VALID|CE_MATCH_IGNORE_SKIP_WORKTREE|CE_MATCH_RACY_IS_DIRTY; @@ -794,12 +863,22 @@ int add_to_index(struct index_state *istate, const char *path, struct stat *st, !ce_stage(alias) && oideq(&alias->oid, &ce->oid) && ce->ce_mode == alias->ce_mode); + logical_same = same_persistent_add_entry(alias, ce); + + if (!pretend && (flags & ADD_CACHE_TRACK_CLEAN_HISTORY) && + !logical_same) + clean_status_invalidate_current_proof(istate); if (pretend) discard_cache_entry(ce); - else if (add_index_entry(istate, ce, add_option)) { - discard_cache_entry(ce); - return error(_("unable to add '%s' to index"), path); + else { + if (add_index_entry(istate, ce, add_option)) { + discard_cache_entry(ce); + return error(_("unable to add '%s' to index"), path); + } + if ((flags & ADD_CACHE_TRACK_CLEAN_HISTORY) && + cache_nr != istate->cache_nr) + clean_status_invalidate_current_proof(istate); } if (verbose && !was_same) printf("add '%s'\n", path); @@ -1415,7 +1494,8 @@ static struct cache_entry *refresh_cache_ent(struct index_state *istate, */ if (!S_ISGITLINK(ce->ce_mode)) { ce_mark_uptodate(ce); - mark_fsmonitor_valid(istate, ce); + if (fsmonitor_stat_can_be_valid(&st)) + mark_fsmonitor_valid(istate, ce); } return ce; } @@ -1429,19 +1509,7 @@ static struct cache_entry *refresh_cache_ent(struct index_state *istate, return NULL; } - updated = make_empty_cache_entry(istate, ce_namelen(ce)); - copy_cache_entry(updated, ce); - memcpy(updated->name, ce->name, ce->ce_namelen + 1); - fill_stat_cache_info(istate, updated, &st); - /* - * If ignore_valid is not set, we should leave CE_VALID bit - * alone. Otherwise, paths marked with --no-assume-unchanged - * (i.e. things to be edited) will reacquire CE_VALID bit - * automatically, which is not really what we want. - */ - if (!ignore_valid && assume_unchanged && - !(ce->ce_flags & CE_VALID)) - updated->ce_flags &= ~CE_VALID; + updated = make_refreshed_cache_entry(istate, ce, &st, !ignore_valid); /* istate->cache_changed is updated in the caller */ return updated; @@ -1521,7 +1589,7 @@ int refresh_index(struct index_state *istate, unsigned int flags, * cache entries quickly then in the single threaded loop below, * we only have to do the special cases that are left. */ - preload_index(istate, pathspec, 0); + preload_index(istate, pathspec, flags & REFRESH_DEFER_BULK_DIRTY); trace2_region_enter("index", "refresh", NULL); for (i = 0; i < istate->cache_nr; i++) { @@ -1565,6 +1633,23 @@ int refresh_index(struct index_state *istate, unsigned int flags, if (filtered) continue; + if ((flags & REFRESH_DEFER_BULK_DIRTY) && + istate->preload_bulk_tracked_nr == istate->cache_nr) { + unsigned char state = + istate->preload_bulk_tracked_state[i]; + + if (istate->preload_bulk_provider_pending && + state == PRELOAD_BULK_TRACKED_CLEAN) + continue; + if (state == PRELOAD_BULK_TRACKED_CONTENT_CHECK) { + ce->ce_flags |= CE_CONTENT_CHECK_REQUIRED; + continue; + } + if (state == PRELOAD_BULK_TRACKED_DEFINITIVE_MODIFIED || + state == PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED) + continue; + } + new_entry = refresh_cache_ent(istate, ce, options, &cache_errno, &changed, &t2_did_lstat, &t2_did_scan); @@ -1602,7 +1687,17 @@ int refresh_index(struct index_state *istate, unsigned int flags, continue; } - replace_index_entry(istate, i, new_entry); + { + int baseline_valid = + clean_status_fsmonitor_semantic_baseline_pending( + istate) && + (new_entry->ce_flags & CE_FSMONITOR_VALID); + + replace_index_entry(istate, i, new_entry); + if (baseline_valid) + mark_fsmonitor_valid(istate, + istate->cache[i]); + } } trace2_data_intmax("index", NULL, "refresh/sum_lstat", t2_sum_lstat); trace2_data_intmax("index", NULL, "refresh/sum_scan", t2_sum_scan); @@ -1749,6 +1844,12 @@ static int read_index_extension(struct index_state *istate, case CACHE_EXT_FSMONITOR: read_fsmonitor_extension(istate, data, sz); break; + case CACHE_EXT_FSMONITOR_CONFIG: + clean_status_read_fsmonitor_config(istate, data, sz); + break; + case CACHE_EXT_FSMONITOR_UNTRACKED: + read_fsmonitor_untracked_extension(istate, data, sz); + break; case CACHE_EXT_ENDOFINDEXENTRIES: case CACHE_EXT_INDEXENTRYOFFSETTABLE: /* already handled in do_read_index() */ @@ -1950,6 +2051,8 @@ static void post_read_index_from(struct index_state *istate) check_ce_order(istate); tweak_untracked_cache(istate); tweak_split_index(istate); + prepare_fsmonitor_untracked(istate); + clean_status_prepare_fsmonitor_config(istate); tweak_fsmonitor(istate); } @@ -1994,30 +2097,149 @@ struct load_index_extensions const char *mmap; size_t mmap_size; unsigned long src_offset; + int allow_parallel; + int force_parallel; +}; + +struct load_index_extension { + pthread_t pthread; + struct index_state *istate; + const char *ext; + const char *data; + unsigned long size; + int result; }; +#define PARALLEL_INDEX_EXTENSION_THRESHOLD (1024 * 1024) + +static void *load_one_index_extension(void *_data) +{ + struct load_index_extension *p = _data; + + trace2_thread_start("index-extension"); + trace2_data_intmax("index", p->istate->repo, + "extension/parallel/tree-untracked", 1); + p->result = read_index_extension(p->istate, p->ext, p->data, p->size); + trace2_thread_exit(); + return NULL; +} + +/* + * TREE and UNTR are usually the two largest index extensions. They read the + * same immutable mmap but publish to separate index_state fields, so they can + * be decoded concurrently. Keep split indexes on the established serial + * path because LINK changes how the completed index is assembled. + */ +static int find_parallel_index_extensions(struct load_index_extensions *p, + struct load_index_extension *tree) +{ + size_t offset = p->src_offset; + size_t end = p->mmap_size - the_hash_algo->rawsz; + int tree_nr = 0, untracked_nr = 0, link_nr = 0; + uint32_t untracked_size = 0; + + if (!p->allow_parallel) + return 0; + + while (offset <= end - 8) { + const char *ext = p->mmap + offset; + uint32_t size = get_be32(ext + 4); + + if (size > end - offset - 8) + return 0; + + switch (CACHE_EXT(ext)) { + case CACHE_EXT_TREE: + tree_nr++; + tree->istate = p->istate; + tree->ext = ext; + tree->data = ext + 8; + tree->size = size; + break; + case CACHE_EXT_UNTRACKED: + untracked_nr++; + untracked_size = size; + break; + case CACHE_EXT_LINK: + link_nr++; + break; + } + + offset += 8 + size; + } + + return offset == end && tree_nr == 1 && untracked_nr == 1 && !link_nr && + (p->force_parallel || + (tree->size >= PARALLEL_INDEX_EXTENSION_THRESHOLD && + untracked_size >= PARALLEL_INDEX_EXTENSION_THRESHOLD)); +} + static void *load_index_extensions(void *_data) { struct load_index_extensions *p = _data; - unsigned long src_offset = p->src_offset; + size_t src_offset = p->src_offset; + size_t end; + struct load_index_extension tree = { 0 }; + int tree_thread = 0; + int extension_error = 0; + + if (p->mmap_size < the_hash_algo->rawsz) { + extension_error = 1; + goto join_tree; + } + end = p->mmap_size - the_hash_algo->rawsz; + if (src_offset > end) { + extension_error = 1; + goto join_tree; + } + if (find_parallel_index_extensions(p, &tree) && + !pthread_create(&tree.pthread, NULL, load_one_index_extension, &tree)) { + tree_thread = 1; + } - while (src_offset <= p->mmap_size - the_hash_algo->rawsz - 8) { + while (src_offset < end) { /* After an array of active_nr index entries, * there can be arbitrary number of extended * sections, each of which is prefixed with * extension name (4-byte) and section length * in 4-byte network byte order. */ - uint32_t extsize = get_be32(p->mmap + src_offset + 4); - if (read_index_extension(p->istate, - p->mmap + src_offset, - p->mmap + src_offset + 8, - extsize) < 0) { - munmap((void *)p->mmap, p->mmap_size); - die(_("index file corrupt")); + uint32_t extsize; + const char *ext = p->mmap + src_offset; + + if (end - src_offset < 8) { + extension_error = 1; + break; } - src_offset += 8; - src_offset += extsize; + extsize = get_be32(ext + 4); + if (extsize > end - src_offset - 8) { + extension_error = 1; + break; + } + if ((!tree_thread || CACHE_EXT(ext) != CACHE_EXT_TREE) && + read_index_extension(p->istate, ext, ext + 8, extsize) < 0) { + extension_error = 1; + break; + } + src_offset += 8 + extsize; + } + if (src_offset != end) + extension_error = 1; + +join_tree: + if (tree_thread) { + int err = pthread_join(tree.pthread, NULL); + + if (err) + die(_("unable to join load_index_extension thread: %s"), + strerror(err)); + if (tree.result < 0) + extension_error = 1; + } + + if (extension_error) { + munmap((void *)p->mmap, p->mmap_size); + die(_("index file corrupt")); } return NULL; @@ -2208,6 +2430,7 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) int nr_threads, cpus; struct index_entry_offset_table *ieot = NULL; + clean_status_attach_config(istate); if (istate->initialized) return istate->cache_nr; @@ -2225,6 +2448,7 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) if (fstat(fd, &st)) die_errno(_("%s: cannot stat the open index"), path); + clean_status_record_source_identity(istate, &st); mmap_size = xsize_t(st.st_size); if (mmap_size < sizeof(struct cache_header) + the_hash_algo->rawsz) @@ -2251,6 +2475,8 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) p.istate = istate; p.mmap = mmap; p.mmap_size = mmap_size; + p.allow_parallel = 0; + p.force_parallel = git_env_bool("GIT_TEST_PARALLEL_INDEX_EXTENSIONS", 0); src_offset = sizeof(*hdr); @@ -2272,13 +2498,21 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) extension_offset = read_eoie_extension(mmap, mmap_size); if (extension_offset) { int err; + struct load_index_extension tree = { 0 }; p.src_offset = extension_offset; + /* Keep at least two workers available for cache entries. */ + p.allow_parallel = nr_threads > 3; + if (p.allow_parallel) + p.allow_parallel = + find_parallel_index_extensions(&p, &tree); err = pthread_create(&p.pthread, NULL, load_index_extensions, &p); if (err) die(_("unable to create load_index_extensions thread: %s"), strerror(err)); nr_threads--; + if (p.allow_parallel) + nr_threads--; } } @@ -2438,6 +2672,11 @@ void release_index(struct index_state *istate) free_name_hash(istate); cache_tree_free(&(istate->cache_tree)); free(istate->fsmonitor_last_update); + free(istate->fsmonitor_last_update_pending); + free(istate->fsmonitor_untracked_token); + clean_status_release(istate); + free(istate->preload_bulk_tracked_state); + free(istate->preload_bulk_stat_updates); free(istate->cache); discard_split_index(istate); free_untracked_cache(istate->untracked); @@ -2711,6 +2950,9 @@ static int verify_index_from(const struct index_state *istate, const char *path) if (st.st_size < sizeof(struct cache_header) + the_hash_algo->rawsz) goto out; + if (is_null_oid(&istate->oid) && + !clean_status_verify_null_index(istate, &st)) + goto out; n = pread_in_full(fd, hash, the_hash_algo->rawsz, st.st_size - the_hash_algo->rawsz); if (n != the_hash_algo->rawsz) @@ -2793,6 +3035,7 @@ enum write_extensions { WRITE_RESOLVE_UNDO_EXTENSION = 1<<2, WRITE_UNTRACKED_CACHE_EXTENSION = 1<<3, WRITE_FSMONITOR_EXTENSION = 1<<4, + WRITE_FSCF_EXTENSION = 1<<5, }; #define WRITE_ALL_EXTENSIONS ((enum write_extensions)-1) @@ -3044,6 +3287,35 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, goto out; } } + if (write_extensions & WRITE_FSMONITOR_EXTENSION && + istate->untracked && + istate->fsmonitor_last_update && + istate->fsmonitor_untracked_valid) { + strbuf_reset(&sb); + + write_fsmonitor_untracked_extension(&sb, istate); + err = write_index_ext_header(f, eoie_c, + CACHE_EXT_FSMONITOR_UNTRACKED, + sb.len) < 0; + hashwrite(f, sb.buf, sb.len); + if (err) { + ret = -1; + goto out; + } + } + if (write_extensions & WRITE_FSCF_EXTENSION && + clean_status_should_write_fsmonitor_config(istate)) { + strbuf_reset(&sb); + clean_status_write_fsmonitor_config(&sb, istate); + err = write_index_ext_header(f, eoie_c, + CACHE_EXT_FSMONITOR_CONFIG, + sb.len) < 0; + hashwrite(f, sb.buf, sb.len); + if (err) { + ret = -1; + goto out; + } + } if (istate->sparse_index) { if (write_index_ext_header(f, eoie_c, CACHE_EXT_SPARSE_DIRECTORIES, 0) < 0) { ret = -1; @@ -3498,6 +3770,7 @@ void *read_blob_data_from_index(struct index_state *istate, void move_index_extensions(struct index_state *dst, struct index_state *src) { + clean_status_copy_fsmonitor_history(dst, src); dst->untracked = src->untracked; src->untracked = NULL; dst->cache_tree = src->cache_tree; @@ -4001,8 +4274,12 @@ static void update_callback(struct diff_queue_struct *q, case DIFF_STATUS_DELETED: if (data->flags & ADD_CACHE_IGNORE_REMOVAL) break; - if (!(data->flags & ADD_CACHE_PRETEND)) + if (!(data->flags & ADD_CACHE_PRETEND)) { + if (data->flags & ADD_CACHE_TRACK_CLEAN_HISTORY) + clean_status_invalidate_current_proof( + data->index); remove_file_from_index(data->index, path); + } if (data->flags & (ADD_CACHE_PRETEND|ADD_CACHE_VERBOSE)) printf(_("remove '%s'\n"), path); break; diff --git a/refs.c b/refs.c index 92d5df5b71fa4b..845a5fd3f7587a 100644 --- a/refs.c +++ b/refs.c @@ -2350,6 +2350,23 @@ static struct ref_store *ref_store_init(struct repository *repo, return refs; } +int refs_for_each_replace_ref_uncached(struct repository *repo, + refs_for_each_cb cb, void *cb_data) +{ + struct ref_store *refs; + int ret; + + if (!repo->gitdir) + BUG("attempting to get uncached refs outside of repository"); + + refs = ref_store_init(repo, repo->ref_storage_format, repo->gitdir, + REF_STORE_READ); + ret = refs_for_each_replace_ref(refs, cb, cb_data); + ref_store_release(refs); + free(refs); + return ret; +} + void ref_store_release(struct ref_store *ref_store) { ref_store->be->release(ref_store); diff --git a/refs.h b/refs.h index 9979446d15fd3b..1f80233c5fc472 100644 --- a/refs.h +++ b/refs.h @@ -520,6 +520,12 @@ int refs_for_each_remote_ref(struct ref_store *refs, refs_for_each_cb fn, void *cb_data); int refs_for_each_replace_ref(struct ref_store *refs, refs_for_each_cb fn, void *cb_data); +/* + * Iterate replacement refs through a fresh read-only ref store, without + * consulting caches held by the repository's main ref store. + */ +int refs_for_each_replace_ref_uncached(struct repository *repo, + refs_for_each_cb fn, void *cb_data); /** * Iterate all refs in "prefixes" by partitioning prefixes into disjoint sets diff --git a/replace-object.c b/replace-object.c index 03d0f1f083bed9..29f7c4903e20de 100644 --- a/replace-object.c +++ b/replace-object.c @@ -107,3 +107,16 @@ int replace_refs_enabled(struct repository *r) /* repository has no objects or refs. */ return 0; } + +static int has_replace_ref(const struct reference *ref UNUSED, + void *data UNUSED) +{ + return 1; +} + +int repo_has_replace_refs_uncached(struct repository *r) +{ + if (!replace_refs_enabled(r)) + return 0; + return refs_for_each_replace_ref_uncached(r, has_replace_ref, NULL) != 0; +} diff --git a/replace-object.h b/replace-object.h index 4c9f2a2383d577..595b4598e7e629 100644 --- a/replace-object.h +++ b/replace-object.h @@ -31,6 +31,13 @@ const struct object_id *do_lookup_replace_object(struct repository *r, */ int replace_refs_enabled(struct repository *r); +/* + * Return whether the repository currently has any replacement objects that + * would be honored by lookup_replace_object(). Do not consult the cached + * replacement map. + */ +int repo_has_replace_refs_uncached(struct repository *r); + /* * If object sha1 should be replaced, return the replacement object's * name (replaced recursively, if necessary). The return value is diff --git a/semantic-verify-file.c b/semantic-verify-file.c new file mode 100644 index 00000000000000..e2cfa3ca94d1e0 --- /dev/null +++ b/semantic-verify-file.c @@ -0,0 +1,258 @@ +#include "git-compat-util.h" +#include "convert.h" +#include "environment.h" +#include "object-file.h" +#include "path-namespace.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "semantic-verify.h" +#include "semantic-verify-internal.h" + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +static int mode_matches_ce(struct repository *repo, + const struct cache_entry *ce, + const struct stat *st) +{ + if (!S_ISREG(st->st_mode)) + return 0; + if (repo_trust_executable_bit(repo) && + ((ce->ce_mode ^ st->st_mode) & 0100)) + return 0; + return 1; +} + +static int hash_raw_blob(int fd, size_t size, + const struct git_hash_algo *algo, + struct object_id *oid, void *buffer, + size_t *bytes_hashed) +{ + struct git_hash_ctx ctx; + char header[MAX_HEADER_LEN]; + int header_len; + size_t remaining = size; + + header_len = format_object_header(header, sizeof(header), OBJ_BLOB, size); + git_hash_init(&ctx, algo); + git_hash_update(&ctx, header, header_len); + + while (remaining) { + size_t want = remaining < SEMANTIC_VERIFY_HASH_BUFFER_SIZE ? + remaining : SEMANTIC_VERIFY_HASH_BUFFER_SIZE; + ssize_t nr = xread(fd, buffer, want); + + if (nr < 0) + return -1; + if (!nr) { + errno = EIO; + return -1; + } + git_hash_update(&ctx, buffer, nr); + remaining -= nr; + *bytes_hashed += nr; + } + + /* Do not silently omit an append which raced with the declared size. */ + { + char extra; + ssize_t nr = xread(fd, &extra, 1); + + if (nr < 0) + return -1; + if (nr) { + errno = EAGAIN; + return -1; + } + } + + git_hash_final_oid(oid, &ctx); + return 0; +} + +static unsigned int classify_resolve_error(int error) +{ + if (error == ENOENT) + return SEMANTIC_VERIFY_RAW_MODIFIED; + if (error == ELOOP || error == ENOTDIR || error == EXDEV || + error == EINVAL) + return SEMANTIC_VERIFY_STRUCTURAL; + return SEMANTIC_VERIFY_ERROR; +} +#endif + +int semantic_verify_classify_entry(struct index_state *istate, + const struct cache_entry *ce, + struct attr_check *check, + struct semantic_verify_file_result *result) +{ + memset(result, 0, sizeof(*result)); + if (ce_skip_worktree(ce) || (ce->ce_flags & CE_VALID)) { + result->kind = SEMANTIC_VERIFY_SKIPPED; + return 0; + } + if (ce_stage(ce) || ce_intent_to_add(ce) || + S_ISSPARSEDIR(ce->ce_mode)) { + result->kind = SEMANTIC_VERIFY_STRUCTURAL; + return 0; + } + if (!S_ISREG(ce->ce_mode) || + !convert_attrs_is_raw_safe(istate, ce->name, check)) { + result->kind = SEMANTIC_VERIFY_SENSITIVE; + return 0; + } + return 1; +} + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +static int observed_stat_equal(const struct stat *a, const struct stat *b, + int has_platform_identity) +{ + if (a->st_dev != b->st_dev || a->st_ino != b->st_ino || + a->st_mode != b->st_mode || a->st_nlink != b->st_nlink || + a->st_uid != b->st_uid || a->st_gid != b->st_gid || + a->st_size != b->st_size || a->st_mtime != b->st_mtime || + ST_MTIME_NSEC(*a) != ST_MTIME_NSEC(*b) || + a->st_ctime != b->st_ctime || + ST_CTIME_NSEC(*a) != ST_CTIME_NSEC(*b)) + return 0; +#ifdef __APPLE__ + if (has_platform_identity && + (a->st_birthtimespec.tv_sec != b->st_birthtimespec.tv_sec || + a->st_birthtimespec.tv_nsec != b->st_birthtimespec.tv_nsec || + a->st_gen != b->st_gen)) + return 0; +#else + (void)has_platform_identity; +#endif + return 1; +} + +void semantic_verify_file_at(int parent_fd, const char *basename, + const struct stat *observed, + int observed_has_platform_identity, dev_t root_dev, + const struct cache_entry *ce, + struct repository *repo, void *buffer, + struct semantic_verify_file_result *result) +{ + struct stat path_before = *observed, fd_before, fd_after, path_after; + struct object_id oid; + int fd = -1; + int saved_errno; + + memset(result, 0, sizeof(*result)); + if (!mode_matches_ce(repo, ce, &path_before)) { + result->kind = SEMANTIC_VERIFY_RAW_MODIFIED; + return; + } + if (path_before.st_size < 0 || + (uintmax_t)path_before.st_size > (uintmax_t)SIZE_MAX) { + result->kind = SEMANTIC_VERIFY_SENSITIVE; + return; + } + + fd = semantic_verify_openat(parent_fd, basename, + O_RDONLY | O_NONBLOCK | O_NOFOLLOW); + if (fd < 0) { + result->error = errno; + result->kind = errno == ENOENT || errno == ENOTDIR || + errno == ELOOP ? + SEMANTIC_VERIFY_UNSTABLE : SEMANTIC_VERIFY_ERROR; + return; + } + if (fstat(fd, &fd_before)) + goto unstable; + if (fd_before.st_dev != root_dev || + !observed_stat_equal(&path_before, &fd_before, + observed_has_platform_identity)) { + errno = EAGAIN; + goto unstable; + } + if (hash_raw_blob(fd, (size_t)fd_before.st_size, repo->hash_algo, &oid, + buffer, &result->bytes_hashed)) + goto unstable; + if (fstat(fd, &fd_after)) + goto unstable; + if (fstatat(parent_fd, basename, &path_after, AT_SYMLINK_NOFOLLOW)) + goto unstable; + if (!path_namespace_stat_equal(&fd_before, &fd_after) || + !path_namespace_stat_equal(&fd_after, &path_after)) { + errno = EAGAIN; + goto unstable; + } + if (path_namespace_reopen_component( + parent_fd, basename, O_RDONLY | O_NONBLOCK | O_NOFOLLOW, + semantic_verify_openat, &fd_after)) + goto unstable; + close(fd); + + if (!oideq(&oid, &ce->oid)) { + result->kind = SEMANTIC_VERIFY_RAW_MODIFIED; + return; + } + result->kind = SEMANTIC_VERIFY_RAW_CLEAN; + result->persistable = fd_after.st_nlink == 1; + fill_stat_data(&result->stat_data, &fd_after); + return; + +unstable: + saved_errno = errno; + close(fd); + result->kind = SEMANTIC_VERIFY_UNSTABLE; + result->error = saved_errno; +} + +void semantic_verify_file(struct semantic_verify_root *root, + struct semantic_verify_path *path, + const struct cache_entry *ce, size_t cache_pos, + struct repository *repo, void *buffer, + struct semantic_verify_file_result *result) +{ + struct stat path_before; + const char *basename; + int parent_fd; + + memset(result, 0, sizeof(*result)); + if (semantic_verify_resolve_parent(path, ce->name, cache_pos, + &parent_fd, &basename)) { + result->error = errno; + result->kind = classify_resolve_error(errno); + return; + } + if (fstatat(parent_fd, basename, &path_before, AT_SYMLINK_NOFOLLOW)) { + result->error = errno; + result->kind = errno == ENOENT || errno == ENOTDIR ? + SEMANTIC_VERIFY_RAW_MODIFIED : SEMANTIC_VERIFY_ERROR; + return; + } + semantic_verify_file_at(parent_fd, basename, &path_before, 1, + root->stat.st_dev, ce, repo, buffer, result); +} +#else +static void semantic_verify_file_unavailable( + struct semantic_verify_file_result *result) +{ + memset(result, 0, sizeof(*result)); + result->kind = SEMANTIC_VERIFY_ERROR; + result->error = ENOSYS; +} + +void semantic_verify_file_at( + int parent_fd UNUSED, const char *basename UNUSED, + const struct stat *observed UNUSED, + int observed_has_platform_identity UNUSED, dev_t root_dev UNUSED, + const struct cache_entry *ce UNUSED, + struct repository *repo UNUSED, void *buffer UNUSED, + struct semantic_verify_file_result *result) +{ + semantic_verify_file_unavailable(result); +} + +void semantic_verify_file( + struct semantic_verify_root *root UNUSED, + struct semantic_verify_path *path UNUSED, + const struct cache_entry *ce UNUSED, size_t cache_pos UNUSED, + struct repository *repo UNUSED, void *buffer UNUSED, + struct semantic_verify_file_result *result) +{ + semantic_verify_file_unavailable(result); +} +#endif diff --git a/semantic-verify-internal.h b/semantic-verify-internal.h new file mode 100644 index 00000000000000..8d48ce92a83100 --- /dev/null +++ b/semantic-verify-internal.h @@ -0,0 +1,150 @@ +#ifndef SEMANTIC_VERIFY_INTERNAL_H +#define SEMANTIC_VERIFY_INTERNAL_H + +#include "hash.h" +#include "statinfo.h" +#include "thread-utils.h" + +#ifdef __linux__ +#include +#if !defined(SYS_openat2) && defined(__NR_openat2) +#define SYS_openat2 __NR_openat2 +#elif !defined(SYS_openat2) && \ + (defined(__x86_64__) || defined(__i386__)) +#define SYS_openat2 437 +#endif +#endif + +#if defined(__APPLE__) && defined(O_NONBLOCK) && \ + defined(O_NOFOLLOW) && defined(O_DIRECTORY) && \ + defined(AT_SYMLINK_NOFOLLOW) +#define SEMANTIC_VERIFY_HAS_ANCHORED_OPEN 1 +#elif defined(__linux__) && defined(SYS_openat2) && \ + defined(O_CLOEXEC) && defined(O_NONBLOCK) && \ + defined(O_NOFOLLOW) && defined(O_DIRECTORY) && \ + defined(AT_SYMLINK_NOFOLLOW) +#define SEMANTIC_VERIFY_HAS_ANCHORED_OPEN 1 +#else +#define SEMANTIC_VERIFY_HAS_ANCHORED_OPEN 0 +#endif + +struct attr_check; +struct repository; +struct clean_status_proof_epoch; +struct cache_entry; +struct git_hash_algo; +struct index_state; +struct semantic_verify_result; +struct semantic_verify_path; + +#define SEMANTIC_VERIFY_HASH_BUFFER_SIZE (256 * 1024) + +struct semantic_verify_root { + int fd; + char *path; + struct stat stat; +}; + +int semantic_verify_root_init(struct repository *repo, + struct semantic_verify_root **root_out); +int semantic_verify_root_stable(const struct semantic_verify_root *root); +void semantic_verify_root_clear(struct semantic_verify_root *root); + +int semantic_verify_openat(int dirfd, const char *path, int flags); + +struct semantic_verify_path *semantic_verify_path_new( + struct semantic_verify_root *root); +int semantic_verify_resolve_parent(struct semantic_verify_path *path, + const char *name, size_t cache_pos, + int *parent_fd, const char **basename); +void semantic_verify_path_free(struct semantic_verify_path *path, + unsigned int *namespace_unstable, + size_t *namespace_unstable_from); + +struct semantic_verify_file_result { + struct stat_data stat_data; + size_t bytes_hashed; + int error; + unsigned int kind; + unsigned int persistable; +}; + +int semantic_verify_classify_entry(struct index_state *istate, + const struct cache_entry *ce, + struct attr_check *check, + struct semantic_verify_file_result *result); +void semantic_verify_file(struct semantic_verify_root *root, + struct semantic_verify_path *path, + const struct cache_entry *ce, size_t cache_pos, + struct repository *repo, void *buffer, + struct semantic_verify_file_result *result); +void semantic_verify_file_at(int parent_fd, const char *basename, + const struct stat *observed, + int observed_has_platform_identity, dev_t root_dev, + const struct cache_entry *ce, + struct repository *repo, void *buffer, + struct semantic_verify_file_result *result); + +struct semantic_verify_stat_update { + uint32_t cache_pos; + struct stat_data stat_data; +}; + +struct semantic_verify_entry_identity { + const struct cache_entry *entry; + struct object_id oid; + struct stat_data stat_data; + char *name; + unsigned int mode; + unsigned int flags; +}; + +struct semantic_verify_worker { + pthread_t pthread; + unsigned int started; + struct index_state *istate; + struct semantic_verify_root *root; + struct semantic_verify_result *results; + struct attr_check *check; + size_t start; + size_t end; + struct semantic_verify_stat_update *updates; + size_t updates_nr; + size_t updates_alloc; + size_t bytes_hashed; + size_t raw_clean; + size_t raw_modified; + size_t sensitive; + size_t structural; + size_t skipped; + size_t unstable; + size_t errors; + size_t hardlinks; + unsigned int namespace_unstable; +}; + +void semantic_verify_worker_run(struct semantic_verify_worker *worker); + +struct semantic_verify_proof { + struct index_state *istate; + struct semantic_verify_root *root; + struct clean_status_proof_epoch *epoch; + struct semantic_verify_result *results; + struct semantic_verify_entry_identity *entry_identities; + struct semantic_verify_stat_update *stat_updates; + size_t cache_nr; + size_t stat_updates_nr; + size_t bytes_hashed; + size_t raw_clean; + size_t raw_modified; + size_t sensitive; + size_t structural; + size_t skipped; + size_t unstable; + size_t errors; + size_t hardlinks; + unsigned int namespace_unstable; + unsigned int epoch_required; +}; + +#endif /* SEMANTIC_VERIFY_INTERNAL_H */ diff --git a/semantic-verify-path.c b/semantic-verify-path.c new file mode 100644 index 00000000000000..db7fd874084c42 --- /dev/null +++ b/semantic-verify-path.c @@ -0,0 +1,186 @@ +#include "git-compat-util.h" +#include "path-namespace.h" +#include "semantic-verify-internal.h" +#include "strbuf.h" + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +struct anchored_dir { + char *component; + int fd; + struct stat stat; + size_t first_cache_pos; +}; + +struct semantic_verify_path { + struct semantic_verify_root *root; + struct anchored_dir *dirs; + size_t dirs_nr; + size_t dirs_alloc; + size_t namespace_unstable_from; + unsigned int namespace_unstable; + struct strbuf component; +}; + +static void note_namespace_unstable(struct semantic_verify_path *path, + size_t from) +{ + path->namespace_unstable = 1; + if (from < path->namespace_unstable_from) + path->namespace_unstable_from = from; +} + +static void pop_anchored_dir(struct semantic_verify_path *path) +{ + struct anchored_dir *dir = &path->dirs[path->dirs_nr - 1]; + int parent_fd = path->dirs_nr == 1 ? path->root->fd : + path->dirs[path->dirs_nr - 2].fd; + int named_fd; + struct stat named_stat; + + named_fd = semantic_verify_openat(parent_fd, dir->component, + O_RDONLY | O_DIRECTORY | O_NOFOLLOW); + if (named_fd < 0 || fstat(named_fd, &named_stat) || + !path_namespace_stat_equal(&dir->stat, &named_stat)) + note_namespace_unstable(path, dir->first_cache_pos); + if (named_fd >= 0) + close(named_fd); + close(dir->fd); + free(dir->component); + path->dirs_nr--; +} + +struct semantic_verify_path *semantic_verify_path_new( + struct semantic_verify_root *root) +{ + struct semantic_verify_path *path; + + CALLOC_ARRAY(path, 1); + path->root = root; + path->namespace_unstable_from = SIZE_MAX; + path->component = (struct strbuf)STRBUF_INIT; + return path; +} + +int semantic_verify_resolve_parent(struct semantic_verify_path *path, + const char *name, size_t cache_pos, + int *parent_fd, const char **basename) +{ + const char *slash = strrchr(name, '/'); + size_t parent_len = slash ? (size_t)(slash - name) : 0; + size_t begin = 0, depth = 0; + + *basename = slash ? slash + 1 : name; + if (!**basename) { + errno = EINVAL; + return -1; + } + + /* Find the component-aligned prefix already pinned by this worker. */ + while (begin < parent_len && depth < path->dirs_nr) { + size_t end = begin; + struct anchored_dir *dir = &path->dirs[depth]; + + while (end < parent_len && name[end] != '/') + end++; + if (strlen(dir->component) != end - begin || + memcmp(dir->component, name + begin, end - begin)) + break; + depth++; + begin = end + 1; + } + while (path->dirs_nr > depth) + pop_anchored_dir(path); + + while (begin < parent_len) { + size_t end = begin; + struct anchored_dir *dir; + int dirfd, fd; + struct stat st; + + while (end < parent_len && name[end] != '/') + end++; + if (end == begin || + (end - begin == 1 && name[begin] == '.') || + (end - begin == 2 && name[begin] == '.' && + name[begin + 1] == '.')) { + errno = EINVAL; + return -1; + } + strbuf_reset(&path->component); + strbuf_add(&path->component, name + begin, end - begin); + dirfd = path->dirs_nr ? path->dirs[path->dirs_nr - 1].fd : + path->root->fd; + fd = semantic_verify_openat(dirfd, path->component.buf, + O_RDONLY | O_DIRECTORY | O_NOFOLLOW); + if (fd < 0) + return -1; + if (fstat(fd, &st)) { + int saved_errno = errno; + + close(fd); + errno = saved_errno; + return -1; + } + if (!S_ISDIR(st.st_mode) || st.st_dev != path->root->stat.st_dev) { + close(fd); + errno = EXDEV; + return -1; + } + ALLOC_GROW(path->dirs, path->dirs_nr + 1, path->dirs_alloc); + dir = &path->dirs[path->dirs_nr++]; + dir->component = xstrdup(path->component.buf); + dir->fd = fd; + memcpy(&dir->stat, &st, sizeof(st)); + dir->first_cache_pos = cache_pos; + begin = end + 1; + } + + *parent_fd = path->dirs_nr ? path->dirs[path->dirs_nr - 1].fd : + path->root->fd; + return 0; +} + +void semantic_verify_path_free(struct semantic_verify_path *path, + unsigned int *namespace_unstable, + size_t *namespace_unstable_from) +{ + if (!path) + return; + while (path->dirs_nr) + pop_anchored_dir(path); + if (namespace_unstable) + *namespace_unstable = path->namespace_unstable; + if (namespace_unstable_from) + *namespace_unstable_from = path->namespace_unstable_from; + free(path->dirs); + strbuf_release(&path->component); + free(path); +} +#else +struct semantic_verify_path *semantic_verify_path_new( + struct semantic_verify_root *root UNUSED) +{ + errno = ENOSYS; + return NULL; +} + +int semantic_verify_resolve_parent( + struct semantic_verify_path *path UNUSED, + const char *name UNUSED, size_t cache_pos UNUSED, + int *parent_fd UNUSED, const char **basename UNUSED) +{ + errno = ENOSYS; + return -1; +} + +void semantic_verify_path_free( + struct semantic_verify_path *path UNUSED, + unsigned int *namespace_unstable, + size_t *namespace_unstable_from) +{ + if (namespace_unstable) + *namespace_unstable = 0; + if (namespace_unstable_from) + *namespace_unstable_from = SIZE_MAX; +} +#endif diff --git a/semantic-verify-root.c b/semantic-verify-root.c new file mode 100644 index 00000000000000..fbec93ac147607 --- /dev/null +++ b/semantic-verify-root.c @@ -0,0 +1,168 @@ +#include "git-compat-util.h" +#include "path-namespace.h" +#include "repository.h" +#include "semantic-verify-internal.h" +#include "wrapper.h" + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN && defined(__linux__) +struct semantic_open_how { + uint64_t flags; + uint64_t mode; + uint64_t resolve; +}; + +#define SEMANTIC_RESOLVE_NO_XDEV 0x01 +#define SEMANTIC_RESOLVE_NO_MAGICLINKS 0x02 +#define SEMANTIC_RESOLVE_NO_SYMLINKS 0x04 +#define SEMANTIC_RESOLVE_BENEATH 0x08 +#endif + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN && !defined(__linux__) +static int set_fd_cloexec(int fd) +{ +#if defined(F_GETFD) && defined(F_SETFD) && defined(FD_CLOEXEC) + int flags = fcntl(fd, F_GETFD); + + if (flags < 0 || fcntl(fd, F_SETFD, flags | FD_CLOEXEC) < 0) + return -1; +#endif + return 0; +} +#endif + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +int semantic_verify_openat(int dirfd, const char *path, int flags) +{ +#ifdef __linux__ + struct semantic_open_how how = { + .flags = flags | O_CLOEXEC, + .resolve = SEMANTIC_RESOLVE_BENEATH | + SEMANTIC_RESOLVE_NO_SYMLINKS | + SEMANTIC_RESOLVE_NO_MAGICLINKS | + SEMANTIC_RESOLVE_NO_XDEV, + }; + + return syscall(SYS_openat2, dirfd, path, &how, sizeof(how)); +#else + int fd; + int saved_errno; + +#ifdef O_CLOEXEC + fd = openat(dirfd, path, flags | O_CLOEXEC); + if (fd >= 0) + return fd; + if (errno != EINVAL) + return -1; +#endif + fd = openat(dirfd, path, flags); + if (fd < 0) + return -1; + if (!set_fd_cloexec(fd)) + return fd; + saved_errno = errno; + close(fd); + errno = saved_errno; + return -1; +#endif +} +#else +int semantic_verify_openat(int dirfd UNUSED, const char *path UNUSED, + int flags UNUSED) +{ + errno = ENOSYS; + return -1; +} +#endif + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +int semantic_verify_root_init(struct repository *repo, + struct semantic_verify_root **root_out) +{ + struct semantic_verify_root *root; + const char *path = repo_get_work_tree(repo); + + if (!path) { + errno = ENOENT; + return -1; + } + CALLOC_ARRAY(root, 1); + root->fd = -1; + root->path = xstrdup(path); + root->fd = git_open_cloexec(root->path, + O_RDONLY | O_DIRECTORY | O_NOFOLLOW); + if (root->fd < 0 || fstat(root->fd, &root->stat) || + !S_ISDIR(root->stat.st_mode)) { + int saved_errno = errno ? errno : ENOTDIR; + + semantic_verify_root_clear(root); + errno = saved_errno; + return -1; + } +#ifdef __linux__ + { + struct stat probe_stat; + int probe_fd = semantic_verify_openat( + root->fd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW); + int saved_errno; + + if (probe_fd < 0) { + saved_errno = errno; + semantic_verify_root_clear(root); + errno = saved_errno; + return -1; + } + if (fstat(probe_fd, &probe_stat)) { + saved_errno = errno; + close(probe_fd); + semantic_verify_root_clear(root); + errno = saved_errno; + return -1; + } + if (!path_namespace_stat_equal(&root->stat, &probe_stat)) { + close(probe_fd); + semantic_verify_root_clear(root); + errno = EAGAIN; + return -1; + } + if (close(probe_fd)) { + saved_errno = errno; + semantic_verify_root_clear(root); + errno = saved_errno; + return -1; + } + } +#endif + *root_out = root; + return 0; +} +#else +int semantic_verify_root_init(struct repository *repo UNUSED, + struct semantic_verify_root **root_out UNUSED) +{ + errno = ENOSYS; + return -1; +} +#endif + +int semantic_verify_root_stable(const struct semantic_verify_root *root) +{ + struct stat fd_stat, path_stat; + + if (!root || root->fd < 0) + return 0; + if (fstat(root->fd, &fd_stat) || lstat(root->path, &path_stat) || + !S_ISDIR(path_stat.st_mode)) + return 0; + return path_namespace_stat_equal(&root->stat, &fd_stat) && + path_namespace_stat_equal(&fd_stat, &path_stat); +} + +void semantic_verify_root_clear(struct semantic_verify_root *root) +{ + if (!root) + return; + if (root->fd >= 0) + close(root->fd); + free(root->path); + free(root); +} diff --git a/semantic-verify-worker.c b/semantic-verify-worker.c new file mode 100644 index 00000000000000..81c33ee59fabe2 --- /dev/null +++ b/semantic-verify-worker.c @@ -0,0 +1,115 @@ +#define USE_THE_REPOSITORY_VARIABLE + +#include "git-compat-util.h" +#include "attr.h" +#include "convert.h" +#include "object.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "semantic-verify.h" +#include "semantic-verify-internal.h" + +static void record_stat_update(struct semantic_verify_worker *worker, + uint32_t cache_pos, + const struct stat_data *stat_data) +{ + struct semantic_verify_stat_update *update; + + ALLOC_GROW(worker->updates, worker->updates_nr + 1, + worker->updates_alloc); + update = &worker->updates[worker->updates_nr++]; + update->cache_pos = cache_pos; + memcpy(&update->stat_data, stat_data, sizeof(*stat_data)); +} + +static void count_result(struct semantic_verify_worker *worker, + enum semantic_verify_kind kind) +{ + switch (kind) { + case SEMANTIC_VERIFY_SKIPPED: + worker->skipped++; + break; + case SEMANTIC_VERIFY_RAW_CLEAN: + worker->raw_clean++; + break; + case SEMANTIC_VERIFY_RAW_MODIFIED: + worker->raw_modified++; + break; + case SEMANTIC_VERIFY_SENSITIVE: + worker->sensitive++; + break; + case SEMANTIC_VERIFY_STRUCTURAL: + worker->structural++; + break; + case SEMANTIC_VERIFY_UNSTABLE: + worker->unstable++; + break; + case SEMANTIC_VERIFY_ERROR: + worker->errors++; + break; + case SEMANTIC_VERIFY_UNCHECKED: + BUG("cannot count an unchecked semantic result"); + } +} + +void semantic_verify_worker_run(struct semantic_verify_worker *worker) +{ + struct semantic_verify_path *path = + semantic_verify_path_new(worker->root); + struct attr_check *check = worker->check; + void *buffer = xmalloc(SEMANTIC_VERIFY_HASH_BUFFER_SIZE); + size_t unstable_from = SIZE_MAX; + + worker->check = NULL; + if (!check) + check = convert_attrs_check_alloc(); + + for (size_t i = worker->start; i < worker->end; i++) { + struct cache_entry *ce = worker->istate->cache[i]; + struct semantic_verify_result *result = &worker->results[i]; + struct semantic_verify_file_result file; + + if (!semantic_verify_classify_entry(worker->istate, ce, check, + &file)) { + result->kind = file.kind; + count_result(worker, result->kind); + continue; + } + + semantic_verify_file(worker->root, path, ce, i, + worker->istate->repo, + buffer, &file); + result->kind = file.kind; + result->error = file.error > UINT16_MAX ? EIO : file.error; + worker->bytes_hashed += file.bytes_hashed; + if (result->kind == SEMANTIC_VERIFY_RAW_CLEAN) { + if (file.persistable) + result->flags |= SEMANTIC_VERIFY_PERSISTABLE; + else + worker->hardlinks++; + if (memcmp(&file.stat_data, &ce->ce_stat_data, + sizeof(file.stat_data))) + record_stat_update(worker, i, &file.stat_data); + } + count_result(worker, result->kind); + } + + semantic_verify_path_free(path, &worker->namespace_unstable, + &unstable_from); + if (worker->namespace_unstable) { + for (size_t i = unstable_from; i < worker->end; i++) { + struct semantic_verify_result *result = &worker->results[i]; + + if (result->kind != SEMANTIC_VERIFY_RAW_CLEAN) + continue; + result->kind = SEMANTIC_VERIFY_UNSTABLE; + result->flags = 0; + result->error = EAGAIN; + worker->raw_clean--; + worker->unstable++; + } + } + + free(buffer); + attr_check_free(check); +} diff --git a/semantic-verify.c b/semantic-verify.c new file mode 100644 index 00000000000000..7852dcf2ad630c --- /dev/null +++ b/semantic-verify.c @@ -0,0 +1,404 @@ +#define USE_THE_REPOSITORY_VARIABLE + +#include "git-compat-util.h" +#include "attr.h" +#include "clean-status.h" +#include "convert.h" +#include "fsmonitor.h" +#include "object.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "semantic-verify.h" +#include "semantic-verify-internal.h" +#include "trace2.h" + +#define SEMANTIC_VERIFY_ENTRY_FLAGS \ + (CE_VALID | CE_STAGEMASK | CE_INTENT_TO_ADD | CE_SKIP_WORKTREE | \ + CE_UPTODATE | CE_FSMONITOR_VALID | CE_CONTENT_CHECK_REQUIRED) +#define SEMANTIC_VERIFY_MAX_THREADS 32 + +static void *run_worker(void *data) +{ + semantic_verify_worker_run(data); + return NULL; +} + +static unsigned int select_thread_count( + size_t cache_nr, + const struct semantic_verify_options *options) +{ + unsigned int nr; + + if (!HAVE_THREADS) + return 1; + if (options && options->nr_threads) { + nr = options->nr_threads; + } else { + unsigned int cpus = online_cpus(); + + nr = cpus > SEMANTIC_VERIFY_MAX_THREADS / 2 ? + SEMANTIC_VERIFY_MAX_THREADS : cpus * 2; + } + if (nr > SEMANTIC_VERIFY_MAX_THREADS) + nr = SEMANTIC_VERIFY_MAX_THREADS; + if (nr > cache_nr && cache_nr) + nr = cache_nr; + return nr; +} + +static void combine_worker(struct semantic_verify_proof *proof, + struct semantic_verify_worker *worker) +{ + size_t base = proof->stat_updates_nr; + + if (worker->updates_nr) + COPY_ARRAY(proof->stat_updates + base, worker->updates, + worker->updates_nr); + for (size_t i = 0; i < worker->updates_nr; i++) { + uint32_t cache_pos = worker->updates[i].cache_pos; + + proof->results[cache_pos].stat_update_index = base + i; + } + proof->stat_updates_nr += worker->updates_nr; + proof->bytes_hashed += worker->bytes_hashed; + proof->raw_clean += worker->raw_clean; + proof->raw_modified += worker->raw_modified; + proof->sensitive += worker->sensitive; + proof->structural += worker->structural; + proof->skipped += worker->skipped; + proof->unstable += worker->unstable; + proof->errors += worker->errors; + proof->hardlinks += worker->hardlinks; + proof->namespace_unstable |= worker->namespace_unstable; + free(worker->updates); +} + +int semantic_verify_prepare(struct index_state *istate, + const struct semantic_verify_options *options, + struct semantic_verify_proof **proof_out) +{ + struct semantic_verify_proof *proof; + struct semantic_verify_worker *workers; + unsigned int nr_threads; + size_t updates_nr = 0; + int create_threads = 1; + + if (!istate || !proof_out) + BUG("semantic_verify_prepare requires an index and output"); + if (sizeof(struct semantic_verify_result) != 8) + BUG("semantic verify result unexpectedly grew to %"PRIuMAX" bytes", + (uintmax_t)sizeof(struct semantic_verify_result)); + CALLOC_ARRAY(proof, 1); + proof->istate = istate; + proof->epoch_required = options && options->require_proof_epoch; + proof->cache_nr = istate->cache_nr; + CALLOC_ARRAY(proof->results, proof->cache_nr); + CALLOC_ARRAY(proof->entry_identities, proof->cache_nr); + for (size_t i = 0; i < proof->cache_nr; i++) { + const struct cache_entry *ce = istate->cache[i]; + struct semantic_verify_entry_identity *identity = + &proof->entry_identities[i]; + + proof->results[i].stat_update_index = UINT32_MAX; + identity->entry = ce; + oidcpy(&identity->oid, &ce->oid); + identity->stat_data = ce->ce_stat_data; + identity->name = xstrdup(ce->name); + identity->mode = ce->ce_mode; + identity->flags = ce->ce_flags & SEMANTIC_VERIFY_ENTRY_FLAGS; + } + *proof_out = proof; + if (!proof->cache_nr && !proof->epoch_required) + return 0; + if (istate->sparse_index != INDEX_EXPANDED) { + for (size_t i = 0; i < proof->cache_nr; i++) { + proof->results[i].kind = SEMANTIC_VERIFY_STRUCTURAL; + proof->structural++; + } + return 0; + } + if (semantic_verify_root_init(istate->repo, &proof->root)) { + int saved_errno = errno; + + for (size_t i = 0; i < proof->cache_nr; i++) { + proof->results[i].kind = SEMANTIC_VERIFY_ERROR; + proof->results[i].error = saved_errno > UINT16_MAX ? + EIO : saved_errno; + } + proof->errors = proof->cache_nr; + return -1; + } + if (proof->epoch_required) { + proof->epoch = clean_status_capture_proof_epoch( + istate, options->attr_snapshot); + if (!proof->epoch) { + for (size_t i = 0; i < proof->cache_nr; i++) { + proof->results[i].kind = SEMANTIC_VERIFY_ERROR; + proof->results[i].error = EAGAIN; + } + proof->errors = proof->cache_nr; + return -1; + } + } + if (!proof->cache_nr) + return 0; + + /* Initialize conversion config and default attribute state serially. */ + convert_attrs_prepare(istate); + nr_threads = select_thread_count(proof->cache_nr, options); + CALLOC_ARRAY(workers, nr_threads); + if (proof->epoch_required) { + /* + * Load each worker's system, global, root, and info + * attribute frames before closing the proof epoch. + */ + for (unsigned int i = 0; i < nr_threads; i++) { + workers[i].check = convert_attrs_check_alloc(); + git_check_attr(istate, "", workers[i].check); + } + if (!clean_status_proof_epoch_prime_matches( + istate, proof->epoch)) { + for (unsigned int i = 0; i < nr_threads; i++) + attr_check_free(workers[i].check); + free(workers); + for (size_t i = 0; i < proof->cache_nr; i++) { + proof->results[i].kind = SEMANTIC_VERIFY_ERROR; + proof->results[i].error = EAGAIN; + } + proof->errors = proof->cache_nr; + git_attr_invalidate_all(); + return -1; + } + } + trace2_region_enter("semantic_verify", "prepare", istate->repo); + trace2_data_intmax("semantic_verify", istate->repo, + "threads", nr_threads); + trace2_data_intmax("semantic_verify", istate->repo, + "result-bytes", sizeof(struct semantic_verify_result)); + + for (unsigned int i = 0; i < nr_threads; i++) { + struct semantic_verify_worker *worker = &workers[i]; + int err; + + worker->istate = istate; + worker->root = proof->root; + worker->results = proof->results; + worker->start = st_mult(proof->cache_nr, i) / nr_threads; + worker->end = st_mult(proof->cache_nr, i + 1) / nr_threads; + if (nr_threads == 1 || !create_threads) { + semantic_verify_worker_run(worker); + continue; + } + err = pthread_create(&worker->pthread, NULL, run_worker, worker); + if (!err) { + worker->started = 1; + continue; + } + create_threads = 0; + trace2_data_intmax("semantic_verify", istate->repo, + "thread-failure", err); + semantic_verify_worker_run(worker); + } + for (unsigned int i = 0; i < nr_threads; i++) { + int err; + + if (!workers[i].started) + continue; + err = pthread_join(workers[i].pthread, NULL); + if (err) + die("could not join semantic verifier thread: %s", + strerror(err)); + } + + for (unsigned int i = 0; i < nr_threads; i++) + updates_nr += workers[i].updates_nr; + ALLOC_ARRAY(proof->stat_updates, updates_nr); + for (unsigned int i = 0; i < nr_threads; i++) + combine_worker(proof, &workers[i]); + free(workers); + + trace2_data_intmax("semantic_verify", istate->repo, + "raw-clean", proof->raw_clean); + trace2_data_intmax("semantic_verify", istate->repo, + "raw-modified", proof->raw_modified); + trace2_data_intmax("semantic_verify", istate->repo, + "sensitive", proof->sensitive); + trace2_data_intmax("semantic_verify", istate->repo, + "structural", proof->structural); + trace2_data_intmax("semantic_verify", istate->repo, + "unstable", proof->unstable); + trace2_data_intmax("semantic_verify", istate->repo, + "errors", proof->errors); + trace2_data_intmax("semantic_verify", istate->repo, + "bytes-hashed", proof->bytes_hashed); + trace2_region_leave("semantic_verify", "prepare", istate->repo); + return 0; +} + +int semantic_verify_root_is_stable(const struct semantic_verify_proof *proof) +{ + return proof && semantic_verify_root_stable(proof->root); +} + +int semantic_verify_start_token_is_current( + struct index_state *istate, + const struct semantic_verify_proof *proof) +{ + return proof && proof->istate == istate && + (!proof->epoch_required || + clean_status_proof_epoch_start_token_matches( + istate, proof->epoch)); +} + +int semantic_verify_proof_is_current( + struct index_state *istate, + const struct semantic_verify_proof *proof) +{ + return istate && proof && proof->istate == istate && + proof->cache_nr == istate->cache_nr && + !proof->namespace_unstable && + semantic_verify_root_is_stable(proof) && + (!proof->epoch_required || + clean_status_proof_epoch_content_matches( + istate, proof->epoch)); +} + +void semantic_verify_get_stats(const struct semantic_verify_proof *proof, + struct semantic_verify_stats *stats) +{ + if (!proof || !stats) + BUG("semantic_verify_get_stats requires proof and output"); + stats->cache_nr = proof->cache_nr; + stats->stat_updates_nr = proof->stat_updates_nr; + stats->bytes_hashed = proof->bytes_hashed; + stats->raw_clean = proof->raw_clean; + stats->raw_modified = proof->raw_modified; + stats->sensitive = proof->sensitive; + stats->structural = proof->structural; + stats->skipped = proof->skipped; + stats->unstable = proof->unstable; + stats->errors = proof->errors; + stats->hardlinks = proof->hardlinks; + stats->namespace_unstable = proof->namespace_unstable; +} + +const struct semantic_verify_result *semantic_verify_result_at( + const struct semantic_verify_proof *proof, size_t cache_pos) +{ + if (!proof || cache_pos >= proof->cache_nr) + BUG("semantic verifier result position out of range"); + return &proof->results[cache_pos]; +} + +int semantic_verify_apply_after_closure( + struct index_state *istate, + const struct semantic_verify_proof *proof) +{ + int applied = 0; + int poisoned = 0; + size_t validated_updates = 0; + + if (!semantic_verify_proof_is_current(istate, proof)) + return -1; + + for (size_t i = 0; i < proof->cache_nr; i++) { + const struct semantic_verify_entry_identity *identity = + &proof->entry_identities[i]; + const struct cache_entry *ce = istate->cache[i]; + + if (ce != identity->entry || + !oideq(&ce->oid, &identity->oid) || + memcmp(&ce->ce_stat_data, &identity->stat_data, + sizeof(ce->ce_stat_data)) || + strcmp(ce->name, identity->name) || + ce->ce_mode != identity->mode || + (ce->ce_flags & SEMANTIC_VERIFY_ENTRY_FLAGS) != + identity->flags) + return -1; + } + + /* Validate the complete proof before changing any cache entry. */ + for (size_t i = 0; i < proof->cache_nr; i++) { + const struct semantic_verify_result *result = &proof->results[i]; + + if (result->kind > SEMANTIC_VERIFY_ERROR || + (result->flags & ~SEMANTIC_VERIFY_PERSISTABLE)) + return -1; + if (result->kind == SEMANTIC_VERIFY_UNCHECKED || + result->kind == SEMANTIC_VERIFY_STRUCTURAL || + result->kind == SEMANTIC_VERIFY_UNSTABLE || + result->kind == SEMANTIC_VERIFY_ERROR) + return -1; + if (result->kind != SEMANTIC_VERIFY_RAW_CLEAN) { + if (result->flags || + result->stat_update_index != UINT32_MAX) + return -1; + continue; + } + if (result->stat_update_index != UINT32_MAX) { + const struct semantic_verify_stat_update *update; + + if (result->stat_update_index >= proof->stat_updates_nr) + return -1; + update = &proof->stat_updates[result->stat_update_index]; + if (update->cache_pos != i) + return -1; + validated_updates++; + } + } + if (validated_updates != proof->stat_updates_nr) + return -1; + + for (size_t i = 0; i < proof->cache_nr; i++) { + const struct semantic_verify_result *result = &proof->results[i]; + struct cache_entry *ce = istate->cache[i]; + + /* Force the ordinary refresh tail to preserve mismatches. */ + if (result->kind == SEMANTIC_VERIFY_RAW_MODIFIED || + (result->kind == SEMANTIC_VERIFY_RAW_CLEAN && + !(result->flags & SEMANTIC_VERIFY_PERSISTABLE))) { + fsmonitor_invalidate_cache_entry(ce); + mark_fsmonitor_invalid(istate, ce); + ce->ce_flags |= CE_UPDATE_IN_BASE; + istate->cache_changed |= CE_ENTRY_CHANGED; + poisoned++; + if (result->kind == SEMANTIC_VERIFY_RAW_CLEAN) + applied++; + continue; + } + if (result->kind != SEMANTIC_VERIFY_RAW_CLEAN) + continue; + if (result->stat_update_index != UINT32_MAX) { + const struct semantic_verify_stat_update *update = + &proof->stat_updates[result->stat_update_index]; + + memcpy(&ce->ce_stat_data, &update->stat_data, + sizeof(ce->ce_stat_data)); + ce->ce_flags |= CE_UPDATE_IN_BASE; + istate->cache_changed |= CE_ENTRY_CHANGED; + } + ce_mark_uptodate(ce); + if (result->flags & SEMANTIC_VERIFY_PERSISTABLE) + mark_fsmonitor_valid(istate, ce); + applied++; + } + trace2_data_intmax("semantic_verify", istate->repo, + "applied", applied); + trace2_data_intmax("semantic_verify", istate->repo, + "poisoned-for-tail", poisoned); + return applied; +} + +void semantic_verify_proof_clear(struct semantic_verify_proof *proof) +{ + if (!proof) + return; + clean_status_release_proof_epoch(proof->epoch); + semantic_verify_root_clear(proof->root); + for (size_t i = 0; i < proof->cache_nr; i++) + free(proof->entry_identities[i].name); + free(proof->entry_identities); + free(proof->stat_updates); + free(proof->results); + free(proof); +} diff --git a/semantic-verify.h b/semantic-verify.h new file mode 100644 index 00000000000000..21d4acb6295158 --- /dev/null +++ b/semantic-verify.h @@ -0,0 +1,78 @@ +#ifndef SEMANTIC_VERIFY_H +#define SEMANTIC_VERIFY_H + +struct index_state; +struct attr_source_snapshot; +struct semantic_verify_proof; + +struct semantic_verify_options { + unsigned int nr_threads; + const struct attr_source_snapshot *attr_snapshot; + unsigned int require_proof_epoch : 1; +}; + +#define SEMANTIC_VERIFY_OPTIONS_INIT { 0 } + +enum semantic_verify_kind { + SEMANTIC_VERIFY_UNCHECKED = 0, + SEMANTIC_VERIFY_SKIPPED, + SEMANTIC_VERIFY_RAW_CLEAN, + SEMANTIC_VERIFY_RAW_MODIFIED, + SEMANTIC_VERIFY_SENSITIVE, + SEMANTIC_VERIFY_STRUCTURAL, + SEMANTIC_VERIFY_UNSTABLE, + SEMANTIC_VERIFY_ERROR, +}; + +enum semantic_verify_result_flags { + /* The clean result may receive persistent fsmonitor validity. */ + SEMANTIC_VERIFY_PERSISTABLE = (1u << 0), +}; + +/* Exactly eight bytes per cache entry. */ +struct semantic_verify_result { + uint32_t stat_update_index; + uint16_t error; + uint8_t kind; + uint8_t flags; +}; + +struct semantic_verify_stats { + size_t cache_nr; + size_t stat_updates_nr; + size_t bytes_hashed; + size_t raw_clean; + size_t raw_modified; + size_t sensitive; + size_t structural; + size_t skipped; + size_t unstable; + size_t errors; + size_t hardlinks; + unsigned int namespace_unstable; +}; + +/* Build a proof candidate without changing the index. */ +int semantic_verify_prepare(struct index_state *istate, + const struct semantic_verify_options *options, + struct semantic_verify_proof **proof_out); +int semantic_verify_apply_after_closure( + struct index_state *istate, + const struct semantic_verify_proof *proof); +int semantic_verify_root_is_stable( + const struct semantic_verify_proof *proof); +int semantic_verify_start_token_is_current( + struct index_state *istate, + const struct semantic_verify_proof *proof); +int semantic_verify_proof_is_current( + struct index_state *istate, + const struct semantic_verify_proof *proof); +void semantic_verify_proof_clear(struct semantic_verify_proof *proof); + +/* Introspection used by the semantic verifier test helper. */ +void semantic_verify_get_stats(const struct semantic_verify_proof *proof, + struct semantic_verify_stats *stats); +const struct semantic_verify_result *semantic_verify_result_at( + const struct semantic_verify_proof *proof, size_t cache_pos); + +#endif /* SEMANTIC_VERIFY_H */ diff --git a/t/README b/t/README index 4252774f86c8ce..31ead13c82aeb5 100644 --- a/t/README +++ b/t/README @@ -422,6 +422,22 @@ overridden by the --no-path-walk command-line argument. GIT_TEST_PRELOAD_INDEX= exercises the preload-index code path by overriding the minimum number of cache entries required per thread. +GIT_TEST_PRELOAD_INDEX_BULK= overrides the +`core.preloadIndexBulk` setting. + +GIT_TEST_PRELOAD_INDEX_BULK_DIRENT_TYPE=:, when +GIT_TEST_PRELOAD_INDEX_BULK is enabled, overrides the Linux directory +entry type for one worktree-relative path. is `dir` or `reg`. + +GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_PATH=, +GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_READY=, and +GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_RESUME=, when +GIT_TEST_PRELOAD_INDEX_BULK is enabled, pause a bulk preload before +scanning the named directory. An empty directory path pauses after the +complete walk. Git writes `ready` to the ready path, then waits until it +can read from the resume path. Tests which set one barrier variable must +set all three. + GIT_TEST_INDEX_THREADS= enables exercising the multi-threaded loading of the index for the whole test suite by bypassing the default number of cache entries and thread minimums. Setting this to 1 will make the diff --git a/t/helper/meson.build b/t/helper/meson.build index 3235f10ab8aae1..7c97bfb1e6ec51 100644 --- a/t/helper/meson.build +++ b/t/helper/meson.build @@ -59,6 +59,7 @@ test_tool_sources = [ 'test-rot13-filter.c', 'test-run-command.c', 'test-scrap-cache-tree.c', + 'test-semantic-verify.c', 'test-serve-v2.c', 'test-sha1.c', 'test-sha256.c', diff --git a/t/helper/test-read-cache.c b/t/helper/test-read-cache.c index 6b08ba8f078d00..5228c2065e4404 100644 --- a/t/helper/test-read-cache.c +++ b/t/helper/test-read-cache.c @@ -1,17 +1,413 @@ #define USE_THE_REPOSITORY_VARIABLE #include "test-tool.h" +#include "attr.h" +#include "attr-fingerprint.h" +#include "attr-manifest.h" +#include "clean-status.h" +#include "clean-status-internal.h" #include "config.h" +#include "dir.h" #include "environment.h" +#include "ewah/ewok.h" +#include "ewah/ewok_rlw.h" +#include "fsmonitor.h" +#include "fsmonitor-clean-proof.h" +#include "fsmonitor-ll.h" +#include "lockfile.h" #include "read-cache-ll.h" #include "repository.h" #include "setup.h" +#include "strbuf.h" + +static int test_fsmonitor_content_recovery(const char *path) +{ + struct index_state *istate; + struct cache_entry *ce; + struct stat_data empty = { 0 }; + struct stat st; + int pos; + + setup_git_directory(the_repository); + repo_config(the_repository, git_default_config, NULL); + if (repo_read_index(the_repository) < 0) + return error("unable to read test index"); + istate = the_repository->index; + pos = index_name_pos(istate, path, strlen(path)); + if (pos < 0) + return error("path is not indexed: %s", path); + ce = istate->cache[pos]; + if (lstat(path, &st)) + return error_errno("unable to stat indexed path"); + + fsmonitor_invalidate_cache_entry(ce); + if (memcmp(&ce->ce_stat_data, &empty, sizeof(empty))) + return error("invalidation did not poison cached stat data"); + if (ie_match_stat_with_content_check(istate, ce, &st, 0)) + return error("clean content did not match"); + if (!memcmp(&ce->ce_stat_data, &empty, sizeof(empty))) + return error("verified clean entry retained poisoned stat data"); + if (!(ce->ce_flags & CE_UPDATE_IN_BASE) || + !(istate->cache_changed & CE_ENTRY_CHANGED)) + return error("verified stat refresh was not marked for persistence"); + return 0; +} + +static int fsuc_failed_closed(const struct index_state *istate) +{ + return istate->fsmonitor_untracked_extension_seen && + istate->fsmonitor_untracked_extension_invalid && + !istate->fsmonitor_untracked_token; +} + +static int test_fsuc_parser(void) +{ + struct index_state duplicate = INDEX_STATE_INIT(the_repository); + struct index_state truncated = INDEX_STATE_INIT(the_repository); + struct untracked_cache untracked = { 0 }; + struct untracked_cache_dir root = { 0 }; + struct strbuf encoded = STRBUF_INIT; + struct strbuf written = STRBUF_INIT; + uint32_t version; + + put_be32(&version, 1); + strbuf_add(&encoded, &version, sizeof(version)); + strbuf_addstr(&encoded, "token"); + strbuf_addch(&encoded, '\0'); + read_fsmonitor_untracked_extension( + &duplicate, encoded.buf, encoded.len); + if (duplicate.fsmonitor_untracked_extension_invalid || + !duplicate.fsmonitor_untracked_token || + strcmp(duplicate.fsmonitor_untracked_token, "token")) + return error("valid FSUC was not published"); + + duplicate.fsmonitor_last_update = xstrdup("token"); + write_fsmonitor_untracked_extension(&written, &duplicate); + if (written.len != encoded.len || + memcmp(written.buf, encoded.buf, encoded.len)) + return error("FSUC did not round-trip"); + duplicate.fsmonitor_token_valid = 1; + duplicate.untracked = &untracked; + untracked.root = &root; + prepare_fsmonitor_untracked(&duplicate); + if (!duplicate.fsmonitor_untracked_valid) + return error("matching FSMN and FSUC tokens were not paired"); + free(duplicate.fsmonitor_last_update); + duplicate.fsmonitor_last_update = xstrdup("other"); + prepare_fsmonitor_untracked(&duplicate); + if (duplicate.fsmonitor_untracked_valid) + return error("mismatched FSMN and FSUC tokens were paired"); + read_fsmonitor_untracked_extension( + &duplicate, encoded.buf, encoded.len); + if (!fsuc_failed_closed(&duplicate)) + return error("duplicate FSUC did not fail closed"); + + truncated.fsmonitor_untracked_token = xstrdup("old"); + read_fsmonitor_untracked_extension( + &truncated, encoded.buf, sizeof(version)); + if (!fsuc_failed_closed(&truncated)) + return error("truncated FSUC was partially published"); + + free(duplicate.fsmonitor_last_update); + strbuf_release(&written); + strbuf_release(&encoded); + return 0; +} + +static void wrap_fsmn_ewah(struct strbuf *out, const struct strbuf *ewah) +{ + uint32_t value; + + put_be32(&value, 2); + strbuf_add(out, &value, sizeof(value)); + strbuf_addstr(out, "token"); + strbuf_addch(out, '\0'); + put_be32(&value, ewah->len); + strbuf_add(out, &value, sizeof(value)); + strbuf_addbuf(out, ewah); +} + +static void make_valid_fsmn(struct strbuf *out) +{ + struct ewah_bitmap *dirty = ewah_new(); + struct strbuf ewah = STRBUF_INIT; + + ewah_set(dirty, 0); + ewah_serialize_strbuf(dirty, &ewah); + wrap_fsmn_ewah(out, &ewah); + ewah_free(dirty); + strbuf_release(&ewah); +} + +static void make_raw_fsmn(struct strbuf *out, uint32_t bit_size, + const eword_t *words, uint32_t word_count, + uint32_t rlw) +{ + struct strbuf ewah = STRBUF_INIT; + uint32_t value; + uint32_t i; + + put_be32(&value, bit_size); + strbuf_add(&ewah, &value, sizeof(value)); + put_be32(&value, word_count); + strbuf_add(&ewah, &value, sizeof(value)); + for (i = 0; i < word_count; i++) { + eword_t word = htonll(words[i]); + + strbuf_add(&ewah, &word, sizeof(word)); + } + put_be32(&value, rlw); + strbuf_add(&ewah, &value, sizeof(value)); + wrap_fsmn_ewah(out, &ewah); + strbuf_release(&ewah); +} + +static int fsmn_failed_closed(const struct index_state *istate) +{ + return istate->fsmonitor_extension_seen && + !istate->fsmonitor_last_update && !istate->fsmonitor_dirty && + !istate->fsmonitor_token_valid; +} + +static int check_invalid_fsmn(const struct strbuf *encoded, + const char *description) +{ + struct index_state invalid = INDEX_STATE_INIT(the_repository); + + invalid.cache_nr = 1; + invalid.fsmonitor_last_update = xstrdup("old"); + invalid.fsmonitor_dirty = ewah_new(); + invalid.fsmonitor_token_valid = 1; + read_fsmonitor_extension(&invalid, encoded->buf, encoded->len); + if (!fsmn_failed_closed(&invalid)) + return error("%s FSMN was published", description); + return 0; +} + +static int test_fsmn_parser(void) +{ + struct index_state duplicate = INDEX_STATE_INIT(the_repository); + struct index_state truncated = INDEX_STATE_INIT(the_repository); + struct strbuf encoded = STRBUF_INIT; + struct strbuf malformed = STRBUF_INIT; + eword_t words[2] = { 0 }; + + duplicate.cache_nr = truncated.cache_nr = 1; + make_valid_fsmn(&encoded); + read_fsmonitor_extension(&duplicate, encoded.buf, encoded.len); + if (!duplicate.fsmonitor_token_valid || + !duplicate.fsmonitor_last_update || + strcmp(duplicate.fsmonitor_last_update, "token") || + !duplicate.fsmonitor_dirty) + return error("valid FSMN was not published"); + read_fsmonitor_extension(&duplicate, encoded.buf, encoded.len); + if (!fsmn_failed_closed(&duplicate)) + return error("duplicate FSMN did not fail closed"); + + truncated.fsmonitor_last_update = xstrdup("old"); + truncated.fsmonitor_dirty = ewah_new(); + truncated.fsmonitor_token_valid = 1; + read_fsmonitor_extension(&truncated, encoded.buf, encoded.len - 1); + if (!fsmn_failed_closed(&truncated)) + return error("truncated FSMN was partially published"); + + rlw_set_literal_words(&words[0], 1); + make_raw_fsmn(&malformed, 1, words, 1, 0); + if (check_invalid_fsmn(&malformed, "out-of-bounds literal")) + return 1; + strbuf_reset(&malformed); + + words[0] = 0; + rlw_set_run_bit(&words[0], 1); + rlw_set_running_len(&words[0], 1); + make_raw_fsmn(&malformed, 1, words, 1, 0); + if (check_invalid_fsmn(&malformed, "oversized set-bit run")) + return 1; + strbuf_reset(&malformed); + + words[0] = words[1] = 0; + rlw_set_literal_words(&words[0], 1); + words[1] = 2; + make_raw_fsmn(&malformed, 1, words, 2, 0); + if (check_invalid_fsmn(&malformed, "set padding bit")) + return 1; + strbuf_reset(&malformed); + + words[1] = 1; + make_raw_fsmn(&malformed, 1, words, 2, 1); + if (check_invalid_fsmn(&malformed, "non-final RLW")) + return 1; + + strbuf_release(&malformed); + strbuf_release(&encoded); + return 0; +} + +static int write_test_index(void) +{ + struct lock_file index_lock = LOCK_INIT; + + repo_hold_locked_index(the_repository, &index_lock, LOCK_DIE_ON_ERROR); + if (write_locked_index(the_repository->index, &index_lock, COMMIT_LOCK)) + return error("unable to write test index"); + return 0; +} + +static int test_fscf_history_is_coherent(const struct index_state *istate) +{ + const struct clean_status_state *state = istate->clean_status; + + return state && state->disk_config_valid && + !state->disk_config_invalid && state->disk_semantic_valid && + state->disk_attr_valid && state->manifest.disk_valid && + (state->manifest.disk_flags & FSMONITOR_CLEAN_PROOF_ALL) == + FSMONITOR_CLEAN_PROOF_ALL && + state->disk_config_raw.len && state->initial_coherent; +} + +static int test_fscf_config(const char *key, const char *value, + const struct config_context *ctx, void *cb) +{ + struct clean_status_config_digest *config = cb; + + clean_status_config_add(config, key, value, ctx); + return git_default_config(key, value, ctx, NULL); +} + +static int test_fscf_history(void) +{ + struct clean_status_config_digest config; + struct attr_fingerprint attrs; + struct attr_manifest_writer writer; + struct strbuf manifest = STRBUF_INIT; + struct strbuf encoded = STRBUF_INIT; + unsigned char index_hash[GIT_MAX_RAWSZ] = { 0 }; + const char *token; + struct fsmonitor_clean_proof proof = { + .flags = FSMONITOR_CLEAN_PROOF_ALL, + }; + const struct git_hash_algo *algo; + int ret = 1; + + setup_git_directory(the_repository); + algo = the_repository->hash_algo; + clean_status_config_init(&config, algo); + repo_config(the_repository, test_fscf_config, &config); + clean_status_config_final(&config); + clean_status_set_config_digest(the_repository, &config); + if (repo_read_index(the_repository) < 0) + return error("unable to read test index"); + token = "fscf-test-token"; + if (attr_fingerprint_repository(the_repository, &attrs)) + return error("unable to fingerprint attribute sources"); + + attr_manifest_writer_init(&writer, &manifest, algo); + if (attr_manifest_writer_add(&writer, ".gitattributes", + ATTR_MANIFEST_INDEX, index_hash)) + return error("unable to write test attribute manifest"); + proof.config_hash = config.hash; + proof.semantic_hash = config.semantic_hash; + proof.attr_hash = attrs.content_hash; + proof.token = (const unsigned char *)token; + proof.token_len = strlen(token); + proof.attr_manifest = (const unsigned char *)manifest.buf; + proof.attr_manifest_len = manifest.len; + if (fsmonitor_clean_proof_write(&encoded, &proof, algo)) + return error("unable to write test clean proof"); + + FREE_AND_NULL(the_repository->index->fsmonitor_last_update); + the_repository->index->fsmonitor_last_update = xstrdup(token); + the_repository->index->fsmonitor_token_valid = 1; + clean_status_read_fsmonitor_config(the_repository->index, + encoded.buf, encoded.len); + clean_status_prepare_fsmonitor_config(the_repository->index); + if (!test_fscf_history_is_coherent(the_repository->index)) + return error("test clean proof was not coherent"); + if (write_test_index()) + goto done; + + discard_index(the_repository->index); + if (repo_read_index(the_repository) < 0) + return error("unable to reread test index"); + if (!test_fscf_history_is_coherent(the_repository->index)) + return error("FSCF did not survive an index round trip"); + + clean_status_invalidate_current_manifest(the_repository->index); + if (write_test_index()) + goto done; + discard_index(the_repository->index); + if (repo_read_index(the_repository) < 0) + return error("unable to reread preserved test index"); + if (clean_status_has_persistent_fsmonitor_semantic_history( + the_repository->index)) + return error("generic rewrite retained FSCF epoch bindings"); + if (!clean_status_has_worktree_manifest_history(the_repository->index)) + return error("generic rewrite discarded FSCF manifest history"); + ret = 0; + +done: + strbuf_release(&encoded); + strbuf_release(&manifest); + return ret; +} + +static int test_fsmonitor_directory_attributes(void) +{ + struct attr_check *check; + int ret = 1; + + setup_git_directory(the_repository); + repo_config(the_repository, git_default_config, NULL); + if (repo_read_index(the_repository) < 0) + return error("unable to read test index"); + + check = attr_check_initl("marker", NULL); + git_check_attr(the_repository->index, "tracked-dir/tracked", check); + if (!check->items[0].value || + strcmp(check->items[0].value, "old")) { + error("initial attribute value was not cached"); + goto done; + } + + write_file("tracked-dir/.gitattributes", "tracked marker=new\n"); + /* + * repo_read_index() consumed the normal refresh. Re-arm it after + * caching the pre-event attribute value. + */ + the_repository->index->fsmonitor_has_run_once = 0; + refresh_fsmonitor(the_repository->index); + git_check_attr(the_repository->index, "tracked-dir/tracked", check); + if (!check->items[0].value || + strcmp(check->items[0].value, "new")) { + error("directory event did not invalidate cached attributes"); + goto done; + } + ret = 0; + +done: + attr_check_free(check); + discard_index(the_repository->index); + return ret; +} int cmd__read_cache(int argc, const char **argv) { int i, cnt = 1; const char *name = NULL; + if (argc == 3 && + !strcmp(argv[1], "--test-fsmonitor-content-recovery")) + return test_fsmonitor_content_recovery(argv[2]); + if (argc == 2 && !strcmp(argv[1], "--test-fsuc-parser")) + return test_fsuc_parser(); + if (argc == 2 && !strcmp(argv[1], "--test-fsmn-parser")) + return test_fsmn_parser(); + if (argc == 2 && !strcmp(argv[1], "--test-fscf-round-trip")) + return test_fscf_history(); + if (argc == 2 && + !strcmp(argv[1], "--test-fsmonitor-directory-attributes")) + return test_fsmonitor_directory_attributes(); + if (argc > 1 && skip_prefix(argv[1], "--print-and-refresh=", &name)) { argc--; argv++; diff --git a/t/helper/test-semantic-verify.c b/t/helper/test-semantic-verify.c new file mode 100644 index 00000000000000..9e0926633607c0 --- /dev/null +++ b/t/helper/test-semantic-verify.c @@ -0,0 +1,140 @@ +#define USE_THE_REPOSITORY_VARIABLE + +#include "test-tool.h" +#include "config.h" +#include "parse-options.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "semantic-verify.h" +#include "setup.h" + +static const char *kind_name(enum semantic_verify_kind kind) +{ + switch (kind) { + case SEMANTIC_VERIFY_UNCHECKED: + return "unchecked"; + case SEMANTIC_VERIFY_SKIPPED: + return "skipped"; + case SEMANTIC_VERIFY_RAW_CLEAN: + return "raw-clean"; + case SEMANTIC_VERIFY_RAW_MODIFIED: + return "raw-modified"; + case SEMANTIC_VERIFY_SENSITIVE: + return "sensitive"; + case SEMANTIC_VERIFY_STRUCTURAL: + return "structural"; + case SEMANTIC_VERIFY_UNSTABLE: + return "unstable"; + case SEMANTIC_VERIFY_ERROR: + return "error"; + } + BUG("unknown semantic verification kind"); +} + +int cmd__semantic_verify(int argc, const char **argv) +{ + struct semantic_verify_options options = SEMANTIC_VERIFY_OPTIONS_INIT; + struct semantic_verify_proof *proof = NULL; + struct semantic_verify_stats stats; + int thread_count = 0; + int show_results = 0; + int apply = 0; + int applied = -2; + int before_uptodate = 0, after_uptodate = 0; + int before_valid = 0, after_valid = 0; + int ret; + const char *replace_after_prepare = NULL; + const char * const usage[] = { + "test-tool semantic-verify []", + NULL + }; + struct option opts[] = { + OPT_INTEGER(0, "threads", &thread_count, + "number of verifier threads"), + OPT_BOOL(0, "show-results", &show_results, + "show one result per cache entry"), + OPT_BOOL(0, "apply", &apply, "apply the completed proof"), + OPT_STRING(0, "replace-after-prepare", &replace_after_prepare, + "path", "replace an entry after preparing the proof"), + OPT_END() + }; + + argc = parse_options(argc, argv, NULL, opts, usage, 0); + if (argc) + usage_with_options(usage, opts); + if (thread_count < 0) + die("negative semantic verifier thread count"); + options.nr_threads = thread_count; + + setup_git_directory(the_repository); + repo_config(the_repository, git_default_config, NULL); + prepare_repo_settings(the_repository); + the_repository->settings.command_requires_full_index = 0; + if (repo_read_index(the_repository) < 0) + die("unable to read index"); + ret = semantic_verify_prepare(the_repository->index, &options, &proof); + semantic_verify_get_stats(proof, &stats); + if (replace_after_prepare) { + struct index_state *istate = the_repository->index; + struct cache_entry *replacement; + int pos = index_name_pos(istate, replace_after_prepare, + strlen(replace_after_prepare)); + + if (pos < 0) + die("%s not in index", replace_after_prepare); + replacement = dup_cache_entry(istate->cache[pos], istate); + replacement->oid.hash[0] ^= 1; + replacement->ce_flags &= + ~(CE_UPTODATE | CE_FSMONITOR_VALID); + if (add_index_entry(istate, replacement, + ADD_CACHE_OK_TO_REPLACE | + ADD_CACHE_KEEP_CACHE_TREE)) + die("unable to replace %s", replace_after_prepare); + } + for (size_t i = 0; i < stats.cache_nr; i++) { + struct cache_entry *ce = the_repository->index->cache[i]; + + before_uptodate += !!ce_uptodate(ce); + before_valid += !!(ce->ce_flags & CE_FSMONITOR_VALID); + } + if (show_results) { + for (size_t i = 0; i < stats.cache_nr; i++) { + const struct semantic_verify_result *result = + semantic_verify_result_at(proof, i); + + printf("%s %s persist=%d error=%u\n", + the_repository->index->cache[i]->name, + kind_name(result->kind), + !!(result->flags & SEMANTIC_VERIFY_PERSISTABLE), + result->error); + } + } + if (apply) + applied = semantic_verify_apply_after_closure( + the_repository->index, proof); + for (size_t i = 0; i < stats.cache_nr; i++) { + struct cache_entry *ce = the_repository->index->cache[i]; + + after_uptodate += !!ce_uptodate(ce); + after_valid += !!(ce->ce_flags & CE_FSMONITOR_VALID); + } + printf("entries=%"PRIuMAX" clean=%"PRIuMAX + " modified=%"PRIuMAX" sensitive=%"PRIuMAX + " structural=%"PRIuMAX" unstable=%"PRIuMAX + " errors=%"PRIuMAX" hardlinks=%"PRIuMAX + " bytes=%"PRIuMAX" stat_updates=%"PRIuMAX + " root_stable=%d namespace_stable=%d applied=%d" + " before_uptodate=%d before_valid=%d" + " after_uptodate=%d after_valid=%d\n", + (uintmax_t)stats.cache_nr, (uintmax_t)stats.raw_clean, + (uintmax_t)stats.raw_modified, (uintmax_t)stats.sensitive, + (uintmax_t)stats.structural, (uintmax_t)stats.unstable, + (uintmax_t)stats.errors, (uintmax_t)stats.hardlinks, + (uintmax_t)stats.bytes_hashed, + (uintmax_t)stats.stat_updates_nr, + semantic_verify_root_is_stable(proof), + !stats.namespace_unstable, applied, + before_uptodate, before_valid, after_uptodate, after_valid); + semantic_verify_proof_clear(proof); + return !!ret; +} diff --git a/t/helper/test-simple-ipc.c b/t/helper/test-simple-ipc.c index 442ad6b16f18d8..3be92e4fbd01ca 100644 --- a/t/helper/test-simple-ipc.c +++ b/t/helper/test-simple-ipc.c @@ -159,9 +159,40 @@ static int app__sendbytes_command(const char *received, size_t received_len, * data is handled properly. */ static int my_app_data = 42; +static int fsmonitor_legacy; +static int fsmonitor_capability_superset; static ipc_server_application_cb test_app_cb; +static int app__fsmonitor_capability_superset( + const char *command, size_t command_len, + ipc_server_reply_cb *reply_cb, + struct ipc_server_reply_data *reply_data) +{ + static const char capability_command[] = "get-capabilities"; + static const char capabilities[] = "query-v1\nquery-v2\n"; + static const char query_prefix[] = "query-v1 "; + static const char token[] = "builtin:test-capable:0"; + const char *query; + size_t query_len; + int ret; + + if (command_len == sizeof(capability_command) - 1 && + !memcmp(command, capability_command, command_len)) + return reply_cb(reply_data, capabilities, + sizeof(capabilities) - 1); + + query = memchr(command, '\n', command_len); + query_len = query ? command_len - (query + 1 - command) : 0; + ret = reply_cb(reply_data, token, sizeof(token)); + if (!ret && + (!starts_with(command, query_prefix) || + query_len != sizeof(token) - 1 || + memcmp(query + 1, token, query_len))) + ret = reply_cb(reply_data, "/", 2); + return ret; +} + /* * This is the "application callback" that sits on top of the * "ipc-server". It completely defines the set of commands supported @@ -201,6 +232,20 @@ static int test_app_cb(void *application_data, return SIMPLE_IPC_QUIT; } + if (fsmonitor_capability_superset) + return app__fsmonitor_capability_superset( + command, command_len, reply_cb, reply_data); + + if (fsmonitor_legacy) { + static const char token[] = "builtin:test-legacy:0"; + int ret; + + ret = reply_cb(reply_data, token, sizeof(token)); + if (!ret && !starts_with(command, "builtin:")) + ret = reply_cb(reply_data, "/", 2); + return ret; + } + if (command_len == 4 && !strncmp(command, "ping", 4)) { const char *answer = "pong"; return reply_cb(reply_data, answer, strlen(answer)); @@ -310,6 +355,10 @@ static int daemon__start_server(void) strvec_push(&cp.args, "run-daemon"); strvec_pushf(&cp.args, "--name=%s", cl_args.path); strvec_pushf(&cp.args, "--threads=%d", cl_args.nr_threads); + if (fsmonitor_legacy) + strvec_push(&cp.args, "--fsmonitor-legacy"); + if (fsmonitor_capability_superset) + strvec_push(&cp.args, "--fsmonitor-capability-superset"); cp.no_stdin = 1; cp.no_stdout = 1; @@ -602,6 +651,11 @@ int cmd__simple_ipc(int argc, const char **argv) OPT_INTEGER(0, "bytecount", &cl_args.bytecount, N_("number of bytes")), OPT_INTEGER(0, "batchsize", &cl_args.batchsize, N_("number of requests per thread")), + OPT_BOOL(0, "fsmonitor-legacy", &fsmonitor_legacy, + N_("emulate the legacy fsmonitor query protocol")), + OPT_BOOL(0, "fsmonitor-capability-superset", + &fsmonitor_capability_superset, + N_("advertise multiple fsmonitor query versions")), /* * The "byte" string here is not marked for translation and diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index b71a22b43bbc9e..5ccf3864beb235 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -70,6 +70,7 @@ static struct test_cmd cmds[] = { { "revision-walking", cmd__revision_walking }, { "run-command", cmd__run_command }, { "scrap-cache-tree", cmd__scrap_cache_tree }, + { "semantic-verify", cmd__semantic_verify }, { "serve-v2", cmd__serve_v2 }, { "sha1", cmd__sha1 }, { "sha1-is-sha1dc", cmd__sha1_is_sha1dc }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index f2885b33d58aa8..d1044198247ae4 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -63,6 +63,7 @@ int cmd__repository(int argc, const char **argv); int cmd__revision_walking(int argc, const char **argv); int cmd__run_command(int argc, const char **argv); int cmd__scrap_cache_tree(int argc, const char **argv); +int cmd__semantic_verify(int argc, const char **argv); int cmd__serve_v2(int argc, const char **argv); int cmd__sha1(int argc, const char **argv); int cmd__sha1_is_sha1dc(int argc, const char **argv); diff --git a/t/lib-semantic-verify.sh b/t/lib-semantic-verify.sh new file mode 100644 index 00000000000000..b46fd064711daf --- /dev/null +++ b/t/lib-semantic-verify.sh @@ -0,0 +1,9 @@ +test_lazy_prereq SEMANTIC_VERIFY_ANCHORED_OPEN ' + test_create_repo semantic-anchored-open-probe && + test_commit -C semantic-anchored-open-probe base tracked && + ( + cd semantic-anchored-open-probe && + test-tool semantic-verify --show-results >actual && + test_grep "^tracked raw-clean " actual + ) +' diff --git a/t/meson.build b/t/meson.build index a25f37d2f5ae7d..6beb78851c887f 100644 --- a/t/meson.build +++ b/t/meson.build @@ -1,7 +1,20 @@ clar_test_suites = [ + 'unit-tests/u-attr-fingerprint.c', + 'unit-tests/u-attr-manifest.c', + 'unit-tests/u-clean-status-config.c', + 'unit-tests/u-clean-status-history.c', + 'unit-tests/u-clean-status-identity.c', + 'unit-tests/u-clean-status-index.c', + 'unit-tests/u-clean-status-manifest.c', + 'unit-tests/u-clean-status-sidecar.c', + 'unit-tests/u-clean-status-store.c', 'unit-tests/u-ctype.c', 'unit-tests/u-dir.c', 'unit-tests/u-example-decorate.c', + 'unit-tests/u-exclude-source-proof.c', + 'unit-tests/u-fsmonitor-attributes.c', + 'unit-tests/u-fsmonitor-clean-proof.c', + 'unit-tests/u-fsmonitor-response.c', 'unit-tests/u-hash.c', 'unit-tests/u-hashmap.c', 'unit-tests/u-list-objects-filter-options.c', @@ -10,6 +23,8 @@ clar_test_suites = [ 'unit-tests/u-oid-array.c', 'unit-tests/u-oidmap.c', 'unit-tests/u-oidtree.c', + 'unit-tests/u-path-namespace.c', + 'unit-tests/u-preload-index-bulk-darwin.c', 'unit-tests/u-prio-queue.c', 'unit-tests/u-reftable-basics.c', 'unit-tests/u-reftable-block.c', @@ -27,6 +42,7 @@ clar_test_suites = [ 'unit-tests/u-trailer.c', 'unit-tests/u-urlmatch-normalization.c', 'unit-tests/u-utf8-width.c', + 'unit-tests/u-worktree-attr-source.c', ] clar_sources = [ @@ -943,6 +959,10 @@ integration_tests = [ 't7526-commit-pathspec-file.sh', 't7527-builtin-fsmonitor.sh', 't7528-signed-commit-ssh.sh', + 't7529-preload-index-apfs.sh', + 't7530-status-clean-sidecar.sh', + 't7531-semantic-verify.sh', + 't7532-preload-index-linux.sh', 't7600-merge.sh', 't7601-merge-pull-config.sh', 't7602-merge-octopus-many.sh', diff --git a/t/t0008-ignores.sh b/t/t0008-ignores.sh index ed95faf3272e60..949897c36aa9b2 100755 --- a/t/t0008-ignores.sh +++ b/t/t0008-ignores.sh @@ -959,7 +959,7 @@ test_expect_success EXPENSIVE 'large exclude file ignored in tree' ' test_when_finished "rm .gitignore" && find . -name .gitignore -exec rm "{}" ";" && dd if=/dev/zero of=.gitignore bs=101M count=1 && - git ls-files -o --exclude-standard 2>err && + GIT_ALLOC_LIMIT=1m git ls-files -o --exclude-standard 2>err && echo "warning: ignoring excessively large pattern file: .gitignore" >expect && test_cmp expect err ' diff --git a/t/t5331-pack-objects-stdin.sh b/t/t5331-pack-objects-stdin.sh index c74b5861af322f..2a983e28ac43e7 100755 --- a/t/t5331-pack-objects-stdin.sh +++ b/t/t5331-pack-objects-stdin.sh @@ -368,7 +368,8 @@ test_expect_success '--stdin-packs does not perform backfill fetch' ' git -C remote config set --local uploadpack.allowfilter 1 && git -C remote config set --local uploadpack.allowanysha1inwant 1 && - git clone --filter=tree:0 "file://$(pwd)/remote" client && + git -c maintenance.auto=false clone --filter=tree:0 \ + "file://$(pwd)/remote" client && ( cd client && ls .git/objects/pack/*.promisor | sed "s|.*/||; s/\.promisor$/.pack/" >packs && diff --git a/t/t7063-status-untracked-cache.sh b/t/t7063-status-untracked-cache.sh index 8929ef481f926c..1cf25b5088284e 100755 --- a/t/t7063-status-untracked-cache.sh +++ b/t/t7063-status-untracked-cache.sh @@ -991,4 +991,220 @@ test_expect_success 'empty repo (no index) and core.untrackedCache' ' git -C emptyrepo -c core.untrackedCache=true write-tree ' +test_expect_success 'directory snapshots ignore weak file-stat configuration' ' + test_create_repo weak-dir && + ( + cd weak-dir && + mkdir nested && + echo tracked >tracked && + echo one >nested/one && + git add tracked && + git commit -m base && + git config core.untrackedCache true && + git config core.fsmonitor false && + git config core.trustCtime false && + git config core.checkStat minimal && + avoid_racy && + git status --porcelain -uall >/dev/null && + git status --porcelain -uall >/dev/null && + dir_mtime=$(test-tool chmtime --get nested) && + mv nested/one nested/two && + test-tool chmtime =$dir_mtime nested && + GIT_OPTIONAL_LOCKS=0 git -c core.untrackedCache=false \ + status --porcelain -uall >.git/expect && + git status --porcelain -uall >.git/actual && + test_cmp .git/expect .git/actual + ) +' + +test_expect_success 'automatic preload observes its directory threshold' ' + test_create_repo auto-preload-threshold && + ( + cd auto-preload-threshold && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor false && + sane_unset GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD && + for i in $(test_seq 1 1998) + do + mkdir "d$i" && + >"d$i/file" || return 1 + done && + git status --porcelain >/dev/null && + GIT_TRACE2_EVENT="$PWD/.git/below-threshold.trace" \ + git status --porcelain >/dev/null && + test_grep ! "preload_untracked_cache/automatic" \ + .git/below-threshold.trace && + + mkdir d1999 && + >d1999/file && + git status --porcelain >/dev/null && + GIT_TRACE2_EVENT="$PWD/.git/at-threshold.trace" \ + git status --porcelain >/dev/null && + test_grep \ + "preload_untracked_cache/automatic.*value.*1" \ + .git/at-threshold.trace + ) +' + +test_expect_success 'preload verifies cached per-directory excludes' ' + test_create_repo auto-exclude && + ( + cd auto-exclude && + test_write_lines hide-a >.gitignore && + test_write_lines tracked >tracked && + git add .gitignore tracked && + git commit -m base && + git config core.untrackedCache true && + git config core.fsmonitor false && + test_write_lines a >hide-a && + test_write_lines b >hide-b && + git status --porcelain >/dev/null && + git status --porcelain >/dev/null && + avoid_racy && + mtime=$(test-tool chmtime --get .gitignore) && + test_write_lines hide-b >.gitignore && + test-tool chmtime =$mtime .gitignore && + GIT_OPTIONAL_LOCKS=0 git -c core.untrackedCache=false \ + status --porcelain >.git/expect && + GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD=1 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=4 \ + GIT_TRACE2_EVENT="$PWD/.git/actual.trace" \ + git status --porcelain >.git/actual && + test_cmp .git/expect .git/actual && + test_grep "preload_untracked_cache/valid.*value.*0" \ + .git/actual.trace + ) +' + +test_expect_success 'recursive preload checks descendant directory mtimes' ' + test_create_repo recursive-preload && + ( + cd recursive-preload && + mkdir -p a/b && + echo tracked >a/b/tracked && + git add a/b/tracked && + git commit -m base && + git config core.untrackedCache true && + git config core.fsmonitor false && + git status --porcelain >/dev/null && + git status --porcelain >/dev/null && + avoid_racy && + git status --porcelain >/dev/null && + git status --porcelain >/dev/null && + GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD=1 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=4 \ + GIT_TRACE2_EVENT="$PWD/.git/pruned.trace" \ + git status --porcelain >.git/pruned && + test_must_be_empty .git/pruned && + test_grep \ + "directories-visited.*value.*0" \ + .git/pruned.trace && + avoid_racy && + echo untracked >a/b/new-untracked && + GIT_OPTIONAL_LOCKS=0 git -c core.untrackedCache=false \ + status --porcelain >.git/expect && + GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD=1 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=4 \ + git status --porcelain >.git/actual && + test_cmp .git/expect .git/actual + ) +' + +test_expect_success 'recursive preload rescans a vanished collapsed witness' ' + test_create_repo collapsed-witness && + ( + cd collapsed-witness && + test_write_lines "*.ignored" >.gitignore && + git add .gitignore && + git commit -m base && + git config core.untrackedCache true && + git config core.fsmonitor false && + for i in 00 01 + do + mkdir -p "scratch/d$i" && + test_write_lines "$i" >"scratch/d$i/file" || return 1 + done && + echo "?? scratch/" >.git/expect && + git status --porcelain >/dev/null && + git status --porcelain >/dev/null && + test-tool dump-untracked-cache >.git/cache && + witness=$(sed -n \ + "s#^/scratch/\\(d[0-9][0-9]*\\)/ .*#\\1#p" \ + .git/cache | sed -n 1p) && + test -n "$witness" && + avoid_racy && + rm "scratch/$witness/file" && + GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD=1 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=4 \ + git status --porcelain >.git/actual && + test_cmp .git/expect .git/actual + ) +' + +test_expect_success 'status preloads cached-directory validation' ' + test_create_repo auto-preload && + ( + cd auto-preload && + mkdir -p nested/deep && + echo tracked >nested/tracked && + git add nested/tracked && + git commit -m base && + git config core.untrackedCache true && + git config core.fsmonitor false && + echo visible >nested/deep/visible && + git -c core.untrackedCache=false status --porcelain \ + >.git/expect && + avoid_racy && + git status --porcelain >/dev/null && + git status --porcelain >/dev/null && + + GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD=1 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=3 \ + GIT_TRACE2_EVENT="$PWD/.git/normal.trace" \ + git status --porcelain >.git/actual && + test_cmp .git/expect .git/actual && + if test_have_prereq PTHREADS + then + expect_threads=3 + else + expect_threads=1 + fi && + test_grep \ + "preload_untracked_cache/threads.*value.*$expect_threads" \ + .git/normal.trace && + test_grep "preload_untracked_cache/valid.*value.*1" \ + .git/normal.trace && + test_grep "opendir.*value.*0" .git/normal.trace && + echo changed >nested/deep/changed && + GIT_OPTIONAL_LOCKS=0 git -c core.untrackedCache=false \ + status --porcelain >.git/expect-changed && + GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD=1 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=3 \ + GIT_TRACE2_EVENT="$PWD/.git/changed.trace" \ + git status --porcelain >.git/actual-changed && + test_cmp .git/expect-changed .git/actual-changed && + test_grep "preload_untracked_cache/valid.*value.*0" \ + .git/changed.trace && + test_grep "opendir.*value.*[1-9][0-9]*" \ + .git/changed.trace && + + GIT_OPTIONAL_LOCKS=0 git -c core.untrackedCache=false \ + status --porcelain -- nested >.git/expect-pathspec && + GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD=1 \ + GIT_TRACE2_EVENT="$PWD/.git/pathspec.trace" \ + git status --porcelain -- nested >.git/actual-pathspec && + test_cmp .git/expect-pathspec .git/actual-pathspec && + test_grep ! 'preload_untracked_cache/threads' \ + .git/pathspec.trace && + GIT_OPTIONAL_LOCKS=0 git -c core.untrackedCache=false \ + status --porcelain -uall >.git/expect-uall && + GIT_TEST_UNTRACKED_CACHE_AUTO_PRELOAD=1 \ + GIT_TRACE2_EVENT="$PWD/.git/uall.trace" \ + git status --porcelain -uall >.git/actual-uall && + test_cmp .git/expect-uall .git/actual-uall && + test_grep ! 'preload_untracked_cache/threads' .git/uall.trace + ) +' + test_done diff --git a/t/t7508-status.sh b/t/t7508-status.sh index 0fd7c79911572e..8059c64940f165 100755 --- a/t/t7508-status.sh +++ b/t/t7508-status.sh @@ -1789,4 +1789,44 @@ test_expect_success EXPENSIVE,SIZE_T_IS_64BIT 'status does not re-read unchanged ) ' +test_expect_success 'status uses only a matching effective cache-tree' ' + test_when_finished "rm -rf cache-tree-status" && + test_create_repo cache-tree-status && + ( + cd cache-tree-status && + sane_unset GIT_TEST_SPLIT_INDEX && + test_write_lines base >tracked && + git add tracked && + git commit -m base && + + GIT_TRACE2_EVENT="$PWD/.git/clean.trace" \ + git status >.git/clean && + test_grep "nothing to commit, working tree clean" \ + .git/clean && + test_trace2_data status index/cache-tree-match 1 \ + <.git/clean.trace && + + test_write_lines staged >tracked && + git add tracked && + git write-tree >.git/staged-tree && + GIT_TRACE2_EVENT="$PWD/.git/staged.trace" \ + git status >.git/staged && + test_grep "Changes to be committed:" .git/staged && + test_grep "modified:.*tracked" .git/staged && + ! test_trace2_data status index/cache-tree-match 1 \ + <.git/staged.trace && + + replacement_tree=$(cat .git/staged-tree) && + git reset --hard HEAD && + head_tree=$(git rev-parse HEAD^{tree}) && + git replace "$head_tree" "$replacement_tree" && + GIT_TRACE2_EVENT="$PWD/.git/replaced.trace" \ + git status >.git/replaced && + test_grep "Changes to be committed:" .git/replaced && + test_grep "modified:.*tracked" .git/replaced && + ! test_trace2_data status index/cache-tree-match 1 \ + <.git/replaced.trace + ) +' + test_done diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 93973ed25a448b..e6eb39f5c93b6b 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -3,6 +3,7 @@ test_description='git status with file system watcher' . ./test-lib.sh +. "$TEST_DIRECTORY"/lib-semantic-verify.sh # Note, after "git reset --hard HEAD" no extensions exist other than 'TREE' # "git update-index --fsmonitor" can be used to get the extension written @@ -55,6 +56,809 @@ test_lazy_prereq UNTRACKED_CACHE ' test $ret -ne 1 ' +test_lazy_prereq HARDLINKS ' + : >hardlink-a && + ln hardlink-a hardlink-b +' + +test_expect_success 'FSMN parser fails closed' ' + test-tool read-cache --test-fsmn-parser +' + +test_expect_success 'FSUC parser fails closed' ' + test-tool read-cache --test-fsuc-parser +' + +test_expect_success !SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'unsupported identity preserves an ordinary provider token' ' + test_when_finished "rm -rf unsupported-provider-token" && + test_create_repo unsupported-provider-token && + ( + cd unsupported-provider-token && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual && + test-tool dump-fsmonitor >.git/fsmonitor && + test_grep "^fsmonitor last update builtin:test:" \ + .git/fsmonitor && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor semantic/strong-invalidation 1 \ + <.git/status.trace + ) +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'FSCF survives index I/O and generic rewrites' ' + test_when_finished "rm -rf fscf-round-trip" && + test_create_repo fscf-round-trip && + ( + cd fscf-round-trip && + test_commit base tracked && + test-tool read-cache --test-fscf-round-trip && + test_grep FSCF .git/index + ) +' + +test_expect_success 'hook parser ignores empty path records' ' + test_when_finished "rm -rf empty-hook-record" && + test_create_repo empty-hook-record && + ( + cd empty-hook-record && + test_commit base tracked && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token\0" + printf "\0" + printf "tracked\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + echo changed >tracked && + git status --porcelain --untracked-files=no >actual && + echo " M tracked" >expect && + test_cmp expect actual + ) +' + +test_expect_success UNTRACKED_CACHE 'trivial hook clears a paired UNTR token' ' + test_when_finished "rm -rf hook-token-pair" && + test_create_repo hook-token-pair && + ( + cd hook-token-pair && + test_commit base tracked && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token1\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + test_grep ! FSUC .git/index && + git status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual && + test_grep FSUC .git/index && + test_hook --clobber fsmonitor-test <<-\EOF && + printf "token2\0/\0" + EOF + git status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual && + test_grep ! FSUC .git/index + ) +' + +test_expect_success UNTRACKED_CACHE 'failed hook clears a paired UNTR token' ' + test_when_finished "rm -rf hook-token-error" && + test_create_repo hook-token-error && + ( + cd hook-token-error && + test_commit base tracked && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain=v2 >/dev/null && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token1\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual && + test_grep FSUC .git/index && + test_hook --clobber fsmonitor-test <<-\EOF && + exit 1 + EOF + git status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual && + test_grep ! FSUC .git/index + ) +' + +test_expect_success UNTRACKED_CACHE \ + 'paired fsmonitor cache prunes recursively valid empty subtrees' ' + test_when_finished "rm -rf fsmonitor-untracked-prune" && + test_create_repo fsmonitor-untracked-prune && + ( + cd fsmonitor-untracked-prune && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir -p cached/empty/deep && + test_write_lines tracked >cached/empty/deep/tracked && + git add cached/empty/deep/tracked && + git commit -m base && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain=v2 \ + >.git/prime && + test_must_be_empty .git/prime && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime-fsmonitor && + test_must_be_empty .git/prime-fsmonitor && + test_grep FSUC .git/index && + + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/clean.trace" \ + git status --porcelain=v2 >.git/clean && + test_must_be_empty .git/clean && + test_trace2_data read_directory directories-visited 0 \ + <.git/clean.trace && + + test_write_lines untracked >cached/empty/deep/new && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=cached/empty/deep/new \ + git status --porcelain=v2 >.git/changed && + test_grep "^? cached/empty/deep/new$" .git/changed + ) +' + +test_expect_success UNTRACKED_CACHE,HARDLINKS \ + 'fsmonitor pruning rechecks cached per-directory excludes' ' + test_when_finished "rm -rf fsmonitor-untracked-exclude" && + test_when_finished "rm -f fsmonitor-untracked-exclude-alias" && + test_create_repo fsmonitor-untracked-exclude && + ( + cd fsmonitor-untracked-exclude && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir cached cached2 cached3 && + test_write_lines ignored >cached/.gitignore && + test_write_lines hidden >cached/ignored && + test_write_lines ignored >cached2/.gitignore && + test_write_lines hidden >cached2/ignored && + test_write_lines ignored >cached3/.gitignore && + test_write_lines hidden >cached3/ignored && + git add cached/.gitignore cached2/.gitignore \ + cached3/.gitignore && + git commit -m base && + test-tool chmtime +60 cached2/.gitignore && + test-tool chmtime =-60 cached3/.gitignore && + git update-index --refresh && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain=v2 \ + >.git/prime && + test_must_be_empty .git/prime && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime-fsmonitor && + test_must_be_empty .git/prime-fsmonitor && + test_grep FSUC .git/index && + ln cached/.gitignore ../fsmonitor-untracked-exclude-alias && + + if test_have_prereq PTHREADS + then + threads=2 + else + threads=1 + fi && + if test_have_prereq MINGW || test_have_prereq CYGWIN + then + index_excludes=0 + else + index_excludes=1 + fi && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=2 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT_NESTING=10 \ + GIT_TRACE2_EVENT="$PWD/.git/clean.trace" \ + git status --porcelain >.git/clean && + test_must_be_empty .git/clean && + test_trace2_data dir \ + preload_untracked_cache/fsmonitor-excludes-only 1 \ + <.git/clean.trace && + test_trace2_data dir preload_untracked_cache/threads \ + $threads \ + <.git/clean.trace && + test_trace2_data dir preload_untracked_cache/dirs 3 \ + <.git/clean.trace && + test_trace2_data dir \ + preload_untracked_cache/index-excludes "$index_excludes" \ + <.git/clean.trace && + test_trace2_data read_directory directories-visited 0 \ + <.git/clean.trace && + + if test_have_prereq FILEMODE + then + chmod +x ../fsmonitor-untracked-exclude-alias && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=2 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/mode.trace" \ + git status --porcelain >.git/mode && + test_trace2_data dir \ + preload_untracked_cache/index-uptodate 2 \ + <.git/mode.trace && + chmod -x ../fsmonitor-untracked-exclude-alias + else + : + fi && + + mtime=$(test-tool chmtime --get cached/.gitignore) && + test_write_lines visible \ + >../fsmonitor-untracked-exclude-alias && + test-tool chmtime =$mtime cached/.gitignore && + GIT_TEST_UNTRACKED_CACHE_THREADS=2 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/changed.trace" \ + git status >.git/changed && + test_grep "modified:.*cached/.gitignore" .git/changed && + test_grep "cached/ignored" .git/changed && + test_trace2_data status \ + fsmonitor/exclude-index-invalidated 1 \ + <.git/changed.trace && + + test_write_lines ignored \ + >../fsmonitor-untracked-exclude-alias && + test-tool chmtime =$mtime cached/.gitignore && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=2 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/restored.trace" \ + git status >.git/restored && + test_grep "nothing to commit, working tree clean" \ + .git/restored && + + test_write_lines "?? cached/ignored" >.git/flagged.expect && + + git update-index --assume-unchanged cached/.gitignore && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain >.git/assume-prime && + test_must_be_empty .git/assume-prime && + mtime=$(test-tool chmtime --get cached/.gitignore) && + test_write_lines visible \ + >../fsmonitor-untracked-exclude-alias && + test-tool chmtime =$mtime cached/.gitignore && + GIT_TEST_UNTRACKED_CACHE_THREADS=2 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/assume-changed.trace" \ + git status --porcelain >.git/assume-changed && + test_cmp .git/flagged.expect .git/assume-changed && + test_write_lines ignored \ + >../fsmonitor-untracked-exclude-alias && + test-tool chmtime =$mtime cached/.gitignore && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=2 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/assume-restored.trace" \ + git status --porcelain >.git/assume-restored && + test_must_be_empty .git/assume-restored && + git update-index --no-assume-unchanged cached/.gitignore && + + git update-index --skip-worktree cached/.gitignore && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain >.git/skip-prime && + test_must_be_empty .git/skip-prime && + mtime=$(test-tool chmtime --get cached/.gitignore) && + test_write_lines visible \ + >../fsmonitor-untracked-exclude-alias && + test-tool chmtime =$mtime cached/.gitignore && + GIT_TEST_UNTRACKED_CACHE_THREADS=2 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/skip-changed.trace" \ + git status --porcelain >.git/skip-changed && + test_cmp .git/flagged.expect .git/skip-changed && + test_write_lines ignored \ + >../fsmonitor-untracked-exclude-alias && + test-tool chmtime =$mtime cached/.gitignore && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_UNTRACKED_CACHE_THREADS=2 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/skip-restored.trace" \ + git status --porcelain >.git/skip-restored && + test_must_be_empty .git/skip-restored && + git update-index --no-skip-worktree cached/.gitignore + ) +' + +test_expect_success UNTRACKED_CACHE \ + 'converted cached excludes retain a stable index proof' ' + test_when_finished "rm -rf fsmonitor-converted-exclude" && + test_create_repo fsmonitor-converted-exclude && + ( + cd fsmonitor-converted-exclude && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir cached && + printf "ignored\r\n" >cached/.gitignore && + test_write_lines hidden >cached/ignored && + test_write_lines "cached/.gitignore text eol=lf" \ + >.gitattributes && + git add .gitattributes cached/.gitignore && + git commit -m base && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain >.git/prime && + test_must_be_empty .git/prime && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain >.git/prime-fsmonitor && + test_must_be_empty .git/prime-fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status >.git/settle && + test_grep "nothing to commit, working tree clean" \ + .git/settle && + + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/clean.trace" \ + git status >.git/clean && + test_grep "nothing to commit, working tree clean" \ + .git/clean && + test_trace2_data dir \ + preload_untracked_cache/index-uptodate 1 \ + <.git/clean.trace && + test_trace2_data dir \ + preload_untracked_cache/index-invalidated 0 \ + <.git/clean.trace && + test_trace2_data dir \ + preload_untracked_cache/normalized-excludes 0 \ + <.git/clean.trace && + test_trace2_data read_directory directories-visited 0 \ + <.git/clean.trace + ) +' + +test_expect_success UNTRACKED_CACHE,HARDLINKS,POSIXPERM,SANITY \ + 'fsmonitor rechecks cached unreadable per-directory excludes' ' + test_when_finished "rm -rf fsmonitor-unreadable-exclude" && + test_when_finished "rm -f fsmonitor-unreadable-exclude-alias" && + test_create_repo fsmonitor-unreadable-exclude && + ( + cd fsmonitor-unreadable-exclude && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir cached && + test_write_lines hidden >cached/.gitignore && + test_write_lines untracked >cached/hidden && + test_write_lines tracked >cached/tracked && + git add cached/tracked && + git commit -m base && + chmod 000 cached/.gitignore && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain \ + >.git/prime 2>.git/prime.err && + test_grep "^?? cached/hidden$" .git/prime && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain >.git/prime-fsmonitor \ + 2>.git/prime-fsmonitor.err && + test_grep "^?? cached/hidden$" .git/prime-fsmonitor && + empty_tree=$(git mktree .git/untracked-cache && + test_grep "cached/ $empty_tree" .git/untracked-cache && + ln cached/.gitignore \ + ../fsmonitor-unreadable-exclude-alias && + + chmod 644 ../fsmonitor-unreadable-exclude-alias && + test_write_lines "?? cached/.gitignore" \ + >.git/readable.expect && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/readable.trace" \ + git status --porcelain >.git/readable && + test_cmp .git/readable.expect .git/readable && + test_trace2_data dir \ + preload_untracked_cache/dirs "[1-9]" \ + <.git/readable.trace + ) +' + +check_weak_exclude_stat () { + repo=$1 && + key=$2 && + value=$3 && + test_create_repo "$repo" && + ( + cd "$repo" && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir cached && + test_write_lines ignored >cached/.gitignore && + test_write_lines hidden >cached/ignored && + git add cached/.gitignore && + git commit -m base && + git config "$key" "$value" && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain=v2 \ + >.git/prime && + test_must_be_empty .git/prime && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime-fsmonitor && + test_must_be_empty .git/prime-fsmonitor && + + mtime=$(test-tool chmtime --get cached/.gitignore) && + test_write_lines visible >cached/.gitignore && + test-tool chmtime =$mtime cached/.gitignore && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + GIT_TRACE2_EVENT="$PWD/.git/changed.trace" \ + git status --porcelain >.git/changed && + test_grep "^?? cached/ignored$" .git/changed && + test_trace2_data dir \ + preload_untracked_cache/index-excludes 0 \ + <.git/changed.trace + ) +} + +test_expect_success UNTRACKED_CACHE \ + 'weak stat settings retain exclude content checks' ' + test_when_finished "rm -rf weak-exclude-ctime weak-exclude-stat" && + check_weak_exclude_stat weak-exclude-ctime \ + core.trustctime false && + check_weak_exclude_stat weak-exclude-stat \ + core.checkStat minimal +' + +test_expect_success UNTRACKED_CACHE \ + 'root untracked events preserve cached descendant excludes' ' + test_when_finished "rm -rf root-untracked-event" && + test_create_repo root-untracked-event && + ( + cd root-untracked-event && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir -p cached/deep && + test_write_lines "*.ignored" >.gitignore && + test_write_lines "*.ignored" >cached/.gitignore && + test_write_lines tracked >cached/deep/tracked && + test_write_lines ignored >cached/deep/junk.ignored && + git add .gitignore cached/.gitignore cached/deep/tracked && + git commit -m base && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain=v2 \ + >.git/prime && + test_must_be_empty .git/prime && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime-fsmonitor && + test_must_be_empty .git/prime-fsmonitor && + + test_write_lines visible >root-probe && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=root-probe \ + GIT_TRACE2_EVENT="$PWD/.git/created.trace" \ + git status >.git/created && + test_grep "root-probe" .git/created && + test_trace2_data read_directory directories-visited 1 \ + <.git/created.trace && + test_trace2_data read_directory gitignore-invalidation 0 \ + <.git/created.trace && + + rm root-probe && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=root-probe \ + GIT_TRACE2_EVENT="$PWD/.git/removed.trace" \ + git status >.git/removed && + test_grep "nothing to commit, working tree clean" \ + .git/removed && + test_trace2_data read_directory directories-visited 1 \ + <.git/removed.trace && + test_trace2_data read_directory gitignore-invalidation 0 \ + <.git/removed.trace + ) +' + +prepare_builtin_closure_repo () { + test_create_repo "$1" && + ( + cd "$1" && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + if test "${2-}" = untracked + then + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain=v2 \ + >.git/actual && + test_must_be_empty .git/actual && + test_grep UNTR .git/index && + test_grep ! FSUC .git/index + else + : + fi && + git config core.fsmonitor true && + test_grep ! FSMN .git/index + ) +} + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'bare status reuses a current tracked fsmonitor proof' ' + test_when_finished "rm -rf builtin-tracked-clean" && + prepare_builtin_closure_repo builtin-tracked-clean && + ( + cd builtin-tracked-clean && + sane_unset GIT_TEST_SPLIT_INDEX && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain >.git/prime && + test_must_be_empty .git/prime && + + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + GIT_TRACE2_EVENT="$PWD/.git/clean.trace" \ + git status >.git/clean && + test_grep "nothing to commit, working tree clean" \ + .git/clean && + test_trace2_data status fsmonitor/tracked-clean 1 \ + <.git/clean.trace && + test_trace2_data status index/cache-tree-match 1 \ + <.git/clean.trace && + + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/exact.trace" \ + git status --porcelain=v2 >.git/exact && + test_must_be_empty .git/exact && + ! test_trace2_data status fsmonitor/tracked-clean 1 \ + <.git/exact.trace && + test_grep \ + "\"category\":\"index\",\"label\":\"refresh\"" \ + .git/exact.trace && + + test_write_lines changed >tracked && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$PWD/.git/dirty.trace" \ + git status >.git/dirty && + test_grep "modified:.*tracked" .git/dirty && + ! test_trace2_data status fsmonitor/tracked-clean 1 \ + <.git/dirty.trace && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git checkout -- tracked && + test_write_lines staged >tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git add tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git write-tree >.git/staged-tree && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain >.git/staged-prime && + test_grep "^M tracked$" .git/staged-prime && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + GIT_TRACE2_EVENT="$PWD/.git/staged.trace" \ + git status >.git/staged && + test_grep "Changes to be committed:" .git/staged && + test_grep "modified:.*tracked" .git/staged && + test_trace2_data status fsmonitor/tracked-clean 1 \ + <.git/staged.trace && + ! test_trace2_data status index/cache-tree-match 1 \ + <.git/staged.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'builtin closure initializes a new untracked cache' ' + test_when_finished "rm -rf builtin-closure-new-uc" && + test_create_repo builtin-closure-new-uc && + ( + cd builtin-closure-new-uc && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + test_grep UNTR .git/index && + test_grep ! FSUC .git/index && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual && + test_grep \ + "\"event\":\"region_enter\".*\"category\":\"dir\",\"label\":\"read_directory\"" \ + .git/status.trace >.git/read-directory && + test_line_count = 1 .git/read-directory && + test_trace2_data fsmonitor semantic/manifest-scan-count 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep FSUC .git/index + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'builtin clean closure publishes its proof' ' + test_when_finished "rm -rf builtin-closure-clean" && + prepare_builtin_closure_repo builtin-closure-clean untracked && + ( + cd builtin-closure-clean && + sane_unset GIT_TEST_SPLIT_INDEX && + test_write_lines visible >visible && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_grep "^? visible$" .git/actual && + test_grep \ + "\"event\":\"region_enter\".*\"category\":\"dir\",\"label\":\"read_directory\"" \ + .git/status.trace >.git/read-directory && + test_line_count = 1 .git/read-directory && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep FSMN .git/index && + test_grep FSUC .git/index + ) +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'builtin changed closure rescans before acceptance' ' + test_when_finished "rm -rf builtin-closure-changed" && + prepare_builtin_closure_repo builtin-closure-changed && + ( + cd builtin-closure-changed && + sane_unset GIT_TEST_SPLIT_INDEX && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CDC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + GIT_TRACE_FSMONITOR="$PWD/.git/fsmonitor.trace" \ + git status --porcelain=v2 \ + --untracked-files=no >.git/actual && + test_must_be_empty .git/actual && + test_grep "fsmonitor_refresh_callback.*tracked" \ + .git/fsmonitor.trace && + test_trace2_data fsmonitor token_closure/apply_count 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace + ) +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'builtin initial trivial response anchors a closure' ' + test_when_finished "rm -rf builtin-initial-trivial" && + prepare_builtin_closure_repo builtin-initial-trivial && + ( + cd builtin-initial-trivial && + sane_unset GIT_TEST_SPLIT_INDEX && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 \ + --untracked-files=no >.git/actual && + test_must_be_empty .git/actual && + test_trace2_data fsm_client query/trivial-response 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/status.trace && + test-tool dump-fsmonitor >.git/fsmonitor && + test_grep "^fsmonitor last update builtin:test:2" \ + .git/fsmonitor + ) +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'builtin trivial closure can rescan and accept' ' + test_when_finished "rm -rf builtin-closure-trivial" && + prepare_builtin_closure_repo builtin-closure-trivial && + ( + cd builtin-closure-trivial && + sane_unset GIT_TEST_SPLIT_INDEX && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CTC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 \ + --untracked-files=no >.git/actual && + test_must_be_empty .git/actual && + test_trace2_data fsmonitor token_closure/trivial 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/status.trace + ) +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'builtin closure rejects three intervening changes' ' + test_when_finished "rm -rf builtin-closure-exhausted" && + prepare_builtin_closure_repo builtin-closure-exhausted && + ( + cd builtin-closure-exhausted && + sane_unset GIT_TEST_SPLIT_INDEX && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CDDD \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 \ + --untracked-files=no >.git/actual && + test_must_be_empty .git/actual && + test_trace2_data fsmonitor token_closure/apply_count 1 \ + <.git/status.trace >.git/applied && + test_line_count = 3 .git/applied && + test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep ! FSUC .git/index + ) +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'builtin closure query errors fall back completely' ' + test_when_finished "rm -rf builtin-closure-error" && + prepare_builtin_closure_repo builtin-closure-error untracked && + ( + cd builtin-closure-error && + sane_unset GIT_TEST_SPLIT_INDEX && + test_write_lines visible >visible && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCE \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_grep "^? visible$" .git/actual && + test_grep \ + "\"event\":\"region_enter\".*\"category\":\"dir\",\"label\":\"read_directory\"" \ + .git/status.trace >.git/read-directory && + test_line_count = 2 .git/read-directory && + test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep ! FSUC .git/index + ) +' + +test_expect_success PERL_TEST_HELPERS \ + 'index reader rejects an out-of-bounds extension size' ' + test_when_finished "rm -rf oversized-index-extension" && + test_create_repo oversized-index-extension && + ( + cd oversized-index-extension && + test_commit base tracked && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + test_grep FSMN .git/index >/dev/null && + perl -0777 -pe " + \$pos = index(\$_, q(FSMN)); + die q(FSMN-not-found) if \$pos < 0; + substr(\$_, \$pos + 4, 4) = pack(q(N), 0xffffffff); + " .git/index >.git/index.bad && + mv .git/index.bad .git/index && + test_must_fail git status --porcelain=v2 2>err && + test_grep "index file corrupt" err + ) +' + # Test that we detect and disallow repos that are incompatible with FSMonitor. test_expect_success 'incompatible bare repo' ' test_when_finished "rm -rf ./bare-clone actual expect" && @@ -477,4 +1281,368 @@ test_expect_success 'status succeeds with sparse index' ' ) ' +test_expect_success 'reported events poison weak stat-cache matches' ' + test_create_repo reported-event && + ( + cd reported-event && + printf "aaaa\n" >tracked && + printf "clean\n" >clean && + git add tracked clean && + git commit -m base && + git config core.trustctime false && + git config core.checkStat minimal && + test-tool chmtime =-60 tracked clean && + git update-index --refresh && + mtime=$(test-tool chmtime --get tracked) && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token\0tracked\0clean\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git status --porcelain=v2 >/dev/null && + git status --porcelain=v2 >/dev/null && + printf "bbbb\n" >tracked && + test-tool chmtime =$mtime tracked && + + GIT_OPTIONAL_LOCKS=0 git diff-index --name-status HEAD \ + >.git/diff-index && + test_grep "^M.*tracked$" .git/diff-index && + test_grep ! "clean$" .git/diff-index && + git status --porcelain=v2 >.git/status && + test_grep "^1 \.M .* tracked$" .git/status && + test_grep ! " clean$" .git/status + ) +' + +test_expect_success 'reported path permits apply --index content match' ' + test_create_repo apply-marker && + ( + cd apply-marker && + test_commit base tracked && + test_write_lines next >tracked && + git diff >../apply-marker.patch && + git checkout -- tracked && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token\0tracked\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git apply --index ../apply-marker.patch + ) +' + +test_expect_success 'reported path permits checkout-index content match' ' + test_create_repo checkout-marker && + ( + cd checkout-marker && + test_commit base tracked && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token\0tracked\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git checkout-index tracked + ) +' + +test_expect_success CASE_INSENSITIVE_FS \ + 'reported path permits case-folded unpack match' ' + test_create_repo icase-marker && + ( + cd icase-marker && + test_write_lines same >foo && + git add foo && + git commit -m base && + base=$(git rev-parse HEAD) && + git mv foo intermediate && + git mv intermediate FOO && + git commit -m target && + target=$(git rev-parse HEAD) && + git checkout "$base" && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token\0foo\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git read-tree -m -u "$target" && + echo FOO >expect && + git ls-files >actual && + test_cmp expect actual + ) +' + +test_expect_success 'reported unchanged path avoids reset checkout' ' + test_create_repo reset-marker && + ( + cd reset-marker && + test_commit base tracked && + git config core.trustctime false && + git config core.checkStat minimal && + test-tool chmtime =-60 tracked && + git update-index --refresh && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token\0tracked\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + before=$(test-tool chmtime --get tracked) && + git reset --hard HEAD && + after=$(test-tool chmtime --get tracked) && + test "$before" = "$after" + ) +' + +test_expect_success 'ordinary zero-stat entries retain diff-index behavior' ' + test_create_repo ordinary-zero-stat && + ( + cd ordinary-zero-stat && + echo content >tracked && + git add tracked && + git commit -m base && + oid=$(git rev-parse :tracked) && + git update-index --cacheinfo 100644,$oid,tracked && + git -c core.fsmonitor=false diff-index --name-status HEAD >actual && + test_grep "^M.*tracked$" actual + ) +' + +test_expect_success 'verified reported paths restore poisoned stat data' ' + test_create_repo fsmonitor-stat-recovery && + ( + cd fsmonitor-stat-recovery && + echo content >tracked && + git add tracked && + git commit -m base && + test-tool read-cache \ + --test-fsmonitor-content-recovery tracked + ) +' + +test_expect_success 'provider global marker invalidates every tracked entry' ' + test_create_repo global-invalidate && + ( + cd global-invalidate && + printf "aaaa\n" >tracked && + git add tracked && + git commit -m base && + git config core.trustctime false && + git config core.checkStat minimal && + test-tool chmtime =-60 tracked && + git update-index --refresh && + mtime=$(test-tool chmtime --get tracked) && + test_hook --setup fsmonitor-test <<-\EOF && + if test -f .git/global + then + printf "token1\0//\0" + else + printf "token0\0" + fi + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git status --porcelain=v2 >/dev/null && + git status --porcelain=v2 >/dev/null && + printf "bbbb\n" >tracked && + test-tool chmtime =$mtime tracked && + > .git/global && + git status --porcelain=v2 >.git/actual && + test_grep "^1 \.M .* tracked$" .git/actual + ) +' + +test_expect_success \ + 'directory attribute invalidation requires an indexed cone' ' + test_create_repo directory-attributes && + ( + cd directory-attributes && + mkdir tracked-dir && + test_commit base tracked-dir/tracked && + test_hook --setup fsmonitor-test <<-\EOF && + if test -f .git/report-cone + then + printf "cone-token\0tracked-dir/\0" + elif test -f .git/report-unmatched + then + printf "unmatched-token\0untracked/\0" + else + printf "base-token\0" + fi + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git status --porcelain=v2 >/dev/null && + git status --porcelain=v2 >/dev/null && + + > .git/report-cone && + GIT_TRACE2_EVENT="$PWD/.git/cone.trace" \ + GIT_TRACE_FSMONITOR="$PWD/.git/cone.fsm" \ + git status --porcelain=v2 >.git/cone.actual && + test_must_be_empty .git/cone.actual && + test_grep "fsmonitor_refresh_callback.*tracked-dir/" \ + .git/cone.fsm && + test_trace2_data fsmonitor semantic/attributes-cone 1 \ + <.git/cone.trace && + + rm .git/report-cone && + > .git/report-unmatched && + GIT_TRACE2_EVENT="$PWD/.git/unmatched.trace" \ + GIT_TRACE_FSMONITOR="$PWD/.git/unmatched.fsm" \ + git status --porcelain=v2 >.git/unmatched.actual && + test_must_be_empty .git/unmatched.actual && + test_grep "fsmonitor_refresh_callback.*untracked/" \ + .git/unmatched.fsm && + test_grep ! \ + "\"category\":\"fsmonitor\",\"key\":\"semantic/attributes-cone\"" \ + .git/unmatched.trace + ) +' + +test_expect_success 'directory events invalidate cached attributes' ' + test_create_repo directory-attribute-cache && + ( + cd directory-attribute-cache && + mkdir tracked-dir && + test_write_lines "tracked marker=old" \ + >tracked-dir/.gitattributes && + test_write_lines tracked >tracked-dir/tracked && + git add tracked-dir && + git commit -m base && + test_hook --setup fsmonitor-test <<-\EOF && + printf "new-token\0tracked-dir/\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + test-tool read-cache \ + --test-fsmonitor-directory-attributes + ) +' + +test_expect_success HARDLINKS,!MINGW,!CYGWIN \ + 'multiply-linked files stay fsmonitor-invalid' ' + test_when_finished "rm -f hardlink-alias" && + test_create_repo hardlink-validity && + ( + cd hardlink-validity && + echo content >tracked && + git add tracked && + git commit -m base && + ln tracked ../hardlink-alias && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git status --porcelain=v2 >/dev/null && + git status --porcelain=v2 >/dev/null && + git ls-files -f >.git/flags && + test_grep "^H tracked$" .git/flags && + echo changed >>../hardlink-alias && + git status --porcelain=v2 >.git/actual && + test_grep "^1 \.M .* tracked$" .git/actual + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'second closing-query change reprimes untracked cache' ' + test_when_finished "rm -rf second-query-changed" && + test_create_repo second-query-changed && + ( + cd second-query-changed && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir cached && + test_write_lines "*.ignored" >.gitignore && + test_write_lines "*.ignored" >cached/.gitignore && + printf "aaaa\n" >cached/tracked && + test_write_lines ignored >cached/junk.ignored && + git add .gitignore cached/.gitignore cached/tracked && + git commit -m base && + git config core.trustctime false && + git config core.checkStat minimal && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain=v2 \ + >.git/prime && + test_must_be_empty .git/prime && + test-tool chmtime =-60 cached/tracked && + git update-index --refresh && + mtime=$(test-tool chmtime --get cached/tracked) && + printf "bbbb\n" >cached/tracked && + test-tool chmtime =$mtime cached/tracked && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid cached/tracked && + test_grep ! FSCF .git/index && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCDC \ + GIT_TEST_FSMONITOR_QUERY_PATH=cached/tracked \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_line_count = 1 .git/actual && + test_grep "^1 \.M .* cached/tracked$" .git/actual && + test_trace2_data status fsmonitor_token/semantic-closed 1 \ + <.git/status.trace && + test_trace2_data status \ + fsmonitor_token/untracked-after-semantic 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/apply_count 1 \ + <.git/status.trace && + test_trace2_data status \ + fsmonitor_token/untracked-after-retry 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep FSCF .git/index && + test_grep FSUC .git/index + ) +' + +test_expect_success PTHREADS,UNTRACKED_CACHE,SHA1 'load cache-tree and untracked-cache extensions in parallel' ' + test_create_repo parallel-extensions && + ( + cd parallel-extensions && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir dir && + echo tracked >dir/tracked && + git config index.threads 4 && + git add dir/tracked && + git commit -m initial && + git config core.untrackedCache true && + git status --porcelain >/dev/null && + echo modified >>dir/tracked && + echo untracked >dir/untracked && + GIT_TEST_INDEX_THREADS=1 \ + git --no-optional-locks status --porcelain >"$TRASH_DIRECTORY/parallel-serial.status" && + GIT_TEST_INDEX_THREADS=1 \ + test-tool dump-cache-tree >"$TRASH_DIRECTORY/parallel-serial.tree" && + GIT_TEST_INDEX_THREADS=1 \ + test-tool dump-untracked-cache >"$TRASH_DIRECTORY/parallel-serial.untracked" && + GIT_TEST_INDEX_THREADS=4 \ + GIT_TEST_PARALLEL_INDEX_EXTENSIONS=1 \ + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/parallel-extensions.trace" \ + git --no-optional-locks status --porcelain >"$TRASH_DIRECTORY/parallel-parallel.status" && + GIT_TEST_INDEX_THREADS=4 GIT_TEST_PARALLEL_INDEX_EXTENSIONS=1 \ + test-tool dump-cache-tree >"$TRASH_DIRECTORY/parallel-parallel.tree" && + GIT_TEST_INDEX_THREADS=4 GIT_TEST_PARALLEL_INDEX_EXTENSIONS=1 \ + test-tool dump-untracked-cache >"$TRASH_DIRECTORY/parallel-parallel.untracked" && + test_grep "extension/parallel/tree-untracked" \ + "$TRASH_DIRECTORY/parallel-extensions.trace" && + test_cmp "$TRASH_DIRECTORY/parallel-serial.status" \ + "$TRASH_DIRECTORY/parallel-parallel.status" && + test_cmp "$TRASH_DIRECTORY/parallel-serial.tree" \ + "$TRASH_DIRECTORY/parallel-parallel.tree" && + test_cmp "$TRASH_DIRECTORY/parallel-serial.untracked" \ + "$TRASH_DIRECTORY/parallel-parallel.untracked" + ) +' + test_done diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index 86195770e97779..5c3735fef41315 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -3,6 +3,7 @@ test_description='built-in file system watcher' . ./test-lib.sh +. "$TEST_DIRECTORY"/lib-semantic-verify.sh if ! test_have_prereq FSMONITOR_DAEMON then @@ -53,6 +54,11 @@ test_lazy_prereq FSMONITOR_WORKS ' return $ret ' +test_lazy_prereq HARDLINKS ' + : >hardlink-a && + ln hardlink-a hardlink-b +' + if ! test_have_prereq FSMONITOR_WORKS then skip_all="filesystem does not deliver fsmonitor events (container/overlayfs?)" @@ -384,7 +390,8 @@ test_expect_success 'update-index implicitly starts daemon' ' # Confirm that the trace2 log contains a record of the # daemon starting. - test_subcommand git fsmonitor--daemon start <.git/trace_implicit_1 + test_grep "\"argv\":.*\"fsmonitor--daemon\",\"run\",\"--detach\"" \ + .git/trace_implicit_1 ' test_expect_success 'status implicitly starts daemon' ' @@ -400,7 +407,8 @@ test_expect_success 'status implicitly starts daemon' ' # Confirm that the trace2 log contains a record of the # daemon starting. - test_subcommand git fsmonitor--daemon start <.git/trace_implicit_2 + test_grep "\"argv\":.*\"fsmonitor--daemon\",\"run\",\"--detach\"" \ + .git/trace_implicit_2 ' edit_files () { @@ -978,7 +986,8 @@ test_expect_success "submodule absorbgitdirs implicitly starts daemon" ' # Confirm that the trace2 log contains a record of the # daemon starting. - test_subcommand git fsmonitor--daemon start "$PWD/file_case_wrong-lsf2.out" && - test_grep -q "h dir1/dir2/dir3/file-3-a" "$PWD/file_case_wrong-lsf2.out" && - test_grep -q "h dir1/dir2/dir4/FILE-4-A" "$PWD/file_case_wrong-lsf2.out" && + if test_grep -E -q \ + "fsmonitor_refresh_callback .dir1(/dir2(/dir3)?)?/?. .*pos " \ + "$PWD/file_case_wrong-try2.log" >/dev/null 2>&1 4>/dev/null + then + expected_3=H + else + expected_3=h + fi && + test_grep -q "$expected_3 dir1/dir2/dir3/file-3-a" \ + "$PWD/file_case_wrong-lsf2.out" && + if test_grep -E -q \ + "fsmonitor_refresh_callback .dir1(/dir2(/dir4)?)?/?. .*pos " \ + "$PWD/file_case_wrong-try2.log" >/dev/null 2>&1 4>/dev/null + then + expected_4=H + else + expected_4=h + fi && + test_grep -q "$expected_4 dir1/dir2/dir4/FILE-4-A" \ + "$PWD/file_case_wrong-lsf2.out" && # We now have files with clean content, but with case-incorrect @@ -1389,4 +1417,1471 @@ test_expect_success CASE_INSENSITIVE_FS 'fsmonitor file case wrong on disk' ' test_grep -q " M dir1/dir2/dir4/FILE-4-A" "$PWD/file_case_wrong-try3.out" ' +test_expect_success MACOS,HARDLINKS 'hardlink events invalidate all tracked paths' ' + test_when_finished "git -C hardlink-event fsmonitor--daemon stop 2>/dev/null || :" && + test_create_repo hardlink-event && + ( + cd hardlink-event && + printf "AAAA\\n" >tracked && + git add tracked && + git commit -m base && + git config core.fsmonitor true && + git config core.untrackedCache true && + git config core.trustctime false && + git config core.checkStat minimal && + cp -p tracked .git/mtime-reference && + start_daemon --tf "$PWD/../hardlink-event.trace" && + git update-index --fsmonitor && + git status --porcelain=v2 >/dev/null && + git status --porcelain=v2 >/dev/null && + printf "ignore\n" >.git/hardlink-source && + ln .git/hardlink-source .git/hardlink-alias && + printf "still-ignore\n" >.git/hardlink-alias && + test-tool fsmonitor-client query >.git/gitdir-query && + test_grep ! "^event: //$" ../hardlink-event.trace && + ln tracked alias && + printf "BBBB\\n" >alias && + GIT_OPTIONAL_LOCKS=0 git -c core.fsmonitor=false \ + -c core.untrackedCache=false status --porcelain=v2 \ + >.git/expect && + touch -r .git/mtime-reference alias && + GIT_OPTIONAL_LOCKS=0 git status --porcelain=v2 >.git/actual && + test_cmp .git/expect .git/actual && + test_grep "^event: //$" ../hardlink-event.trace && + git fsmonitor--daemon stop + ) +' + +test_expect_success MACOS 'implicit daemon reuses the invoking Git executable' ' + test_create_repo same-executable-spawn && + mkdir fake-exec-path && + write_script fake-exec-path/git <<-EOF && + echo invoked >"$TRASH_DIRECTORY/fake-git-used" + exit 1 + EOF + ( + cd same-executable-spawn && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_EXEC_PATH="$TRASH_DIRECTORY/fake-exec-path" \ + GIT_TRACE2_EVENT="$PWD/.git/spawn.trace" \ + "$GIT_BUILD_DIR/git" status --porcelain=v2 \ + >.git/actual && + test_must_be_empty .git/actual && + test_path_is_missing "$TRASH_DIRECTORY/fake-git-used" && + test_grep -F "\"argv\":[\"$GIT_BUILD_DIR/git\",\"fsmonitor--daemon\",\"run\",\"--detach\"]" \ + .git/spawn.trace && + git fsmonitor--daemon stop + ) +' + +test_expect_success MACOS 'implicit daemon rediscovers a linked worktree' ' + test_when_finished " + git -C reexec-linked-wt fsmonitor--daemon stop 2>/dev/null || : + git -C reexec-linked-main worktree remove --force \ + ../reexec-linked-wt 2>/dev/null || : + " && + test_create_repo reexec-linked-main && + ( + cd reexec-linked-main && + test_commit base tracked && + git worktree add ../reexec-linked-wt && + git -C ../reexec-linked-wt config core.untrackedCache true && + git -C ../reexec-linked-wt config core.fsmonitor true && + linked_worktree=$(test-tool path-utils real_path \ + ../reexec-linked-wt) && + GIT_TRACE2_EVENT="$PWD/../reexec-linked.trace" \ + git -C ../reexec-linked-wt status --porcelain=v2 \ + >../reexec-linked.actual && + test_must_be_empty ../reexec-linked.actual && + test_grep "\"argv\":.*\"fsmonitor--daemon\",\"run\",\"--detach\"" \ + ../reexec-linked.trace && + test_grep \ + "\"child_class\":\"fsmonitor\",\"cd\":\"$linked_worktree\"" \ + ../reexec-linked.trace && + git -C ../reexec-linked-wt fsmonitor--daemon stop && + git worktree remove ../reexec-linked-wt + ) +' + +test_expect_success MACOS 'implicit startup treats a bad timeout as best effort' ' + test_create_repo reexec-timeout && + ( + cd reexec-timeout && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + git config fsmonitor.starttimeout nonsense && + git status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual && + git config --unset fsmonitor.starttimeout && + git fsmonitor--daemon stop && + git config fsmonitor.starttimeout nonsense && + test_must_fail git fsmonitor--daemon start 2>.git/err && + test_grep "bad numeric config value" .git/err + ) +' + +test_expect_success 'bound query replaces a legacy daemon' ' + test_when_finished \ + "stop_daemon_delete_repo legacy-daemon-upgrade" && + test_create_repo legacy-daemon-upgrade && + ( + cd legacy-daemon-upgrade && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.preloadIndex false && + git config core.untrackedCache true && + git status --porcelain=v2 >/dev/null && + git config core.fsmonitor true && + ipc_path=$(git rev-parse --path-format=absolute \ + --git-path fsmonitor--daemon.ipc) && + test-tool simple-ipc start-daemon \ + --name="$ipc_path" --threads=1 --fsmonitor-legacy && + + GIT_TRACE2_EVENT="$PWD/.git/upgrade.trace" \ + git status >.git/upgrade.out && + test_trace2_data fsm_client query/incompatible-daemon 1 \ + <.git/upgrade.trace && + test-tool dump-fsmonitor >.git/fsmonitor && + test_grep ! "builtin:test-legacy:0" .git/fsmonitor && + test_grep \ + "\"argv\":.*\"fsmonitor--daemon\",\"run\",\"--detach\"" \ + .git/upgrade.trace && + + GIT_TRACE2_EVENT="$PWD/.git/warm.trace" \ + git status >.git/warm.out && + ! test_trace2_data index refresh/sum_lstat \ + "[1-9][0-9]*" <.git/warm.trace && + ! test_trace2_data fsm_client query/trivial-response 1 \ + <.git/warm.trace && + test_grep ! \ + "\"argv\":.*\"fsmonitor--daemon\",\"run\",\"--detach\"" \ + .git/warm.trace + ) +' + +test_expect_success MACOS 'daemon token reset closes a skipHash index' ' + test_when_finished \ + "stop_daemon_delete_repo daemon-token-reset" && + test_create_repo daemon-token-reset && + ( + cd daemon-token-reset && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test_commit remove removed && + test_commit keep clean && + git config core.preloadIndexBulk true && + git config core.untrackedCache true && + git config index.skipHash true && + test-tool chmtime =-60 tracked removed clean && + git update-index --refresh && + git config core.fsmonitor true && + start_daemon && + + git update-index --force-write-index && + git status --porcelain=v2 >.git/prime.out && + test_must_be_empty .git/prime.out && + test_grep FSMN .git/index && + test_grep FSCF .git/index && + test_grep FSUC .git/index && + test_trailing_hash .git/index >.git/index.hash && + test_oid zero >.git/zero && + test_cmp .git/zero .git/index.hash && + test-tool dump-fsmonitor >.git/token.before && + token_before=$(sed -n \ + "s/^fsmonitor last update //p" .git/token.before) && + + git fsmonitor--daemon stop && + start_daemon && + GIT_TRACE2_EVENT="$PWD/.git/reset.trace" \ + git status --porcelain=v2 >.git/reset.out && + test_must_be_empty .git/reset.out && + test_trace2_data fsm_client query/trivial-response 1 \ + <.git/reset.trace && + test_trace2_data index preload/bulk_provider_applied \ + "[1-9][0-9]*" \ + <.git/reset.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/reset.trace && + test-tool dump-fsmonitor >.git/token.after && + token_after=$(sed -n \ + "s/^fsmonitor last update //p" .git/token.after) && + test -n "$token_before" && + test -n "$token_after" && + test "$token_before" != "$token_after" && + + GIT_TRACE2_EVENT="$PWD/.git/warm.trace" \ + git status --porcelain=v2 >.git/warm.out && + test_must_be_empty .git/warm.out && + ! test_trace2_data index refresh/sum_lstat \ + "[1-9][0-9]*" <.git/warm.trace && + ! test_trace2_data fsm_client query/trivial-response 1 \ + <.git/warm.trace && + + git fsmonitor--daemon stop && + echo changed >>tracked && + rm removed && + start_daemon && + GIT_TRACE2_EVENT="$PWD/.git/dirty-reset.trace" \ + git status --porcelain=v2 >.git/dirty-reset.out && + test_line_count = 2 .git/dirty-reset.out && + test_grep "^1 \.M .* tracked$" .git/dirty-reset.out && + test_grep "^1 \.D .* removed$" .git/dirty-reset.out && + test_trace2_data fsm_client query/trivial-response 1 \ + <.git/dirty-reset.trace && + test_trace2_data index preload/bulk_provider_applied 1 \ + <.git/dirty-reset.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/dirty-reset.trace && + + GIT_TRACE2_EVENT="$PWD/.git/dirty-warm.trace" \ + git status --porcelain=v2 >.git/dirty-warm.out && + test_cmp .git/dirty-reset.out .git/dirty-warm.out && + test_trace2_data index preload/bulk_provider_applied 1 \ + <.git/dirty-warm.trace && + test_trace2_data index refresh/sum_lstat 0 \ + <.git/dirty-warm.trace && + ! test_trace2_data fsm_client query/trivial-response 1 \ + <.git/dirty-warm.trace + ) +' + +test_expect_success 'bound query accepts a capability superset' ' + test_when_finished \ + "stop_daemon_delete_repo capability-superset" && + test_create_repo capability-superset && + ( + cd capability-superset && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.preloadIndex false && + git config core.untrackedCache true && + git status --porcelain=v2 >/dev/null && + git config core.fsmonitor true && + ipc_path=$(git rev-parse --path-format=absolute \ + --git-path fsmonitor--daemon.ipc) && + test-tool simple-ipc start-daemon \ + --name="$ipc_path" --threads=1 \ + --fsmonitor-capability-superset && + + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status >.git/status.out && + test-tool dump-fsmonitor >.git/fsmonitor && + test_grep \ + "^fsmonitor last update builtin:test-capable:0" \ + .git/fsmonitor && + test_grep ! \ + "\"key\":\"query/incompatible-daemon\"" \ + .git/status.trace && + test_grep ! \ + "\"argv\":.*\"fsmonitor--daemon\",\"run\",\"--detach\"" \ + .git/status.trace + ) +' + +test_expect_success MACOS 'worktree binding rejects same-gitdir aliases' ' + test_when_finished "git -C binding-a fsmonitor--daemon stop 2>/dev/null || :" && + git init --separate-git-dir="$PWD/binding-gitdir" binding-a && + mkdir binding-b && + cp binding-a/.git binding-b/.git && + ( + cd binding-a && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TRACE2_EVENT="$PWD/../binding-daemon.trace" \ + git status --porcelain=v2 >/dev/null && + git status --porcelain=v2 >/dev/null + ) && + cp binding-a/tracked binding-b/tracked && + echo changed >>binding-b/tracked && + GIT_OPTIONAL_LOCKS=0 git -c core.fsmonitor=false \ + -c core.untrackedCache=false -C binding-b \ + status --porcelain=v2 >binding.expect && + GIT_OPTIONAL_LOCKS=0 git -C binding-b \ + status --porcelain=v2 >binding.actual && + test_cmp binding.expect binding.actual && + test_grep "^1 \.M .* tracked$" binding.actual && + test_grep "\"key\":\"query/worktree-mismatch\",\"value\":\"1\"" \ + binding-daemon.trace && + git -C binding-a fsmonitor--daemon stop +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'ordinary deltas advance only attribute-stable proofs' ' + test_when_finished "rm -rf token-carry" && + test_create_repo token-carry && + ( + cd token-carry && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test_write_lines "*.txt text" >.gitattributes && + git add .gitattributes && + git commit -m attributes && + git config core.untrackedCache true && + git -c core.fsmonitor=false status --porcelain=v2 \ + >.git/initial && + test_must_be_empty .git/initial && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSUC .git/index && + test_grep FSCF .git/index && + + touch x && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=x \ + GIT_TRACE2_EVENT="$PWD/.git/created.trace" \ + git status --porcelain=v2 >.git/created && + test_grep "^? x$" .git/created && + test_trace2_data fsmonitor config/token-advanced 1 \ + <.git/created.trace && + + rm x && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=x \ + GIT_TRACE2_EVENT="$PWD/.git/deleted.trace" \ + git status --porcelain=v2 >.git/deleted && + test_must_be_empty .git/deleted && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/deleted.trace && + test_trace2_data fsmonitor config/token-advanced 1 \ + <.git/deleted.trace && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DDCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=.gitattributes \ + GIT_TRACE2_EVENT="$PWD/.git/attributes.trace" \ + git status --porcelain=v2 >.git/attributes && + test_must_be_empty .git/attributes && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/attributes.trace && + test_trace2_data fsmonitor apply_count 1 \ + <.git/attributes.trace && + ! test_trace2_data fsmonitor config/token-advanced 1 \ + <.git/attributes.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'worktree-only checkout preserves closed semantic history' ' + test_when_finished "rm -rf checkout-history" && + test_create_repo checkout-history && + ( + cd checkout-history && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test_commit other other && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + test_write_lines changed >tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git status >.git/dirty && + test_grep "modified:.*tracked" .git/dirty && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git checkout -- tracked && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status >.git/actual && + test_grep "nothing to commit, working tree clean" .git/actual && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/status.trace && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/status.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'source-tree checkout preserves closed semantic history' ' + test_when_finished "rm -rf checkout-source-history" && + test_create_repo checkout-source-history && + ( + cd checkout-source-history && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + test_write_lines changed >tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git status >.git/dirty && + test_grep "modified:.*tracked" .git/dirty && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git checkout HEAD -- tracked && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status >.git/actual && + test_grep "nothing to commit, working tree clean" .git/actual && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/status.trace && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/status.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'source-tree checkout drops history after an index change' ' + test_when_finished "rm -rf checkout-source-changed" && + test_create_repo checkout-source-changed && + ( + cd checkout-source-changed && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test_write_lines next >tracked && + git add tracked && + git commit -m next && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git checkout HEAD^ -- tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_grep "^1 M\." .git/actual && + test_trace2_data fsmonitor config/coherent 0 \ + <.git/status.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'checkout-index -u preserves closed semantic history' ' + test_when_finished "rm -rf checkout-index-history" && + test_create_repo checkout-index-history && + ( + cd checkout-index-history && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + test_write_lines changed >tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git checkout-index -f -u tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status >.git/actual && + test_grep "nothing to commit, working tree clean" .git/actual && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/status.trace && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/status.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'stat-only update-index preserves closed semantic history' ' + test_when_finished "rm -rf update-index-history" && + test_create_repo update-index-history && + ( + cd update-index-history && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + test-tool chmtime +1 tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git update-index --refresh --force-write-index && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status >.git/actual && + test_grep "nothing to commit, working tree clean" .git/actual && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/status.trace && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/status.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'add --refresh preserves closed semantic history' ' + test_when_finished "rm -rf add-refresh-history" && + test_create_repo add-refresh-history && + ( + cd add-refresh-history && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + test-tool chmtime +1 tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git add --refresh tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status >.git/actual && + test_grep "nothing to commit, working tree clean" .git/actual && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/status.trace && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/status.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'mtime-only ordinary add preserves closed semantic history' ' + test_when_finished "rm -rf add-ordinary-history" && + test_create_repo add-ordinary-history && + ( + cd add-ordinary-history && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + test-tool chmtime +1 tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git add tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/path.trace" \ + git status >.git/path && + test_grep "nothing to commit, working tree clean" .git/path && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/path.trace && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/path.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'ordinary add drops history after a logical index change' ' + test_when_finished "rm -rf add-ordinary-changed" && + test_create_repo add-ordinary-changed && + ( + cd add-ordinary-changed && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + test_write_lines changed >tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git add tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_grep "^1 M\." .git/actual && + test_trace2_data fsmonitor config/coherent 0 \ + <.git/status.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'ordinary add drops history after ITA resolution' ' + test_when_finished "rm -rf add-ordinary-ita" && + test_create_repo add-ordinary-ita && + ( + cd add-ordinary-ita && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + touch empty && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=empty \ + git add -N empty && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=empty \ + git status --porcelain=v2 >.git/ita && + test_grep FSCF .git/index && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git add empty && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_grep "^1 A\\." .git/actual && + test_trace2_data fsmonitor config/coherent 0 \ + <.git/status.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'describe --dirty preserves closed semantic history' ' + test_when_finished "rm -rf describe-dirty-history" && + test_create_repo describe-dirty-history && + ( + cd describe-dirty-history && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + test-tool chmtime +1 tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git describe --always --dirty >.git/describe && + test_grep ! dirty .git/describe && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status >.git/actual && + test_grep "nothing to commit, working tree clean" .git/actual && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/status.trace && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/status.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'clean stash push preserves closed semantic history' ' + test_when_finished "rm -rf stash-clean-history" && + test_create_repo stash-clean-history && + ( + cd stash-clean-history && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git stash push >.git/stash && + test_grep "No local changes to save" .git/stash && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status >.git/actual && + test_grep "nothing to commit, working tree clean" .git/actual && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/status.trace && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/status.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'dirty stash push drops closed semantic history' ' + test_when_finished "rm -rf stash-dirty-history" && + test_create_repo stash-dirty-history && + ( + cd stash-dirty-history && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + test_write_lines changed >tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git stash push >.git/stash && + test_grep "Saved working directory" .git/stash && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status >.git/actual && + test_grep "nothing to commit, working tree clean" .git/actual && + test_trace2_data fsmonitor config/coherent 0 \ + <.git/status.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'mixed reset to a same-tree commit preserves closed history' ' + test_when_finished "rm -rf reset-mixed-same-tree" && + test_create_repo reset-mixed-same-tree && + ( + cd reset-mixed-same-tree && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git commit --allow-empty -m same-tree && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + test-tool chmtime +1 tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git reset --mixed HEAD^ >.git/reset && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status >.git/actual && + test_grep "nothing to commit, working tree clean" .git/actual && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/status.trace && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/status.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'mixed reset drops history after a logical index change' ' + test_when_finished "rm -rf reset-mixed-changed" && + test_create_repo reset-mixed-changed && + ( + cd reset-mixed-changed && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + test_write_lines staged >tracked && + git add tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/staged && + test_grep "^1 M\." .git/staged && + test_grep FSCF .git/index && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git reset --mixed HEAD >.git/reset && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status >.git/actual && + test_grep "modified:.*tracked" .git/actual && + test_trace2_data fsmonitor config/coherent 0 \ + <.git/status.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'hard reset to a same-tree commit preserves closed history' ' + test_when_finished "rm -rf reset-hard-same-tree" && + test_create_repo reset-hard-same-tree && + ( + cd reset-hard-same-tree && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git commit --allow-empty -m same-tree && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + test_write_lines changed >tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git status >.git/dirty && + test_grep "modified:.*tracked" .git/dirty && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git reset --hard HEAD^ >.git/reset && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status >.git/actual && + test_grep "nothing to commit, working tree clean" .git/actual && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/status.trace && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/status.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'hard reset to a different tree drops closed semantic history' ' + test_when_finished "rm -rf reset-hard-changed" && + test_create_repo reset-hard-changed && + ( + cd reset-hard-changed && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + test_write_lines next >tracked && + git add tracked && + git commit -m next && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git reset --hard HEAD^ >.git/reset && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status >.git/actual && + test_grep "nothing to commit, working tree clean" .git/actual && + test_trace2_data fsmonitor config/coherent 0 \ + <.git/status.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'forced same-tree checkout preserves closed semantic history' ' + test_when_finished "rm -rf checkout-same-tree" && + test_create_repo checkout-same-tree && + ( + cd checkout-same-tree && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git branch same && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + test_write_lines changed >tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git status >.git/dirty && + test_grep "modified:.*tracked" .git/dirty && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git checkout -f same >.git/checkout && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status >.git/actual && + test_grep "nothing to commit, working tree clean" .git/actual && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/status.trace && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/status.trace + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'one-tree read-tree reset preserves closed semantic history' ' + test_when_finished "rm -rf read-tree-reset-history" && + test_create_repo read-tree-reset-history && + ( + cd read-tree-reset-history && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + test_write_lines changed >tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git status >.git/dirty && + test_grep "modified:.*tracked" .git/dirty && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=tracked \ + git read-tree --reset -u HEAD >.git/read-tree && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status >.git/actual && + test_grep "nothing to commit, working tree clean" .git/actual && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/status.trace && + test_trace2_data index refresh/sum_lstat 1 \ + <.git/status.trace && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/status.trace + ) +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'missing semantic history seeds a forward baseline' ' + test_when_finished \ + "stop_daemon_delete_repo missing-semantic-baseline" && + test_create_repo missing-semantic-baseline && + ( + cd missing-semantic-baseline && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid tracked && + test_grep ! FSCF .git/index && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual && + test_grep FSCF .git/index && + test_trace2_data fsmonitor semantic/adoption-baseline 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor semantic/strong-invalidation 1 \ + <.git/status.trace && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace + ) +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'missing semantic history with weak stat identity forces content verification' ' + test_when_finished \ + "stop_daemon_delete_repo missing-semantic-history" && + test_create_repo missing-semantic-history && + ( + cd missing-semantic-history && + printf "aaaa\n" >tracked && + git add tracked && + git commit -m base && + git config core.trustctime false && + git config core.checkStat minimal && + test-tool chmtime =-60 tracked && + git update-index --refresh && + mtime=$(test-tool chmtime --get tracked) && + printf "bbbb\n" >tracked && + test-tool chmtime =$mtime tracked && + git config core.fsmonitor true && + git update-index --fsmonitor && + git update-index --fsmonitor-valid tracked && + test_grep FSMN .git/index && + test_grep ! FSCF .git/index && + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_line_count = 1 .git/actual && + test_grep "^1 \.M .* tracked$" .git/actual && + test_trace2_data fsmonitor semantic/strong-invalidation 1 \ + <.git/status.trace + ) +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'token closure refresh starts inside its proof epoch' ' + test_when_finished "rm -rf proof-epoch-refresh" && + test_create_repo proof-epoch-refresh && + ( + cd proof-epoch-refresh && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.untrackedCache true && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_grep FSCF .git/index && + + # Leave the next refresh with untracked history to bootstrap. + git update-index --no-untracked-cache 2>.git/no-uc.err && + test_grep FSCF .git/index && + test_grep ! FSUC .git/index && + + test_env GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + test_must_fail git commit --dry-run --porcelain \ + >.git/actual && + test_must_be_empty .git/actual && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + captured=$(test_grep -n \ + "\"key\":\"semantic/proof-epoch-captured\"" \ + .git/status.trace | sed -n "1s/:.*//p") && + refreshed=$(test_grep -n \ + "\"category\":\"index\",\"label\":\"refresh\"" \ + .git/status.trace | sed -n "\$s/:.*//p") && + test -n "$captured" && + test -n "$refreshed" && + test "$captured" -lt "$refreshed" + ) +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'trivial query closes unbound semantic history' ' + test_when_finished "rm -rf unbound-trivial" && + test_create_repo unbound-trivial && + ( + cd unbound-trivial && + sane_unset GIT_TEST_SPLIT_INDEX && + test_commit base tracked && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + test-tool read-cache --test-fscf-round-trip && + test_grep FSMN .git/index && + test_grep FSCF .git/index && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TC \ + GIT_TRACE2_EVENT="$PWD/.git/recovery.trace" \ + git status --porcelain=v2 --untracked-files=no \ + >.git/recovery.out && + test_must_be_empty .git/recovery.out && + test_trace2_data fsm_client query/trivial-response 1 \ + <.git/recovery.trace && + test_trace2_data fsmonitor semantic/manifest-scan-count \ + "[1-9]" <.git/recovery.trace && + test_trace2_data fsmonitor semantic/proof-epoch-captured 1 \ + <.git/recovery.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/recovery.trace && + ! test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/recovery.trace && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + GIT_TRACE2_EVENT="$PWD/.git/warm.trace" \ + git status --porcelain=v2 --untracked-files=no \ + >.git/warm.out && + test_must_be_empty .git/warm.out && + test_trace2_data fsmonitor config/coherent 1 \ + <.git/warm.trace && + ! test_trace2_data fsmonitor semantic/manifest-scan-count \ + "[1-9]" <.git/warm.trace && + ! test_trace2_data fsm_client query/trivial-response 1 \ + <.git/warm.trace && + ! test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/warm.trace + ) +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'sparse index rebuilds semantic history without expansion' ' + test_when_finished "rm -rf sparse-semantic" && + test_create_repo sparse-semantic && + ( + cd sparse-semantic && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir in outside && + printf "aaaa\n" >in/tracked && + printf "outside\n" >outside/file && + git add . && + git commit -m base && + git sparse-checkout set --cone --sparse-index in && + git ls-files --sparse >.git/sparse.before && + test_grep "^outside/$" .git/sparse.before && + git config core.trustctime false && + git config core.checkStat minimal && + git config core.untrackedCache true && + test-tool chmtime =-60 in/tracked && + git update-index --refresh && + mtime=$(test-tool chmtime --get in/tracked) && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=TCCC \ + GIT_TRACE2_EVENT="$PWD/.git/prime.trace" \ + git status --porcelain=v2 >.git/prime && + test_must_be_empty .git/prime && + test_trace2_data fsm_client query/trivial-response 1 \ + <.git/prime.trace && + test_trace2_data fsmonitor semantic/manifest-scan-count \ + "[1-9]" <.git/prime.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/prime.trace && + test_grep FSMN .git/index && + git -c core.fsmonitor=false ls-files --sparse \ + >.git/sparse.after-prime && + test_grep "^outside/$" .git/sparse.after-prime && + printf "bbbb\n" >in/tracked && + test-tool chmtime =$mtime in/tracked && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=D \ + GIT_TEST_FSMONITOR_QUERY_PATH=in/tracked \ + GIT_TRACE2_EVENT="$PWD/.git/change.trace" \ + git status --porcelain=v2 >.git/actual && + test_grep "^1 \.M .* in/tracked$" .git/actual && + test_trace2_data fsmonitor apply_count 1 \ + <.git/change.trace && + test_grep FSMN .git/index && + git -c core.fsmonitor=false ls-files --sparse \ + >.git/sparse.after-change && + test_grep "^outside/$" .git/sparse.after-change + ) +' + +prepare_semantic_untracked_repo () { + r=$1 && + test_create_repo "$r" && + ( + cd "$r" && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir cached && + test_write_lines "*.ignored" >.gitignore && + test_write_lines "*.ignored" >cached/.gitignore && + printf "aaaa\n" >cached/tracked && + printf "cccc\n" >cached/hook-tracked && + git add .gitignore cached/.gitignore cached/hook-tracked \ + cached/tracked && + git commit -m base && + git config core.trustctime false && + git config core.checkStat minimal && + git config core.untrackedCache true && + test_write_lines ignored >cached/junk.ignored && + git status --porcelain=v2 >.git/prime.actual && + test_must_be_empty .git/prime.actual && + test_grep UNTR .git/index && + test-tool chmtime =-60 cached/tracked && + git update-index --refresh && + mtime=$(test-tool chmtime --get cached/tracked) && + printf "bbbb\n" >cached/tracked && + test-tool chmtime =$mtime cached/tracked && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid cached/tracked && + test_grep FSMN .git/index && + test_grep ! FSCF .git/index + ) +} + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'semantic adoption closes the untracked scan' ' + test_when_finished "rm -rf semantic-untracked" && + prepare_semantic_untracked_repo semantic-untracked && + ( + cd semantic-untracked && + sane_unset GIT_TEST_SPLIT_INDEX && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_line_count = 1 .git/actual && + test_grep "^1 \.M .* cached/tracked$" .git/actual && + test_trace2_data status fsmonitor_token/untracked-deferred 1 \ + <.git/status.trace && + test_trace2_data status fsmonitor_token/semantic-closed 1 \ + <.git/status.trace && + test_trace2_data status \ + fsmonitor_token/untracked-after-semantic 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/apply_count \ + "[0-9][0-9]*" <.git/status.trace >.git/apply-count && + test_line_count = 2 .git/apply-count && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep FSCF .git/index && + test_grep FSUC .git/index + ) +' + +test_expect_success UNTRACKED_CACHE,SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'failed untracked closure discards semantic adoption' ' + test_when_finished "rm -rf failed-semantic-untracked" && + prepare_semantic_untracked_repo failed-semantic-untracked && + ( + cd failed-semantic-untracked && + sane_unset GIT_TEST_SPLIT_INDEX && + GIT_DISABLE_UNTRACKED_CACHE=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_line_count = 1 .git/actual && + test_grep "^1 \.M .* cached/tracked$" .git/actual && + test_trace2_data status fsmonitor_token/semantic-closed 1 \ + <.git/status.trace && + test_trace2_data status \ + fsmonitor_token/untracked-after-semantic 0 \ + <.git/status.trace && + test_trace2_data fsmonitor semantic/strong-invalidation 1 \ + <.git/status.trace >.git/strong-invalidations && + test_line_count = 2 .git/strong-invalidations && + test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep ! FSCF .git/index + ) +' + +test_expect_success UNTRACKED_CACHE,!MINGW,!CYGWIN \ + 'commit closes hook changes without an untracked cache' ' + test_when_finished "rm -rf commit-hook-closure" && + prepare_semantic_untracked_repo commit-hook-closure && + ( + cd commit-hook-closure && + sane_unset GIT_TEST_SPLIT_INDEX && + git config core.untrackedCache false && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --no-untracked-cache && + test_grep ! UNTR .git/index && + write_script .git/hooks/pre-commit <<-\EOF && + mtime=$(test-tool chmtime --get cached/hook-tracked) && + printf "dddd\n" >cached/hook-tracked && + test-tool chmtime =$mtime cached/hook-tracked + EOF + GIT_EDITOR=: \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCDC \ + GIT_TEST_FSMONITOR_QUERY_PATH=cached/hook-tracked \ + GIT_TRACE2_EVENT="$PWD/.git/commit.trace" \ + git commit --allow-empty --edit -m adoption && + test_trace2_data status fsmonitor_token/semantic-closed 1 \ + <.git/commit.trace && + test_trace2_data fsmonitor token_closure/apply_count "[1-9]" \ + <.git/commit.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/commit.trace >.git/accepted && + test_line_count = 2 .git/accepted && + test_grep FSCF .git/index && + test_grep ! FSUC .git/index && + GIT_OPTIONAL_LOCKS=0 git -c core.fsmonitor=false \ + -c core.untrackedCache=false status --porcelain=v2 \ + >.git/status.actual && + test_grep "^1 \.M .* cached/hook-tracked$" \ + .git/status.actual && + test_grep ! UNTR .git/index + ) +' + +test_expect_success UNTRACKED_CACHE,!MINGW,!CYGWIN \ + 'failed hook closure refreshes the worktree' ' + test_when_finished "rm -rf commit-hook-fallback" && + prepare_semantic_untracked_repo commit-hook-fallback && + ( + cd commit-hook-fallback && + sane_unset GIT_TEST_SPLIT_INDEX && + write_script .git/hooks/pre-commit <<-\EOF && + mtime=$(test-tool chmtime --get cached/hook-tracked) && + printf "dddd\n" >cached/hook-tracked && + test-tool chmtime =$mtime cached/hook-tracked + EOF + GIT_EDITOR=: \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCE \ + GIT_TRACE2_EVENT="$PWD/.git/commit.trace" \ + git commit --allow-empty --edit -m adoption && + test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/commit.trace && + test_trace2_data status count/changed 2 <.git/commit.trace && + test_grep "cached/hook-tracked$" .git/COMMIT_EDITMSG && + GIT_OPTIONAL_LOCKS=0 git -c core.fsmonitor=false \ + -c core.untrackedCache=false status --porcelain=v2 \ + >.git/status.actual && + test_grep "cached/hook-tracked$" .git/status.actual + ) +' + +test_expect_success UNTRACKED_CACHE,!MINGW,!CYGWIN \ + 'post-hook refresh preserves hook index updates' ' + test_when_finished "rm -rf commit-hook-index" && + prepare_semantic_untracked_repo commit-hook-index && + ( + cd commit-hook-index && + sane_unset GIT_TEST_SPLIT_INDEX && + write_script .git/hooks/pre-commit <<-\EOF && + printf "hook update\n" >cached/hook-tracked && + oid=$(git hash-object -w cached/hook-tracked) && + git update-index --cacheinfo \ + 100644,$oid,cached/hook-tracked + EOF + GIT_EDITOR=: \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git commit --allow-empty --edit -m adoption && + printf "hook update\n" >expect && + git show HEAD:cached/hook-tracked >actual && + test_cmp expect actual + ) +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'tracked attribute events reopen semantic history' ' + test_when_finished "rm -rf tracked-attr-change" && + test_create_repo tracked-attr-change && + ( + cd tracked-attr-change && + sane_unset GIT_TEST_SPLIT_INDEX && + printf "*.txt text eol=crlf\n" >.gitattributes && + printf "alpha\r\n" >tracked.txt && + git add .gitattributes tracked.txt && + git commit -m base && + git config core.fsmonitor true && + git config core.untrackedCache true && + git config core.trustctime false && + git config core.checkStat minimal && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + git status --porcelain=v2 >.git/prime.actual && + test_must_be_empty .git/prime.actual && + test_grep FSCF .git/index && + test-tool chmtime =-60 tracked.txt && + + printf "*.txt -text\n" >.gitattributes && + test-tool chmtime +1 tracked.txt && + GIT_OPTIONAL_LOCKS=0 git -c core.fsmonitor=false \ + -c core.untrackedCache=false status --porcelain=v2 \ + >.git/expect && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DCCC \ + GIT_TEST_FSMONITOR_QUERY_PATH=.gitattributes \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_cmp .git/expect .git/actual && + test_grep "^1 \.M .* tracked.txt$" .git/actual && + test_trace2_data fsmonitor semantic/manifest-scan-count \ + "[1-9]" <.git/status.trace + ) +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'tracked-only status adopts missing semantic history' ' + test_when_finished "rm -rf tracked-semantic-adoption" && + test_create_repo tracked-semantic-adoption && + ( + cd tracked-semantic-adoption && + sane_unset GIT_TEST_SPLIT_INDEX && + printf "aaaa\n" >tracked && + git add tracked && + git commit -m base && + git config core.trustctime false && + git config core.checkStat minimal && + git config core.untrackedCache true && + test-tool chmtime -60 tracked && + git update-index --refresh && + mtime=$(test-tool chmtime --get tracked) && + printf "bbbb\n" >tracked && + test-tool chmtime =$mtime tracked && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid tracked && + test_grep ! FSCF .git/index && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CCCC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 --untracked-files=no \ + >.git/actual && + test_grep "^1 \.M .* tracked$" .git/actual && + test_trace2_data status fsmonitor_token/semantic-closed 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep FSCF .git/index + ) +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'collapsed sparse index uses ordinary token closure' ' + test_when_finished "rm -rf sparse-tracked-only" && + test_create_repo sparse-tracked-only && + ( + cd sparse-tracked-only && + sane_unset GIT_TEST_SPLIT_INDEX && + mkdir in outside && + printf "aaaa\n" >in/tracked && + printf "outside\n" >outside/file && + git add . && + git commit -m base && + git sparse-checkout set --cone --sparse-index in && + git config core.trustctime false && + git config core.checkStat minimal && + git config core.untrackedCache true && + test-tool chmtime =-60 in/tracked && + git update-index --refresh && + mtime=$(test-tool chmtime --get in/tracked) && + printf "bbbb\n" >in/tracked && + test-tool chmtime =$mtime in/tracked && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid in/tracked && + test_grep ! FSCF .git/index && + + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=DDC \ + GIT_TEST_FSMONITOR_QUERY_PATH=in/tracked \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 --untracked-files=no \ + >.git/actual && + test_grep "^1 \.M .* in/tracked$" .git/actual && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/apply_count 1 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep FSCF .git/index && + git -c core.fsmonitor=false ls-files --sparse \ + >.git/sparse.after && + test_grep "^outside/$" .git/sparse.after + ) +' + test_done diff --git a/t/t7529-preload-index-apfs.sh b/t/t7529-preload-index-apfs.sh new file mode 100755 index 00000000000000..9c9177b0f46ec5 --- /dev/null +++ b/t/t7529-preload-index-apfs.sh @@ -0,0 +1,817 @@ +#!/bin/sh + +test_description='APFS bulk index preload' + +. ./test-lib.sh + +test_lazy_prereq APFS_BULK_PRELOAD ' + test_have_prereq MACOS && + darwin_major=$(uname -r) && + darwin_major=${darwin_major%%.*} && + test "$darwin_major" -ge 20 && + /bin/df -t apfs "$TRASH_DIRECTORY" >/dev/null +' + +if ! test_have_prereq APFS_BULK_PRELOAD +then + skip_all='bulk index preload requires macOS on APFS' + test_done +fi + +setup_repo () { + repo=$1 && + git init "$repo" && + mkdir -p "$repo/nested/deep" "$repo/other" && + test_write_lines root >"$repo/root" && + test_write_lines root-peer >"$repo/root-peer" && + test_write_lines nested >"$repo/nested/tracked" && + test_write_lines nested-peer >"$repo/nested/peer" && + test_write_lines deep >"$repo/nested/deep/tracked" && + test_write_lines deep-peer >"$repo/nested/deep/peer" && + test_write_lines other >"$repo/other/tracked" && + test_write_lines other-peer >"$repo/other/peer" && + git -C "$repo" add . && + git -C "$repo" commit -m base && + git -C "$repo" config core.fsmonitor false && + test-tool chmtime -120 \ + "$repo/root" "$repo/root-peer" \ + "$repo/nested/tracked" "$repo/nested/peer" \ + "$repo/nested/deep/tracked" "$repo/nested/deep/peer" \ + "$repo/other/tracked" "$repo/other/peer" && + git -C "$repo" update-index --refresh +} + +ordinary_status () { + GIT_OPTIONAL_LOCKS=0 \ + git -C "$1" -c core.preloadIndex=false \ + status --porcelain=v2 >"$2" +} + +bulk_status () { + repo=$1 && + output=$2 && + bulk_trace=$TRASH_DIRECTORY/$3 && + rm -f "$bulk_trace" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TRACE2_EVENT="$bulk_trace" \ + git -C "$repo" status --porcelain=v2 >"$output" +} + +writable_ordinary_status () { + git -C "$1" -c core.preloadIndex=false \ + status --porcelain=v2 >"$2" +} + +writable_bulk_status () { + repo=$1 && + output=$2 && + writable_trace=$TRASH_DIRECTORY/$3 && + rm -f "$writable_trace" && + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TRACE2_EVENT="$writable_trace" \ + git -C "$repo" status --porcelain=v2 >"$output" +} + +check_data () { + test_trace2_data index "$2" "$3" <"$TRASH_DIRECTORY/$1" +} + +check_lstat_data () { + test_have_prereq !PTHREADS || + check_data "$1" preload/sum_lstat "$2" +} + +compare_status () { + ordinary_status "$1" expect && + bulk_status "$1" actual "$2" && + test_cmp expect actual +} + +compare_fallback_status () { + repo=$1 && + fallback_trace=$TRASH_DIRECTORY/$2 && + shift 2 && + GIT_OPTIONAL_LOCKS=0 \ + git -C "$repo" -c core.preloadIndex=false \ + status --porcelain=v2 "$@" >expect && + rm -f "$fallback_trace" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TRACE2_EVENT="$fallback_trace" \ + git -C "$repo" status --porcelain=v2 "$@" >actual && + test_cmp expect actual && + test_grep "\"category\":\"read_directory\"" "$fallback_trace" +} + +configured_bulk_status () { + repo=$1 && + output=$2 && + bulk_trace=$TRASH_DIRECTORY/$3 && + bulk=${4-true} && + preload=${5-true} && + rm -f "$bulk_trace" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TRACE2_EVENT="$bulk_trace" \ + git -C "$repo" \ + -c core.preloadIndex="$preload" \ + -c core.preloadIndexBulk="$bulk" \ + status --porcelain=v2 >"$output" +} + +setup_provider_proof_repo () { + setup_repo "$1" && + ( + cd "$1" && + git config core.trustctime false && + git config core.checkStat minimal && + mtime=$(test-tool chmtime --get root) && + printf "dirt\n" >root && + test-tool chmtime =$mtime root && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid root && + test_grep ! FSCF .git/index + ) +} + +setup_provider_proof_repo_with_untracked_cache () { + setup_repo "$1" && + ( + cd "$1" && + git config core.untrackedCache true && + git status --porcelain=2 >.git/prime && + test_must_be_empty .git/prime && + test_grep UNTR .git/index && + git config core.trustctime false && + git config core.checkStat minimal && + mtime=$(test-tool chmtime --get root) && + printf "dirt\n" >root && + test-tool chmtime =$mtime root && + test_write_lines visible >visible && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid root && + test_grep ! FSCF .git/index + ) +} + +test_expect_success 'bulk preload follows its configuration' ' + setup_repo opt-in && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/default.trace" \ + git -C opt-in status --porcelain=v2 >actual && + test_must_be_empty actual && + test_grep ! "\"key\":\"preload/bulk_result\"" default.trace && + configured_bulk_status opt-in actual enabled.trace && + test_must_be_empty actual && + test_grep "\"key\":\"preload/bulk_result\"" enabled.trace && + configured_bulk_status opt-in actual preload-disabled.trace true false && + test_must_be_empty actual && + test_grep ! "\"key\":\"preload/bulk_result\"" preload-disabled.trace +' + +test_expect_success 'test variable overrides bulk preload configuration' ' + test_env GIT_TEST_PRELOAD_INDEX_BULK=0 \ + configured_bulk_status opt-in actual disabled.trace && + test_must_be_empty actual && + test_grep ! "\"key\":\"preload/bulk_result\"" disabled.trace && + test_env GIT_TEST_PRELOAD_INDEX_BULK=1 \ + configured_bulk_status opt-in actual forced.trace false && + test_must_be_empty actual && + check_data forced.trace preload/bulk_applied 8 +' + +test_expect_success 'bulk preload waits for fsmonitor provider closure' ' + write_script opt-in/.git/hooks/fsmonitor-test <<-\EOF && + printf "token\\0" + EOF + git -C opt-in config core.fsmonitor .git/hooks/fsmonitor-test && + configured_bulk_status opt-in actual fsmonitor.trace && + test_must_be_empty actual && + test_grep ! "\"key\":\"preload/bulk_result\"" fsmonitor.trace +' + +test_expect_success 'provider closure accepts bulk content proofs' ' + setup_provider_proof_repo provider-proof && + ( + cd provider-proof && + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_grep "^1 \.M .* root$" .git/actual && + test_trace2_data status semantic_verify/bulk_scan 1 \ + <.git/status.trace && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/status.trace && + test_trace2_data index preload/bulk_content_verify 1 \ + <.git/status.trace && + test_trace2_data index preload/bulk_bytes_hashed \ + "[1-9][0-9]*" \ + <.git/status.trace && + test_trace2_data index preload/bulk_provider_applied 7 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep FSCF .git/index + ) +' + +test_expect_success \ + 'provider bulk preserves an existing untracked-cache binding' ' + setup_provider_proof_repo_with_untracked_cache provider-proof-uc && + ( + cd provider-proof-uc && + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=2 >.git/actual && + test_grep "^1 \.M .* root$" .git/actual && + test_grep "^? visible$" .git/actual && + ! test_trace2_data index preload/bulk_untracked_complete 1 \ + <.git/status.trace && + test_grep \ + "\"event\":\"region_enter\".*\"category\":\"dir\",\"label\":\"read_directory\"" \ + .git/status.trace >.git/read-directory && + test_line_count = 1 .git/read-directory && + test_grep FSUC .git/index + ) +' + +test_expect_success 'provider failure discards bulk content proofs' ' + setup_provider_proof_repo provider-failure && + ( + cd provider-failure && + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CE \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_grep "^1 \.M .* root$" .git/actual && + test_trace2_data status semantic_verify/bulk_scan 1 \ + <.git/status.trace && + test_trace2_data index preload/bulk_content_verify 1 \ + <.git/status.trace && + ! test_trace2_data index preload/bulk_provider_applied \ + "[0-9][0-9]*" <.git/status.trace && + test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep ! FSCF .git/index + ) +' + +cleanup_race () { + exec 9>&- + if test -n "$status_pid" + then + kill "$status_pid" 2>/dev/null || : + wait "$status_pid" 2>/dev/null || : + fi + status_pid= && + rm -f "$ready" "$resume" +} + +wait_for_ready () { + for i in $(test_seq 1 1000) + do + test "$(cat "$ready" 2>/dev/null)" = ready && return 0 + kill -0 "$status_pid" 2>/dev/null || return 1 + sleep 0.01 + done + return 1 +} + +start_raced_status () { + repo=$1 && + barrier=$2 && + ready=$TRASH_DIRECTORY/$repo.ready && + resume=$TRASH_DIRECTORY/$repo.resume && + race_trace=$TRASH_DIRECTORY/$repo.trace && + status_pid= && + rm -f "$ready" "$resume" "$race_trace" && + mkfifo "$resume" && + exec 9<>"$resume" && + { + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_PATH="$barrier" \ + GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_READY="$ready" \ + GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_RESUME="$resume" \ + GIT_TRACE2_EVENT="$race_trace" \ + git -C "$repo" status --porcelain=v2 >actual 9>&- & + status_pid=$! + } && + wait_for_ready +} + +finish_raced_status () { + printf "resume\n" >&9 && + exec 9>&- && + wait "$status_pid" && + status_pid= && + ordinary_status "$1" expect && + test_cmp expect actual && + test_trace2_data index preload/bulk_applied 0 <"$race_trace" +} + +finish_raced_untracked_status () { + printf "resume\n" >&9 && + exec 9>&- && + wait "$status_pid" && + status_pid= && + ordinary_status "$1" expect && + test_cmp expect actual && + test_trace2_data index preload/bulk_applied "$2" <"$race_trace" +} + +test_expect_success 'clean entries are published without lstat' ' + setup_repo clean && + bulk_status clean actual clean.trace && + test_must_be_empty actual && + check_data clean.trace preload/bulk_applied 8 && + check_data clean.trace preload/bulk_untracked_complete 1 && + check_data clean.trace preload/bulk_untracked_count 0 && + check_lstat_data clean.trace 0 +' + +test_expect_success 'definitive size changes are not restated' ' + setup_repo dirty && + test_write_lines changed-content >dirty/root && + compare_status dirty dirty.trace && + test_file_not_empty actual && + check_data dirty.trace preload/bulk_applied 7 && + check_data dirty.trace preload/bulk_definitive_modified 1 && + test_trace2_data status preload/direct_modified 1 \ + <"$TRASH_DIRECTORY/dirty.trace" && + check_lstat_data dirty.trace 0 && + check_data dirty.trace refresh/sum_lstat 0 +' + +test_expect_success 'missing entries bypass speculative lstat' ' + setup_repo missing && + rm missing/root && + rm -rf missing/nested && + compare_status missing missing.trace && + test_line_count = 5 actual && + check_data missing.trace preload/bulk_applied 3 && + check_data missing.trace preload/bulk_definitive_deleted 5 && + test_trace2_data status preload/direct_deleted 5 \ + <"$TRASH_DIRECTORY/missing.trace" && + check_lstat_data missing.trace 0 && + check_data missing.trace refresh/sum_lstat 0 +' + +test_expect_success 'metadata-only mismatches are checked by diff' ' + setup_repo metadata && + test-tool chmtime +60 metadata/root && + compare_status metadata metadata.trace && + test_must_be_empty actual && + check_data metadata.trace preload/bulk_content_check 1 && + check_lstat_data metadata.trace 0 && + check_data metadata.trace refresh/sum_lstat 0 +' + +test_expect_success PIPE \ + 'tracked directories and unsupported vnodes fall back' ' + setup_repo tracked-types && + rm tracked-types/root tracked-types/root-peer && + mkdir tracked-types/root && + mkfifo tracked-types/root-peer && + compare_status tracked-types tracked-types.trace && + test_line_count = 2 actual && + check_data tracked-types.trace preload/bulk_applied 6 && + check_data tracked-types.trace preload/bulk_fallback 2 && + check_lstat_data tracked-types.trace 2 && + check_data tracked-types.trace refresh/sum_lstat 2 +' + +test_expect_success 'writable status retains refreshed stat data' ' + setup_repo writable-stat && + test-tool chmtime +60 writable-stat/root && + cp writable-stat/.git/index before.index && + writable_ordinary_status writable-stat expect && + cp writable-stat/.git/index ordinary.index && + cp before.index writable-stat/.git/index && + writable_bulk_status writable-stat actual writable-stat.trace && + test_cmp expect actual && + test_cmp ordinary.index writable-stat/.git/index && + check_data writable-stat.trace preload/bulk_content_check 1 && + check_data writable-stat.trace refresh/sum_lstat 0 +' + +test_expect_success CASE_INSENSITIVE_FS \ + 'case aliases retain parallel preload' ' + setup_repo case-alias && + mv case-alias/root case-alias/ROOT && + mv case-alias/nested case-alias/NESTED && + compare_status case-alias case-alias.trace && + test_must_be_empty actual && + check_data case-alias.trace preload/bulk_applied 3 && + check_lstat_data case-alias.trace 5 && + { + test_have_prereq !PTHREADS || + check_data case-alias.trace refresh/sum_lstat 0 + } +' + +test_expect_success 'visible paths are returned by the bulk walk' ' + setup_repo visible-output && + test_write_lines "*.ignored" >visible-output/.gitignore && + git -C visible-output add .gitignore && + git -C visible-output commit -m ignore && + test_write_lines root >visible-output/root-untracked && + test_write_lines nested >visible-output/nested/untracked && + mkdir -p visible-output/collapsed/deep \ + visible-output/ignored-only/deep \ + visible-output/empty && + test_write_lines collapsed >visible-output/collapsed/deep/file && + test_write_lines ignored >visible-output/ignored-only/deep/file.ignored && + compare_status visible-output visible-output.trace && + test_grep "^? root-untracked$" actual && + test_grep "^? nested/untracked$" actual && + test_grep "^? collapsed/$" actual && + test_grep ! "ignored-only" actual && + test_grep ! "empty" actual && + check_data visible-output.trace preload/bulk_untracked_complete 1 && + check_data visible-output.trace preload/bulk_untracked_count 3 && + test_grep ! "\"category\":\"read_directory\"" \ + visible-output.trace +' + +test_expect_success PIPE 'special files are ignored' ' + setup_repo special-file && + mkfifo special-file/fifo && + compare_status special-file special-file.trace && + test_must_be_empty actual && + check_data special-file.trace preload/bulk_untracked_complete 1 && + check_data special-file.trace preload/bulk_untracked_count 0 +' + +test_expect_success 'activation guards retain ordinary traversal' ' + setup_repo activation-guards && + test_write_lines "*.ignored" >activation-guards/.gitignore && + git -C activation-guards add .gitignore && + git -C activation-guards commit -m ignore && + test_write_lines visible >activation-guards/visible && + test_write_lines ignored >activation-guards/file.ignored && + compare_fallback_status activation-guards all.trace \ + --untracked-files=all && + compare_fallback_status activation-guards ignored.trace \ + --ignored && + compare_fallback_status activation-guards pathspec.trace \ + -- nested && + git -C activation-guards update-index --untracked-cache && + compare_fallback_status activation-guards untracked-cache.trace +' + +test_expect_success CASE_INSENSITIVE_FS 'case aliases fall back' ' + setup_repo untracked-case-alias && + mv untracked-case-alias/root untracked-case-alias/ROOT && + compare_status untracked-case-alias untracked-case-alias.trace && + check_data untracked-case-alias.trace preload/bulk_untracked_complete 0 && + check_data untracked-case-alias.trace preload/bulk_untracked_count 0 +' + +test_expect_success 'nested repositories fall back' ' + setup_repo nested-repo && + test_write_lines "embedded/**" >nested-repo/.gitignore && + git -C nested-repo add .gitignore && + git -C nested-repo commit -m ignore && + mkdir nested-repo/embedded && + git -C nested-repo/embedded init && + test_write_lines ignored >nested-repo/embedded/file && + compare_status nested-repo nested-repo.trace && + test_grep "^? embedded/$" actual && + check_data nested-repo.trace preload/bulk_untracked_complete 0 +' + +test_expect_success 'tracked submodules retain collected paths' ' + git init submodule-child && + git -C submodule-child commit --allow-empty -m base && + setup_repo submodule-parent && + git -C submodule-parent -c protocol.file.allow=always \ + submodule add ../submodule-child embedded && + git -C submodule-parent commit -m submodule && + git -C submodule-parent update-index --refresh && + test_write_lines visible >submodule-parent/visible && + compare_status submodule-parent submodule-parent.trace && + test_grep "^? visible$" actual && + check_data submodule-parent.trace preload/bulk_untracked_complete 1 +' + +test_expect_success 'exclude changes discard collected paths' ' + setup_repo exclude-race && + exclude=$TRASH_DIRECTORY/exclude-race.patterns && + test_write_lines visible >"$exclude" && + git -C exclude-race config core.excludesFile "$exclude" && + test_write_lines visible >exclude-race/visible && + test_when_finished cleanup_race && + start_raced_status exclude-race "" && + >"$exclude" && + finish_raced_untracked_status exclude-race 8 && + test_grep "^? visible$" actual && + check_data exclude-race.trace preload/bulk_untracked_complete 0 && + check_data exclude-race.trace preload/bulk_untracked_reason exclude-race +' + +test_expect_success 'info exclude changes discard collected paths' ' + setup_repo info-exclude && + test_write_lines visible >info-exclude/.git/info/exclude && + test_write_lines visible >info-exclude/visible && + test_when_finished cleanup_race && + start_raced_status info-exclude "" && + >info-exclude/.git/info/exclude && + finish_raced_untracked_status info-exclude 8 && + test_grep "^? visible$" actual && + check_data info-exclude.trace preload/bulk_untracked_complete 0 && + check_data info-exclude.trace \ + preload/bulk_untracked_reason exclude-race +' + +test_expect_success 'new exclusion source discards collected paths' ' + setup_repo absent-exclude && + exclude_dir=$TRASH_DIRECTORY/absent-exclude-config && + exclude=$exclude_dir/ignore && + rm -rf "$exclude_dir" && + git -C absent-exclude config core.excludesFile "$exclude" && + test_write_lines visible >absent-exclude/visible && + test_when_finished cleanup_race && + test_when_finished "rm -rf \"$exclude_dir\"" && + start_raced_status absent-exclude "" && + mkdir "$exclude_dir" && + test_write_lines visible >"$exclude" && + finish_raced_untracked_status absent-exclude 8 && + test_must_be_empty actual && + check_data absent-exclude.trace preload/bulk_untracked_complete 0 && + check_data absent-exclude.trace \ + preload/bulk_untracked_reason exclude-race +' + +test_expect_success 'separate-git-dir excludes are proven' ' + setup_repo separate-info && + mv separate-info/.git separate-info.git && + printf "gitdir: ../separate-info.git\n" >separate-info/.git && + test_write_lines visible >separate-info.git/info/exclude && + test_write_lines visible >separate-info/visible && + compare_status separate-info separate-info.trace && + test_must_be_empty actual && + check_data separate-info.trace preload/bulk_untracked_complete 1 +' + +test_expect_success 'nested per-directory excludes are closed both ways' ' + test_when_finished cleanup_race && + for direction in visible-to-ignored ignored-to-visible + do + repo=per-dir-$direction && + setup_repo "$repo" && + git -C "$repo" config core.trustctime false && + case "$direction" in + visible-to-ignored) + initial=nomatch && + updated=visible + ;; + ignored-to-visible) + initial=visible && + updated=nomatch + ;; + esac && + test_write_lines "$initial" >"$repo/nested/.gitignore" && + git -C "$repo" add nested/.gitignore && + git -C "$repo" commit -m ignore && + git -C "$repo" update-index --assume-unchanged \ + nested/.gitignore && + test_write_lines visible >"$repo/nested/visible" && + mtime=$(test-tool chmtime --get \ + "$repo/nested/.gitignore") && + start_raced_status "$repo" "" && + test_write_lines "$updated" >"$repo/nested/.gitignore" && + test-tool chmtime "=$mtime" "$repo/nested/.gitignore" && + finish_raced_untracked_status "$repo" 8 && + check_data "$repo.trace" \ + preload/bulk_untracked_complete 0 && + check_data "$repo.trace" \ + preload/bulk_untracked_reason exclude-race && + case "$direction" in + visible-to-ignored) + test_must_be_empty actual + ;; + ignored-to-visible) + test_grep "^? nested/visible$" actual + ;; + esac || + return 1 + done +' + +test_expect_success 'multiply-linked per-directory excludes are proven' ' + setup_repo linked-exclude && + test_write_lines visible >linked-exclude/.gitignore && + git -C linked-exclude add .gitignore && + git -C linked-exclude commit -m ignore && + test_write_lines visible >linked-exclude/visible && + ln linked-exclude/.gitignore linked-exclude-alias && + test_when_finished "rm -f linked-exclude-alias" && + compare_status linked-exclude linked-exclude.trace && + test_must_be_empty actual && + check_data linked-exclude.trace preload/bulk_untracked_complete 1 && + check_data linked-exclude.trace preload/bulk_applied 8 && + check_data linked-exclude.trace preload/bulk_untracked_count 0 +' + +test_expect_success 'multiply-linked exclude changes discard paths' ' + setup_repo linked-exclude-race && + test_write_lines visible >linked-exclude-race/.gitignore && + git -C linked-exclude-race add .gitignore && + git -C linked-exclude-race commit -m ignore && + test_write_lines visible >linked-exclude-race/visible && + ln linked-exclude-race/.gitignore linked-exclude-race-alias && + test_when_finished "rm -f linked-exclude-race-alias" && + test_when_finished cleanup_race && + start_raced_status linked-exclude-race "" && + test_write_lines nomatch >linked-exclude-race-alias && + finish_raced_untracked_status linked-exclude-race 8 && + test_grep "^? visible$" actual && + check_data linked-exclude-race.trace \ + preload/bulk_untracked_complete 0 && + check_data linked-exclude-race.trace \ + preload/bulk_untracked_reason exclude-race +' + +test_expect_success ULIMIT_FILE_DESCRIPTORS \ + 'bulk preload reopens directories under a low descriptor limit' ' + git init low-fd && + for i in $(test_seq 1 64) + do + mkdir "low-fd/$i" && + test_write_lines "$i" >"low-fd/$i/tracked" || + return 1 + done && + git -C low-fd add . && + git -C low-fd commit -m base && + test-tool chmtime -120 low-fd/*/tracked && + git -C low-fd update-index --refresh && + run_with_limited_open_files \ + bulk_status low-fd actual low-fd.trace && + test_must_be_empty actual && + check_data low-fd.trace preload/bulk_applied 64 +' + +test_expect_success 'prefix siblings do not hide tracked descendants' ' + git init prefix-order && + mkdir prefix-order/feather prefix-order/feather-db && + test_write_lines tracked >prefix-order/feather/tracked && + test_write_lines sibling >prefix-order/feather-db/tracked && + git -C prefix-order add . && + git -C prefix-order commit -m base && + test-tool chmtime -120 prefix-order/feather/tracked \ + prefix-order/feather-db/tracked && + git -C prefix-order update-index --refresh && + bulk_status prefix-order actual prefix-order.trace && + test_must_be_empty actual && + check_data prefix-order.trace preload/bulk_applied 2 && + check_lstat_data prefix-order.trace 0 +' + +test_expect_success SYMLINKS \ + 'modified, deleted, typechanged, and symlink entries agree' ' + setup_repo worktree-states && + ln -s root worktree-states/link && + git -C worktree-states add link && + git -C worktree-states commit -m symlink && + mtime=$(test-tool chmtime --get worktree-states/root) && + sleep 1 && + test_write_lines moot >worktree-states/root && + test-tool chmtime "=$mtime" worktree-states/root && + rm worktree-states/nested/tracked && + rm worktree-states/nested/deep/tracked && + mkdir worktree-states/nested/deep/tracked && + rm worktree-states/other/tracked worktree-states/other/peer && + rmdir worktree-states/other && + test_write_lines other >worktree-states/other && + rm worktree-states/link && + ln -s nested/deep/peer worktree-states/link && + compare_status worktree-states worktree-states.trace && + test_file_not_empty actual +' + +test_expect_success 'tracked-file replacement directories are pruned' ' + setup_repo replacement-dir && + rm replacement-dir/root && + mkdir -p replacement-dir/root/deep/embedded && + test_write_lines hidden >replacement-dir/root/deep/untracked && + git -C replacement-dir/root/deep/embedded init && + compare_status replacement-dir replacement-dir.trace && + test_line_count = 1 actual && + check_data replacement-dir.trace preload/bulk_fallback 1 && + check_data replacement-dir.trace preload/bulk_untracked_complete 1 +' + +test_expect_success 'staged and unmerged entries agree' ' + setup_repo index-states && + test_write_lines staged >index-states/root && + test_write_lines added >index-states/added && + git -C index-states add root added && + git -C index-states rm nested/tracked && + base=$(git -C index-states rev-parse HEAD:nested/peer) && + ours=$(printf "ours\n" | + git -C index-states hash-object -w --stdin) && + theirs=$(printf "theirs\n" | + git -C index-states hash-object -w --stdin) && + { + printf "0 %s\tnested/peer\n" "$(test_oid zero)" && + printf "100644 %s 1\tnested/peer\n" "$base" && + printf "100644 %s 2\tnested/peer\n" "$ours" && + printf "100644 %s 3\tnested/peer\n" "$theirs" + } | git -C index-states update-index --index-info && + compare_status index-states index-states.trace && + test_file_not_empty actual +' + +test_expect_success 'sparse-index entries are left unexpanded' ' + setup_repo sparse && + git -C sparse sparse-checkout init --cone --sparse-index && + git -C sparse sparse-checkout set nested && + git -C sparse ls-files --sparse >before && + test_grep "^other/$" before && + test_write_lines changed >sparse/nested/tracked && + compare_status sparse sparse.trace && + git -C sparse ls-files --sparse >after && + test_cmp before after +' + +test_expect_success UTF8_NFD_TO_NFC \ + 'decomposed Unicode names agree' ' + setup_repo unicode && + nfc=$(printf "\303\244") && + nfd=$(printf "\141\314\210") && + git -C unicode config core.precomposeunicode true && + test_write_lines unicode >"unicode/$nfd" && + git -C unicode add "$nfc" && + git -C unicode commit -m unicode && + test-tool chmtime -120 "unicode/$nfd" && + git -C unicode update-index --refresh && + compare_status unicode unicode.trace && + test_must_be_empty actual && + check_data unicode.trace preload/bulk_applied 9 && + check_lstat_data unicode.trace 0 +' + +test_expect_success 'multiply-linked entries are left to lstat' ' + setup_repo hardlink && + # Mutate through a name outside the watched worktree, then restore + # mtime. The bulk scan must leave the multiply-linked entry to lstat. + ln hardlink/root hardlink-alias && + test-tool chmtime -120 hardlink/root && + git -C hardlink update-index --refresh && + mtime=$(test-tool chmtime --get hardlink/root) && + sleep 1 && + test_write_lines moot >hardlink-alias && + test-tool chmtime "=$mtime" hardlink/root && + compare_status hardlink hardlink.trace && + test_file_not_empty actual && + check_data hardlink.trace preload/bulk_applied 7 && + check_data hardlink.trace preload/bulk_fallback 1 && + check_lstat_data hardlink.trace 1 && + check_data hardlink.trace refresh/sum_lstat 1 +' + +test_expect_success PIPE 'queued child replacement discards observations' ' + setup_repo child-race && + test_when_finished cleanup_race && + start_raced_status child-race nested/deep && + mv child-race/nested/deep child-race/nested/deep-away && + mkdir child-race/nested/deep && + test_write_lines dirty >child-race/nested/deep/tracked && + test_write_lines deep-peer >child-race/nested/deep/peer && + finish_raced_status child-race && + test_file_not_empty actual +' + +test_expect_success PIPE,SYMLINKS \ + 'worktree root replacement discards observations' ' + setup_repo root-race && + test_when_finished cleanup_race && + start_raced_status root-race "" && + mv root-race root-race-away && + ln -s root-race-away root-race && + test_write_lines dirty >root-race-away/root && + finish_raced_status root-race && + test_file_not_empty actual +' + +test_done diff --git a/t/t7530-status-clean-sidecar.sh b/t/t7530-status-clean-sidecar.sh new file mode 100755 index 00000000000000..1616d5faf9a7db --- /dev/null +++ b/t/t7530-status-clean-sidecar.sh @@ -0,0 +1,629 @@ +#!/bin/sh + +test_description='exact clean status sidecars' + +. ./test-lib.sh + +test_lazy_prereq LOCAL_APFS ' + test_have_prereq MACOS && + /bin/df -t apfs "$TRASH_DIRECTORY" >/dev/null +' + +if ! test_have_prereq FSMONITOR_DAEMON,LOCAL_APFS,MACOS +then + skip_all='clean status sidecars require local APFS and the macOS fsmonitor daemon' + test_done +fi + +test_lazy_prereq DURABLE_FSMONITOR ' + test_create_repo durable-fsmonitor-probe || return 1 + ( + cd durable-fsmonitor-probe && + test_commit base tracked && + git config core.fsmonitor true && + git fsmonitor--daemon start --start-timeout=10 && + git status --porcelain=v2 >/dev/null && + test-tool dump-fsmonitor >token && + grep "^fsmonitor last update builtin:" token + result=$? + git fsmonitor--daemon stop >/dev/null 2>&1 || : + exit $result + ) +' + +stop_daemon () { + git -C "$1" fsmonitor--daemon stop 2>/dev/null || : +} + +setup_repo () { + repo=$1 && + test_create_repo "$repo" && + test_commit -C "$repo" base tracked && + test-tool chmtime -120 "$repo/tracked" && + git -C "$repo" update-index --refresh && + git -C "$repo" config core.fsmonitor true && + git -C "$repo" fsmonitor--daemon start --start-timeout=10 +} + +bulk_status () { + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + git "$@" +} + +prime_semantic_history () { + repo=$1 && + bulk_status -C "$repo" status --porcelain=2 >actual.1 && + test_must_be_empty actual.1 && + bulk_status -C "$repo" status --porcelain=2 >actual.2 && + test_must_be_empty actual.2 && + test_grep FSCF "$repo/.git/index" +} + +issue_sidecar () { + repo=$1 && + prime_semantic_history "$repo" && + git -C "$repo" config core.autocrlf false && + bulk_status -C "$repo" status --porcelain=v2 >actual.issue && + test_must_be_empty actual.issue && + test_path_is_file "$repo/.git/index.csts" +} + +assert_fallback_matches_oracle () { + repo=$1 && + sidecar_trace=$2 && + GIT_OPTIONAL_LOCKS=0 git -c core.fsmonitor=false \ + -c core.untrackedCache=false -C "$repo" \ + status --porcelain=v2 >expect && + GIT_OPTIONAL_LOCKS=0 GIT_TRACE2_EVENT="$PWD/$sidecar_trace" \ + git -C "$repo" status --porcelain=v2 >actual && + test_cmp expect actual && + test_grep ! "\"key\":\"clean-proof/hit\"" "$sidecar_trace" +} + +assert_custom_replace_fallback_matches_oracle () { + repo=$1 && + sidecar_trace=$2 && + GIT_REPLACE_REF_BASE=refs/status-replace/ \ + GIT_OPTIONAL_LOCKS=0 git -c core.fsmonitor=false \ + -c core.untrackedCache=false -C "$repo" \ + status --porcelain=v2 >expect && + GIT_REPLACE_REF_BASE=refs/status-replace/ \ + GIT_OPTIONAL_LOCKS=0 GIT_TRACE2_EVENT="$PWD/$sidecar_trace" \ + git -C "$repo" status --porcelain=v2 >actual && + test_cmp expect actual && + test_grep ! "\"key\":\"clean-proof/hit\"" "$sidecar_trace" +} + +replacement_tree () { + repo=$1 && + blob=$(printf "replacement\n" | + git -C "$repo" hash-object -w --stdin) && + printf "100644 blob %s\ttracked\n" "$blob" | + git -C "$repo" mktree +} + +cleanup_fast_race () { + if test -n "$status_pid" + then + kill "$status_pid" 2>/dev/null || : + wait "$status_pid" 2>/dev/null || : + fi && + status_pid= && + exec 9>&- && + rm -f "$ready" "$resume" +} + +wait_for_fast_ready () { + for i in $(test_seq 1 1000) + do + test "$(cat "$ready" 2>/dev/null)" = ready && return 0 + kill -0 "$status_pid" 2>/dev/null || return 1 + sleep 0.01 + done + return 1 +} + +start_fast_raced_status () { + repo=$1 && + ready=$TRASH_DIRECTORY/$repo.fast-ready && + resume=$TRASH_DIRECTORY/$repo.fast-resume && + race_trace=$TRASH_DIRECTORY/$repo.fast-trace && + status_pid= && + rm -f "$ready" "$resume" "$race_trace" && + : >"$ready" && + mkfifo "$resume" && + exec 9<>"$resume" && + { + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_STATUS_CLEAN_SIDECAR_BARRIER_READY="$ready" \ + GIT_TEST_STATUS_CLEAN_SIDECAR_BARRIER_RESUME="$resume" \ + GIT_TRACE2_EVENT="$race_trace" \ + git -C "$repo" status --porcelain=v2 \ + >raced.actual 9>&- & + status_pid=$! + } && + wait_for_fast_ready +} + +start_issue_raced_status () { + repo=$1 && + ready=$TRASH_DIRECTORY/$repo.issue-ready && + resume=$TRASH_DIRECTORY/$repo.issue-resume && + race_trace=$TRASH_DIRECTORY/$repo.issue-trace && + status_pid= && + rm -f "$ready" "$resume" "$race_trace" && + : >"$ready" && + mkfifo "$resume" && + exec 9<>"$resume" && + { + test_env \ + GIT_TEST_STATUS_CLEAN_SIDECAR_ISSUE_BARRIER_READY="$ready" \ + GIT_TEST_STATUS_CLEAN_SIDECAR_ISSUE_BARRIER_RESUME="$resume" \ + GIT_TRACE2_EVENT="$race_trace" \ + bulk_status -C "$repo" status --porcelain=v2 \ + >raced.actual 9>&- & + status_pid=$! + } && + wait_for_fast_ready +} + +stop_after_fast_fallback () { + for i in $(test_seq 1 1000) + do + if grep -q "\"value\":\"fast-excludes-raced\"" \ + "$race_trace" + then + kill "$status_pid" 2>/dev/null || return 1 + wait "$status_pid" 2>/dev/null || : + status_pid= + return 0 + fi + kill -0 "$status_pid" 2>/dev/null || return 1 + sleep 0.01 + done + return 1 +} + +finish_fast_raced_status () { + printf "resume\n" >&9 && + exec 9>&- && + wait "$status_pid" && + status_pid= +} + +test_expect_success DURABLE_FSMONITOR \ + 'exact clean status installs a sidecar without rewriting the index' ' + test_when_finished "stop_daemon sidecar-issue" && + setup_repo sidecar-issue && + test_env GIT_TRACE2_EVENT="$PWD/first-scan.trace" \ + bulk_status -C sidecar-issue status --porcelain=v2 \ + >actual.first && + test_must_be_empty actual.first && + test_path_is_missing sidecar-issue/.git/index.csts && + test_grep "\"value\":\"issue-coherent-history\"" first-scan.trace && + + prime_semantic_history sidecar-issue && + git -C sidecar-issue config core.autocrlf false && + cp sidecar-issue/.git/index index.before && + + test_env GIT_TRACE2_EVENT="$PWD/issue.trace" \ + bulk_status -C sidecar-issue status --porcelain=v2 >actual && + test_must_be_empty actual && + test_cmp index.before sidecar-issue/.git/index && + test_path_is_file sidecar-issue/.git/index.csts && + test_grep \ + "\"key\":\"preload/bulk_untracked_complete\",\"value\":\"1\"" \ + issue.trace && + test_grep "\"key\":\"preload/bulk_provider_applied\"" issue.trace && + test_grep "\"key\":\"clean-proof/sidecar\"" issue.trace && + test_grep ! "\"label\":\"do_write_index\"" issue.trace && + + GIT_TRACE2_EVENT="$PWD/hit.trace" \ + git -C sidecar-issue status --porcelain=v2 >actual && + test_must_be_empty actual && + test_grep "\"key\":\"clean-proof/hit\"" hit.trace && + test_grep ! "\"label\":\"do_read_index\"" hit.trace +' + +test_expect_success DURABLE_FSMONITOR \ + 'exact clean status certifies an existing untracked cache' ' + test_when_finished "stop_daemon sidecar-untracked-cache" && + setup_repo sidecar-untracked-cache && + git -C sidecar-untracked-cache config core.untrackedCache true && + git -C sidecar-untracked-cache config core.autocrlf false && + bulk_status -C sidecar-untracked-cache status --porcelain=2 \ + >actual.1 && + test_must_be_empty actual.1 && + bulk_status -C sidecar-untracked-cache status --porcelain=2 \ + >actual.2 && + test_must_be_empty actual.2 && + test_grep UNTR sidecar-untracked-cache/.git/index && + + test_env GIT_TRACE2_EVENT="$PWD/untracked-cache-issue.trace" \ + bulk_status -C sidecar-untracked-cache \ + status --porcelain=v2 >actual && + test_must_be_empty actual && + test_path_is_file sidecar-untracked-cache/.git/index.csts && + test_grep ! \ + "\"key\":\"preload/bulk_useful\"" \ + untracked-cache-issue.trace && + test_grep \ + "\"event\":\"region_enter\".*\"category\":\"dir\",\"label\":\"read_directory\"" \ + untracked-cache-issue.trace >untracked-cache-read-directory && + test_line_count = 1 untracked-cache-read-directory && + test_grep "\"key\":\"proof_valid\",\"value\":\"1\"" \ + untracked-cache-issue.trace && + test_grep FSUC sidecar-untracked-cache/.git/index && + + GIT_TRACE2_EVENT="$PWD/untracked-cache-hit.trace" \ + git -C sidecar-untracked-cache status --porcelain=v2 \ + >actual && + test_must_be_empty actual && + test_grep "\"key\":\"clean-proof/hit\"" \ + untracked-cache-hit.trace && + test_grep ! "\"label\":\"do_read_index\"" \ + untracked-cache-hit.trace +' + +test_expect_success POSIXPERM,DURABLE_FSMONITOR \ + 'an incomplete untracked traversal cannot issue a sidecar' ' + test_when_finished "stop_daemon sidecar-unreadable" && + setup_repo sidecar-unreadable && + git -C sidecar-unreadable config core.untrackedCache true && + git -C sidecar-unreadable config core.autocrlf false && + prime_semantic_history sidecar-unreadable && + mkdir sidecar-unreadable/hidden && + test_when_finished "chmod u+rwx sidecar-unreadable/hidden" && + test_write_lines untracked >sidecar-unreadable/hidden/untracked && + chmod a-r sidecar-unreadable/hidden && + + test_env GIT_TRACE2_EVENT="$PWD/unreadable.trace" \ + bulk_status -C sidecar-unreadable \ + status --porcelain=v2 >actual 2>err && + test_must_be_empty actual && + test_grep "could not open directory .hidden/." err && + test_path_is_missing sidecar-unreadable/.git/index.csts && + test_grep "\"value\":\"issue-scan-or-index-shape\"" \ + unreadable.trace && + + chmod u+rwx sidecar-unreadable/hidden && + git -C sidecar-unreadable status --porcelain=v2 >actual && + test_grep "^? hidden/" actual +' + +test_expect_success DURABLE_FSMONITOR \ + 'a replaced worktree root cannot inherit a sidecar' ' + test_when_finished "stop_daemon sidecar-root-race" && + test_when_finished "stop_daemon sidecar-root-race.scanned" && + test_when_finished "cleanup_fast_race" && + setup_repo sidecar-root-race && + git -C sidecar-root-race config core.untrackedCache true && + prime_semantic_history sidecar-root-race && + git -C sidecar-root-race config core.autocrlf false && + cp -R sidecar-root-race sidecar-root-race.replacement && + test_write_lines replacement-only \ + >sidecar-root-race.replacement/replacement-only && + rm -f sidecar-root-race.replacement/.git/index.csts && + + start_issue_raced_status sidecar-root-race && + mv sidecar-root-race sidecar-root-race.scanned && + mv sidecar-root-race.replacement sidecar-root-race && + finish_fast_raced_status && + + test_must_be_empty raced.actual && + test_path_is_missing sidecar-root-race/.git/index.csts && + test_grep \ + "\"category\":\"dir\",\"label\":\"read_directory\"" \ + "$race_trace" && + test_grep ! \ + "\"key\":\"preload/bulk_untracked_complete\",\"value\":\"1\"" \ + "$race_trace" && + test_grep "\"key\":\"proof_valid\",\"value\":\"1\"" "$race_trace" && + test_grep "\"value\":\"issue-pinned-inputs\"" "$race_trace" +' + +test_expect_success PIPE,DURABLE_FSMONITOR \ + 'an existing per-directory exclude FIFO cannot block sidecar issuance' ' + test_when_finished "stop_daemon sidecar-issue-fifo" && + setup_repo sidecar-issue-fifo && + test_when_finished "rm -f sidecar-issue-fifo/.gitignore" && + git -C sidecar-issue-fifo config core.untrackedCache true && + git -C sidecar-issue-fifo config core.autocrlf false && + prime_semantic_history sidecar-issue-fifo && + mkfifo sidecar-issue-fifo/.gitignore && + + test_env GIT_TRACE2_EVENT="$PWD/issue-fifo.trace" \ + bulk_status -C sidecar-issue-fifo \ + status --porcelain=v2 >actual && + test_must_be_empty actual && + test_path_is_file sidecar-issue-fifo/.git/index.csts && + test_grep "\"key\":\"clean-proof/sidecar\"" issue-fifo.trace +' + +test_expect_success DURABLE_FSMONITOR \ + 'a clean exact status replaces a stale sidecar' ' + test_when_finished "stop_daemon sidecar-reissue" && + setup_repo sidecar-reissue && + git -C sidecar-reissue config core.untrackedCache true && + issue_sidecar sidecar-reissue && + test_write_lines untracked >sidecar-reissue/untracked && + git -C sidecar-reissue status --porcelain=2 >dirty && + test_grep "^? untracked$" dirty && + rm sidecar-reissue/untracked && + + test_env GIT_TRACE2_EVENT="$PWD/reissue.trace" \ + bulk_status -C sidecar-reissue status --porcelain=v2 \ + >actual && + test_must_be_empty actual && + test_grep "\"key\":\"clean-proof/sidecar\"" reissue.trace && + test_grep ! \ + "\"key\":\"preload/bulk_useful\"" \ + reissue.trace && + test_grep \ + "\"event\":\"region_enter\".*\"category\":\"dir\",\"label\":\"read_directory\"" \ + reissue.trace >reissue-read-directory && + test_line_count = 1 reissue-read-directory && + test_grep ! "\"label\":\"do_write_index\"" reissue.trace && + + GIT_TRACE2_EVENT="$PWD/reissued-hit.trace" \ + git -C sidecar-reissue status --porcelain=v2 >actual && + test_must_be_empty actual && + test_grep "\"key\":\"clean-proof/hit\"" reissued-hit.trace && + test_grep ! "\"label\":\"do_read_index\"" reissued-hit.trace +' + +test_expect_success DURABLE_FSMONITOR \ + 'only an exact empty output installs a sidecar' ' + test_when_finished "stop_daemon sidecar-shape" && + setup_repo sidecar-shape && + prime_semantic_history sidecar-shape && + + bulk_status -C sidecar-shape status --porcelain=2 >actual && + test_must_be_empty actual && + test_path_is_missing sidecar-shape/.git/index.csts && + + bulk_status -C sidecar-shape status --porcelain=v2 --branch >actual && + test_grep "^# branch.oid " actual && + test_path_is_missing sidecar-shape/.git/index.csts && + + echo changed >sidecar-shape/tracked && + bulk_status -C sidecar-shape status --porcelain=v2 >actual && + test_grep "^1 .M " actual && + test_path_is_missing sidecar-shape/.git/index.csts +' + +test_expect_success DURABLE_FSMONITOR \ + 'external attributes and alternate indexes are rejected' ' + test_when_finished "stop_daemon sidecar-inputs" && + setup_repo sidecar-inputs && + prime_semantic_history sidecar-inputs && + + test_write_lines "tracked -text" \ + >sidecar-inputs/.git/info/attributes && + bulk_status -C sidecar-inputs status --porcelain=v2 >actual && + test_must_be_empty actual && + test_path_is_missing sidecar-inputs/.git/index.csts && + + rm sidecar-inputs/.git/info/attributes && + cp sidecar-inputs/.git/index sidecar-inputs/.git/alternate-index && + test_env GIT_INDEX_FILE="$PWD/sidecar-inputs/.git/alternate-index" \ + bulk_status -C sidecar-inputs status --porcelain=v2 >actual && + test_must_be_empty actual && + test_path_is_missing sidecar-inputs/.git/alternate-index.csts +' + +test_expect_success DURABLE_FSMONITOR \ + 'a fast hit remains read-only without optional locks' ' + test_when_finished "stop_daemon sidecar-read-only" && + setup_repo sidecar-read-only && + issue_sidecar sidecar-read-only && + cp sidecar-read-only/.git/index.csts sidecar.before && + + GIT_OPTIONAL_LOCKS=0 GIT_TRACE2_EVENT="$PWD/read-only.trace" \ + git -C sidecar-read-only status --porcelain=v2 >actual && + test_must_be_empty actual && + test_grep "\"key\":\"clean-proof/hit\"" read-only.trace && + test_grep ! "\"label\":\"do_read_index\"" read-only.trace && + test_cmp sidecar.before sidecar-read-only/.git/index.csts +' + +test_expect_success DURABLE_FSMONITOR \ + 'provider changes fall back for each dirty worktree shape' ' + test_when_finished "stop_daemon sidecar-modified" && + test_when_finished "stop_daemon sidecar-deleted" && + test_when_finished "stop_daemon sidecar-renamed" && + test_when_finished "stop_daemon sidecar-untracked" && + + setup_repo sidecar-modified && + issue_sidecar sidecar-modified && + echo changed >sidecar-modified/tracked && + assert_fallback_matches_oracle sidecar-modified modified.trace && + test_grep "^1 .M " actual && + + setup_repo sidecar-deleted && + issue_sidecar sidecar-deleted && + rm sidecar-deleted/tracked && + assert_fallback_matches_oracle sidecar-deleted deleted.trace && + test_grep "^1 .D " actual && + + setup_repo sidecar-renamed && + issue_sidecar sidecar-renamed && + mv sidecar-renamed/tracked sidecar-renamed/renamed && + assert_fallback_matches_oracle sidecar-renamed renamed.trace && + test_grep "^1 .D " actual && + test_grep "^? renamed" actual && + + setup_repo sidecar-untracked && + issue_sidecar sidecar-untracked && + echo untracked >sidecar-untracked/new-file && + assert_fallback_matches_oracle sidecar-untracked untracked.trace && + test_grep "^? new-file" actual +' + +test_expect_success DURABLE_FSMONITOR \ + 'loose and packed replace refs invalidate a sidecar' ' + test_when_finished "stop_daemon sidecar-replace" && + setup_repo sidecar-replace && + issue_sidecar sidecar-replace && + old_tree=$(git -C sidecar-replace rev-parse "HEAD^{tree}") && + new_tree=$(replacement_tree sidecar-replace) && + git -C sidecar-replace replace "$old_tree" "$new_tree" && + + assert_fallback_matches_oracle sidecar-replace replace-loose.trace && + test_grep "^1 M. " actual && + git -C sidecar-replace pack-refs --all && + test_path_is_missing \ + "sidecar-replace/.git/refs/replace/$old_tree" && + assert_fallback_matches_oracle sidecar-replace replace-packed.trace && + test_grep "^1 M. " actual +' + +test_expect_success DURABLE_FSMONITOR \ + 'a custom replace namespace invalidates a sidecar' ' + test_when_finished "stop_daemon sidecar-custom-replace" && + setup_repo sidecar-custom-replace && + issue_sidecar sidecar-custom-replace && + old_tree=$(git -C sidecar-custom-replace rev-parse "HEAD^{tree}") && + new_tree=$(replacement_tree sidecar-custom-replace) && + GIT_REPLACE_REF_BASE=refs/status-replace/ \ + git -C sidecar-custom-replace update-ref \ + "refs/status-replace/$old_tree" "$new_tree" && + + assert_custom_replace_fallback_matches_oracle \ + sidecar-custom-replace replace-custom.trace && + test_grep "^1 M. " actual +' + +test_expect_success PIPE,DURABLE_FSMONITOR \ + 'a sidecar FIFO cannot block or supply a hit' ' + test_when_finished "stop_daemon sidecar-fifo" && + test_when_finished "rm -f sidecar-fifo/.git/index.csts" && + setup_repo sidecar-fifo && + issue_sidecar sidecar-fifo && + rm sidecar-fifo/.git/index.csts && + mkfifo sidecar-fifo/.git/index.csts && + + GIT_OPTIONAL_LOCKS=0 GIT_TRACE2_EVENT="$PWD/fifo.trace" \ + git -C sidecar-fifo status --porcelain=v2 >actual && + test_must_be_empty actual && + test_grep ! "\"key\":\"clean-proof/hit\"" fifo.trace && + test_grep "\"value\":\"fast-sidecar-missing-or-corrupt\"" fifo.trace +' + +test_expect_success DURABLE_FSMONITOR \ + 'non-provider proof inputs invalidate a sidecar' ' + test_when_finished "stop_daemon sidecar-metadata" && + setup_repo sidecar-metadata && + issue_sidecar sidecar-metadata && + + git -C sidecar-metadata config status.relativePaths false && + assert_fallback_matches_oracle sidecar-metadata config.trace && + test_grep "\"value\":\"fast-config-changed\"" config.trace && + git -C sidecar-metadata config --unset status.relativePaths && + + cp sidecar-metadata/.git/info/exclude info-exclude && + test_write_lines ignored >sidecar-metadata/.git/info/exclude && + assert_fallback_matches_oracle sidecar-metadata exclude.trace && + test_grep "\"value\":\"fast-excludes\"" exclude.trace && + mv info-exclude sidecar-metadata/.git/info/exclude && + + test_write_lines "tracked ident" \ + >sidecar-metadata/.git/info/attributes && + assert_fallback_matches_oracle sidecar-metadata attributes.trace && + test_grep "\"value\":\"fast-repository-unavailable\"" attributes.trace && + rm sidecar-metadata/.git/info/attributes && + + blob=$(printf "different\n" | + git -C sidecar-metadata hash-object -w --stdin) && + tree=$(printf "100644 blob %s\ttracked\n" "$blob" | + git -C sidecar-metadata mktree) && + commit=$(printf "different tree\n" | + git -C sidecar-metadata commit-tree "$tree" -p HEAD) && + git -C sidecar-metadata update-ref HEAD "$commit" && + assert_fallback_matches_oracle sidecar-metadata head.trace && + test_grep "\"value\":\"fast-head-changed\"" head.trace && + test_grep "^1 M. " actual +' + +test_expect_success DURABLE_FSMONITOR \ + 'a v4 skipHash index is not certified' ' + test_when_finished "stop_daemon sidecar-v4" && + setup_repo sidecar-v4 && + prime_semantic_history sidecar-v4 && + git -C sidecar-v4 config index.version 4 && + git -C sidecar-v4 config index.skipHash true && + git -C sidecar-v4 update-index --force-write-index && + git -C sidecar-v4 config core.autocrlf false && + + dd if=/dev/zero of=zeros bs=20 count=1 2>/dev/null && + tail -c 20 sidecar-v4/.git/index >trailer && + test_cmp_bin zeros trailer && + test_env GIT_TRACE2_EVENT="$PWD/v4.trace" \ + bulk_status -C sidecar-v4 status --porcelain=v2 >actual && + test_must_be_empty actual && + test_path_is_missing sidecar-v4/.git/index.csts && + test_grep ! "\"key\":\"clean-proof/hit\"" v4.trace +' + +test_expect_success PIPE,DURABLE_FSMONITOR \ + 'an existing exclude FIFO cannot block fast-path capture' ' + test_when_finished "stop_daemon sidecar-exclude-fifo" && + setup_repo sidecar-exclude-fifo && + exclude_file=$(mktemp \ + "${TMPDIR:-/tmp}/git-status-exclude-fifo.XXXXXX") && + test_when_finished "rm -f \"$exclude_file\"" && + git -C sidecar-exclude-fifo config core.excludesFile \ + "$exclude_file" && + issue_sidecar sidecar-exclude-fifo && + rm "$exclude_file" && + mkfifo "$exclude_file" && + + GIT_OPTIONAL_LOCKS=0 GIT_TRACE2_EVENT="$PWD/exclude-fifo.trace" \ + git -C sidecar-exclude-fifo status --porcelain=v2 >actual && + test_must_be_empty actual && + test_grep "\"key\":\"clean-proof/hit\"" exclude-fifo.trace +' + +test_expect_success PIPE,DURABLE_FSMONITOR \ + 'a raced exclude FIFO cannot block sidecar validation' ' + test_when_finished "stop_daemon sidecar-exclude-race" && + test_when_finished "cleanup_fast_race" && + setup_repo sidecar-exclude-race && + exclude_file=$(mktemp \ + "${TMPDIR:-/tmp}/git-status-exclude-race.XXXXXX") && + test_when_finished "rm -f \"$exclude_file\"" && + test_write_lines ignored >"$exclude_file" && + git -C sidecar-exclude-race config core.excludesFile \ + "$exclude_file" && + issue_sidecar sidecar-exclude-race && + + start_fast_raced_status sidecar-exclude-race && + rm "$exclude_file" && + mkfifo "$exclude_file" && + printf "resume\n" >&9 && + exec 9>&- && + stop_after_fast_fallback && + test_must_be_empty raced.actual && + test_grep ! "\"key\":\"clean-proof/hit\"" "$race_trace" && + test_grep "\"value\":\"fast-excludes-raced\"" "$race_trace" +' + +test_expect_success PIPE,DURABLE_FSMONITOR \ + 'a replace ref created after the provider query prevents a hit' ' + test_when_finished "stop_daemon sidecar-replace-race" && + test_when_finished "cleanup_fast_race" && + setup_repo sidecar-replace-race && + issue_sidecar sidecar-replace-race && + old_tree=$(git -C sidecar-replace-race rev-parse "HEAD^{tree}") && + new_tree=$(replacement_tree sidecar-replace-race) && + + start_fast_raced_status sidecar-replace-race && + git -C sidecar-replace-race update-ref \ + "refs/replace/$old_tree" "$new_tree" && + finish_fast_raced_status && + test_grep ! "\"key\":\"clean-proof/hit\"" "$race_trace" && + test_grep "\"value\":\"fast-repository-raced\"" "$race_trace" +' + +test_done diff --git a/t/t7531-semantic-verify.sh b/t/t7531-semantic-verify.sh new file mode 100755 index 00000000000000..8c8293d8d689da --- /dev/null +++ b/t/t7531-semantic-verify.sh @@ -0,0 +1,253 @@ +#!/bin/sh + +test_description='descriptor-anchored semantic verification' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-semantic-verify.sh + +verify_repo () { + repo=$1 && + shift && + ( + cd "$repo" && + test-tool semantic-verify "$@" + ) +} + +test_expect_success !SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'unsupported platforms decline semantic verification' ' + test_create_repo anchored-open-unsupported && + test_commit -C anchored-open-unsupported base tracked && + ( + cd anchored-open-unsupported && + test_must_fail test-tool semantic-verify --show-results + ) >actual && + test_grep "^tracked error persist=0 error=[1-9][0-9]*$" actual && + test_grep "^entries=1 clean=0 modified=0 sensitive=0 structural=0 " \ + actual && + test_grep " unstable=0 errors=1 hardlinks=0 bytes=0 " actual && + test_grep " stat_updates=0 root_stable=0 namespace_stable=1" \ + actual +' + +test_lazy_prereq HARDLINKS ' + rm -f hardlink-source hardlink-alias && + : >hardlink-source && + ln hardlink-source hardlink-alias +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'classifies raw, converted, nested, and modified files' ' + test_create_repo classify && + mkdir -p classify/a/b && + test_write_lines "converted text" >classify/.gitattributes && + for path in raw converted modified deleted a/b/nested + do + test_write_lines original >"classify/$path" || return 1 + done && + test-tool chmtime -120 classify/raw classify/converted \ + classify/modified classify/deleted classify/a/b/nested && + git -C classify add . && + git -C classify commit -m base && + cp -p classify/modified classify/mtime-reference && + test_write_lines replaced >classify/modified && + touch -r classify/mtime-reference classify/modified && + rm classify/deleted classify/mtime-reference && + + verify_repo classify --show-results --apply >actual && + test_grep "^.gitattributes raw-clean persist=1" actual && + test_grep "^raw raw-clean persist=1" actual && + test_grep "^a/b/nested raw-clean persist=1" actual && + test_grep "^converted sensitive" actual && + test_grep "^modified raw-modified" actual && + test_grep "^deleted raw-modified" actual && + test_grep "entries=6 clean=3 modified=2 sensitive=1" actual && + test_grep "applied=3 .* after_uptodate=3" actual +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN,HARDLINKS \ + 'clean hardlinks require the ordinary tail' ' + test_create_repo hardlink && + test_write_lines content >hardlink/tracked && + git -C hardlink add tracked && + git -C hardlink commit -m base && + ln hardlink/tracked hardlink/alias && + + verify_repo hardlink --show-results --apply >actual && + test_grep "^tracked raw-clean persist=0" actual && + test_grep "hardlinks=1" actual && + test_grep "applied=1 .* after_uptodate=0 after_valid=0" actual +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'structural index state rejects the whole proof' ' + test_create_repo structural && + test_write_lines tracked >structural/a-tracked && + git -C structural add a-tracked && + git -C structural commit -m base && + test_write_lines intent >structural/z-intent && + git -C structural add -N z-intent && + + verify_repo structural --show-results --apply >actual && + test_grep "^a-tracked raw-clean persist=1" actual && + test_grep "^z-intent structural" actual && + test_grep "applied=-1 .* after_uptodate=0" actual +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'replaced index entries reject the whole proof' ' + test_create_repo replaced-entry && + test_write_lines tracked >replaced-entry/a-tracked && + test_write_lines replaced >replaced-entry/z-replaced && + git -C replaced-entry add . && + git -C replaced-entry commit -m base && + + verify_repo replaced-entry --show-results --apply \ + --replace-after-prepare=z-replaced >actual && + test_grep "^a-tracked raw-clean persist=1" actual && + test_grep "^z-replaced raw-clean persist=1" actual && + test_grep "applied=-1 before_uptodate=0 before_valid=0 " actual && + test_grep "after_uptodate=0 after_valid=0" actual +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN,PTHREADS \ + 'parallel verification preserves index-order results' ' + test_create_repo parallel && + i=0 && + while test $i -lt 16 + do + mkdir "parallel/d$i" && + printf "*.txt -text attr_%s=value\n" "$i" \ + >"parallel/d$i/.gitattributes" && + printf "content %s\n" "$i" >"parallel/d$i/file.txt" && + i=$((i + 1)) || return 1 + done && + git -C parallel add . && + git -C parallel commit -m base && + + verify_repo parallel --threads=1 --show-results >expect && + verify_repo parallel --threads=4 --show-results >actual && + test_cmp expect actual +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'raw hashing uses the repository object format' ' + git init --object-format=sha256 sha256 && + test_write_lines sha256 >sha256/tracked && + git -C sha256 add tracked && + git -C sha256 commit -m base && + verify_repo sha256 --show-results >actual && + test_grep "^tracked raw-clean persist=1" actual +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'immutable attribute sources preserve file parsing' ' + attrs="$TRASH_DIRECTORY/attribute-parser-file" && + printf "\357\273\277*.dat text\nignored\0junk\n*.txt text\r\n*.bin text\r" \ + >"$attrs" && + test_create_repo attribute-parser && + git -C attribute-parser config core.attributesFile "$attrs" && + git -C attribute-parser config core.fsmonitor false && + git -C attribute-parser config core.untrackedCache false && + for extension in dat txt bin + do + printf "alpha\r\n" \ + >"attribute-parser/tracked.$extension" || return 1 + done && + git -C attribute-parser add . && + git -C attribute-parser commit -m base && + + GIT_OPTIONAL_LOCKS=0 GIT_TEST_COLD_BULK_STATUS=0 \ + git -C attribute-parser status --porcelain=v2 >actual && + test_must_be_empty actual +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'missing attribute history invalidates cached stats' ' + attrs="$TRASH_DIRECTORY/external-attributes-file" && + printf "*.txt text eol=crlf\n" >"$attrs" && + test_create_repo external-attributes && + git -C external-attributes config core.attributesFile "$attrs" && + git -C external-attributes config core.fsmonitor false && + git -C external-attributes config core.untrackedCache false && + printf "alpha\r\n" >external-attributes/tracked.txt && + git -C external-attributes add tracked.txt && + git -C external-attributes commit -m base && + + printf "*.txt -text\n" >"$attrs" && + GIT_OPTIONAL_LOCKS=0 GIT_TEST_COLD_BULK_STATUS=0 \ + git -C external-attributes status --porcelain=v2 >actual && + test_grep "^1 \.M .* tracked.txt$" actual +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'hook cannot hide an observed external attribute change' ' + attrs="$TRASH_DIRECTORY/hook-attribute-change.rules" && + marker="$TRASH_DIRECTORY/hook-attribute-change.marker" && + printf "*.txt text eol=crlf\n" >"$attrs" && + test_create_repo hook-attribute-change && + ( + cd hook-attribute-change && + git config core.attributesFile "$attrs" && + git config core.untrackedCache false && + printf "alpha\r\n" >tracked.txt && + git add tracked.txt && + git commit -m base && + test_hook --setup fsmonitor-test <<-\EOF && + if test -n "$GIT_TEST_ATTR_FILE" + then + printf "*.txt -text\n" >"$GIT_TEST_ATTR_FILE" + : >"$GIT_TEST_ATTR_MARKER" + fi + printf "token\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git status --porcelain=v2 >/dev/null && + git status --porcelain=v2 >/dev/null && + + GIT_TEST_ATTR_FILE="$attrs" \ + GIT_TEST_ATTR_MARKER="$marker" \ + git status --porcelain=v2 >actual && + test_path_is_file "$marker" && + test_grep "^1 \.M .* tracked.txt$" actual + ) +' + +test_expect_success SEMANTIC_VERIFY_ANCHORED_OPEN \ + 'hook missing-history exception preserves reported paths' ' + attrs="$TRASH_DIRECTORY/hook-missing-history.rules" && + printf "*.txt -text\n" >"$attrs" && + test_create_repo hook-missing-history && + ( + cd hook-missing-history && + git config core.attributesFile "$attrs" && + git config core.untrackedCache false && + git config core.trustctime false && + git config core.checkStat minimal && + printf "aaaa\n" >tracked.txt && + git add tracked.txt && + git commit -m base && + test-tool chmtime =-60 tracked.txt && + git update-index --refresh && + mtime=$(test-tool chmtime --get tracked.txt) && + test_hook --setup fsmonitor-test <<-\EOF && + printf "token\0" + EOF + git config core.fsmonitor .git/hooks/fsmonitor-test && + git config core.fsmonitorHookVersion 2 && + git update-index --fsmonitor && + git status --porcelain=v2 >/dev/null && + git status --porcelain=v2 >/dev/null && + test_grep ! FSCF .git/index && + + printf "bbbb\n" >tracked.txt && + test-tool chmtime =$mtime tracked.txt && + GIT_OPTIONAL_LOCKS=0 \ + git status --porcelain=v2 >.git/actual && + test_must_be_empty .git/actual + ) +' + +test_done diff --git a/t/t7532-preload-index-linux.sh b/t/t7532-preload-index-linux.sh new file mode 100755 index 00000000000000..4d4063809f884c --- /dev/null +++ b/t/t7532-preload-index-linux.sh @@ -0,0 +1,421 @@ +#!/bin/sh + +test_description='Linux bulk index preload' + +. ./test-lib.sh + +if test "$(uname -s)" != Linux +then + skip_all='Linux getdents64/statx backend required' + test_done +fi + +case "$(stat -f -c %t "$TRASH_DIRECTORY")" in +ef53) + filesystem=ext-family + ;; +58465342) + filesystem=xfs + ;; +*) + skip_all='tests require an ext-family filesystem or XFS' + test_done + ;; +esac + +setup_repo () { + repo=$1 && + git init "$repo" && + mkdir -p "$repo/nested/deep" && + test_write_lines root >"$repo/root" && + test_write_lines peer >"$repo/peer" && + test_write_lines nested >"$repo/nested/tracked" && + test_write_lines deep >"$repo/nested/deep/tracked" && + git -C "$repo" add . && + git -C "$repo" commit -m base && + git -C "$repo" config core.fsmonitor false && + test-tool chmtime -120 "$repo/root" "$repo/peer" \ + "$repo/nested/tracked" "$repo/nested/deep/tracked" && + git -C "$repo" update-index --refresh +} + +setup_provider_proof_repo () { + setup_repo "$1" && + ( + cd "$1" && + git config core.trustctime false && + git config core.checkStat minimal && + size=$(test_file_size root) && + mtime=$(test-tool chmtime --get root) && + printf "dirt\n" >root && + test "$(test_file_size root)" = "$size" && + test-tool chmtime =$mtime root && + test "$(test-tool chmtime --get root)" = "$mtime" && + git config core.fsmonitor true && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor && + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=C \ + git update-index --fsmonitor-valid root && + test_grep ! FSCF .git/index + ) +} + +test_lazy_prereq LINUX_BULK_PRELOAD ' + setup_repo linux-bulk-prereq && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/linux-bulk-prereq.trace" \ + git -C linux-bulk-prereq \ + -c core.preloadIndexBulk=true \ + status --porcelain=v2 >actual && + test_must_be_empty actual && + test_trace2_data index preload/bulk_result complete \ + <"$TRASH_DIRECTORY/linux-bulk-prereq.trace" +' + +if ! test_have_prereq LINUX_BULK_PRELOAD +then + skip_all="Linux bulk preload backend unavailable at runtime" + test_done +fi + +ordinary_status () { + GIT_OPTIONAL_LOCKS=0 \ + git -C "$1" -c core.preloadIndex=false \ + status --porcelain=v2 >"$2" +} + +bulk_status () { + repo=$1 && + output=$2 && + bulk_trace=$TRASH_DIRECTORY/$3 && + rm -f "$bulk_trace" && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TRACE2_EVENT="$bulk_trace" \ + git -C "$repo" status --porcelain=v2 >"$output" +} + +check_data () { + test_trace2_data index "$2" "$3" <"$TRASH_DIRECTORY/$1" +} + +check_lstat_data () { + test_have_prereq !PTHREADS || + check_data "$1" preload/sum_lstat "$2" +} + +compare_status () { + ordinary_status "$1" expect && + bulk_status "$1" actual "$2" && + test_cmp expect actual +} + +cleanup_race () { + exec 9>&- + if test -n "$status_pid" + then + kill "$status_pid" 2>/dev/null || : + wait "$status_pid" 2>/dev/null || : + fi + status_pid= && + rm -f "$ready" "$resume" +} + +wait_for_ready () { + for i in $(test_seq 1 1000) + do + test "$(cat "$ready" 2>/dev/null)" = ready && return 0 + kill -0 "$status_pid" 2>/dev/null || return 1 + sleep 0.01 + done + return 1 +} + +start_raced_status () { + repo=$1 && + barrier=$2 && + ready=$TRASH_DIRECTORY/$repo.ready && + resume=$TRASH_DIRECTORY/$repo.resume && + race_trace=$TRASH_DIRECTORY/$repo.trace && + status_pid= && + rm -f "$ready" "$resume" "$race_trace" && + mkfifo "$resume" && + exec 9<>"$resume" && + { + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_PATH="$barrier" \ + GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_READY="$ready" \ + GIT_TEST_PRELOAD_INDEX_BULK_BARRIER_RESUME="$resume" \ + GIT_TRACE2_EVENT="$race_trace" \ + git -C "$repo" status --porcelain=v2 >actual 9>&- & + status_pid=$! + } && + wait_for_ready +} + +finish_raced_status () { + printf "resume\n" >&9 && + exec 9>&- && + wait "$status_pid" && + status_pid= && + ordinary_status "$1" expect && + test_cmp expect actual && + test_trace2_data index preload/bulk_applied 0 <"$race_trace" +} + +test_expect_success 'clean entries are published without lstat' ' + setup_repo clean && + bulk_status clean actual clean.trace && + test_must_be_empty actual && + check_data clean.trace preload/bulk_filesystem "$filesystem" && + check_data clean.trace preload/bulk_applied 4 && + check_data clean.trace preload/bulk_untracked_complete 1 && + check_lstat_data clean.trace 0 +' + +test_expect_success 'provider closure accepts bulk content proofs' ' + ( + sane_unset GIT_TEST_SPLIT_INDEX && + setup_provider_proof_repo provider-proof && + cd provider-proof && + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CC \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_grep "^1 \.M .* root$" .git/actual && + test_trace2_data status semantic_verify/bulk_scan 1 \ + <.git/status.trace && + ! test_trace2_data status semantic_verify/prepared 1 \ + <.git/status.trace && + test_trace2_data index preload/bulk_content_verify 1 \ + <.git/status.trace && + test_trace2_data index preload/bulk_bytes_hashed \ + "[1-9][0-9]*" <.git/status.trace && + test_trace2_data index preload/bulk_provider_applied 3 \ + <.git/status.trace && + test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep FSCF .git/index + ) +' + +test_expect_success 'provider failure discards bulk content proofs' ' + ( + sane_unset GIT_TEST_SPLIT_INDEX && + setup_provider_proof_repo provider-failure && + cd provider-failure && + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_FSMONITOR_QUERY_SEQUENCE=CE \ + GIT_TRACE2_EVENT="$PWD/.git/status.trace" \ + git status --porcelain=v2 >.git/actual && + test_grep "^1 \.M .* root$" .git/actual && + test_trace2_data status semantic_verify/bulk_scan 1 \ + <.git/status.trace && + test_trace2_data index preload/bulk_content_verify 1 \ + <.git/status.trace && + test_trace2_data index preload/bulk_bytes_hashed \ + "[1-9][0-9]*" <.git/status.trace && + ! test_trace2_data index preload/bulk_provider_applied \ + "[0-9][0-9]*" <.git/status.trace && + test_trace2_data fsmonitor token_closure/rejected 1 \ + <.git/status.trace && + ! test_trace2_data fsmonitor token_closure/accepted 1 \ + <.git/status.trace && + test_grep ! FSCF .git/index + ) +' + +test_expect_success 'tracked files ignore a directory type hint' ' + setup_repo dirent-file && + ordinary_status dirent-file expect && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_PRELOAD_INDEX_BULK_DIRENT_TYPE=dir:root \ + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/dirent-file.trace" \ + git -C dirent-file status --porcelain=v2 >actual && + test_cmp expect actual && + check_data dirent-file.trace preload/bulk_result complete && + check_data dirent-file.trace preload/bulk_applied 4 && + check_data dirent-file.trace preload/bulk_fallback 0 +' + +test_expect_success 'tracked subtrees ignore a regular-file type hint' ' + setup_repo dirent-prefix && + test_write_lines visible >dirent-prefix/nested/untracked && + ordinary_status dirent-prefix expect && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_PRELOAD_INDEX_BULK_DIRENT_TYPE=reg:nested \ + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/dirent-prefix.trace" \ + git -C dirent-prefix status --porcelain=v2 >actual && + test_cmp expect actual && + check_data dirent-prefix.trace preload/bulk_result complete && + check_data dirent-prefix.trace preload/bulk_applied 4 && + check_data dirent-prefix.trace preload/bulk_definitive_deleted 0 && + check_data dirent-prefix.trace preload/bulk_fallback 0 && + check_data dirent-prefix.trace preload/bulk_untracked_complete 1 +' + +test_expect_success 'untracked directories ignore a regular-file type hint' ' + setup_repo dirent-untracked-directory && + mkdir -p dirent-untracked-directory/collapsed/deep && + test_write_lines visible \ + >dirent-untracked-directory/collapsed/deep/file && + ordinary_status dirent-untracked-directory expect && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_PRELOAD_INDEX_BULK_DIRENT_TYPE=reg:collapsed \ + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/dirent-untracked-directory.trace" \ + git -C dirent-untracked-directory \ + status --porcelain=v2 >actual && + test_cmp expect actual && + test_grep "^? collapsed/$" actual && + check_data dirent-untracked-directory.trace \ + preload/bulk_result complete && + check_data dirent-untracked-directory.trace \ + preload/bulk_untracked_complete 1 && + check_data dirent-untracked-directory.trace \ + preload/bulk_untracked_count 1 +' + +test_expect_success 'ignored directories ignore a regular-file type hint' ' + setup_repo dirent-ignored-directory && + test_write_lines "*.ignored" >dirent-ignored-directory/.gitignore && + git -C dirent-ignored-directory add .gitignore && + git -C dirent-ignored-directory commit -m ignore && + mkdir dirent-ignored-directory/ignored-only && + test_write_lines ignored \ + >dirent-ignored-directory/ignored-only/file.ignored && + ordinary_status dirent-ignored-directory expect && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_PRELOAD_INDEX_BULK_DIRENT_TYPE=reg:ignored-only \ + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/dirent-ignored-directory.trace" \ + git -C dirent-ignored-directory \ + status --porcelain=v2 >actual && + test_cmp expect actual && + test_must_be_empty actual && + check_data dirent-ignored-directory.trace \ + preload/bulk_result complete && + check_data dirent-ignored-directory.trace \ + preload/bulk_untracked_complete 1 && + check_data dirent-ignored-directory.trace \ + preload/bulk_untracked_count 0 +' + +test_expect_success 'untracked files ignore a directory type hint' ' + setup_repo dirent-untracked-file && + test_write_lines "visible/" >dirent-untracked-file/.gitignore && + git -C dirent-untracked-file add .gitignore && + git -C dirent-untracked-file commit -m ignore && + test_write_lines visible >dirent-untracked-file/visible && + ordinary_status dirent-untracked-file expect && + GIT_OPTIONAL_LOCKS=0 \ + GIT_TEST_PRELOAD_INDEX=1 \ + GIT_TEST_PRELOAD_INDEX_BULK=1 \ + GIT_TEST_PRELOAD_INDEX_BULK_DIRENT_TYPE=dir:visible \ + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/dirent-untracked-file.trace" \ + git -C dirent-untracked-file status --porcelain=v2 >actual && + test_cmp expect actual && + test_grep "^? visible$" actual && + check_data dirent-untracked-file.trace \ + preload/bulk_result complete && + check_data dirent-untracked-file.trace \ + preload/bulk_untracked_complete 1 && + check_data dirent-untracked-file.trace \ + preload/bulk_untracked_count 1 +' + +test_expect_success 'visible and ignored paths match ordinary status' ' + setup_repo visible && + test_write_lines "*.ignored" >visible/.gitignore && + git -C visible add .gitignore && + git -C visible commit -m ignore && + test_write_lines root >visible/untracked && + mkdir -p visible/collapsed/deep visible/ignored-only && + test_write_lines nested >visible/collapsed/deep/file && + test_write_lines ignored >visible/ignored-only/file.ignored && + compare_status visible visible.trace && + test_grep "^? untracked$" actual && + test_grep "^? collapsed/$" actual && + test_grep ! "ignored-only" actual && + check_data visible.trace preload/bulk_untracked_complete 1 && + check_data visible.trace preload/bulk_untracked_count 2 && + test_grep ! "\"category\":\"read_directory\"" visible.trace +' + +test_expect_success 'tracked changes match ordinary status' ' + for mode in modified deleted metadata + do + setup_repo "$mode" || return 1 && + case "$mode" in + modified) test_write_lines changed-content >"$mode/root" ;; + deleted) rm "$mode/nested/tracked" ;; + metadata) test-tool chmtime +60 "$mode/root" ;; + esac && + compare_status "$mode" "$mode.trace" || return 1 + done && + check_data modified.trace preload/bulk_definitive_modified 1 && + check_data modified.trace refresh/sum_lstat 0 && + check_data deleted.trace preload/bulk_definitive_deleted 1 && + check_data deleted.trace refresh/sum_lstat 0 && + check_data metadata.trace preload/bulk_content_check 1 && + check_data metadata.trace refresh/sum_lstat 0 +' + +test_expect_success 'tracked-file replacement directories are pruned' ' + setup_repo replacement-dir && + rm replacement-dir/root && + mkdir -p replacement-dir/root/deep/embedded && + test_write_lines hidden >replacement-dir/root/deep/untracked && + git -C replacement-dir/root/deep/embedded init && + compare_status replacement-dir replacement-dir.trace && + test_line_count = 1 actual && + check_data replacement-dir.trace preload/bulk_fallback 1 && + check_data replacement-dir.trace preload/bulk_untracked_complete 1 +' + +test_expect_success PIPE 'tracked FIFO replacements fall back' ' + setup_repo tracked-fifo && + rm tracked-fifo/root && + mkfifo tracked-fifo/root && + compare_status tracked-fifo tracked-fifo.trace && + test_grep "^1 \\.M .* root$" actual && + check_data tracked-fifo.trace preload/bulk_result complete && + check_data tracked-fifo.trace preload/bulk_applied 3 && + check_data tracked-fifo.trace preload/bulk_fallback 1 && + check_data tracked-fifo.trace preload/bulk_definitive_deleted 0 +' + +test_expect_success PIPE 'queued child replacement discards observations' ' + setup_repo child-race && + test_when_finished cleanup_race && + start_raced_status child-race nested/deep && + mv child-race/nested/deep child-race/deep-away && + mkdir child-race/nested/deep && + test_write_lines dirty >child-race/nested/deep/tracked && + finish_raced_status child-race && + test_file_not_empty actual +' + +test_expect_success PIPE 'fallback shapes retain exact output' ' + setup_repo shapes && + ln shapes/root shapes/linked && + mkfifo shapes/fifo && + git init shapes/embedded && + compare_status shapes shapes.trace && + check_data shapes.trace preload/bulk_fallback 1 && + check_data shapes.trace preload/bulk_untracked_complete 0 +' + +test_done diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index a8d691719da062..fb5f2d89028738 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -659,6 +659,51 @@ test_expect_success 'geometric repacking task' ' ) ' +objdir=.git/objects +packdir=$objdir/pack + +pack_promisor () { + p="$(echo "$@" | git pack-objects --revs $packdir/pack)" && + touch "$packdir/pack-$p.promisor" && + echo "$p" +} + +test_expect_success 'geometric repacking task handles promisor packs' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + git config set maintenance.auto false && + git remote add promisor garbage && + git config set remote.promisor.promisor true && + + for n in $(test_seq 6) + do + test_commit $n || return 1 + done && + + A="$(pack_promisor 1)" && + B="$(pack_promisor 1..2)" && + C="$(pack_promisor 2..6)" && + git prune-packed && + + ls $packdir/pack-*.promisor | sort >promisors.before && + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ + git maintenance run --quiet --task=geometric-repack && + ls $packdir/pack-*.promisor | sort >promisors.after && + + test_subcommand git repack -d -l --geometric=2 \ + --quiet --write-midx expect && + comm -23 promisors.before promisors.after >actual && + + test_cmp expect actual + ) +' + test_geometric_repack_needed () { NEEDED="$1" GEOMETRIC_CONFIG="$2" && @@ -714,6 +759,29 @@ test_expect_success 'geometric repacking with --auto' ' ) ' +test_expect_success 'geometric repacking with --auto handles promisor packs' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + git config set maintenance.auto false && + git remote add promisor garbage && + git config set remote.promisor.promisor true && + + for n in $(test_seq 6) + do + test_commit $n || return 1 + done && + + pack_promisor 1 >/dev/null && + pack_promisor 1..2 >/dev/null && + pack_promisor 2..6 >/dev/null && + git prune-packed && + + test_geometric_repack_needed true auto=9000 + ) +' + test_expect_success 'geometric repacking honors configured split factor' ' test_when_finished "rm -rf repo" && git init repo && diff --git a/t/unit-tests/u-attr-fingerprint.c b/t/unit-tests/u-attr-fingerprint.c new file mode 100644 index 00000000000000..e7b61b687c788f --- /dev/null +++ b/t/unit-tests/u-attr-fingerprint.c @@ -0,0 +1,127 @@ +#include "unit-test.h" +#include "attr-fingerprint.h" +#include "dir.h" +#include "strbuf.h" +#include "wrapper.h" + +static char *create_directory(void) +{ + const char *tmp = getenv("TMPDIR"); + char *path = xstrfmt("%s/attr-fingerprint.XXXXXX", + tmp ? tmp : "/tmp"); + + cl_assert(mkdtemp(path) != NULL); + return path; +} + +static void remove_directory(char *path) +{ + struct strbuf cleanup = STRBUF_INIT; + + strbuf_addstr(&cleanup, path); + cl_assert_equal_i(remove_dir_recursively(&cleanup, 0), 0); + strbuf_release(&cleanup); + free(path); +} + +static void fingerprint(const char *path, int enabled, + const struct git_hash_algo *algo, + struct attr_fingerprint *result) +{ + struct attr_fingerprint_source source = { + .path = path, + .enabled = enabled, + }; + + cl_assert_equal_i(attr_fingerprint_sources( + &source, 1, algo, result), 0); +} + +void test_attr_fingerprint__separates_contents_from_namespace(void) +{ +#ifndef O_NONBLOCK + cl_skip(); +#else + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *directory = create_directory(); + struct strbuf path = STRBUF_INIT; + struct attr_fingerprint initial, metadata, changed; + struct stat st; + + strbuf_addf(&path, "%s/attributes", directory); + write_file(path.buf, "*.txt text\n"); + fingerprint(path.buf, 1, algo, &initial); + cl_assert(initial.sources_present); + cl_assert_equal_i(stat(path.buf, &st), 0); + cl_assert_equal_i(chmod(path.buf, st.st_mode ^ S_IXUSR), 0); + fingerprint(path.buf, 1, algo, &metadata); + cl_assert(!memcmp(initial.content_hash, metadata.content_hash, + algo->rawsz)); + cl_assert(memcmp(initial.namespace_hash, metadata.namespace_hash, + algo->rawsz)); + write_file(path.buf, "*.txt -text\n"); + fingerprint(path.buf, 1, algo, &changed); + cl_assert(memcmp(metadata.content_hash, changed.content_hash, + algo->rawsz)); + + strbuf_release(&path); + remove_directory(directory); +#endif +} + +void test_attr_fingerprint__records_missing_parent_namespaces(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA256]; + char *directory; + struct strbuf path = STRBUF_INIT; + struct attr_fingerprint before, after; + struct stat st; + + if (!fstat_is_reliable()) { + struct attr_fingerprint_source source = { + .path = "missing/attributes", + .enabled = 1, + }; + + cl_assert(attr_fingerprint_sources( + &source, 1, algo, &before) < 0); + cl_assert_equal_i(errno, EAGAIN); + return; + } + directory = create_directory(); + + strbuf_addf(&path, "%s/missing/attributes", directory); + fingerprint(path.buf, 1, algo, &before); + cl_assert(!before.sources_present); + cl_assert_equal_i(stat(directory, &st), 0); + cl_assert_equal_i(chmod(directory, st.st_mode ^ S_IXGRP), 0); + fingerprint(path.buf, 1, algo, &after); + cl_assert(!after.sources_present); + cl_assert(!memcmp(before.content_hash, after.content_hash, algo->rawsz)); + cl_assert(memcmp(before.namespace_hash, after.namespace_hash, + algo->rawsz)); + + strbuf_release(&path); + remove_directory(directory); +} + +void test_attr_fingerprint__does_not_observe_disabled_sources(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *directory = create_directory(); + struct strbuf path = STRBUF_INIT; + struct attr_fingerprint before, after; + + strbuf_addf(&path, "%s/attributes", directory); + fingerprint(path.buf, 0, algo, &before); + write_file(path.buf, "*.txt text\n"); + fingerprint(path.buf, 0, algo, &after); + cl_assert(!before.sources_present); + cl_assert(!after.sources_present); + cl_assert(!memcmp(before.content_hash, after.content_hash, algo->rawsz)); + cl_assert(!memcmp(before.namespace_hash, after.namespace_hash, + algo->rawsz)); + + strbuf_release(&path); + remove_directory(directory); +} diff --git a/t/unit-tests/u-attr-manifest.c b/t/unit-tests/u-attr-manifest.c new file mode 100644 index 00000000000000..a41eb2a7975fac --- /dev/null +++ b/t/unit-tests/u-attr-manifest.c @@ -0,0 +1,614 @@ +#include "unit-test.h" +#include "attr-manifest.h" +#include "dir.h" +#include "hash.h" +#include "odb.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "semantic-verify-internal.h" +#include "setup.h" +#include "strbuf.h" +#include "thread-utils.h" +#include "worktree-attr-manifest.h" +#include "wrapper.h" + +#define ATTR_MANIFEST_TEST_THREADS "GIT_TEST_ATTR_MANIFEST_THREADS" +#define ATTR_MANIFEST_TEST_THREAD_FAIL_AT \ + "GIT_TEST_ATTR_MANIFEST_THREAD_FAIL_AT" +#define ATTR_MANIFEST_TEST_SOURCE_NR 257 + +static void fill_hash(unsigned char *hash, unsigned char value, + const struct git_hash_algo *algo) +{ + memset(hash, value, algo->rawsz); +} + +static void add_entry(struct attr_manifest_writer *writer, const char *path, + enum attr_manifest_source source, unsigned char value) +{ + unsigned char hash[GIT_MAX_RAWSZ]; + + fill_hash(hash, value, writer->algo); + cl_assert_equal_i(attr_manifest_writer_add(writer, path, source, hash), 0); +} + +void test_attr_manifest__writer_serializes_sorted_entries(void) +{ + struct attr_manifest_writer writer; + struct strbuf manifest = STRBUF_INIT; + + attr_manifest_writer_init(&writer, &manifest, &hash_algos[GIT_HASH_SHA256]); + add_entry(&writer, ".gitattributes", ATTR_MANIFEST_INDEX, 1); + add_entry(&writer, "a/.gitattributes", ATTR_MANIFEST_WORKTREE, 2); + cl_assert_equal_i(get_be32(manifest.buf), 2); + cl_assert_equal_i(writer.nr, 2); + strbuf_release(&manifest); +} + +void test_attr_manifest__writer_rejects_invalid_or_unsorted_paths(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct attr_manifest_writer writer; + struct strbuf manifest = STRBUF_INIT; + unsigned char hash[GIT_MAX_RAWSZ]; + + fill_hash(hash, 1, algo); + attr_manifest_writer_init(&writer, &manifest, algo); + add_entry(&writer, "b/.gitattributes", ATTR_MANIFEST_INDEX, 1); + cl_assert_equal_i(attr_manifest_writer_add(&writer, "a/.gitattributes", + ATTR_MANIFEST_INDEX, hash), -1); + cl_assert_equal_i(attr_manifest_writer_add(&writer, "b/.gitattributes", + ATTR_MANIFEST_INDEX, hash), -1); + cl_assert_equal_i(attr_manifest_writer_add(&writer, "/.gitattributes", + ATTR_MANIFEST_INDEX, hash), -1); + cl_assert_equal_i(attr_manifest_writer_add(&writer, "b/not-attributes", + ATTR_MANIFEST_INDEX, hash), -1); + cl_assert_equal_i(writer.nr, 1); + strbuf_release(&manifest); +} + +void test_attr_manifest__reader_round_trips_entries(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA256]; + struct attr_manifest_writer writer; + struct attr_manifest_cursor cursor; + struct attr_manifest_entry entry; + struct strbuf manifest = STRBUF_INIT; + + attr_manifest_writer_init(&writer, &manifest, algo); + add_entry(&writer, ".gitattributes", ATTR_MANIFEST_INDEX, 1); + add_entry(&writer, "a/.gitattributes", ATTR_MANIFEST_WORKTREE, 2); + cl_assert(attr_manifest_valid(manifest.buf, manifest.len, algo)); + cl_assert_equal_i(attr_manifest_cursor_init(&cursor, manifest.buf, + manifest.len, algo), 0); + cl_assert_equal_i(attr_manifest_cursor_next(&cursor, &entry), 1); + cl_assert_equal_i(entry.source, ATTR_MANIFEST_INDEX); + cl_assert_equal_i(entry.hash[0], 1); + cl_assert_equal_i(attr_manifest_cursor_next(&cursor, &entry), 1); + cl_assert_equal_i(entry.source, ATTR_MANIFEST_WORKTREE); + cl_assert_equal_i(entry.hash[0], 2); + cl_assert_equal_i(attr_manifest_cursor_next(&cursor, &entry), 0); + strbuf_release(&manifest); +} + +void test_attr_manifest__reader_rejects_corrupt_encoding(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct attr_manifest_writer writer; + struct strbuf manifest = STRBUF_INIT; + size_t metadata_offset = 2 * sizeof(uint32_t); + unsigned char saved; + + attr_manifest_writer_init(&writer, &manifest, algo); + add_entry(&writer, ".gitattributes", ATTR_MANIFEST_INDEX, 1); + cl_assert(!attr_manifest_valid(manifest.buf, manifest.len - 1, algo)); + strbuf_addch(&manifest, 0); + cl_assert(!attr_manifest_valid(manifest.buf, manifest.len, algo)); + strbuf_setlen(&manifest, manifest.len - 1); + + saved = manifest.buf[metadata_offset + 1]; + manifest.buf[metadata_offset + 1] = 1; + cl_assert(!attr_manifest_valid(manifest.buf, manifest.len, algo)); + manifest.buf[metadata_offset + 1] = saved; + put_be32(manifest.buf, 2); + cl_assert(!attr_manifest_valid(manifest.buf, manifest.len, algo)); + strbuf_release(&manifest); +} + +void test_attr_manifest__reader_accepts_empty_manifest(void) +{ + struct attr_manifest_writer writer; + struct strbuf manifest = STRBUF_INIT; + + attr_manifest_writer_init(&writer, &manifest, &hash_algos[GIT_HASH_SHA1]); + cl_assert(attr_manifest_valid(manifest.buf, manifest.len, + &hash_algos[GIT_HASH_SHA1])); + strbuf_release(&manifest); +} + +static int record_changed_path(const struct attr_manifest_entry *entry, + void *data) +{ + struct strbuf *paths = data; + + if (paths->len) + strbuf_addch(paths, ' '); + strbuf_add(paths, entry->path, entry->path_len); + return 0; +} + +void test_attr_manifest__iterates_added_removed_and_modified_entries(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct attr_manifest_writer old_writer, new_writer; + struct strbuf old = STRBUF_INIT, new = STRBUF_INIT, changed = STRBUF_INIT; + + attr_manifest_writer_init(&old_writer, &old, algo); + add_entry(&old_writer, ".gitattributes", ATTR_MANIFEST_INDEX, 1); + add_entry(&old_writer, "a/.gitattributes", ATTR_MANIFEST_INDEX, 2); + add_entry(&old_writer, "c/.gitattributes", ATTR_MANIFEST_INDEX, 3); + attr_manifest_writer_init(&new_writer, &new, algo); + add_entry(&new_writer, ".gitattributes", ATTR_MANIFEST_INDEX, 1); + add_entry(&new_writer, "a/.gitattributes", ATTR_MANIFEST_WORKTREE, 2); + add_entry(&new_writer, "b/.gitattributes", ATTR_MANIFEST_INDEX, 4); + + cl_assert_equal_i(attr_manifest_for_each_changed( + old.buf, old.len, new.buf, new.len, algo, + record_changed_path, &changed), 0); + cl_assert_equal_s(changed.buf, + "a/.gitattributes b/.gitattributes c/.gitattributes"); + strbuf_release(&changed); + strbuf_release(&new); + strbuf_release(&old); +} + +void test_attr_manifest__does_not_report_identical_entries(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct attr_manifest_writer old_writer, new_writer; + struct strbuf old = STRBUF_INIT, new = STRBUF_INIT, changed = STRBUF_INIT; + + attr_manifest_writer_init(&old_writer, &old, algo); + add_entry(&old_writer, ".gitattributes", ATTR_MANIFEST_INDEX, 1); + attr_manifest_writer_init(&new_writer, &new, algo); + add_entry(&new_writer, ".gitattributes", ATTR_MANIFEST_INDEX, 1); + cl_assert_equal_i(attr_manifest_for_each_changed( + old.buf, old.len, new.buf, new.len, algo, + record_changed_path, &changed), 0); + cl_assert_equal_i(changed.len, 0); + strbuf_release(&changed); + strbuf_release(&new); + strbuf_release(&old); +} + +void test_attr_manifest__rejects_malformed_tail_before_callbacks(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct attr_manifest_writer old_writer, new_writer; + struct strbuf old = STRBUF_INIT, new = STRBUF_INIT, changed = STRBUF_INIT; + + attr_manifest_writer_init(&old_writer, &old, algo); + add_entry(&old_writer, ".gitattributes", ATTR_MANIFEST_INDEX, 1); + attr_manifest_writer_init(&new_writer, &new, algo); + add_entry(&new_writer, ".gitattributes", ATTR_MANIFEST_INDEX, 2); + strbuf_addch(&new, 0); + + cl_assert_equal_i(attr_manifest_for_each_changed( + old.buf, old.len, new.buf, new.len, algo, + record_changed_path, &changed), -1); + cl_assert_equal_i(changed.len, 0); + + strbuf_release(&changed); + strbuf_release(&new); + strbuf_release(&old); +} + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +static char *create_worktree(void) +{ + const char *tmp = getenv("TMPDIR"); + char *path = xstrfmt("%s/attr-manifest.XXXXXX", tmp ? tmp : "/tmp"); + + cl_assert(mkdtemp(path) != NULL); + return path; +} + +static void remove_worktree(char *worktree) +{ + struct strbuf path = STRBUF_INIT; + + strbuf_addstr(&path, worktree); + cl_assert_equal_i(remove_dir_recursively(&path, 0), 0); + strbuf_release(&path); + free(worktree); +} + +static struct cache_entry *add_index_path(struct index_state *istate, + size_t pos, const char *path, + unsigned int stage) +{ + size_t len = strlen(path); + struct cache_entry *ce = make_empty_cache_entry(istate, len); + + ce->ce_mode = S_IFREG | 0644; + ce->ce_flags = create_ce_flags(stage); + ce->ce_namelen = len; + memcpy(ce->name, path, len + 1); + istate->cache[pos] = ce; + return ce; +} + +static void init_object_store(struct repository *repo, const char *worktree) +{ + struct strbuf object_dir = STRBUF_INIT; + + strbuf_addf(&object_dir, "%s/objects", worktree); + repo->objects = odb_new(repo, object_dir.buf, ""); + strbuf_release(&object_dir); +} +#endif + +void test_attr_manifest__builds_sources_for_tracked_scopes(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *worktree = create_worktree(); + struct repository repo = { .worktree = worktree, .hash_algo = algo }; + struct index_state istate = INDEX_STATE_INIT(&repo); + struct worktree_attr_manifest_stats stats; + struct attr_manifest_cursor cursor; + struct attr_manifest_entry entry; + struct git_hash_ctx ctx; + struct strbuf path = STRBUF_INIT, manifest = STRBUF_INIT; + char root_source[] = "*.root text\n"; + unsigned char expected[GIT_MAX_RAWSZ], hash[GIT_MAX_RAWSZ]; + + strbuf_addf(&path, "%s/a", worktree); + cl_assert_equal_i(mkdir(path.buf, 0777), 0); + strbuf_reset(&path); + strbuf_addf(&path, "%s/b", worktree); + cl_assert_equal_i(mkdir(path.buf, 0777), 0); + strbuf_reset(&path); + strbuf_addf(&path, "%s/.gitattributes", worktree); + write_file_buf(path.buf, root_source, strlen(root_source)); + git_hash_init(&ctx, algo); + git_hash_update(&ctx, root_source, strlen(root_source)); + git_hash_final(expected, &ctx); + strbuf_reset(&path); + strbuf_addf(&path, "%s/a/.gitattributes", worktree); + write_file(path.buf, "*.dat -text\n"); + + CALLOC_ARRAY(istate.cache, 2); + istate.cache_alloc = istate.cache_nr = 2; + add_index_path(&istate, 0, "a/file", 0); + add_index_path(&istate, 1, "b/file", 0); + cl_assert_equal_i(worktree_attr_manifest_build( + &istate, &manifest, hash, &stats), 0); + cl_assert_equal_i(stats.candidates, 3); + cl_assert_equal_i(stats.worktree_sources, 2); + cl_assert_equal_i(stats.index_sources, 0); + cl_assert(attr_manifest_valid(manifest.buf, manifest.len, algo)); + + cl_assert_equal_i(attr_manifest_cursor_init( + &cursor, manifest.buf, manifest.len, algo), 0); + cl_assert_equal_i(attr_manifest_cursor_next(&cursor, &entry), 1); + cl_assert_equal_i(entry.source, ATTR_MANIFEST_WORKTREE); + cl_assert_equal_i(entry.path_len, strlen(GITATTRIBUTES_FILE)); + cl_assert(!memcmp(entry.path, GITATTRIBUTES_FILE, entry.path_len)); + cl_assert(!memcmp(entry.hash, expected, algo->rawsz)); + cl_assert_equal_i(attr_manifest_cursor_next(&cursor, &entry), 1); + cl_assert_equal_i(entry.source, ATTR_MANIFEST_WORKTREE); + cl_assert_equal_i(attr_manifest_cursor_next(&cursor, &entry), 0); + + strbuf_release(&manifest); + strbuf_release(&path); + release_index(&istate); + remove_worktree(worktree); +#endif +} + +void test_attr_manifest__falls_back_to_index_source(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *worktree = create_worktree(); + char source[] = "*.dat text\n"; + struct repository repo = { .worktree = worktree, .hash_algo = algo }; + struct index_state istate = INDEX_STATE_INIT(&repo); + struct worktree_attr_manifest_stats stats; + struct attr_manifest_cursor cursor; + struct attr_manifest_entry entry; + struct cache_entry *attributes; + struct strbuf manifest = STRBUF_INIT; + unsigned char hash[GIT_MAX_RAWSZ]; + int have_repository, ret; + + init_object_store(&repo, worktree); + CALLOC_ARRAY(istate.cache, 1); + istate.cache_alloc = istate.cache_nr = 1; + attributes = add_index_path(&istate, 0, GITATTRIBUTES_FILE, 0); + cl_must_pass(odb_pretend_object( + repo.objects, source, strlen(source), OBJ_BLOB, + &attributes->oid)); + + have_repository = startup_info->have_repository; + startup_info->have_repository = 1; + ret = worktree_attr_manifest_build( + &istate, &manifest, hash, &stats); + startup_info->have_repository = have_repository; + cl_assert_equal_i(ret, 0); + cl_assert_equal_i(stats.candidates, 1); + cl_assert_equal_i(stats.worktree_sources, 0); + cl_assert_equal_i(stats.index_sources, 1); + cl_assert_equal_i(attr_manifest_cursor_init( + &cursor, manifest.buf, manifest.len, algo), 0); + cl_assert_equal_i(attr_manifest_cursor_next(&cursor, &entry), 1); + cl_assert_equal_i(entry.source, ATTR_MANIFEST_INDEX); + cl_assert_equal_i(entry.path_len, strlen(GITATTRIBUTES_FILE)); + cl_assert(!memcmp(entry.path, GITATTRIBUTES_FILE, entry.path_len)); + cl_assert(!memcmp(entry.hash, attributes->oid.hash, algo->rawsz)); + cl_assert_equal_i(attr_manifest_cursor_next(&cursor, &entry), 0); + + strbuf_release(&manifest); + release_index(&istate); + odb_free(repo.objects); + remove_worktree(worktree); +#endif +} + +void test_attr_manifest__rejects_hardlinked_source_over_index(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *worktree = create_worktree(); + char indexed_source[] = "*.dat text\n"; + struct repository repo = { .worktree = worktree, .hash_algo = algo }; + struct index_state istate = INDEX_STATE_INIT(&repo); + struct worktree_attr_manifest_stats stats; + struct cache_entry *attributes; + struct strbuf source = STRBUF_INIT, alias = STRBUF_INIT; + struct strbuf manifest = STRBUF_INIT; + unsigned char hash[GIT_MAX_RAWSZ]; + int have_repository, ret; + + init_object_store(&repo, worktree); + strbuf_addf(&source, "%s/a", worktree); + cl_assert_equal_i(mkdir(source.buf, 0777), 0); + strbuf_addstr(&source, "/" GITATTRIBUTES_FILE); + write_file(source.buf, "*.dat -text\n"); + strbuf_addf(&alias, "%s/attributes-alias", worktree); + cl_assert_equal_i(link(source.buf, alias.buf), 0); + + CALLOC_ARRAY(istate.cache, 2); + istate.cache_alloc = istate.cache_nr = 2; + attributes = add_index_path( + &istate, 0, "a/" GITATTRIBUTES_FILE, 0); + add_index_path(&istate, 1, "a/file", 0); + cl_must_pass(odb_pretend_object( + repo.objects, indexed_source, strlen(indexed_source), OBJ_BLOB, + &attributes->oid)); + + have_repository = startup_info->have_repository; + startup_info->have_repository = 1; + ret = worktree_attr_manifest_build( + &istate, &manifest, hash, &stats); + startup_info->have_repository = have_repository; + cl_assert_equal_i(ret, -1); + cl_assert_equal_i(manifest.len, 0); + cl_assert_equal_i(stats.worktree_sources, 0); + cl_assert_equal_i(stats.index_sources, 0); + + strbuf_release(&manifest); + strbuf_release(&alias); + strbuf_release(&source); + release_index(&istate); + odb_free(repo.objects); + remove_worktree(worktree); +#endif +} + +void test_attr_manifest__rejects_missing_index_source(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *worktree = create_worktree(); + struct repository repo = { .worktree = worktree, .hash_algo = algo }; + struct index_state istate = INDEX_STATE_INIT(&repo); + struct worktree_attr_manifest_stats stats; + struct cache_entry *attributes; + struct strbuf manifest = STRBUF_INIT; + unsigned char hash[GIT_MAX_RAWSZ]; + unsigned char missing[GIT_MAX_RAWSZ]; + + init_object_store(&repo, worktree); + CALLOC_ARRAY(istate.cache, 1); + istate.cache_alloc = istate.cache_nr = 1; + attributes = add_index_path(&istate, 0, GITATTRIBUTES_FILE, 0); + fill_hash(missing, 0x42, algo); + oidread(&attributes->oid, missing, algo); + strbuf_addstr(&manifest, "discard me"); + + cl_assert_equal_i(worktree_attr_manifest_build( + &istate, &manifest, hash, &stats), -1); + cl_assert_equal_i(manifest.len, 0); + + strbuf_release(&manifest); + release_index(&istate); + odb_free(repo.objects); + remove_worktree(worktree); +#endif +} + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +struct many_sources_fixture { + const struct git_hash_algo *algo; + char *worktree; + struct repository repo; + struct index_state istate; +}; + +static void many_sources_fixture_init(struct many_sources_fixture *fixture) +{ + struct strbuf path = STRBUF_INIT; + size_t i; + + memset(fixture, 0, sizeof(*fixture)); + fixture->algo = &hash_algos[GIT_HASH_SHA1]; + fixture->worktree = create_worktree(); + fixture->repo.worktree = fixture->worktree; + fixture->repo.hash_algo = fixture->algo; + index_state_init(&fixture->istate, &fixture->repo); + CALLOC_ARRAY(fixture->istate.cache, ATTR_MANIFEST_TEST_SOURCE_NR); + fixture->istate.cache_alloc = fixture->istate.cache_nr = + ATTR_MANIFEST_TEST_SOURCE_NR; + + for (i = 0; i < ATTR_MANIFEST_TEST_SOURCE_NR; i++) { + strbuf_reset(&path); + strbuf_addf(&path, "%s/d%03" PRIuMAX, + fixture->worktree, (uintmax_t)i); + cl_assert_equal_i(mkdir(path.buf, 0777), 0); + strbuf_addstr(&path, "/" GITATTRIBUTES_FILE); + write_file(path.buf, "source %" PRIuMAX "\n", (uintmax_t)i); + strbuf_reset(&path); + strbuf_addf(&path, "d%03" PRIuMAX "/file", (uintmax_t)i); + add_index_path(&fixture->istate, i, path.buf, 0); + } + strbuf_release(&path); +} + +static void many_sources_fixture_release(struct many_sources_fixture *fixture) +{ + release_index(&fixture->istate); + remove_worktree(fixture->worktree); +} + +static void clear_attr_manifest_thread_env(void *unused UNUSED) +{ + unsetenv(ATTR_MANIFEST_TEST_THREADS); + unsetenv(ATTR_MANIFEST_TEST_THREAD_FAIL_AT); +} +#endif + +void test_attr_manifest__parallel_probes_match_serial_output(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + struct many_sources_fixture fixture; + struct worktree_attr_manifest_stats serial_stats, parallel_stats; + struct strbuf serial = STRBUF_INIT, parallel = STRBUF_INIT; + unsigned char serial_hash[GIT_MAX_RAWSZ]; + unsigned char parallel_hash[GIT_MAX_RAWSZ]; + + if (!HAVE_THREADS) + return; + cl_set_cleanup(clear_attr_manifest_thread_env, NULL); + many_sources_fixture_init(&fixture); + + xsetenv(ATTR_MANIFEST_TEST_THREADS, "1", 1); + cl_assert_equal_i(worktree_attr_manifest_build( + &fixture.istate, &serial, serial_hash, &serial_stats), 0); + cl_assert_equal_i(serial_stats.candidates, + ATTR_MANIFEST_TEST_SOURCE_NR + 1); + cl_assert_equal_i(serial_stats.threads, 1); + cl_assert_equal_i(serial_stats.worktree_sources, + ATTR_MANIFEST_TEST_SOURCE_NR); + + xsetenv(ATTR_MANIFEST_TEST_THREADS, "2", 1); + cl_assert_equal_i(worktree_attr_manifest_build( + &fixture.istate, ¶llel, parallel_hash, ¶llel_stats), 0); + cl_assert_equal_i(parallel_stats.candidates, + ATTR_MANIFEST_TEST_SOURCE_NR + 1); + cl_assert_equal_i(parallel_stats.threads, 2); + cl_assert_equal_i(parallel_stats.thread_failures, 0); + cl_assert_equal_i(parallel_stats.worktree_sources, + ATTR_MANIFEST_TEST_SOURCE_NR); + cl_assert_equal_i(serial.len, parallel.len); + cl_assert(!memcmp(serial.buf, parallel.buf, serial.len)); + cl_assert(!memcmp(serial_hash, parallel_hash, fixture.algo->rawsz)); + + strbuf_release(¶llel); + strbuf_release(&serial); + many_sources_fixture_release(&fixture); + clear_attr_manifest_thread_env(NULL); +#endif +} + +void test_attr_manifest__thread_failure_completes_remaining_ranges(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + struct many_sources_fixture fixture; + struct worktree_attr_manifest_stats serial_stats, fallback_stats; + struct strbuf serial = STRBUF_INIT, fallback = STRBUF_INIT; + unsigned char serial_hash[GIT_MAX_RAWSZ]; + unsigned char fallback_hash[GIT_MAX_RAWSZ]; + + if (!HAVE_THREADS) + return; + cl_set_cleanup(clear_attr_manifest_thread_env, NULL); + many_sources_fixture_init(&fixture); + + xsetenv(ATTR_MANIFEST_TEST_THREADS, "1", 1); + cl_assert_equal_i(worktree_attr_manifest_build( + &fixture.istate, &serial, serial_hash, &serial_stats), 0); + xsetenv(ATTR_MANIFEST_TEST_THREADS, "2", 1); + xsetenv(ATTR_MANIFEST_TEST_THREAD_FAIL_AT, "1", 1); + cl_assert_equal_i(worktree_attr_manifest_build( + &fixture.istate, &fallback, fallback_hash, &fallback_stats), 0); + cl_assert_equal_i(fallback_stats.candidates, + ATTR_MANIFEST_TEST_SOURCE_NR + 1); + cl_assert_equal_i(fallback_stats.threads, 2); + cl_assert_equal_i(fallback_stats.thread_failures, 1); + cl_assert_equal_i(fallback_stats.worktree_sources, + ATTR_MANIFEST_TEST_SOURCE_NR); + cl_assert_equal_i(serial.len, fallback.len); + cl_assert(!memcmp(serial.buf, fallback.buf, serial.len)); + cl_assert(!memcmp(serial_hash, fallback_hash, fixture.algo->rawsz)); + + strbuf_release(&fallback); + strbuf_release(&serial); + many_sources_fixture_release(&fixture); + clear_attr_manifest_thread_env(NULL); +#endif +} + +void test_attr_manifest__builder_rejects_structural_indexes(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *worktree = create_worktree(); + struct repository repo = { .worktree = worktree, .hash_algo = algo }; + struct index_state istate = INDEX_STATE_INIT(&repo); + struct worktree_attr_manifest_stats stats; + struct strbuf manifest = STRBUF_INIT; + unsigned char hash[GIT_MAX_RAWSZ]; + + CALLOC_ARRAY(istate.cache, 1); + istate.cache_alloc = istate.cache_nr = 1; + add_index_path(&istate, 0, "file", 1); + cl_assert_equal_i(worktree_attr_manifest_build( + &istate, &manifest, hash, &stats), -1); + cl_assert_equal_i(manifest.len, 0); + istate.cache[0]->ce_flags = create_ce_flags(0); + istate.sparse_index = INDEX_COLLAPSED; + cl_assert_equal_i(worktree_attr_manifest_build( + &istate, &manifest, hash, &stats), -1); + cl_assert_equal_i(manifest.len, 0); + + strbuf_release(&manifest); + release_index(&istate); + remove_worktree(worktree); +#endif +} diff --git a/t/unit-tests/u-clean-status-config.c b/t/unit-tests/u-clean-status-config.c new file mode 100644 index 00000000000000..4d836e10d15315 --- /dev/null +++ b/t/unit-tests/u-clean-status-config.c @@ -0,0 +1,193 @@ +#include "unit-test.h" +#include "attr-fingerprint.h" +#include "clean-status.h" +#include "clean-status-config.h" +#include "clean-status-internal.h" +#include "config.h" +#include "dir.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "strbuf.h" +#include "wrapper.h" + +static void digest_one(struct clean_status_config_digest *digest, + const char *key, const char *value, + const struct config_context *ctx) +{ + clean_status_config_init(digest, &hash_algos[GIT_HASH_SHA1]); + clean_status_config_add(digest, key, value, ctx); + clean_status_config_final(digest); +} + +static int hashes_equal(const unsigned char *a, const unsigned char *b) +{ + return hasheq(a, b, &hash_algos[GIT_HASH_SHA1]); +} + +void test_clean_status_config__non_semantic_values_only_change_full_hash(void) +{ + struct clean_status_config_digest a, b; + + digest_one(&a, "status.showuntrackedfiles", "normal", NULL); + digest_one(&b, "status.showuntrackedfiles", "all", NULL); + cl_assert(!hashes_equal(a.hash, b.hash)); + cl_assert(hashes_equal(a.semantic_hash, b.semantic_hash)); + cl_assert(!a.semantic_config_explicit); + cl_assert(!b.semantic_config_explicit); +} + +void test_clean_status_config__semantic_values_change_semantic_hash(void) +{ + struct clean_status_config_digest a, b; + + digest_one(&a, "core.autocrlf", "true", NULL); + digest_one(&b, "core.autocrlf", "false", NULL); + cl_assert(!hashes_equal(a.semantic_hash, b.semantic_hash)); + cl_assert(a.semantic_config_explicit); + cl_assert(b.semantic_config_explicit); +} + +void test_clean_status_config__origin_only_affects_full_hash(void) +{ + struct key_value_info global_kvi = KVI_INIT; + struct key_value_info local_kvi = KVI_INIT; + struct config_context global_ctx = { .kvi = &global_kvi }; + struct config_context local_ctx = { .kvi = &local_kvi }; + struct clean_status_config_digest global, local; + + global_kvi.scope = CONFIG_SCOPE_GLOBAL; + global_kvi.origin_type = CONFIG_ORIGIN_FILE; + global_kvi.filename = "/global"; + local_kvi.scope = CONFIG_SCOPE_LOCAL; + local_kvi.origin_type = CONFIG_ORIGIN_FILE; + local_kvi.filename = "/local"; + digest_one(&global, "core.eol", "lf", &global_ctx); + digest_one(&local, "core.eol", "lf", &local_ctx); + cl_assert(!hashes_equal(global.hash, local.hash)); + cl_assert(hashes_equal(global.semantic_hash, local.semantic_hash)); +} + +void test_clean_status_config__clean_filters_are_unsafe(void) +{ + struct clean_status_config_digest clean, smudge; + + digest_one(&clean, "filter.demo.clean", "command", NULL); + digest_one(&smudge, "filter.demo.smudge", "command", NULL); + cl_assert(clean.unsafe_filter); + cl_assert(clean.semantic_config_explicit); + cl_assert(!smudge.unsafe_filter); + cl_assert(!smudge.semantic_config_explicit); +} + +#if defined(O_NONBLOCK) && !defined(GIT_WINDOWS_NATIVE) +static char *create_gitdir(int with_attributes) +{ + const char *tmp = getenv("TMPDIR"); + char *gitdir = xstrfmt("%s/clean-status-config.XXXXXX", + tmp ? tmp : "/tmp"); + struct strbuf path = STRBUF_INIT; + + cl_assert(mkdtemp(gitdir) != NULL); + if (with_attributes) { + strbuf_addf(&path, "%s/info", gitdir); + cl_assert_equal_i(mkdir(path.buf, 0777), 0); + strbuf_addstr(&path, "/attributes"); + write_file(path.buf, "*.txt text\n"); + } + strbuf_release(&path); + return gitdir; +} + +static void remove_gitdir(char *gitdir) +{ + struct strbuf path = STRBUF_INIT; + + strbuf_addstr(&path, gitdir); + cl_assert_equal_i(remove_dir_recursively(&path, 0), 0); + strbuf_release(&path); + free(gitdir); +} + +static void clear_staged_config(void *unused UNUSED) +{ + clean_status_set_config_digest(NULL, NULL); +} +#endif + +void test_clean_status_config__attaches_only_to_the_staged_repository(void) +{ +#if !defined(O_NONBLOCK) || defined(GIT_WINDOWS_NATIVE) + cl_skip(); +#else + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *gitdir_a = create_gitdir(1); + char *gitdir_b = create_gitdir(0); + struct repository repo_a = { + .gitdir = gitdir_a, + .commondir = gitdir_a, + .hash_algo = algo, + }; + struct repository repo_b = { + .gitdir = gitdir_b, + .commondir = gitdir_b, + .hash_algo = algo, + }; + struct index_state istate_a = INDEX_STATE_INIT(&repo_a); + struct index_state istate_b = INDEX_STATE_INIT(&repo_b); + struct clean_status_config_digest digest, replacement; + struct clean_status_state *state; + struct attr_fingerprint attrs; + + cl_set_cleanup(clear_staged_config, NULL); + digest_one(&digest, "filter.demo.clean", "cat", NULL); + digest_one(&replacement, "core.autocrlf", "false", NULL); + cl_assert_equal_i(attr_fingerprint_repository(&repo_a, &attrs), 0); + cl_assert(attrs.sources_present); + + clean_status_set_config_digest(&repo_a, &digest); + clean_status_attach_config(&istate_b); + cl_assert_equal_p(istate_b.clean_status, NULL); + clean_status_attach_config(&istate_a); + state = istate_a.clean_status; + cl_assert(state != NULL); + cl_assert(state->current_config_valid); + cl_assert(state->current_semantic_valid); + cl_assert(state->current_attr_valid); + cl_assert(state->config_enforced); + cl_assert(state->unsafe_filter); + cl_assert(state->current_semantic_explicit); + cl_assert_equal_i(state->current_attr_sources_present, + attrs.sources_present); + cl_assert(hashes_equal(state->current_config_hash, digest.hash)); + cl_assert(hashes_equal(state->current_semantic_hash, + digest.semantic_hash)); + cl_assert(!memcmp(state->current_attr_hash, attrs.content_hash, + algo->rawsz)); + cl_assert(!memcmp(state->current_attr_namespace_hash, + attrs.namespace_hash, algo->rawsz)); + + clean_status_set_config_digest(&repo_a, &replacement); + clean_status_attach_config(&istate_a); + cl_assert(hashes_equal(state->current_config_hash, digest.hash)); + cl_assert(hashes_equal(state->current_semantic_hash, + digest.semantic_hash)); + cl_assert(state->unsafe_filter); + + release_index(&istate_a); + cl_assert_equal_p(istate_a.clean_status, NULL); + release_index(&istate_b); + clear_staged_config(NULL); + if (repo_a.config) { + git_configset_clear(repo_a.config); + FREE_AND_NULL(repo_a.config); + } + if (repo_b.config) { + git_configset_clear(repo_b.config); + FREE_AND_NULL(repo_b.config); + } + repo_settings_clear(&repo_a); + repo_settings_clear(&repo_b); + remove_gitdir(gitdir_b); + remove_gitdir(gitdir_a); +#endif +} diff --git a/t/unit-tests/u-clean-status-history.c b/t/unit-tests/u-clean-status-history.c new file mode 100644 index 00000000000000..3d196c73c8236e --- /dev/null +++ b/t/unit-tests/u-clean-status-history.c @@ -0,0 +1,339 @@ +#include "unit-test.h" +#include "attr-manifest.h" +#include "clean-status.h" +#include "clean-status-internal.h" +#include "fsmonitor-clean-proof.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "strbuf.h" + +struct history_fixture { + struct repository repo; + struct index_state istate; + struct strbuf manifest; + struct strbuf encoded; + unsigned char config_hash[GIT_MAX_RAWSZ]; + unsigned char semantic_hash[GIT_MAX_RAWSZ]; + unsigned char attr_hash[GIT_MAX_RAWSZ]; +}; + +static void fixture_init(struct history_fixture *fixture, + const struct git_hash_algo *algo) +{ + static const unsigned char token[] = "builtin:1:2"; + struct attr_manifest_writer writer; + struct fsmonitor_clean_proof proof; + unsigned char hash[GIT_MAX_RAWSZ]; + + memset(fixture, 0, sizeof(*fixture)); + fixture->repo.hash_algo = algo; + repo_config_values_init(&fixture->repo.config_values_private_); + index_state_init(&fixture->istate, &fixture->repo); + fixture->manifest = (struct strbuf)STRBUF_INIT; + fixture->encoded = (struct strbuf)STRBUF_INIT; + memset(hash, 1, algo->rawsz); + memset(fixture->config_hash, 2, algo->rawsz); + memset(fixture->semantic_hash, 3, algo->rawsz); + memset(fixture->attr_hash, 4, algo->rawsz); + attr_manifest_writer_init(&writer, &fixture->manifest, algo); + cl_assert_equal_i(attr_manifest_writer_add( + &writer, ".gitattributes", ATTR_MANIFEST_INDEX, hash), 0); + memset(&proof, 0, sizeof(proof)); + proof.flags = FSMONITOR_CLEAN_PROOF_ALL; + proof.token = token; + proof.token_len = sizeof(token) - 1; + proof.config_hash = fixture->config_hash; + proof.semantic_hash = fixture->semantic_hash; + proof.attr_hash = fixture->attr_hash; + proof.attr_manifest = (const unsigned char *)fixture->manifest.buf; + proof.attr_manifest_len = fixture->manifest.len; + cl_assert_equal_i(fsmonitor_clean_proof_write( + &fixture->encoded, &proof, algo), 0); +} + +static void fixture_release(struct history_fixture *fixture) +{ + clean_status_release(&fixture->istate); + free(fixture->istate.fsmonitor_last_update); + repo_config_values_clear(&fixture->repo.config_values_private_); + strbuf_release(&fixture->encoded); + strbuf_release(&fixture->manifest); +} + +static struct clean_status_state *install_current( + struct history_fixture *fixture) +{ + struct clean_status_state *state = + clean_status_get_state(&fixture->istate); + const struct git_hash_algo *algo = fixture->repo.hash_algo; + + memcpy(state->current_config_hash, fixture->config_hash, algo->rawsz); + memcpy(state->current_semantic_hash, fixture->semantic_hash, algo->rawsz); + memcpy(state->current_attr_hash, fixture->attr_hash, algo->rawsz); + state->current_config_valid = 1; + state->current_semantic_valid = 1; + state->current_attr_valid = 1; + state->config_enforced = 1; + FREE_AND_NULL(fixture->istate.fsmonitor_last_update); + fixture->istate.fsmonitor_last_update = xstrdup("builtin:1:2"); + fixture->istate.fsmonitor_token_valid = 1; + return state; +} + +void test_clean_status_history__reads_valid_history_once(void) +{ + struct history_fixture fixture; + struct clean_status_state *state; + + fixture_init(&fixture, &hash_algos[GIT_HASH_SHA1]); + cl_assert_equal_i(clean_status_read_fsmonitor_config( + &fixture.istate, fixture.encoded.buf, fixture.encoded.len), 0); + state = fixture.istate.clean_status; + cl_assert(state->disk_config_valid); + cl_assert(state->manifest.disk_valid); + cl_assert_equal_s(state->disk_config_token, "builtin:1:2"); + + cl_assert_equal_i(clean_status_read_fsmonitor_config( + &fixture.istate, fixture.encoded.buf, fixture.encoded.len), 0); + cl_assert(state->disk_config_invalid); + cl_assert(!state->disk_config_valid); + cl_assert(!state->manifest.disk_valid); + fixture_release(&fixture); +} + +void test_clean_status_history__adopts_only_coherent_proofs(void) +{ + struct history_fixture fixture; + struct clean_status_state *state; + + fixture_init(&fixture, &hash_algos[GIT_HASH_SHA256]); + clean_status_read_fsmonitor_config( + &fixture.istate, fixture.encoded.buf, fixture.encoded.len); + state = install_current(&fixture); + clean_status_prepare_fsmonitor_config(&fixture.istate); + cl_assert(state->initial_coherent); + cl_assert(state->manifest.current_valid); + cl_assert_equal_i(state->manifest.current.len, fixture.manifest.len); + + state->current_semantic_hash[0] ^= 1; + clean_status_prepare_fsmonitor_config(&fixture.istate); + cl_assert(clean_status_fsmonitor_strong_mismatch(&fixture.istate)); + cl_assert(!state->initial_coherent); + fixture_release(&fixture); +} + +void test_clean_status_history__distinguishes_available_history(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct history_fixture fixture; + struct clean_status_state *state; + struct strbuf manifest_only = STRBUF_INIT; + + fixture_init(&fixture, algo); + clean_status_read_fsmonitor_config( + &fixture.istate, fixture.encoded.buf, fixture.encoded.len); + cl_assert(clean_status_has_persistent_fsmonitor_semantic_history( + &fixture.istate)); + cl_assert(clean_status_has_worktree_manifest_history(&fixture.istate)); + state = install_current(&fixture); + cl_assert(!clean_status_fsmonitor_semantic_adoption_needed( + &fixture.istate)); + cl_assert(!clean_status_fsmonitor_semantic_baseline_needed( + &fixture.istate)); + state->strong_mismatch = 1; + cl_assert(clean_status_fsmonitor_semantic_adoption_needed( + &fixture.istate)); + cl_assert(!clean_status_fsmonitor_semantic_baseline_needed( + &fixture.istate)); + fixture_release(&fixture); + + fixture_init(&fixture, algo); + cl_assert_equal_i(fsmonitor_clean_proof_copy_without_bindings( + &manifest_only, fixture.encoded.buf, fixture.encoded.len, + algo), 0); + clean_status_read_fsmonitor_config( + &fixture.istate, manifest_only.buf, manifest_only.len); + cl_assert(!clean_status_has_persistent_fsmonitor_semantic_history( + &fixture.istate)); + cl_assert(clean_status_has_worktree_manifest_history(&fixture.istate)); + install_current(&fixture); + cl_assert(!clean_status_fsmonitor_semantic_adoption_needed( + &fixture.istate)); + cl_assert(!clean_status_fsmonitor_semantic_baseline_needed( + &fixture.istate)); + fixture_release(&fixture); + + fixture_init(&fixture, algo); + state = install_current(&fixture); + fixture.istate.fsmonitor_token_valid = 1; + FREE_AND_NULL(fixture.istate.fsmonitor_last_update); + fixture.istate.fsmonitor_last_update = xstrdup("builtin:test:1"); + cl_assert(clean_status_fsmonitor_semantic_adoption_needed( + &fixture.istate)); + cl_assert(clean_status_fsmonitor_semantic_baseline_needed( + &fixture.istate)); + clean_status_begin_fsmonitor_semantic_baseline(&fixture.istate); + cl_assert(!clean_status_fsmonitor_semantic_adoption_needed( + &fixture.istate)); + state->semantic_baseline_pending = 0; + state->disk_config_seen = 1; + cl_assert(clean_status_fsmonitor_semantic_adoption_needed( + &fixture.istate)); + cl_assert(!clean_status_fsmonitor_semantic_baseline_needed( + &fixture.istate)); + state->disk_config_seen = 0; + fixture.repo.config_values_private_.trust_ctime = 0; + cl_assert(clean_status_fsmonitor_semantic_adoption_needed( + &fixture.istate)); + cl_assert(!clean_status_fsmonitor_semantic_baseline_needed( + &fixture.istate)); + fixture.repo.config_values_private_.trust_ctime = 1; + state->strong_mismatch = 1; + cl_assert(clean_status_fsmonitor_semantic_adoption_needed( + &fixture.istate)); + cl_assert(!clean_status_fsmonitor_semantic_baseline_needed( + &fixture.istate)); + fixture_release(&fixture); + strbuf_release(&manifest_only); +} + +void test_clean_status_history__preserves_unbound_manifests(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct history_fixture fixture; + struct clean_status_state *state; + struct fsmonitor_clean_proof parsed; + struct strbuf rewritten = STRBUF_INIT; + + fixture_init(&fixture, algo); + clean_status_read_fsmonitor_config( + &fixture.istate, fixture.encoded.buf, fixture.encoded.len); + state = install_current(&fixture); + clean_status_prepare_fsmonitor_config(&fixture.istate); + cl_assert(clean_status_should_write_fsmonitor_config(&fixture.istate)); + clean_status_write_fsmonitor_config(&rewritten, &fixture.istate); + cl_assert_equal_i(fsmonitor_clean_proof_parse( + &parsed, rewritten.buf, rewritten.len, algo), 0); + cl_assert_equal_i(parsed.flags, FSMONITOR_CLEAN_PROOF_ALL); + + FREE_AND_NULL(fixture.istate.fsmonitor_last_update); + fixture.istate.fsmonitor_last_update = xstrdup("builtin:1:3"); + clean_status_write_fsmonitor_config(&rewritten, &fixture.istate); + cl_assert_equal_i(fsmonitor_clean_proof_parse( + &parsed, rewritten.buf, rewritten.len, algo), 0); + cl_assert_equal_i(parsed.flags, + FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | + FSMONITOR_CLEAN_PROOF_FULL_INDEX); + + FREE_AND_NULL(fixture.istate.fsmonitor_last_update); + fixture.istate.fsmonitor_last_update = xstrdup("builtin:1:2"); + state->current_config_valid = 0; + clean_status_write_fsmonitor_config(&rewritten, &fixture.istate); + cl_assert_equal_i(fsmonitor_clean_proof_parse( + &parsed, rewritten.buf, rewritten.len, algo), 0); + cl_assert_equal_i(parsed.flags, + FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | + FSMONITOR_CLEAN_PROOF_FULL_INDEX); + strbuf_release(&rewritten); + fixture_release(&fixture); +} +void test_clean_status_history__advances_only_current_proofs(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct history_fixture fixture; + struct clean_status_state *state; + struct fsmonitor_clean_proof parsed; + struct strbuf rewritten = STRBUF_INIT; + + fixture_init(&fixture, algo); + clean_status_read_fsmonitor_config( + &fixture.istate, fixture.encoded.buf, fixture.encoded.len); + state = install_current(&fixture); + clean_status_prepare_fsmonitor_config(&fixture.istate); + + clean_status_advance_fsmonitor_config_token( + &fixture.istate, "builtin:1:3"); + cl_assert_equal_s(state->config_revalidated_token, "builtin:1:3"); + FREE_AND_NULL(fixture.istate.fsmonitor_last_update); + fixture.istate.fsmonitor_last_update = xstrdup("builtin:1:3"); + cl_assert(clean_status_should_write_fsmonitor_config(&fixture.istate)); + + clean_status_invalidate_current_proof(&fixture.istate); + cl_assert(!state->config_revalidated); + cl_assert(!state->initial_coherent); + clean_status_advance_fsmonitor_config_token( + &fixture.istate, "builtin:1:4"); + cl_assert_equal_s(state->config_revalidated_token, "builtin:1:3"); + FREE_AND_NULL(fixture.istate.fsmonitor_last_update); + fixture.istate.fsmonitor_last_update = xstrdup("builtin:1:4"); + clean_status_write_fsmonitor_config(&rewritten, &fixture.istate); + cl_assert_equal_i(fsmonitor_clean_proof_parse( + &parsed, rewritten.buf, rewritten.len, algo), 0); + cl_assert_equal_i(parsed.flags, + FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | + FSMONITOR_CLEAN_PROOF_FULL_INDEX); + + strbuf_release(&rewritten); + fixture_release(&fixture); +} + +void test_clean_status_history__expires_invalidated_proofs(void) +{ + struct history_fixture fixture; + struct clean_status_state *state; + + fixture_init(&fixture, &hash_algos[GIT_HASH_SHA1]); + clean_status_read_fsmonitor_config( + &fixture.istate, fixture.encoded.buf, fixture.encoded.len); + state = install_current(&fixture); + clean_status_prepare_fsmonitor_config(&fixture.istate); + cl_assert(state->manifest.current_valid); + cl_assert(state->config_revalidated); + cl_assert(state->initial_coherent); + + clean_status_invalidate_current_manifest(&fixture.istate); + cl_assert(!state->manifest.current_valid); + cl_assert(!state->config_revalidated); + cl_assert(!state->initial_coherent); + cl_assert(clean_status_fsmonitor_config_mismatch(&fixture.istate)); + fixture_release(&fixture); +} + +void test_clean_status_history__copies_validated_history(void) +{ + struct history_fixture fixture; + struct repository dst_repo = { + .hash_algo = &hash_algos[GIT_HASH_SHA1], + }; + struct index_state dst = INDEX_STATE_INIT(&dst_repo); + struct index_state invalid_dst = INDEX_STATE_INIT(&dst_repo); + struct clean_status_state *dst_state; + + fixture_init(&fixture, &hash_algos[GIT_HASH_SHA1]); + clean_status_read_fsmonitor_config( + &fixture.istate, fixture.encoded.buf, fixture.encoded.len); + move_index_extensions(&dst, &fixture.istate); + dst_state = dst.clean_status; + cl_assert(dst_state != NULL); + cl_assert(dst_state->disk_config_valid); + cl_assert(!dst_state->disk_config_invalid); + cl_assert(dst_state->disk_semantic_valid); + cl_assert(dst_state->disk_attr_valid); + cl_assert(dst_state->manifest.disk_valid); + cl_assert_equal_i(dst_state->manifest.disk_flags, + FSMONITOR_CLEAN_PROOF_ALL); + cl_assert_equal_i(dst_state->disk_config_raw.len, + fixture.encoded.len); + + clean_status_read_fsmonitor_config( + &fixture.istate, fixture.encoded.buf, fixture.encoded.len); + move_index_extensions(&invalid_dst, &fixture.istate); + cl_assert(!invalid_dst.clean_status); + cl_assert(dst_state->disk_config_valid); + cl_assert_equal_i(dst_state->disk_config_raw.len, + fixture.encoded.len); + + release_index(&invalid_dst); + release_index(&dst); + fixture_release(&fixture); +} diff --git a/t/unit-tests/u-clean-status-identity.c b/t/unit-tests/u-clean-status-identity.c new file mode 100644 index 00000000000000..6875d594c58c5c --- /dev/null +++ b/t/unit-tests/u-clean-status-identity.c @@ -0,0 +1,64 @@ +#include "unit-test.h" +#include "clean-status-identity.h" +#include "strbuf.h" + +void test_clean_status_identity__round_trips_fixed_width_encoding(void) +{ + struct clean_status_identity expected = { + .stat.fields = { + 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, + }, + }; + struct clean_status_identity actual; + struct strbuf encoded = STRBUF_INIT; + const unsigned char *p; + + clean_status_identity_write(&encoded, &expected); + cl_assert_equal_i(encoded.len, CLEAN_STATUS_IDENTITY_SIZE); + p = (const unsigned char *)encoded.buf; + cl_assert_equal_i(clean_status_identity_read( + &p, (const unsigned char *)encoded.buf + encoded.len, &actual), 0); + cl_assert_equal_i(p - (const unsigned char *)encoded.buf, encoded.len); + cl_assert(clean_status_identity_equal(&expected, &actual)); + strbuf_release(&encoded); +} + +void test_clean_status_identity__rejects_every_truncation(void) +{ + struct clean_status_identity identity = { 0 }, parsed; + struct strbuf encoded = STRBUF_INIT; + + clean_status_identity_write(&encoded, &identity); + for (size_t len = 0; len < encoded.len; len++) { + const unsigned char *p = (const unsigned char *)encoded.buf; + + cl_assert_equal_i(clean_status_identity_read( + &p, (const unsigned char *)encoded.buf + len, &parsed), -1); + } + strbuf_release(&encoded); +} + +void test_clean_status_identity__requires_a_single_link_regular_file(void) +{ + struct clean_status_identity identity; + struct stat st = { 0 }; + + st.st_mode = S_IFDIR | 0755; + st.st_nlink = 1; + cl_assert_equal_i(clean_status_identity_from_stat(&identity, &st), -1); + st.st_mode = S_IFREG | 0644; + st.st_nlink = 2; + cl_assert_equal_i(clean_status_identity_from_stat(&identity, &st), -1); + st.st_nlink = 1; + cl_assert_equal_i(clean_status_identity_from_stat(&identity, &st), 0); +} + +void test_clean_status_identity__durability_is_platform_specific(void) +{ +#ifdef __APPLE__ + cl_assert(clean_status_identity_is_durable()); +#else + cl_assert(!clean_status_identity_is_durable()); +#endif +} diff --git a/t/unit-tests/u-clean-status-index.c b/t/unit-tests/u-clean-status-index.c new file mode 100644 index 00000000000000..145d06175cbf0b --- /dev/null +++ b/t/unit-tests/u-clean-status-index.c @@ -0,0 +1,382 @@ +#include "unit-test.h" +#include "clean-status.h" +#include "clean-status-index.h" +#include "clean-status-internal.h" +#include "dir.h" +#include "object.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "strbuf.h" +#include "wrapper.h" + +struct index_fixture { + char *path; + int fd; + struct stat st; + struct object_id checksum; +}; + +static void fixture_init(struct index_fixture *fixture, + const struct git_hash_algo *algo) +{ + const char *tmp = getenv("TMPDIR"); + unsigned char header[12] = "DIRC"; + unsigned char hash[GIT_MAX_RAWSZ]; + static const char payload[] = "payload"; + + memset(fixture, 0, sizeof(*fixture)); + fixture->path = xstrfmt("%s/index-snapshot.XXXXXX", + tmp ? tmp : "/tmp"); + fixture->fd = mkstemp(fixture->path); + cl_assert(fixture->fd >= 0); + put_be32(header + 4, 4); + put_be32(header + 8, 7); + memset(hash, 1, algo->rawsz); + oidread(&fixture->checksum, hash, algo); + cl_assert_equal_i(write_in_full(fixture->fd, header, sizeof(header)), + sizeof(header)); + cl_assert_equal_i(write_in_full(fixture->fd, payload, sizeof(payload)), + sizeof(payload)); + cl_assert_equal_i(write_in_full(fixture->fd, hash, algo->rawsz), + algo->rawsz); + cl_assert_equal_i(fstat(fixture->fd, &fixture->st), 0); +} + +static void fixture_release(struct index_fixture *fixture) +{ + cl_assert_equal_i(close(fixture->fd), 0); + cl_assert_equal_i(unlink(fixture->path), 0); + free(fixture->path); +} + +static void write_at(int fd, const void *data, size_t len, off_t offset) +{ + cl_assert_equal_i(lseek(fd, offset, SEEK_SET), offset); + cl_assert_equal_i(write_in_full(fd, data, len), len); +} + +static void fixture_clear_checksum(struct index_fixture *fixture, + const struct git_hash_algo *algo) +{ + unsigned char null_hash[GIT_MAX_RAWSZ] = { 0 }; + + write_at(fixture->fd, null_hash, algo->rawsz, + fixture->st.st_size - algo->rawsz); + oidclr(&fixture->checksum, algo); + cl_assert_equal_i(fstat(fixture->fd, &fixture->st), 0); +} + +static void assert_reads_snapshot(const struct git_hash_algo *algo) +{ + struct index_fixture fixture; + struct clean_status_index_snapshot snapshot; + struct repository repo = { 0 }; + struct index_state istate = INDEX_STATE_INIT(&repo); + + fixture_init(&fixture, algo); + repo.hash_algo = algo; + repo.index_file = fixture.path; + istate.version = 4; + istate.cache_nr = 7; + oidcpy(&istate.oid, &fixture.checksum); + cl_assert_equal_i(clean_status_index_snapshot_pin( + &snapshot, &istate), 0); + cl_assert_equal_i(snapshot.version, 4); + cl_assert_equal_i(snapshot.cache_nr, 7); + cl_assert(oideq(&snapshot.checksum, &fixture.checksum)); + cl_assert(clean_status_index_snapshot_still_matches( + &snapshot, &istate)); + clean_status_index_snapshot_release(&snapshot); + fixture_release(&fixture); +} + +void test_clean_status_index__reads_both_object_formats(void) +{ + assert_reads_snapshot(&hash_algos[GIT_HASH_SHA1]); + assert_reads_snapshot(&hash_algos[GIT_HASH_SHA256]); +} + +void test_clean_status_index__rejects_invalid_headers(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct index_fixture fixture; + struct clean_status_index_snapshot snapshot; + struct repository repo = { 0 }; + struct index_state istate = INDEX_STATE_INIT(&repo); + uint32_t value; + + fixture_init(&fixture, algo); + repo.hash_algo = algo; + repo.index_file = fixture.path; + istate.version = 4; + istate.cache_nr = 7; + oidcpy(&istate.oid, &fixture.checksum); + write_at(fixture.fd, "NOPE", 4, 0); + cl_assert_equal_i(clean_status_index_snapshot_pin( + &snapshot, &istate), -1); + write_at(fixture.fd, "DIRC", 4, 0); + put_be32(&value, 1); + write_at(fixture.fd, &value, sizeof(value), 4); + cl_assert_equal_i(clean_status_index_snapshot_pin( + &snapshot, &istate), -1); + fixture_release(&fixture); +} + +static void assert_rejects_null_checksum(const struct git_hash_algo *algo) +{ + struct clean_status_index_snapshot snapshot; + struct index_fixture fixture; + struct repository repo = { 0 }; + struct index_state istate = INDEX_STATE_INIT(&repo); + + fixture_init(&fixture, algo); + repo.hash_algo = algo; + repo.index_file = fixture.path; + istate.version = 4; + istate.cache_nr = 7; + fixture_clear_checksum(&fixture, algo); + oidcpy(&istate.oid, &fixture.checksum); + cl_assert_equal_i(clean_status_index_snapshot_open( + &snapshot, fixture.path, algo), -1); + cl_assert_equal_i(clean_status_index_snapshot_pin( + &snapshot, &istate), -1); + fixture_release(&fixture); +} + +void test_clean_status_index__rejects_null_checksums(void) +{ + assert_rejects_null_checksum(&hash_algos[GIT_HASH_SHA1]); + assert_rejects_null_checksum(&hash_algos[GIT_HASH_SHA256]); +} + +static void assert_pins_null_checksum_source( + const struct git_hash_algo *algo) +{ + struct clean_status_index_snapshot snapshot; + struct index_fixture fixture, replacement; + struct repository repo = { 0 }; + struct index_state istate = INDEX_STATE_INIT(&repo); + struct index_state parsed = INDEX_STATE_INIT(&repo); + char *moved; + + fixture_init(&fixture, algo); + fixture_init(&replacement, algo); + fixture_clear_checksum(&fixture, algo); + fixture_clear_checksum(&replacement, algo); + repo.hash_algo = algo; + repo.index_file = fixture.path; + istate.version = 4; + istate.cache_nr = 7; + oidcpy(&istate.oid, &fixture.checksum); + parsed.version = 4; + parsed.cache_nr = 7; + oidcpy(&parsed.oid, &replacement.checksum); + clean_status_get_state(&istate); + clean_status_record_source_identity(&istate, &fixture.st); + clean_status_get_state(&parsed); + clean_status_record_source_identity(&parsed, &replacement.st); + + if (clean_status_identity_is_durable()) { + cl_assert_equal_i(clean_status_index_snapshot_pin( + &snapshot, &istate), 0); + cl_assert(clean_status_index_snapshot_still_matches( + &snapshot, &istate)); + + /* + * Model an A-to-B-to-A replacement while a second index state + * parses B. The named path and held descriptor are back on A, + * while the parsed state's source identity still binds it to B. + */ + cl_assert(!clean_status_index_snapshot_still_matches( + &snapshot, &parsed)); + clean_status_index_snapshot_release(&snapshot); + } else { + cl_assert_equal_i(clean_status_index_snapshot_pin( + &snapshot, &istate), -1); + clean_status_release(&istate); + clean_status_release(&parsed); + fixture_release(&fixture); + fixture_release(&replacement); + return; + } + + moved = xstrfmt("%s.old", fixture.path); + cl_assert_equal_i(rename(fixture.path, moved), 0); + cl_assert_equal_i(rename(replacement.path, fixture.path), 0); + cl_assert_equal_i(clean_status_index_snapshot_pin( + &snapshot, &istate), -1); + + clean_status_release(&istate); + clean_status_release(&parsed); + cl_assert_equal_i(close(fixture.fd), 0); + cl_assert_equal_i(close(replacement.fd), 0); + cl_assert_equal_i(unlink(fixture.path), 0); + cl_assert_equal_i(unlink(moved), 0); + free(moved); + free(fixture.path); + free(replacement.path); +} + +void test_clean_status_index__pins_null_checksum_source_identity(void) +{ + assert_pins_null_checksum_source(&hash_algos[GIT_HASH_SHA1]); + assert_pins_null_checksum_source(&hash_algos[GIT_HASH_SHA256]); +} + +void test_clean_status_index__pins_named_index_identity(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct clean_status_index_snapshot snapshot; + struct index_fixture fixture; + struct repository repo = { 0 }; + struct index_state istate = INDEX_STATE_INIT(&repo); + struct index_state parsed = INDEX_STATE_INIT(&repo); + unsigned char replacement_hash[GIT_MAX_RAWSZ]; +#ifndef GIT_WINDOWS_NATIVE + char *moved; + int replacement; +#endif + + fixture_init(&fixture, algo); + repo.hash_algo = algo; + repo.index_file = fixture.path; + istate.version = 4; + istate.cache_nr = 7; + oidcpy(&istate.oid, &fixture.checksum); + + cl_assert_equal_i(clean_status_index_snapshot_pin( + &snapshot, &istate), 0); + cl_assert(clean_status_index_snapshot_still_matches( + &snapshot, &istate)); + + /* + * Model an A-to-B-to-A replacement while a consumer parses B. + * Matching the restored named path is insufficient unless the parsed + * state is also bound to the pinned A contents. + */ + memset(replacement_hash, 2, algo->rawsz); + parsed.version = 4; + parsed.cache_nr = 7; + oidread(&parsed.oid, replacement_hash, algo); + cl_assert(!clean_status_index_snapshot_still_matches( + &snapshot, &parsed)); + + istate.cache_nr++; + cl_assert(!clean_status_index_snapshot_still_matches( + &snapshot, &istate)); + istate.cache_nr--; + +#ifndef GIT_WINDOWS_NATIVE + moved = xstrfmt("%s.old", fixture.path); + cl_assert_equal_i(rename(fixture.path, moved), 0); + replacement = open(fixture.path, O_WRONLY | O_CREAT | O_EXCL, 0600); + cl_assert(replacement >= 0); + cl_assert_equal_i(close(replacement), 0); + cl_assert(!clean_status_index_snapshot_still_matches( + &snapshot, &istate)); + cl_assert_equal_i(unlink(fixture.path), 0); + cl_assert_equal_i(rename(moved, fixture.path), 0); + free(moved); +#endif + clean_status_index_snapshot_release(&snapshot); + +#ifndef GIT_WINDOWS_NATIVE + { + char *symlink_path = xstrfmt("%s.link", fixture.path); + + cl_assert_equal_i(symlink(fixture.path, symlink_path), 0); + repo.index_file = symlink_path; + cl_assert_equal_i(clean_status_index_snapshot_pin( + &snapshot, &istate), -1); + repo.index_file = fixture.path; + cl_assert_equal_i(unlink(symlink_path), 0); + free(symlink_path); + } +#endif + + fixture_release(&fixture); +} + +void test_clean_status_index__binds_the_parsed_source(void) +{ + const char *tmp = getenv("TMPDIR"); + char *worktree = xstrfmt("%s/status-source.XXXXXX", + tmp ? tmp : "/tmp"); + struct index_state istate = { 0 }; + struct strbuf path = STRBUF_INIT, replacement = STRBUF_INIT; + struct strbuf cleanup = STRBUF_INIT; + struct stat original, current; + + cl_assert(mkdtemp(worktree) != NULL); + strbuf_addf(&path, "%s/index", worktree); + strbuf_addf(&replacement, "%s/replacement", worktree); + write_file(path.buf, "original"); + write_file(replacement.buf, "replacement"); + cl_assert_equal_i(stat(path.buf, &original), 0); + clean_status_get_state(&istate); + clean_status_record_source_identity(&istate, &original); + cl_assert(clean_status_verify_null_index(&istate, &original)); + + cl_assert_equal_i(rename(replacement.buf, path.buf), 0); + cl_assert_equal_i(stat(path.buf, ¤t), 0); + if (clean_status_identity_is_durable()) + cl_assert(!clean_status_verify_null_index(&istate, ¤t)); + else + cl_assert(clean_status_verify_null_index(&istate, ¤t)); + + clean_status_release(&istate); + strbuf_addstr(&cleanup, worktree); + cl_assert_equal_i(remove_dir_recursively(&cleanup, 0), 0); + strbuf_release(&cleanup); + strbuf_release(&replacement); + strbuf_release(&path); + free(worktree); +} + +void test_clean_status_index__recognizes_certifiable_entries(void) +{ + struct repository repo = { .hash_algo = &hash_algos[GIT_HASH_SHA1] }; + struct index_state istate = INDEX_STATE_INIT(&repo); + struct cache_entry *ce; + static const char path[] = "tracked"; + + CALLOC_ARRAY(istate.cache, 1); + istate.cache_alloc = istate.cache_nr = 1; + memset(istate.oid.hash, 1, repo.hash_algo->rawsz); + oid_set_algo(&istate.oid, repo.hash_algo); + ce = make_empty_cache_entry(&istate, sizeof(path) - 1); + ce->ce_mode = S_IFREG | 0644; + ce->ce_namelen = sizeof(path) - 1; + memcpy(ce->name, path, sizeof(path)); + istate.cache[0] = ce; + + ce->ce_flags = CE_FSMONITOR_VALID; + cl_assert(clean_status_index_is_certifiable(&istate)); + ce->ce_flags |= CE_UPTODATE | CE_HASHED; + cl_assert(clean_status_index_is_certifiable(&istate)); + + oidclr(&istate.oid, repo.hash_algo); + cl_assert(!clean_status_index_is_certifiable(&istate)); + cl_assert(clean_status_index_entries_are_certifiable(&istate)); + memset(istate.oid.hash, 1, repo.hash_algo->rawsz); + + ce->ce_flags = 0; + cl_assert(!clean_status_index_is_certifiable(&istate)); + ce->ce_flags = CE_FSMONITOR_VALID | CE_VALID; + cl_assert(!clean_status_index_is_certifiable(&istate)); + ce->ce_flags = CE_FSMONITOR_VALID | CE_UPDATE_IN_BASE; + cl_assert(clean_status_index_is_certifiable(&istate)); + ce->ce_flags = CE_FSMONITOR_VALID | create_ce_flags(1); + cl_assert(!clean_status_index_is_certifiable(&istate)); + ce->ce_flags = CE_FSMONITOR_VALID | CE_INTENT_TO_ADD; + cl_assert(!clean_status_index_is_certifiable(&istate)); + ce->ce_flags = CE_FSMONITOR_VALID | CE_SKIP_WORKTREE; + cl_assert(!clean_status_index_is_certifiable(&istate)); + ce->ce_flags = CE_FSMONITOR_VALID | CE_WT_REMOVE; + cl_assert(!clean_status_index_is_certifiable(&istate)); + ce->ce_flags = CE_FSMONITOR_VALID; + ce->ce_mode = S_IFGITLINK; + cl_assert(!clean_status_index_is_certifiable(&istate)); + + release_index(&istate); +} diff --git a/t/unit-tests/u-clean-status-manifest.c b/t/unit-tests/u-clean-status-manifest.c new file mode 100644 index 00000000000000..971b4bea973c0c --- /dev/null +++ b/t/unit-tests/u-clean-status-manifest.c @@ -0,0 +1,229 @@ +#include "unit-test.h" +#include "attr-manifest.h" +#include "clean-status.h" +#include "clean-status-internal.h" +#include "clean-status-manifest.h" +#include "dir.h" +#include "fsmonitor-clean-proof.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "semantic-verify-internal.h" +#include "wrapper.h" + +static void make_manifest(struct strbuf *manifest, + const struct git_hash_algo *algo) +{ + struct attr_manifest_writer writer; + unsigned char hash[GIT_MAX_RAWSZ]; + + memset(hash, 1, algo->rawsz); + attr_manifest_writer_init(&writer, manifest, algo); + cl_assert_equal_i(attr_manifest_writer_add( + &writer, ".gitattributes", ATTR_MANIFEST_INDEX, hash), 0); +} + +void test_clean_status_manifest__loads_and_adopts_valid_history(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct clean_status_manifest_state state; + struct strbuf manifest = STRBUF_INIT; + + clean_status_manifest_init(&state); + make_manifest(&manifest, algo); + cl_assert_equal_i(clean_status_manifest_load( + &state, manifest.buf, manifest.len, + FSMONITOR_CLEAN_PROOF_ALL, algo), 0); + cl_assert(state.disk_valid); + clean_status_manifest_adopt_disk(&state); + cl_assert(state.current_valid); + cl_assert(state.checked); + cl_assert_equal_i(state.current_flags, FSMONITOR_CLEAN_PROOF_ALL); + cl_assert_equal_i(state.current.len, manifest.len); + cl_assert(!memcmp(state.current.buf, manifest.buf, manifest.len)); + clean_status_manifest_invalidate(&state); + cl_assert(!state.current_valid); + cl_assert(state.current_invalidated); + cl_assert_equal_i(state.current_flags, 0); + clean_status_manifest_release(&state); + strbuf_release(&manifest); +} + +void test_clean_status_manifest__rejects_invalid_history(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct clean_status_manifest_state state; + struct strbuf manifest = STRBUF_INIT; + size_t valid_len; + + clean_status_manifest_init(&state); + make_manifest(&manifest, algo); + valid_len = manifest.len; + cl_assert_equal_i(clean_status_manifest_load( + &state, manifest.buf, manifest.len, + FSMONITOR_CLEAN_PROOF_ALL, algo), 0); + cl_assert(state.disk_valid); + cl_assert_equal_i(state.disk_flags, FSMONITOR_CLEAN_PROOF_ALL); + cl_assert_equal_i(state.disk.len, valid_len); + cl_assert(!memcmp(state.disk.buf, manifest.buf, valid_len)); + + strbuf_addch(&manifest, 0); + cl_assert_equal_i(clean_status_manifest_load( + &state, manifest.buf, manifest.len, + FSMONITOR_CLEAN_PROOF_ALL, algo), -1); + cl_assert(!state.disk_valid); + cl_assert_equal_i(state.disk_flags, 0); + cl_assert_equal_i(state.disk.len, 0); + clean_status_manifest_release(&state); + strbuf_release(&manifest); +} + +void test_clean_status_manifest__requires_complete_full_index(void) +{ + const char *current_token = "builtin:1:2"; + const char *closed_token = "builtin:1:3"; + struct repository repo = { .hash_algo = &hash_algos[GIT_HASH_SHA1] }; + struct index_state istate = INDEX_STATE_INIT(&repo); + struct clean_status_state *state = clean_status_get_state(&istate); + + istate.fsmonitor_last_update = xstrdup(current_token); + istate.fsmonitor_token_valid = 1; + state->current_config_valid = 1; + state->current_semantic_valid = 1; + state->current_attr_valid = 1; + state->config_enforced = 1; + state->manifest.current_valid = 1; + state->manifest.checked = 1; + state->manifest.current_flags = + FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE; + + clean_status_mark_fsmonitor_config_valid(&istate, closed_token); + cl_assert(!state->config_revalidated); + cl_assert(!state->config_revalidated_token); + cl_assert(!clean_status_should_write_fsmonitor_config(&istate)); + cl_assert_equal_i(state->manifest.current_flags, + FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE); + + state->manifest.current_flags |= FSMONITOR_CLEAN_PROOF_FULL_INDEX; + clean_status_mark_fsmonitor_config_valid(&istate, closed_token); + cl_assert(state->config_revalidated); + cl_assert_equal_s(state->config_revalidated_token, closed_token); + cl_assert(!clean_status_revalidated_token_matches(&istate)); + cl_assert(!clean_status_should_write_fsmonitor_config(&istate)); + + FREE_AND_NULL(istate.fsmonitor_last_update); + istate.fsmonitor_last_update = xstrdup(closed_token); + cl_assert(clean_status_revalidated_token_matches(&istate)); + cl_assert(clean_status_should_write_fsmonitor_config(&istate)); + cl_assert_equal_i(state->manifest.current_flags, + FSMONITOR_CLEAN_PROOF_ALL); + + clean_status_release(&istate); + FREE_AND_NULL(istate.fsmonitor_last_update); +} + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +static char *create_worktree(void) +{ + const char *tmp = getenv("TMPDIR"); + char *path = xstrfmt("%s/status-manifest.XXXXXX", + tmp ? tmp : "/tmp"); + + cl_assert(mkdtemp(path) != NULL); + return path; +} + +static void add_index_path(struct index_state *istate, size_t pos, + const char *path) +{ + size_t len = strlen(path); + struct cache_entry *ce = make_empty_cache_entry(istate, len); + + ce->ce_mode = S_IFREG | 0644; + ce->ce_namelen = len; + memcpy(ce->name, path, len + 1); + istate->cache[pos] = ce; +} +#endif + +void test_clean_status_manifest__invalidates_only_changed_scopes(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *worktree = create_worktree(); + struct repository repo = { .worktree = worktree, .hash_algo = algo }; + struct index_state istate = INDEX_STATE_INIT(&repo); + struct clean_status_manifest_state state; + struct strbuf path = STRBUF_INIT, old = STRBUF_INIT, cleanup = STRBUF_INIT; + + strbuf_addf(&path, "%s/a", worktree); + cl_assert_equal_i(mkdir(path.buf, 0777), 0); + strbuf_reset(&path); + strbuf_addf(&path, "%s/b", worktree); + cl_assert_equal_i(mkdir(path.buf, 0777), 0); + strbuf_reset(&path); + strbuf_addf(&path, "%s/a/.gitattributes", worktree); + write_file(path.buf, "*.txt text\n"); + + CALLOC_ARRAY(istate.cache, 2); + istate.cache_alloc = istate.cache_nr = 2; + add_index_path(&istate, 0, "a/file"); + add_index_path(&istate, 1, "b/file"); + clean_status_manifest_init(&state); + cl_assert_equal_i(clean_status_manifest_refresh(&istate, &state), 0); + strbuf_addbuf(&old, &state.current); + cl_assert_equal_i(clean_status_manifest_load( + &state, old.buf, old.len, FSMONITOR_CLEAN_PROOF_ALL, algo), 0); + + write_file(path.buf, "*.txt -text\n"); + for (size_t i = 0; i < istate.cache_nr; i++) { + istate.cache[i]->ce_flags = CE_FSMONITOR_VALID | CE_UPTODATE; + memset(&istate.cache[i]->ce_stat_data, 1, + sizeof(istate.cache[i]->ce_stat_data)); + } + cl_assert_equal_i(clean_status_manifest_refresh(&istate, &state), 1); + cl_assert(state.changed); + cl_assert(!state.current_invalidated); + cl_assert(!(istate.cache[0]->ce_flags & CE_FSMONITOR_VALID)); + cl_assert(istate.cache[0]->ce_flags & CE_CONTENT_CHECK_REQUIRED); + cl_assert(istate.cache[1]->ce_flags & CE_FSMONITOR_VALID); + + /* + * Preserve the last complete in-process value when a rebuild fails, + * then return to the on-disk value. The final comparison must use + * the preserved value, not the matching on-disk history. + */ + strbuf_reset(&old); + strbuf_addbuf(&old, &state.current); + clean_status_manifest_invalidate(&state); + cl_assert(state.current_invalidated); + istate.cache[0]->ce_flags = create_ce_flags(1); + cl_assert_equal_i(clean_status_manifest_refresh(&istate, &state), -1); + cl_assert(!state.current_valid); + cl_assert(state.global_fallback); + cl_assert_equal_i(strbuf_cmp(&state.current, &old), 0); + + write_file(path.buf, "*.txt text\n"); + for (size_t i = 0; i < istate.cache_nr; i++) { + istate.cache[i]->ce_flags = CE_FSMONITOR_VALID | CE_UPTODATE; + memset(&istate.cache[i]->ce_stat_data, 1, + sizeof(istate.cache[i]->ce_stat_data)); + } + cl_assert_equal_i(clean_status_manifest_refresh(&istate, &state), 1); + cl_assert(state.changed); + cl_assert(!state.current_invalidated); + cl_assert(!(istate.cache[0]->ce_flags & CE_FSMONITOR_VALID)); + cl_assert(istate.cache[0]->ce_flags & CE_CONTENT_CHECK_REQUIRED); + cl_assert(istate.cache[1]->ce_flags & CE_FSMONITOR_VALID); + + clean_status_manifest_release(&state); + strbuf_release(&old); + strbuf_release(&path); + release_index(&istate); + strbuf_addstr(&cleanup, worktree); + cl_assert_equal_i(remove_dir_recursively(&cleanup, 0), 0); + strbuf_release(&cleanup); + free(worktree); +#endif +} diff --git a/t/unit-tests/u-clean-status-sidecar.c b/t/unit-tests/u-clean-status-sidecar.c new file mode 100644 index 00000000000000..56ea5581b2a059 --- /dev/null +++ b/t/unit-tests/u-clean-status-sidecar.c @@ -0,0 +1,287 @@ +#include "unit-test.h" +#include "clean-status-sidecar.h" +#include "fsmonitor-clean-proof.h" +#include "hash-framing.h" +#include "strbuf.h" + +struct sidecar_fixture { + struct clean_status_sidecar sidecar; + struct strbuf encoded; +}; + +static void fill_oid(struct object_id *oid, unsigned char value, + const struct git_hash_algo *algo) +{ + unsigned char hash[GIT_MAX_RAWSZ]; + + memset(hash, value, algo->rawsz); + oidread(oid, hash, algo); +} + +static void fixture_init(struct sidecar_fixture *fixture, + const struct git_hash_algo *algo) +{ + static const unsigned char token[] = "builtin:1:2"; + struct clean_status_proof *proof; + + memset(fixture, 0, sizeof(*fixture)); + fixture->encoded = (struct strbuf)STRBUF_INIT; + fixture->sidecar.identity.stat.fields[0] = 1; + fixture->sidecar.identity.stat.fields[1] = 2; + proof = &fixture->sidecar.proof; + proof->index_version = 4; + proof->cache_nr = 5; + fill_oid(&proof->index_checksum, 2, algo); + fill_oid(&proof->head_tree, 3, algo); + memset(proof->config_hash, 4, algo->rawsz); + memset(proof->repo_hash, 5, algo->rawsz); + fill_oid(&proof->exclude_source_digest, 6, algo); + fixture->sidecar.token = token; + fixture->sidecar.token_len = sizeof(token) - 1; +} + +static void fixture_encode(struct sidecar_fixture *fixture, + const struct git_hash_algo *algo) +{ + cl_assert_equal_i(clean_status_sidecar_write( + &fixture->encoded, &fixture->sidecar, algo), 0); +} + +static void fixture_release(struct sidecar_fixture *fixture) +{ + strbuf_release(&fixture->encoded); +} + +static void replace_checksum(struct strbuf *encoded, + const struct git_hash_algo *algo) +{ + strbuf_setlen(encoded, encoded->len - algo->rawsz); + hash_append_checksum(encoded, algo); +} + +static size_t flags_offset(void) +{ + return 4 + sizeof(uint32_t); +} + +static size_t proof_offset(void) +{ + return 4 + 2 * sizeof(uint32_t) + CLEAN_STATUS_IDENTITY_SIZE; +} + +static size_t index_checksum_offset(void) +{ + return proof_offset() + 2 * sizeof(uint32_t); +} + +static size_t head_tree_offset(const struct git_hash_algo *algo) +{ + return index_checksum_offset() + algo->rawsz; +} + +static size_t exclude_digest_offset(const struct git_hash_algo *algo) +{ + return index_checksum_offset() + 4 * algo->rawsz; +} + +static size_t token_length_offset(const struct git_hash_algo *algo) +{ + return index_checksum_offset() + 5 * algo->rawsz; +} + +static size_t token_offset(const struct git_hash_algo *algo) +{ + return token_length_offset(algo) + sizeof(uint32_t); +} + +static void assert_parse_fails(struct sidecar_fixture *fixture, + const struct git_hash_algo *algo) +{ + struct clean_status_sidecar parsed; + + replace_checksum(&fixture->encoded, algo); + cl_assert_equal_i(clean_status_sidecar_parse( + &parsed, fixture->encoded.buf, fixture->encoded.len, algo), -1); +} + +static void assert_round_trip(const struct git_hash_algo *algo) +{ + struct sidecar_fixture fixture; + struct clean_status_sidecar parsed; + + fixture_init(&fixture, algo); + fixture_encode(&fixture, algo); + cl_assert_equal_i(clean_status_sidecar_parse( + &parsed, fixture.encoded.buf, fixture.encoded.len, algo), 0); + cl_assert(clean_status_identity_equal(&parsed.identity, + &fixture.sidecar.identity)); + cl_assert_equal_i(parsed.proof.index_version, + fixture.sidecar.proof.index_version); + cl_assert_equal_i(parsed.proof.cache_nr, + fixture.sidecar.proof.cache_nr); + cl_assert(oideq(&parsed.proof.index_checksum, + &fixture.sidecar.proof.index_checksum)); + cl_assert(oideq(&parsed.proof.head_tree, + &fixture.sidecar.proof.head_tree)); + cl_assert(!memcmp(parsed.proof.config_hash, + fixture.sidecar.proof.config_hash, algo->rawsz)); + cl_assert(!memcmp(parsed.proof.repo_hash, + fixture.sidecar.proof.repo_hash, algo->rawsz)); + cl_assert(oideq(&parsed.proof.exclude_source_digest, + &fixture.sidecar.proof.exclude_source_digest)); + cl_assert_equal_i(parsed.token_len, fixture.sidecar.token_len); + cl_assert(!memcmp(parsed.token, fixture.sidecar.token, + parsed.token_len)); + fixture_release(&fixture); +} + +void test_clean_status_sidecar__round_trips_both_object_formats(void) +{ + assert_round_trip(&hash_algos[GIT_HASH_SHA1]); + assert_round_trip(&hash_algos[GIT_HASH_SHA256]); +} + +void test_clean_status_sidecar__rejects_bad_envelopes(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct sidecar_fixture fixture; + struct clean_status_sidecar parsed; + + fixture_init(&fixture, algo); + fixture_encode(&fixture, algo); + + cl_assert_equal_i(clean_status_sidecar_parse( + &parsed, fixture.encoded.buf, fixture.encoded.len - 1, algo), -1); + + fixture.encoded.buf[0] ^= 1; + assert_parse_fails(&fixture, algo); + fixture.encoded.buf[0] ^= 1; + + put_be32(fixture.encoded.buf + 4, CLEAN_STATUS_SIDECAR_VERSION + 1); + assert_parse_fails(&fixture, algo); + put_be32(fixture.encoded.buf + 4, CLEAN_STATUS_SIDECAR_VERSION); + + put_be32(fixture.encoded.buf + flags_offset(), 1); + assert_parse_fails(&fixture, algo); + put_be32(fixture.encoded.buf + flags_offset(), 0); + + fixture.encoded.buf[fixture.encoded.len - 1] ^= 1; + cl_assert_equal_i(clean_status_sidecar_parse( + &parsed, fixture.encoded.buf, fixture.encoded.len, algo), -1); + fixture_release(&fixture); +} + +void test_clean_status_sidecar__rejects_invalid_proofs(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct sidecar_fixture fixture; + + fixture_init(&fixture, algo); + fixture_encode(&fixture, algo); + + put_be32(fixture.encoded.buf + proof_offset(), 1); + assert_parse_fails(&fixture, algo); + put_be32(fixture.encoded.buf + proof_offset(), 4); + + memset(fixture.encoded.buf + index_checksum_offset(), 0, algo->rawsz); + assert_parse_fails(&fixture, algo); + memset(fixture.encoded.buf + index_checksum_offset(), 2, algo->rawsz); + + memset(fixture.encoded.buf + head_tree_offset(algo), 0, algo->rawsz); + assert_parse_fails(&fixture, algo); + memset(fixture.encoded.buf + head_tree_offset(algo), 3, algo->rawsz); + + memset(fixture.encoded.buf + exclude_digest_offset(algo), 0, + algo->rawsz); + assert_parse_fails(&fixture, algo); + fixture_release(&fixture); +} + +void test_clean_status_sidecar__rejects_invalid_tokens(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct sidecar_fixture fixture; + size_t token_len_offset = token_length_offset(algo); + size_t token_start = token_offset(algo); + + fixture_init(&fixture, algo); + fixture_encode(&fixture, algo); + + put_be32(fixture.encoded.buf + token_len_offset, 0); + assert_parse_fails(&fixture, algo); + put_be32(fixture.encoded.buf + token_len_offset, + fixture.sidecar.token_len); + + fixture.encoded.buf[token_start] = 'x'; + assert_parse_fails(&fixture, algo); + fixture.encoded.buf[token_start] = 'b'; + + fixture.encoded.buf[token_start + fixture.sidecar.token_len - 1] = '\0'; + assert_parse_fails(&fixture, algo); + fixture.encoded.buf[token_start + fixture.sidecar.token_len - 1] = '2'; + + put_be32(fixture.encoded.buf + token_len_offset, + FSMONITOR_CLEAN_PROOF_TOKEN_MAX + 1); + assert_parse_fails(&fixture, algo); + fixture_release(&fixture); +} + +void test_clean_status_sidecar__accepts_the_maximum_token(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct sidecar_fixture fixture; + struct clean_status_sidecar parsed; + unsigned char *token; + + fixture_init(&fixture, algo); + token = xmalloc(FSMONITOR_CLEAN_PROOF_TOKEN_MAX); + memset(token, 'x', FSMONITOR_CLEAN_PROOF_TOKEN_MAX); + memcpy(token, "builtin:", strlen("builtin:")); + fixture.sidecar.token = token; + fixture.sidecar.token_len = FSMONITOR_CLEAN_PROOF_TOKEN_MAX; + fixture_encode(&fixture, algo); + cl_assert_equal_i(clean_status_sidecar_parse( + &parsed, fixture.encoded.buf, fixture.encoded.len, algo), 0); + cl_assert_equal_i(parsed.token_len, FSMONITOR_CLEAN_PROOF_TOKEN_MAX); + free(token); + fixture_release(&fixture); +} + +void test_clean_status_sidecar__rejects_trailing_payload(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct sidecar_fixture fixture; + + fixture_init(&fixture, algo); + fixture_encode(&fixture, algo); + strbuf_setlen(&fixture.encoded, fixture.encoded.len - algo->rawsz); + strbuf_addch(&fixture.encoded, 'x'); + hash_append_checksum(&fixture.encoded, algo); + cl_assert_equal_i(clean_status_sidecar_parse( + &fixture.sidecar, fixture.encoded.buf, fixture.encoded.len, algo), + -1); + fixture_release(&fixture); +} + +void test_clean_status_sidecar__rejects_invalid_writes(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct sidecar_fixture fixture; + unsigned char *token; + + fixture_init(&fixture, algo); + oidclr(&fixture.sidecar.proof.index_checksum, algo); + cl_assert_equal_i(clean_status_sidecar_write( + &fixture.encoded, &fixture.sidecar, algo), -1); + fill_oid(&fixture.sidecar.proof.index_checksum, 2, algo); + + token = xmalloc(FSMONITOR_CLEAN_PROOF_TOKEN_MAX + 1); + memset(token, 'x', FSMONITOR_CLEAN_PROOF_TOKEN_MAX + 1); + memcpy(token, "builtin:", strlen("builtin:")); + fixture.sidecar.token = token; + fixture.sidecar.token_len = FSMONITOR_CLEAN_PROOF_TOKEN_MAX + 1; + cl_assert_equal_i(clean_status_sidecar_write( + &fixture.encoded, &fixture.sidecar, algo), -1); + free(token); + fixture_release(&fixture); +} diff --git a/t/unit-tests/u-clean-status-store.c b/t/unit-tests/u-clean-status-store.c new file mode 100644 index 00000000000000..4c2e5739aaceea --- /dev/null +++ b/t/unit-tests/u-clean-status-store.c @@ -0,0 +1,277 @@ +#include "unit-test.h" + +#ifdef __APPLE__ +#include +#endif + +#include "clean-status-index.h" +#include "clean-status-sidecar.h" +#include "dir.h" +#include "strbuf.h" + +struct store_fixture { + char *directory; + struct strbuf index_path; + struct clean_status_sidecar sidecar; +}; + +static void fill_oid(struct object_id *oid, unsigned char value, + const struct git_hash_algo *algo) +{ + unsigned char hash[GIT_MAX_RAWSZ]; + + memset(hash, value, algo->rawsz); + oidread(oid, hash, algo); +} + +static void fixture_init(struct store_fixture *fixture, + const struct git_hash_algo *algo) +{ + static const unsigned char token[] = "builtin:1:2"; + struct strbuf index = STRBUF_INIT; + struct clean_status_proof *proof; + struct stat st; + const char *tmp = getenv("TMPDIR"); + uint32_t value; + + memset(fixture, 0, sizeof(*fixture)); + fixture->index_path = (struct strbuf)STRBUF_INIT; + fixture->directory = xstrfmt("%s/status-store.XXXXXX", + tmp ? tmp : "/tmp"); + cl_assert(mkdtemp(fixture->directory) != NULL); + strbuf_addf(&fixture->index_path, "%s/index", fixture->directory); + strbuf_addstr(&index, "DIRC"); + put_be32(&value, 4); + strbuf_add(&index, &value, sizeof(value)); + put_be32(&value, 5); + strbuf_add(&index, &value, sizeof(value)); + strbuf_addchars(&index, 2, algo->rawsz); + write_file_buf(fixture->index_path.buf, index.buf, index.len); + cl_assert_equal_i(stat(fixture->index_path.buf, &st), 0); + cl_assert_equal_i(clean_status_identity_from_stat( + &fixture->sidecar.identity, &st), 0); + proof = &fixture->sidecar.proof; + proof->index_version = 4; + proof->cache_nr = 5; + fill_oid(&proof->index_checksum, 2, algo); + fill_oid(&proof->head_tree, 3, algo); + memset(proof->config_hash, 4, algo->rawsz); + memset(proof->repo_hash, 5, algo->rawsz); + fill_oid(&proof->exclude_source_digest, 6, algo); + fixture->sidecar.token = token; + fixture->sidecar.token_len = sizeof(token) - 1; + strbuf_release(&index); +} + +static void fixture_release(struct store_fixture *fixture) +{ + struct strbuf cleanup = STRBUF_INIT; + + strbuf_addstr(&cleanup, fixture->directory); + cl_assert_equal_i(remove_dir_recursively(&cleanup, 0), 0); + strbuf_release(&cleanup); + strbuf_release(&fixture->index_path); + free(fixture->directory); +} + +static struct strbuf sidecar_path(struct store_fixture *fixture) +{ + struct strbuf path = STRBUF_INIT; + + strbuf_addf(&path, "%s.csts", fixture->index_path.buf); + return path; +} + +#ifdef O_NONBLOCK +static void write_fixture_sidecar(struct store_fixture *fixture, + const struct git_hash_algo *algo) +{ + struct strbuf encoded = STRBUF_INIT; + struct strbuf path = sidecar_path(fixture); + + cl_assert_equal_i(clean_status_sidecar_write( + &encoded, &fixture->sidecar, algo), 0); + write_file_buf(path.buf, encoded.buf, encoded.len); + strbuf_release(&path); + strbuf_release(&encoded); +} + +static void assert_loads_sidecar(const struct git_hash_algo *algo) +{ + struct clean_status_sidecar_record record = + CLEAN_STATUS_SIDECAR_RECORD_INIT; + struct store_fixture fixture; + + fixture_init(&fixture, algo); + write_fixture_sidecar(&fixture, algo); + cl_assert_equal_i(clean_status_sidecar_load( + fixture.index_path.buf, algo, &record), 0); + cl_assert(clean_status_identity_equal( + &record.sidecar.identity, &fixture.sidecar.identity)); + cl_assert_equal_i(record.sidecar.proof.index_version, + fixture.sidecar.proof.index_version); + cl_assert_equal_i(record.sidecar.proof.cache_nr, + fixture.sidecar.proof.cache_nr); + cl_assert(oideq(&record.sidecar.proof.index_checksum, + &fixture.sidecar.proof.index_checksum)); + cl_assert(oideq(&record.sidecar.proof.head_tree, + &fixture.sidecar.proof.head_tree)); + cl_assert(!memcmp(record.sidecar.proof.config_hash, + fixture.sidecar.proof.config_hash, algo->rawsz)); + cl_assert(!memcmp(record.sidecar.proof.repo_hash, + fixture.sidecar.proof.repo_hash, algo->rawsz)); + cl_assert(oideq(&record.sidecar.proof.exclude_source_digest, + &fixture.sidecar.proof.exclude_source_digest)); + cl_assert_equal_i(record.sidecar.token_len, fixture.sidecar.token_len); + cl_assert(!memcmp(record.sidecar.token, fixture.sidecar.token, + record.sidecar.token_len)); + cl_assert(record.sidecar.token >= + (const unsigned char *)record.storage.buf); + cl_assert(record.sidecar.token + record.sidecar.token_len <= + (const unsigned char *)record.storage.buf + + record.storage.len); + + clean_status_sidecar_record_release(&record); + fixture_release(&fixture); +} +#endif + +void test_clean_status_store__loads_owned_sidecars_in_both_object_formats(void) +{ +#ifdef O_NONBLOCK + assert_loads_sidecar(&hash_algos[GIT_HASH_SHA1]); + assert_loads_sidecar(&hash_algos[GIT_HASH_SHA256]); +#else + cl_skip(); +#endif +} + +void test_clean_status_store__rejects_nonregular_sidecars(void) +{ +#if defined(O_NONBLOCK) && !defined(GIT_WINDOWS_NATIVE) + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct clean_status_sidecar_record record = + CLEAN_STATUS_SIDECAR_RECORD_INIT; + struct store_fixture fixture; + struct strbuf path, target = STRBUF_INIT; + + fixture_init(&fixture, algo); + path = sidecar_path(&fixture); + strbuf_addf(&target, "%s/target", fixture.directory); + write_file(target.buf, "target"); + cl_assert_equal_i(symlink(target.buf, path.buf), 0); + cl_assert_equal_i(clean_status_sidecar_load( + fixture.index_path.buf, algo, &record), -1); + cl_assert_equal_i(unlink(path.buf), 0); + cl_assert_equal_i(mkfifo(path.buf, 0600), 0); + cl_assert_equal_i(clean_status_sidecar_load( + fixture.index_path.buf, algo, &record), -1); + + clean_status_sidecar_record_release(&record); + strbuf_release(&target); + strbuf_release(&path); + fixture_release(&fixture); +#else + cl_skip(); +#endif +} + +void test_clean_status_store__rejects_oversized_sidecars(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct clean_status_sidecar_record record = + CLEAN_STATUS_SIDECAR_RECORD_INIT; + struct store_fixture fixture; + struct strbuf oversized = STRBUF_INIT; + struct strbuf path; + + fixture_init(&fixture, algo); + path = sidecar_path(&fixture); + strbuf_addchars(&oversized, 'x', 8193); + write_file_buf(path.buf, oversized.buf, oversized.len); + cl_assert_equal_i(clean_status_sidecar_load( + fixture.index_path.buf, algo, &record), -1); + + clean_status_sidecar_record_release(&record); + strbuf_release(&oversized); + strbuf_release(&path); + fixture_release(&fixture); +} + +static void require_local_apfs(const char *path MAYBE_UNUSED) +{ +#ifdef __APPLE__ + struct statfs fs; + int fd = git_open_cloexec(path, O_RDONLY); + + if (fd < 0 || fstatfs(fd, &fs) || !(fs.f_flags & MNT_LOCAL) || + strcmp(fs.f_fstypename, "apfs")) { + if (fd >= 0) + close(fd); + cl_skip(); + } + close(fd); +#else + cl_skip(); +#endif +} + +static void assert_installs_against_source(const struct git_hash_algo *algo) +{ + struct clean_status_sidecar parsed; + struct clean_status_index_snapshot snapshot; + struct store_fixture fixture; + struct strbuf encoded = STRBUF_INIT; + struct strbuf path; + + fixture_init(&fixture, algo); + path = sidecar_path(&fixture); + cl_assert_equal_i(clean_status_sidecar_pin_source( + fixture.index_path.buf, &fixture.sidecar, algo, &snapshot), 0); + cl_assert_equal_i(clean_status_sidecar_install( + fixture.index_path.buf, &fixture.sidecar, &snapshot, algo), 0); + cl_assert(strbuf_read_file(&encoded, path.buf, 0) > 0); + cl_assert_equal_i(clean_status_sidecar_parse( + &parsed, encoded.buf, encoded.len, algo), 0); + cl_assert(clean_status_identity_equal( + &parsed.identity, &fixture.sidecar.identity)); + + clean_status_index_snapshot_release(&snapshot); + strbuf_release(&path); + strbuf_release(&encoded); + fixture_release(&fixture); +} + +void test_clean_status_store__installs_both_object_formats(void) +{ + require_local_apfs(getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); + assert_installs_against_source(&hash_algos[GIT_HASH_SHA1]); + assert_installs_against_source(&hash_algos[GIT_HASH_SHA256]); +} + +static void assert_rejects_replaced_source(const struct git_hash_algo *algo) +{ + struct clean_status_index_snapshot snapshot; + struct store_fixture fixture; + struct strbuf replacement = STRBUF_INIT; + + fixture_init(&fixture, algo); + cl_assert_equal_i(clean_status_sidecar_pin_source( + fixture.index_path.buf, &fixture.sidecar, algo, &snapshot), 0); + strbuf_addf(&replacement, "%s/replacement", fixture.directory); + write_file(replacement.buf, "replacement"); + cl_assert_equal_i(rename(replacement.buf, fixture.index_path.buf), 0); + cl_assert_equal_i(clean_status_sidecar_install( + fixture.index_path.buf, &fixture.sidecar, &snapshot, algo), -1); + + clean_status_index_snapshot_release(&snapshot); + strbuf_release(&replacement); + fixture_release(&fixture); +} + +void test_clean_status_store__rejects_a_replaced_source_index(void) +{ + require_local_apfs(getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); + assert_rejects_replaced_source(&hash_algos[GIT_HASH_SHA1]); + assert_rejects_replaced_source(&hash_algos[GIT_HASH_SHA256]); +} diff --git a/t/unit-tests/u-exclude-source-proof.c b/t/unit-tests/u-exclude-source-proof.c new file mode 100644 index 00000000000000..be1f2d046599b1 --- /dev/null +++ b/t/unit-tests/u-exclude-source-proof.c @@ -0,0 +1,456 @@ +#include "unit-test.h" + +#include "dir.h" +#include "exclude-source-proof.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "wrapper.h" + +#if EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN + +static struct repository repo = { + .hash_algo = &hash_algos[GIT_HASH_SHA1], +}; +static struct index_state istate = { + .repo = &repo, +}; +static char *trash; +static int fail_open_parent; + +static int open_parent(void *data UNUSED, const char *path) +{ + if (fail_open_parent) { + errno = EACCES; + return -1; + } + return open(path, O_RDONLY | O_DIRECTORY | O_CLOEXEC); +} + +static struct exclude_source_proof *new_proof(void) +{ + return exclude_source_proof_create( + &istate, NULL, open_parent, 0); +} + +static char *make_path(const char *name) +{ + struct strbuf path = STRBUF_INIT; + + strbuf_addf(&path, "%s/%s", trash, name); + return strbuf_detach(&path, NULL); +} + +static void record_file(struct exclude_source_proof *proof, const char *path) +{ + struct exclude_source_capture *capture = + exclude_source_capture_begin(proof, path, 0); + struct stat before, after; + char *buf; + size_t size; + ssize_t read_size; + int fd; + + cl_assert(capture != NULL); + fd = exclude_source_capture_open(capture); + cl_must_pass(fd); + cl_must_pass(fstat(fd, &before)); + cl_assert(before.st_size >= 0); + size = xsize_t(before.st_size); + buf = xmalloc(size ? size : 1); + read_size = read_in_full(fd, buf, size); + cl_assert(read_size >= 0 && (size_t)read_size == size); + cl_must_pass(fstat(fd, &after)); + exclude_source_capture_record(capture, fd, &after, buf, size); + exclude_source_capture_release(capture); + free(buf); + cl_must_pass(close(fd)); +} + +static void record_absence(struct exclude_source_proof *proof, + const char *path) +{ + struct exclude_source_capture *capture = + exclude_source_capture_begin(proof, path, 0); + + cl_assert(capture != NULL); + cl_assert(exclude_source_capture_absent(capture)); + exclude_source_capture_record(capture, -1, NULL, NULL, 0); + exclude_source_capture_release(capture); +} + +void test_exclude_source_proof__initialize(void) +{ + char template[] = "/tmp/exclude-source-proof-XXXXXX"; + + fail_open_parent = 0; + cl_assert(mkdtemp(template) != NULL); + trash = xstrdup(template); +} + +void test_exclude_source_proof__cleanup(void) +{ + struct strbuf path = STRBUF_INIT; + + strbuf_addstr(&path, trash); + cl_must_pass(remove_dir_recursively( + &path, REMOVE_DIR_PURGE_ORIGINAL_CWD)); + strbuf_release(&path); + FREE_AND_NULL(trash); +} + +void test_exclude_source_proof__accepts_same_content_replacement(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(proof, source); + cl_assert(exclude_source_proof_validate(proof)); + cl_must_pass(unlink(source)); + write_file_buf(source, "content", 7); + cl_assert(exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__rejects_different_content_replacement(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(proof, source); + cl_assert(exclude_source_proof_validate(proof)); + cl_must_pass(unlink(source)); + cl_assert(!exclude_source_proof_validate(proof)); + write_file_buf(source, "changed", 7); + cl_assert(!exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__accepts_repeated_observation(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(proof, source); + record_file(proof, source); + cl_assert(exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__rejects_missing_source_buffer(void) +{ + struct exclude_source_proof *proof = new_proof(); + struct exclude_source_capture *capture; + struct stat st; + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + int fd; + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + capture = exclude_source_capture_begin(proof, source, 0); + cl_assert(capture != NULL); + fd = exclude_source_capture_open(capture); + cl_must_pass(fd); + cl_must_pass(fstat(fd, &st)); + exclude_source_capture_record(capture, fd, &st, NULL, 7); + cl_assert(!exclude_source_proof_validate(proof)); + + cl_must_pass(close(fd)); + exclude_source_capture_release(capture); + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__rejects_conflicting_observations(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(proof, source); + write_file_buf(source, "changed", 7); + record_file(proof, source); + cl_assert(!exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__digest_deduplicates_and_ignores_identity(void) +{ + struct exclude_source_proof *first_proof = + exclude_source_proof_create(&istate, NULL, open_parent, 0); + struct exclude_source_proof *second_proof; + struct object_id first, second; + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(first_proof, source); + cl_must_pass(exclude_source_proof_digest( + first_proof, repo.hash_algo, &first)); + + cl_must_pass(unlink(source)); + write_file_buf(source, "content", 7); + second_proof = + exclude_source_proof_create(&istate, NULL, open_parent, 0); + record_file(second_proof, source); + record_file(second_proof, source); + cl_must_pass(exclude_source_proof_digest( + second_proof, repo.hash_algo, &second)); + cl_assert(oideq(&first, &second)); + + exclude_source_proof_release(second_proof); + exclude_source_proof_release(first_proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__rejects_open_failure(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(proof, source); + fail_open_parent = 1; + cl_assert(!exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__fails_closed_without_parent_opener(void) +{ + struct exclude_source_proof *proof = + exclude_source_proof_create(&istate, NULL, NULL, 0); + + cl_assert(!exclude_source_capture_begin(proof, "/dev/null", 0)); + cl_assert(!exclude_source_proof_validate(proof)); + exclude_source_proof_release(proof); +} + +void test_exclude_source_proof__honors_nofollow(void) +{ + struct exclude_source_proof *proof = new_proof(); + struct exclude_source_capture *capture; + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + char *target = make_path("parent/target"); + int fd; + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(target, "content", 7); + cl_must_pass(symlink("target", source)); + + capture = exclude_source_capture_begin(proof, source, 0); + cl_assert(capture != NULL); + fd = exclude_source_capture_open(capture); + cl_must_pass(fd); + cl_must_pass(close(fd)); + exclude_source_capture_release(capture); + + capture = exclude_source_capture_begin(proof, source, 1); + cl_assert(capture != NULL); + fd = exclude_source_capture_open(capture); + cl_assert(fd < 0 && errno == ELOOP); + exclude_source_capture_release(capture); + + exclude_source_proof_release(proof); + free(target); + free(source); + free(parent); +} + +void test_exclude_source_proof__opens_directory_sources(void) +{ + struct exclude_source_proof *proof = new_proof(); + struct exclude_source_capture *capture; + struct stat st; + char *parent = make_path("parent"); + char *source = make_path("parent/source/"); + int fd; + + cl_must_pass(mkdir(parent, 0700)); + cl_must_pass(mkdir(source, 0700)); + capture = exclude_source_capture_begin(proof, source, 0); + cl_assert(capture != NULL); + fd = exclude_source_capture_open(capture); + cl_must_pass(fd); + cl_must_pass(fstat(fd, &st)); + cl_assert(S_ISDIR(st.st_mode)); + cl_must_pass(close(fd)); + exclude_source_capture_release(capture); + + capture = exclude_source_capture_begin(proof, "/", 0); + cl_assert(capture != NULL); + fd = exclude_source_capture_open(capture); + cl_must_pass(fd); + cl_must_pass(fstat(fd, &st)); + cl_assert(S_ISDIR(st.st_mode)); + cl_must_pass(close(fd)); + exclude_source_capture_release(capture); + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__accepts_same_content_parent_replacement(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *old_parent = make_path("old-parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(proof, source); + cl_must_pass(rename(parent, old_parent)); + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + cl_assert(exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(old_parent); + free(parent); +} + +void test_exclude_source_proof__reresolves_absent_source_parent(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *missing = make_path("parent/missing"); + char *source = make_path("parent/missing/source"); + + cl_must_pass(mkdir(parent, 0700)); + cl_must_pass(mkdir(missing, 0700)); + record_absence(proof, source); + cl_must_pass(rmdir(missing)); + cl_assert(exclude_source_proof_validate(proof)); + cl_must_pass(mkdir(missing, 0700)); + cl_assert(exclude_source_proof_validate(proof)); + write_file_buf(source, "content", 7); + cl_assert(!exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(missing); + free(parent); +} + +void test_exclude_source_proof__accepts_dev_null(void) +{ + struct exclude_source_proof *proof = new_proof(); + + record_file(proof, "/dev/null"); + cl_assert(exclude_source_proof_validate(proof)); + exclude_source_proof_release(proof); +} + +void test_exclude_source_proof__accepts_empty_fifo_replacement(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "", 0); + record_file(proof, source); + cl_must_pass(unlink(source)); + cl_must_pass(mkfifo(source, 0600)); + cl_assert(exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__rejects_nonempty_fifo_replacement(void) +{ + struct exclude_source_proof *proof = new_proof(); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + write_file_buf(source, "content", 7); + record_file(proof, source); + cl_must_pass(unlink(source)); + cl_must_pass(mkfifo(source, 0600)); + cl_assert(!exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +void test_exclude_source_proof__captures_fifo_without_blocking(void) +{ + struct exclude_source_proof *proof = + exclude_source_proof_create( + &istate, NULL, open_parent, + EXCLUDE_SOURCE_PROOF_NONBLOCKING); + char *parent = make_path("parent"); + char *source = make_path("parent/source"); + + cl_must_pass(mkdir(parent, 0700)); + cl_must_pass(mkfifo(source, 0600)); + record_file(proof, source); + cl_assert(exclude_source_proof_validate(proof)); + + exclude_source_proof_release(proof); + free(source); + free(parent); +} + +#else + +#define EMPTY_TEST(name) void name(void) {} +#define SKIP_TEST(name) void name(void) { cl_skip(); } + +EMPTY_TEST(test_exclude_source_proof__initialize) +EMPTY_TEST(test_exclude_source_proof__cleanup) +SKIP_TEST(test_exclude_source_proof__accepts_same_content_replacement) +SKIP_TEST(test_exclude_source_proof__rejects_different_content_replacement) +SKIP_TEST(test_exclude_source_proof__accepts_repeated_observation) +SKIP_TEST(test_exclude_source_proof__rejects_missing_source_buffer) +SKIP_TEST(test_exclude_source_proof__rejects_conflicting_observations) +SKIP_TEST(test_exclude_source_proof__digest_deduplicates_and_ignores_identity) +SKIP_TEST(test_exclude_source_proof__rejects_open_failure) +SKIP_TEST(test_exclude_source_proof__fails_closed_without_parent_opener) +SKIP_TEST(test_exclude_source_proof__honors_nofollow) +SKIP_TEST(test_exclude_source_proof__opens_directory_sources) +SKIP_TEST(test_exclude_source_proof__accepts_same_content_parent_replacement) +SKIP_TEST(test_exclude_source_proof__reresolves_absent_source_parent) +SKIP_TEST(test_exclude_source_proof__accepts_dev_null) +SKIP_TEST(test_exclude_source_proof__accepts_empty_fifo_replacement) +SKIP_TEST(test_exclude_source_proof__rejects_nonempty_fifo_replacement) +SKIP_TEST(test_exclude_source_proof__captures_fifo_without_blocking) + +#endif diff --git a/t/unit-tests/u-fsmonitor-attributes.c b/t/unit-tests/u-fsmonitor-attributes.c new file mode 100644 index 00000000000000..3eedefca7aad0b --- /dev/null +++ b/t/unit-tests/u-fsmonitor-attributes.c @@ -0,0 +1,99 @@ +#include "unit-test.h" +#include "fsmonitor.h" +#include "fsmonitor-ll.h" +#include "fsmonitor-settings.h" +#include "read-cache-ll.h" +#include "repository.h" + +static void add_entry(struct index_state *istate, size_t pos, + const char *path) +{ + size_t len = strlen(path); + struct cache_entry *ce = make_empty_cache_entry(istate, len); + + ce->ce_mode = S_IFREG | 0644; + ce->ce_namelen = len; + ce->ce_flags = CE_FSMONITOR_VALID | CE_UPTODATE; + memset(&ce->ce_stat_data, 1, sizeof(ce->ce_stat_data)); + memcpy(ce->name, path, len + 1); + istate->cache[pos] = ce; +} + +static int stat_data_is_zero(const struct cache_entry *ce) +{ + struct stat_data zero = { 0 }; + + return !memcmp(&ce->ce_stat_data, &zero, sizeof(zero)); +} + +void test_fsmonitor_attributes__invalidates_only_the_affected_scope(void) +{ + struct repository repo = { .hash_algo = &hash_algos[GIT_HASH_SHA1] }; + struct index_state istate = INDEX_STATE_INIT(&repo); + + CALLOC_ARRAY(istate.cache, 3); + istate.cache_alloc = istate.cache_nr = 3; + add_entry(&istate, 0, "a/file"); + add_entry(&istate, 1, "a/sub/file"); + add_entry(&istate, 2, "b/file"); + + cl_assert(!fsmonitor_invalidate_attributes_path( + &istate, "a/not-attributes")); + cl_assert(istate.cache[0]->ce_flags & CE_FSMONITOR_VALID); + cl_assert(fsmonitor_invalidate_attributes_path( + &istate, "a/.gitattributes")); + for (size_t i = 0; i < 2; i++) { + cl_assert(!(istate.cache[i]->ce_flags & CE_FSMONITOR_VALID)); + cl_assert(!(istate.cache[i]->ce_flags & CE_UPTODATE)); + cl_assert(istate.cache[i]->ce_flags & CE_CONTENT_CHECK_REQUIRED); + cl_assert(stat_data_is_zero(istate.cache[i])); + } + cl_assert(istate.cache[2]->ce_flags & CE_FSMONITOR_VALID); + cl_assert(istate.cache[2]->ce_flags & CE_UPTODATE); + cl_assert(!stat_data_is_zero(istate.cache[2])); + cl_assert(istate.cache_changed & FSMONITOR_CHANGED); + release_index(&istate); +} + +void test_fsmonitor_attributes__root_source_invalidates_every_entry(void) +{ + struct repository repo = { .hash_algo = &hash_algos[GIT_HASH_SHA1] }; + struct index_state istate = INDEX_STATE_INIT(&repo); + + CALLOC_ARRAY(istate.cache, 2); + istate.cache_alloc = istate.cache_nr = 2; + add_entry(&istate, 0, "a/file"); + add_entry(&istate, 1, "b/file"); + cl_assert(fsmonitor_invalidate_attributes_path( + &istate, ".gitattributes")); + for (size_t i = 0; i < istate.cache_nr; i++) { + cl_assert(!(istate.cache[i]->ce_flags & CE_FSMONITOR_VALID)); + cl_assert(istate.cache[i]->ce_flags & CE_CONTENT_CHECK_REQUIRED); + } + release_index(&istate); +} + +void test_fsmonitor_attributes__disabled_provider_preserves_skipped_stat(void) +{ + struct repository repo = { .hash_algo = &hash_algos[GIT_HASH_SHA1] }; + struct index_state istate = INDEX_STATE_INIT(&repo); + + fsm_settings__set_disabled(&repo); + CALLOC_ARRAY(istate.cache, 2); + istate.cache_alloc = istate.cache_nr = 2; + add_entry(&istate, 0, "skipped"); + add_entry(&istate, 1, "tracked"); + istate.cache[0]->ce_flags |= CE_SKIP_WORKTREE; + + fsmonitor_invalidate_semantics(&istate); + + cl_assert(!(istate.cache[0]->ce_flags & CE_FSMONITOR_VALID)); + cl_assert(!(istate.cache[0]->ce_flags & CE_CONTENT_CHECK_REQUIRED)); + cl_assert(!stat_data_is_zero(istate.cache[0])); + cl_assert(!(istate.cache[1]->ce_flags & CE_FSMONITOR_VALID)); + cl_assert(istate.cache[1]->ce_flags & CE_CONTENT_CHECK_REQUIRED); + cl_assert(stat_data_is_zero(istate.cache[1])); + + release_index(&istate); + FREE_AND_NULL(repo.settings.fsmonitor); +} diff --git a/t/unit-tests/u-fsmonitor-clean-proof.c b/t/unit-tests/u-fsmonitor-clean-proof.c new file mode 100644 index 00000000000000..b4691221c75f49 --- /dev/null +++ b/t/unit-tests/u-fsmonitor-clean-proof.c @@ -0,0 +1,141 @@ +#include "unit-test.h" +#include "attr-manifest.h" +#include "fsmonitor-clean-proof.h" +#include "strbuf.h" + +struct proof_fixture { + struct strbuf manifest; + struct strbuf encoded; + unsigned char config_hash[GIT_MAX_RAWSZ]; + unsigned char semantic_hash[GIT_MAX_RAWSZ]; + unsigned char attr_hash[GIT_MAX_RAWSZ]; + struct fsmonitor_clean_proof proof; +}; + +static void fixture_init(struct proof_fixture *fixture, + const struct git_hash_algo *algo) +{ + struct attr_manifest_writer writer; + unsigned char hash[GIT_MAX_RAWSZ]; + static const unsigned char token[] = "builtin:1:2"; + + memset(fixture, 0, sizeof(*fixture)); + fixture->manifest = (struct strbuf)STRBUF_INIT; + fixture->encoded = (struct strbuf)STRBUF_INIT; + memset(hash, 1, algo->rawsz); + memset(fixture->config_hash, 2, algo->rawsz); + memset(fixture->semantic_hash, 3, algo->rawsz); + memset(fixture->attr_hash, 4, algo->rawsz); + attr_manifest_writer_init(&writer, &fixture->manifest, algo); + cl_assert_equal_i(attr_manifest_writer_add( + &writer, ".gitattributes", ATTR_MANIFEST_INDEX, hash), 0); + fixture->proof.flags = FSMONITOR_CLEAN_PROOF_ALL; + fixture->proof.token = token; + fixture->proof.token_len = sizeof(token) - 1; + fixture->proof.config_hash = fixture->config_hash; + fixture->proof.semantic_hash = fixture->semantic_hash; + fixture->proof.attr_hash = fixture->attr_hash; + fixture->proof.attr_manifest = + (const unsigned char *)fixture->manifest.buf; + fixture->proof.attr_manifest_len = fixture->manifest.len; +} + +static void fixture_release(struct proof_fixture *fixture) +{ + strbuf_release(&fixture->encoded); + strbuf_release(&fixture->manifest); +} + +static void assert_round_trip(const struct git_hash_algo *algo) +{ + struct proof_fixture fixture; + struct fsmonitor_clean_proof parsed; + + fixture_init(&fixture, algo); + cl_assert_equal_i(fsmonitor_clean_proof_write( + &fixture.encoded, &fixture.proof, algo), 0); + cl_assert_equal_i(fsmonitor_clean_proof_parse( + &parsed, fixture.encoded.buf, fixture.encoded.len, algo), 0); + cl_assert_equal_i(parsed.flags, fixture.proof.flags); + cl_assert_equal_i(parsed.token_len, fixture.proof.token_len); + cl_assert(!memcmp(parsed.token, fixture.proof.token, parsed.token_len)); + cl_assert(!memcmp(parsed.config_hash, fixture.config_hash, algo->rawsz)); + cl_assert(!memcmp(parsed.attr_manifest, fixture.manifest.buf, + parsed.attr_manifest_len)); + fixture_release(&fixture); +} + +static void assert_rejected(struct fsmonitor_clean_proof *parsed, + const struct strbuf *encoded, + const struct git_hash_algo *algo) +{ + memset(parsed, 0xff, sizeof(*parsed)); + cl_assert_equal_i(fsmonitor_clean_proof_parse( + parsed, encoded->buf, encoded->len, algo), -1); + cl_assert_equal_i(parsed->flags, 0); + cl_assert_equal_p(parsed->token, NULL); + cl_assert_equal_i(parsed->token_len, 0); + cl_assert_equal_p(parsed->config_hash, NULL); + cl_assert_equal_p(parsed->semantic_hash, NULL); + cl_assert_equal_p(parsed->attr_hash, NULL); + cl_assert_equal_p(parsed->attr_manifest, NULL); + cl_assert_equal_i(parsed->attr_manifest_len, 0); +} + +void test_fsmonitor_clean_proof__round_trips_both_object_formats(void) +{ + assert_round_trip(&hash_algos[GIT_HASH_SHA1]); + assert_round_trip(&hash_algos[GIT_HASH_SHA256]); +} + +void test_fsmonitor_clean_proof__rejects_corrupt_records(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct proof_fixture fixture; + struct fsmonitor_clean_proof parsed; + size_t token_offset = 5 * sizeof(uint32_t); + uint32_t saved; + unsigned char byte; + + fixture_init(&fixture, algo); + cl_assert_equal_i(fsmonitor_clean_proof_write( + &fixture.encoded, &fixture.proof, algo), 0); + fixture.encoded.len--; + assert_rejected(&parsed, &fixture.encoded, algo); + fixture.encoded.len++; + saved = get_be32(fixture.encoded.buf + 2 * sizeof(uint32_t)); + put_be32(fixture.encoded.buf + 2 * sizeof(uint32_t), 1u << 31); + assert_rejected(&parsed, &fixture.encoded, algo); + put_be32(fixture.encoded.buf + 2 * sizeof(uint32_t), saved); + byte = fixture.encoded.buf[token_offset]; + fixture.encoded.buf[token_offset] = '\0'; + assert_rejected(&parsed, &fixture.encoded, algo); + fixture.encoded.buf[token_offset] = byte; + fixture.encoded.buf[fixture.encoded.len - 1] ^= 1; + assert_rejected(&parsed, &fixture.encoded, algo); + fixture_release(&fixture); +} + +void test_fsmonitor_clean_proof__clears_only_epoch_bindings(void) +{ + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA256]; + struct proof_fixture fixture; + struct fsmonitor_clean_proof parsed; + struct strbuf unbound = STRBUF_INIT; + + fixture_init(&fixture, algo); + cl_assert_equal_i(fsmonitor_clean_proof_write( + &fixture.encoded, &fixture.proof, algo), 0); + cl_assert_equal_i(fsmonitor_clean_proof_copy_without_bindings( + &unbound, fixture.encoded.buf, fixture.encoded.len, algo), 0); + cl_assert_equal_i(fsmonitor_clean_proof_parse( + &parsed, unbound.buf, unbound.len, algo), 0); + cl_assert_equal_i(parsed.flags, + FSMONITOR_CLEAN_PROOF_MANIFEST_COMPLETE | + FSMONITOR_CLEAN_PROOF_FULL_INDEX); + cl_assert_equal_i(parsed.token_len, fixture.proof.token_len); + cl_assert(!memcmp(parsed.attr_manifest, fixture.manifest.buf, + fixture.manifest.len)); + strbuf_release(&unbound); + fixture_release(&fixture); +} diff --git a/t/unit-tests/u-fsmonitor-response.c b/t/unit-tests/u-fsmonitor-response.c new file mode 100644 index 00000000000000..dda747aa764097 --- /dev/null +++ b/t/unit-tests/u-fsmonitor-response.c @@ -0,0 +1,86 @@ +#include "unit-test.h" + +#include "fsmonitor.h" + +static void check_response(const void *data, size_t len, + enum fsmonitor_query_outcome expected, + const char *token, const void *paths, + size_t paths_len) +{ + struct fsmonitor_query_result result = FSMONITOR_QUERY_RESULT_INIT; + struct strbuf raw = STRBUF_INIT; + + strbuf_add(&raw, data, len); + cl_assert_equal_i(fsmonitor_parse_builtin_response(&raw, &result), + expected); + cl_assert_equal_i(result.outcome, expected); + cl_assert_equal_s(result.token.buf, token); + cl_assert_equal_i(result.paths.len, paths_len); + cl_assert(!paths_len || !memcmp(result.paths.buf, paths, paths_len)); + + fsmonitor_query_result_release(&result); + strbuf_release(&raw); +} + +static void check_malformed(const void *data, size_t len) +{ + check_response(data, len, FSMONITOR_QUERY_ERROR, "", NULL, 0); +} + +void test_fsmonitor_response__rejects_malformed_framing(void) +{ + static const char missing_nul[] = "builtin:1"; + static const char empty_token[] = "\0"; + static const char non_builtin[] = "other:1\0"; + static const char unterminated_path[] = "builtin:1\0path"; + static const char empty_path[] = "builtin:1\0\0"; + static const char absolute_path[] = "builtin:1\0/absolute\0"; + static const char parent_path[] = "builtin:1\0../outside\0"; + static const char embedded_parent[] = + "builtin:1\0dir/../tracked\0"; + static const char dot_path[] = "builtin:1\0./tracked\0"; + static const char repeated_separator[] = + "builtin:1\0dir//tracked\0"; + static const char drive_path[] = "builtin:1\0C:/absolute\0"; + static const char backslash_path[] = "builtin:1\0\\absolute\0"; + struct strbuf overlong = STRBUF_INIT; + + check_malformed("", 0); + check_malformed(missing_nul, sizeof(missing_nul) - 1); + check_malformed(empty_token, sizeof(empty_token) - 1); + check_malformed(non_builtin, sizeof(non_builtin) - 1); + check_malformed(unterminated_path, sizeof(unterminated_path) - 1); + check_malformed(empty_path, sizeof(empty_path) - 1); + check_malformed(absolute_path, sizeof(absolute_path) - 1); + check_malformed(parent_path, sizeof(parent_path) - 1); + check_malformed(embedded_parent, sizeof(embedded_parent) - 1); + check_malformed(dot_path, sizeof(dot_path) - 1); + check_malformed(repeated_separator, + sizeof(repeated_separator) - 1); + if (has_dos_drive_prefix(drive_path + sizeof("builtin:1"))) + check_malformed(drive_path, sizeof(drive_path) - 1); + if (is_dir_sep('\\')) + check_malformed(backslash_path, sizeof(backslash_path) - 1); + + strbuf_addstr(&overlong, "builtin:"); + strbuf_addchars(&overlong, 'x', 4096); + strbuf_addch(&overlong, '\0'); + check_malformed(overlong.buf, overlong.len); + strbuf_release(&overlong); +} + +void test_fsmonitor_response__accepts_valid_builtin_responses(void) +{ + static const char delta[] = "builtin:2\0a\0dir/file\0dir/\0"; + static const char global[] = "builtin:3\0//\0"; + static const char trivial[] = "builtin:4\0/\0"; + + check_response(delta, sizeof(delta) - 1, FSMONITOR_QUERY_DELTA, + "builtin:2", delta + sizeof("builtin:2"), + sizeof(delta) - 1 - sizeof("builtin:2")); + check_response(global, sizeof(global) - 1, FSMONITOR_QUERY_DELTA, + "builtin:3", global + sizeof("builtin:3"), + sizeof(global) - 1 - sizeof("builtin:3")); + check_response(trivial, sizeof(trivial) - 1, FSMONITOR_QUERY_TRIVIAL, + "builtin:4", NULL, 0); +} diff --git a/t/unit-tests/u-path-namespace.c b/t/unit-tests/u-path-namespace.c new file mode 100644 index 00000000000000..3c80140a8bbf72 --- /dev/null +++ b/t/unit-tests/u-path-namespace.c @@ -0,0 +1,288 @@ +#include "unit-test.h" + +#include "dir.h" +#include "hash.h" +#include "path-namespace.h" +#include "strbuf.h" +#include "tempfile.h" +#include "wrapper.h" + +#define ASSERT_STAT_FIELD_MATTERS(base, changed, field) do { \ + (changed) = (base); \ + (changed).field = !(base).field; \ + cl_assert(!path_namespace_stat_equal(&(base), &(changed))); \ +} while (0) + +void test_path_namespace__stat_identity(void) +{ + struct stat st = { 0 }; + struct path_stat_identity first, second; + size_t i; + + st.st_dev = 1; + st.st_ino = 2; + st.st_mode = S_IFREG | 0644; + st.st_nlink = 3; + st.st_uid = 4; + st.st_gid = 5; + st.st_size = 6; + st.st_mtime = 7; + st.st_ctime = 8; + + path_stat_identity_init(&first, &st); + path_stat_identity_init(&second, &st); + cl_assert(path_stat_identity_equal(&first, &second)); + cl_assert(path_namespace_stat_equal(&st, &st)); + + for (i = 0; i < PATH_STAT_IDENTITY_FIELDS; i++) { + second = first; + second.fields[i]++; + cl_assert(!path_stat_identity_equal(&first, &second)); + } +} + +void test_path_namespace__stat_fields(void) +{ + struct stat st, changed; + + cl_must_pass(stat(".", &st)); + changed = st; + cl_assert(path_namespace_stat_equal(&st, &changed)); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_dev); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_ino); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_mode); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_nlink); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_uid); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_gid); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_size); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_mtime); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_ctime); +#ifndef NO_NSEC +#ifdef USE_ST_TIMESPEC + ASSERT_STAT_FIELD_MATTERS(st, changed, st_mtimespec.tv_nsec); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_ctimespec.tv_nsec); +#else + ASSERT_STAT_FIELD_MATTERS(st, changed, st_mtim.tv_nsec); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_ctim.tv_nsec); +#endif +#endif +#ifdef __APPLE__ + ASSERT_STAT_FIELD_MATTERS(st, changed, st_birthtimespec.tv_sec); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_birthtimespec.tv_nsec); + ASSERT_STAT_FIELD_MATTERS(st, changed, st_gen); +#endif +} + +static int source_fd = -1; + +static int reopen_source(int dirfd UNUSED, const char *path, int flags UNUSED) +{ + if (strcmp(path, "source")) { + errno = ENOENT; + return -1; + } + return dup(source_fd); +} + +void test_path_namespace__reopen_component(void) +{ + struct tempfile *first = mks_tempfile_t("path-namespace-one-XXXXXX"); + struct tempfile *second = mks_tempfile_t("path-namespace-two-XXXXXX"); + struct stat expected; + + cl_assert(first != NULL); + cl_assert(second != NULL); + cl_must_pass(fstat(get_tempfile_fd(first), &expected)); + + source_fd = get_tempfile_fd(first); + if (!fstat_is_reliable()) { + cl_assert(path_namespace_reopen_component( + -1, "source", O_RDONLY, + reopen_source, &expected) < 0); + cl_assert_equal_i(errno, EAGAIN); + goto invalid_component; + } + cl_must_pass(path_namespace_reopen_component( + -1, "source", O_RDONLY, reopen_source, &expected)); + + source_fd = get_tempfile_fd(second); + cl_assert(path_namespace_reopen_component( + -1, "source", O_RDONLY, reopen_source, &expected) < 0); + cl_assert_equal_i(errno, EAGAIN); + +invalid_component: + cl_assert(path_namespace_reopen_component( + -1, "../source", O_RDONLY, reopen_source, &expected) < 0); + cl_assert_equal_i(errno, EINVAL); + + source_fd = -1; + cl_must_pass(delete_tempfile(&first)); + cl_must_pass(delete_tempfile(&second)); +} + +static char *create_namespace(void) +{ + const char *tmp = getenv("TMPDIR"); + char *path = xstrfmt("%s/path-namespace.XXXXXX", tmp ? tmp : "/tmp"); + + cl_assert(mkdtemp(path) != NULL); + return path; +} + +static void remove_namespace(char *path) +{ + struct strbuf root = STRBUF_INIT; + + strbuf_addstr(&root, path); + cl_must_pass(remove_dir_recursively(&root, 0)); + strbuf_release(&root); + free(path); +} + +static void hash_namespace(const struct path_namespace_snapshot *snapshot, + unsigned char *hash) +{ + struct git_hash_ctx ctx; + + git_hash_init(&ctx, &hash_algos[GIT_HASH_SHA1]); + path_namespace_hash(&ctx, snapshot); + git_hash_final(hash, &ctx); + git_hash_discard(&ctx); +} + +void test_path_namespace__equal_snapshots_have_equal_hashes(void) +{ + struct path_namespace_snapshot *first = NULL, *second = NULL; + struct strbuf directory = STRBUF_INIT, target = STRBUF_INIT; + unsigned char first_hash[GIT_MAX_RAWSZ], second_hash[GIT_MAX_RAWSZ]; + char *root; + + if (!fstat_is_reliable()) + cl_skip(); + root = create_namespace(); + + strbuf_addf(&directory, "%s/a", root); + cl_must_pass(mkdir(directory.buf, 0777)); + strbuf_addf(&target, "%s/target", directory.buf); + write_file(target.buf, "contents\n"); + + cl_must_pass(path_namespace_capture(target.buf, &first)); + cl_must_pass(path_namespace_capture(target.buf, &second)); + cl_assert(path_namespace_target_present(first)); + cl_assert(path_namespace_target_present(second)); + cl_assert(path_namespace_equal(first, second)); + hash_namespace(first, first_hash); + hash_namespace(second, second_hash); + cl_assert(!memcmp(first_hash, second_hash, + hash_algos[GIT_HASH_SHA1].rawsz)); + + path_namespace_clear(second); + path_namespace_clear(first); + strbuf_release(&target); + strbuf_release(&directory); + remove_namespace(root); +} + +void test_path_namespace__captures_missing_and_replaced_components(void) +{ + struct path_namespace_snapshot *missing = NULL, *created = NULL; + struct path_namespace_snapshot *replaced = NULL; + struct strbuf directory = STRBUF_INIT, old_directory = STRBUF_INIT; + struct strbuf target = STRBUF_INIT; + unsigned char missing_hash[GIT_MAX_RAWSZ], created_hash[GIT_MAX_RAWSZ]; + unsigned char replaced_hash[GIT_MAX_RAWSZ]; + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + char *root; + + if (!fstat_is_reliable()) { + cl_assert(path_namespace_capture(".", &missing) < 0); + cl_assert_equal_i(errno, EAGAIN); + return; + } + root = create_namespace(); + + strbuf_addf(&directory, "%s/a", root); + strbuf_addf(&old_directory, "%s/a-old", root); + cl_must_pass(mkdir(directory.buf, 0777)); + strbuf_addf(&target, "%s/target", directory.buf); + + cl_must_pass(path_namespace_capture(target.buf, &missing)); + cl_assert(!path_namespace_target_present(missing)); + hash_namespace(missing, missing_hash); + + write_file(target.buf, "contents\n"); + cl_must_pass(path_namespace_capture(target.buf, &created)); + cl_assert(path_namespace_target_present(created)); + cl_assert(!path_namespace_equal(missing, created)); + hash_namespace(created, created_hash); + cl_assert(memcmp(missing_hash, created_hash, algo->rawsz)); + + cl_must_pass(rename(directory.buf, old_directory.buf)); + cl_must_pass(mkdir(directory.buf, 0777)); + write_file(target.buf, "contents\n"); + cl_must_pass(path_namespace_capture(target.buf, &replaced)); + cl_assert(path_namespace_target_present(replaced)); + cl_assert(!path_namespace_equal(created, replaced)); + hash_namespace(replaced, replaced_hash); + cl_assert(memcmp(created_hash, replaced_hash, algo->rawsz)); + + path_namespace_clear(replaced); + path_namespace_clear(created); + path_namespace_clear(missing); + strbuf_release(&target); + strbuf_release(&old_directory); + strbuf_release(&directory); + remove_namespace(root); +} + +void test_path_namespace__unrelated_ancestor_entries_leave_target_unchanged(void) +{ + struct path_namespace_snapshot *first = NULL, *second = NULL; + struct path_namespace_snapshot *modified = NULL, *permissions = NULL; + struct strbuf directory = STRBUF_INIT, target = STRBUF_INIT; + struct strbuf unrelated = STRBUF_INIT; + unsigned char first_hash[GIT_MAX_RAWSZ], second_hash[GIT_MAX_RAWSZ]; + unsigned char modified_hash[GIT_MAX_RAWSZ]; + struct stat st; + char *root; + + if (!fstat_is_reliable()) + cl_skip(); + root = create_namespace(); + + strbuf_addf(&directory, "%s/a", root); + cl_must_pass(mkdir(directory.buf, 0777)); + strbuf_addf(&target, "%s/target", directory.buf); + write_file(target.buf, "contents\n"); + cl_must_pass(path_namespace_capture(target.buf, &first)); + hash_namespace(first, first_hash); + + strbuf_addf(&unrelated, "%s/unrelated", root); + write_file(unrelated.buf, "unrelated\n"); + cl_must_pass(path_namespace_capture(target.buf, &second)); + hash_namespace(second, second_hash); + cl_assert(path_namespace_equal(first, second)); + cl_assert(!memcmp(first_hash, second_hash, + hash_algos[GIT_HASH_SHA1].rawsz)); + + write_file(target.buf, "changed contents\n"); + cl_must_pass(path_namespace_capture(target.buf, &modified)); + hash_namespace(modified, modified_hash); + cl_assert(!path_namespace_equal(second, modified)); + cl_assert(memcmp(second_hash, modified_hash, + hash_algos[GIT_HASH_SHA1].rawsz)); + + cl_must_pass(stat(directory.buf, &st)); + cl_must_pass(chmod(directory.buf, st.st_mode ^ S_IXGRP)); + cl_must_pass(path_namespace_capture(target.buf, &permissions)); + cl_assert(!path_namespace_equal(modified, permissions)); + + path_namespace_clear(permissions); + path_namespace_clear(modified); + path_namespace_clear(second); + path_namespace_clear(first); + strbuf_release(&unrelated); + strbuf_release(&target); + strbuf_release(&directory); + remove_namespace(root); +} diff --git a/t/unit-tests/u-preload-index-bulk-darwin.c b/t/unit-tests/u-preload-index-bulk-darwin.c new file mode 100644 index 00000000000000..f20a7bbd704ac5 --- /dev/null +++ b/t/unit-tests/u-preload-index-bulk-darwin.c @@ -0,0 +1,193 @@ +#include "unit-test.h" + +#ifdef __APPLE__ + +#include +#include + +#include "compat/preload-index/bulk-darwin.h" + +#define REQUIRED_COMMON \ + (ATTR_CMN_RETURNED_ATTRS | ATTR_CMN_ERROR | ATTR_CMN_NAME | \ + ATTR_CMN_DEVID | ATTR_CMN_OBJTYPE | \ + ATTR_CMN_CRTIME | ATTR_CMN_MODTIME | ATTR_CMN_CHGTIME | \ + ATTR_CMN_OWNERID | ATTR_CMN_GRPID | ATTR_CMN_ACCESSMASK | \ + ATTR_CMN_FLAGS | ATTR_CMN_FILEID) +#define REQUIRED_FILE (ATTR_FILE_LINKCOUNT | ATTR_FILE_DATALENGTH) + +struct test_record { + uint32_t record_len; + attribute_set_t returned; + uint32_t error; + attrreference_t name_ref; + dev_t dev; + fsobj_type_t type; + struct timespec birthtime; + struct timespec mtime; + struct timespec ctime; + uid_t uid; + gid_t gid; + uint32_t access; + uint32_t flags; + uint64_t fileid; + uint32_t linkcount; + off_t size; + char name[8]; +} __attribute__((packed)); + +static struct test_record make_record(void) +{ + struct test_record record = { + .record_len = sizeof(record), + .returned = { + .commonattr = REQUIRED_COMMON, + .fileattr = REQUIRED_FILE, + }, + .name_ref = { + .attr_dataoffset = offsetof(struct test_record, name) - + offsetof(struct test_record, name_ref), + .attr_length = 5, + }, + .dev = 1, + .type = VREG, + .uid = 1, + .gid = 1, + .access = 0644, + .fileid = 1, + .linkcount = 1, + .size = 1, + .name = "file", + }; + + return record; +} + +static void check_malformed(struct test_record *record, size_t len) +{ + cl_assert(preload_bulk_darwin_decode_record((char *)record, len) < 0); +} + +#endif /* __APPLE__ */ + +void test_preload_index_bulk_darwin__accepts_valid_record(void) +{ +#ifndef __APPLE__ + cl_skip(); +#else + struct test_record record = make_record(); + + cl_assert_equal_i(0, preload_bulk_darwin_decode_record((char *)&record, + sizeof(record))); +#endif +} + +void test_preload_index_bulk_darwin__accepts_valid_directory_record(void) +{ +#ifndef __APPLE__ + cl_skip(); +#else + struct test_record record = make_record(); + + record.returned.fileattr = 0; + record.returned.dirattr = ATTR_DIR_MOUNTSTATUS; + record.type = VDIR; + cl_assert_equal_i(0, preload_bulk_darwin_decode_record((char *)&record, + sizeof(record))); +#endif +} + +void test_preload_index_bulk_darwin__rejects_entry_error(void) +{ +#ifndef __APPLE__ + cl_skip(); +#else + struct test_record record = make_record(); + + record.error = EIO; + check_malformed(&record, sizeof(record)); +#endif +} + +void test_preload_index_bulk_darwin__rejects_short_record(void) +{ +#ifndef __APPLE__ + cl_skip(); +#else + struct test_record record = make_record(); + + check_malformed(&record, + sizeof(uint32_t) + sizeof(attribute_set_t) - 1); + + record.record_len = sizeof(record) + sizeof(uint64_t); + check_malformed(&record, sizeof(record)); + + record.record_len = sizeof(uint64_t); + check_malformed(&record, sizeof(record)); + + record.record_len = sizeof(record) - 1; + check_malformed(&record, sizeof(record)); + + record.record_len = (offsetof(struct test_record, type) + + sizeof(uint64_t) - 1) & + ~(sizeof(uint64_t) - 1); + check_malformed(&record, sizeof(record)); +#endif +} + +void test_preload_index_bulk_darwin__rejects_wrong_returned_attributes(void) +{ +#ifndef __APPLE__ + cl_skip(); +#else + struct test_record record; + + record = make_record(); + record.returned.commonattr &= ~ATTR_CMN_NAME; + check_malformed(&record, sizeof(record)); + + record = make_record(); + record.returned.volattr = 1; + check_malformed(&record, sizeof(record)); + + record = make_record(); + record.returned.fileattr &= ~ATTR_FILE_DATALENGTH; + check_malformed(&record, sizeof(record)); +#endif +} + +void test_preload_index_bulk_darwin__rejects_invalid_name_reference(void) +{ +#ifndef __APPLE__ + cl_skip(); +#else + struct test_record record; + + record = make_record(); + record.name_ref.attr_dataoffset = -4; + check_malformed(&record, sizeof(record)); + + record = make_record(); + record.name_ref.attr_dataoffset = 2; + check_malformed(&record, sizeof(record)); + + record = make_record(); + record.name_ref.attr_dataoffset = INT32_MAX; + check_malformed(&record, sizeof(record)); + + record = make_record(); + record.name_ref.attr_length = UINT32_MAX; + check_malformed(&record, sizeof(record)); + + record = make_record(); + record.name_ref.attr_dataoffset = 0; + check_malformed(&record, sizeof(record)); + + record = make_record(); + record.name[1] = '\0'; + check_malformed(&record, sizeof(record)); + + record = make_record(); + record.name[1] = '/'; + check_malformed(&record, sizeof(record)); +#endif +} diff --git a/t/unit-tests/u-worktree-attr-source.c b/t/unit-tests/u-worktree-attr-source.c new file mode 100644 index 00000000000000..59bddc563639a1 --- /dev/null +++ b/t/unit-tests/u-worktree-attr-source.c @@ -0,0 +1,193 @@ +#include "unit-test.h" + +#include "dir.h" +#include "hash.h" +#include "repository.h" +#include "semantic-verify-internal.h" +#include "strbuf.h" +#include "worktree-attr-source.h" +#include "wrapper.h" + +#if SEMANTIC_VERIFY_HAS_ANCHORED_OPEN +struct worktree_attr_source_fixture { + char *worktree; + struct repository repo; + struct semantic_verify_root *root; + struct semantic_verify_path *path; +}; + +static void source_fixture_init(struct worktree_attr_source_fixture *fixture) +{ + const char *tmp = getenv("TMPDIR"); + + memset(fixture, 0, sizeof(*fixture)); + fixture->worktree = xstrfmt( + "%s/worktree-attr-source.XXXXXX", tmp ? tmp : "/tmp"); + cl_assert(mkdtemp(fixture->worktree) != NULL); + fixture->repo.worktree = fixture->worktree; + fixture->repo.hash_algo = &hash_algos[GIT_HASH_SHA1]; + cl_must_pass(semantic_verify_root_init( + &fixture->repo, &fixture->root)); + fixture->path = semantic_verify_path_new(fixture->root); + cl_assert(fixture->path != NULL); +} + +static void source_fixture_release( + struct worktree_attr_source_fixture *fixture, + unsigned int *namespace_unstable, + size_t *namespace_unstable_from) +{ + struct strbuf worktree = STRBUF_INIT; + + semantic_verify_path_free( + fixture->path, namespace_unstable, namespace_unstable_from); + semantic_verify_root_clear(fixture->root); + strbuf_addstr(&worktree, fixture->worktree); + cl_must_pass(remove_dir_recursively(&worktree, 0)); + strbuf_release(&worktree); + free(fixture->worktree); +} + +static void make_directory(struct worktree_attr_source_fixture *fixture, + const char *name) +{ + struct strbuf path = STRBUF_INIT; + + strbuf_addf(&path, "%s/%s", fixture->worktree, name); + cl_must_pass(mkdir(path.buf, 0777)); + strbuf_release(&path); +} +#endif + +void test_worktree_attr_source__hashes_large_regular_file(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + struct worktree_attr_source_fixture fixture; + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct git_hash_ctx ctx; + struct strbuf contents = STRBUF_INIT; + struct strbuf source = STRBUF_INIT; + unsigned char actual[GIT_MAX_RAWSZ], expected[GIT_MAX_RAWSZ]; + unsigned int namespace_unstable; + int found; + + source_fixture_init(&fixture); + make_directory(&fixture, "a"); + strbuf_addchars(&contents, 'x', 64 * 1024 + 17); + strbuf_addf(&source, "%s/a/.gitattributes", fixture.worktree); + write_file_buf(source.buf, contents.buf, contents.len); + git_hash_init(&ctx, algo); + git_hash_update(&ctx, contents.buf, contents.len); + git_hash_final(expected, &ctx); + git_hash_discard(&ctx); + + cl_must_pass(worktree_attr_source_read( + fixture.path, "a/.gitattributes", 7, algo, actual, &found)); + cl_assert_equal_i(found, 1); + cl_assert(!memcmp(actual, expected, algo->rawsz)); + + source_fixture_release(&fixture, &namespace_unstable, NULL); + cl_assert_equal_i(namespace_unstable, 0); + strbuf_release(&source); + strbuf_release(&contents); +#endif +} + +void test_worktree_attr_source__reports_missing_and_non_regular_files(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + struct worktree_attr_source_fixture fixture; + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + unsigned char hash[GIT_MAX_RAWSZ]; + unsigned int namespace_unstable; + int found; + + source_fixture_init(&fixture); + cl_must_pass(worktree_attr_source_read( + fixture.path, ".gitattributes", 0, algo, hash, &found)); + cl_assert_equal_i(found, 0); + cl_must_pass(worktree_attr_source_read( + fixture.path, "missing/.gitattributes", 1, + algo, hash, &found)); + cl_assert_equal_i(found, 0); + + make_directory(&fixture, "attributes-directory"); + cl_must_pass(worktree_attr_source_read( + fixture.path, "attributes-directory", 2, algo, hash, &found)); + cl_assert_equal_i(found, 0); + + source_fixture_release(&fixture, &namespace_unstable, NULL); + cl_assert_equal_i(namespace_unstable, 0); +#endif +} + +void test_worktree_attr_source__rejects_hardlinked_file(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + struct worktree_attr_source_fixture fixture; + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct strbuf alias = STRBUF_INIT, source = STRBUF_INIT; + unsigned char hash[GIT_MAX_RAWSZ]; + unsigned int namespace_unstable; + int found; + + source_fixture_init(&fixture); + make_directory(&fixture, "a"); + strbuf_addf(&source, "%s/a/.gitattributes", fixture.worktree); + strbuf_addf(&alias, "%s/attributes-alias", fixture.worktree); + write_file(source.buf, "*.dat text\n"); + cl_must_pass(link(source.buf, alias.buf)); + + cl_assert_equal_i(worktree_attr_source_read( + fixture.path, "a/.gitattributes", 3, algo, hash, &found), -1); + cl_assert_equal_i(found, 0); + + source_fixture_release(&fixture, &namespace_unstable, NULL); + cl_assert_equal_i(namespace_unstable, 0); + strbuf_release(&source); + strbuf_release(&alias); +#endif +} + +void test_worktree_attr_source__detects_replaced_cached_parent(void) +{ +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + cl_skip(); +#else + struct worktree_attr_source_fixture fixture; + const struct git_hash_algo *algo = &hash_algos[GIT_HASH_SHA1]; + struct strbuf old_parent = STRBUF_INIT, parent = STRBUF_INIT; + struct strbuf source = STRBUF_INIT; + unsigned char hash[GIT_MAX_RAWSZ]; + size_t namespace_unstable_from; + unsigned int namespace_unstable; + int found; + + source_fixture_init(&fixture); + make_directory(&fixture, "a"); + strbuf_addf(&parent, "%s/a", fixture.worktree); + strbuf_addf(&old_parent, "%s/a-old", fixture.worktree); + strbuf_addf(&source, "%s/.gitattributes", parent.buf); + write_file(source.buf, "*.dat text\n"); + cl_must_pass(worktree_attr_source_read( + fixture.path, "a/.gitattributes", 17, algo, hash, &found)); + cl_assert_equal_i(found, 1); + + cl_must_pass(rename(parent.buf, old_parent.buf)); + cl_must_pass(mkdir(parent.buf, 0777)); + source_fixture_release( + &fixture, &namespace_unstable, &namespace_unstable_from); + cl_assert_equal_i(namespace_unstable, 1); + cl_assert_equal_i(namespace_unstable_from, 17); + + strbuf_release(&source); + strbuf_release(&old_parent); + strbuf_release(&parent); +#endif +} diff --git a/unpack-trees.c b/unpack-trees.c index 154d6d40a15934..06bcb0ee9bff0e 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -14,6 +14,7 @@ #include "tree.h" #include "tree-walk.h" #include "cache-tree.h" +#include "clean-status.h" #include "unpack-trees.h" #include "progress.h" #include "refs.h" @@ -2076,6 +2077,9 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options ret = check_updates(o, &o->internal.result) ? (-2) : 0; if (o->dst_index) { + if (!ret) + clean_status_transfer_current_proof_if_same_index( + &o->internal.result, o->src_index); move_index_extensions(&o->internal.result, o->src_index); if (!ret) { if (git_env_bool("GIT_TEST_CHECK_CACHE_TREE", 0) && @@ -2241,7 +2245,8 @@ static int verify_uptodate_1(const struct cache_entry *ce, if (!lstat(ce->name, &st)) { int flags = CE_MATCH_IGNORE_VALID|CE_MATCH_IGNORE_SKIP_WORKTREE; - unsigned changed = ie_match_stat(o->src_index, ce, &st, flags); + unsigned changed = ie_match_stat_with_content_check( + o->src_index, ce, &st, flags); if (submodule_from_ce(ce)) { int r = check_submodule_move_head(ce, @@ -2407,7 +2412,9 @@ static int icase_exists(struct unpack_trees_options *o, const char *name, int le const struct cache_entry *src; src = index_file_exists(o->src_index, name, len, 1); - return src && !ie_match_stat(o->src_index, src, st, CE_MATCH_IGNORE_VALID|CE_MATCH_IGNORE_SKIP_WORKTREE); + return src && !ie_match_stat_with_content_check( + o->src_index, src, st, + CE_MATCH_IGNORE_VALID | CE_MATCH_IGNORE_SKIP_WORKTREE); } enum absent_checking_type { @@ -3038,7 +3045,10 @@ int oneway_merge(const struct cache_entry * const *src, !(old->ce_flags & CE_FSMONITOR_VALID)) { struct stat st; if (lstat(old->name, &st) || - ie_match_stat(o->src_index, old, &st, CE_MATCH_IGNORE_VALID|CE_MATCH_IGNORE_SKIP_WORKTREE)) + ie_match_stat_with_content_check( + o->src_index, old, &st, + CE_MATCH_IGNORE_VALID | + CE_MATCH_IGNORE_SKIP_WORKTREE)) update |= CE_UPDATE; } if (o->update && S_ISGITLINK(old->ce_mode) && diff --git a/worktree-attr-manifest.c b/worktree-attr-manifest.c new file mode 100644 index 00000000000000..1a0234360c67cb --- /dev/null +++ b/worktree-attr-manifest.c @@ -0,0 +1,283 @@ +#include "git-compat-util.h" +#include "attr-manifest.h" +#include "dir.h" +#include "environment.h" +#include "hash-framing.h" +#include "object.h" +#include "odb.h" +#include "parse.h" +#include "read-cache-ll.h" +#include "repository.h" +#include "semantic-verify-internal.h" +#include "string-list.h" +#include "strbuf.h" +#include "thread-utils.h" +#include "worktree-attr-manifest.h" +#include "worktree-attr-source.h" + +#define ATTR_MANIFEST_FILES_PER_THREAD 256 +#define ATTR_MANIFEST_MAX_THREADS 32 + +struct attr_manifest_candidate { + unsigned char worktree_hash[GIT_MAX_RAWSZ]; + unsigned char index_hash[GIT_MAX_RAWSZ]; + unsigned int index_present : 1; + unsigned int worktree_present : 1; + unsigned int error : 1; +}; + +struct attr_manifest_probe_data { + struct string_list *candidates; + struct semantic_verify_root *root; + const struct git_hash_algo *algo; + size_t start; + size_t end; + unsigned int namespace_unstable; +}; + +struct attr_manifest_thread { + struct attr_manifest_probe_data probe; + pthread_t pthread; + unsigned int started : 1; +}; + +static int collect_candidates(struct index_state *istate, + struct string_list *candidates) +{ + struct strbuf candidate = STRBUF_INIT; + const char *previous = NULL; + size_t previous_len = 0; + unsigned int i; + int ret = -1; + + string_list_append(candidates, GITATTRIBUTES_FILE); + for (i = 0; i < istate->cache_nr; i++) { + const struct cache_entry *ce = istate->cache[i]; + const char *slash = ce->name; + + if (ce_stage(ce) || S_ISSPARSEDIR(ce->ce_mode)) + goto done; + while ((slash = strchr(slash, '/')) != NULL) { + size_t len = slash - ce->name; + + if (!previous || previous_len <= len || + !is_dir_sep(previous[len]) || + fspathncmp(previous, ce->name, len)) { + strbuf_reset(&candidate); + strbuf_add(&candidate, ce->name, len + 1); + strbuf_addstr(&candidate, GITATTRIBUTES_FILE); + string_list_append(candidates, candidate.buf); + } + slash++; + } + previous = ce->name; + previous_len = ce->ce_namelen; + } + string_list_sort(candidates); + string_list_remove_duplicates(candidates, 0); + ret = candidates->nr <= UINT32_MAX ? 0 : -1; +done: + strbuf_release(&candidate); + return ret; +} + +static int collect_index_sources(struct index_state *istate, + struct string_list *candidates) +{ + struct strbuf candidate = STRBUF_INIT; + unsigned int i; + int ret = 0; + + for (i = 0; i < candidates->nr; i++) { + struct attr_manifest_candidate *state; + + CALLOC_ARRAY(state, 1); + candidates->items[i].util = state; + } + for (i = 0; i < istate->cache_nr; i++) { + const struct cache_entry *ce = istate->cache[i]; + const char *base = strrchr(ce->name, '/'); + struct string_list_item *item; + struct attr_manifest_candidate *state; + + base = base ? base + 1 : ce->name; + if (fspathcmp(base, GITATTRIBUTES_FILE)) + continue; + strbuf_reset(&candidate); + if (base != ce->name) + strbuf_add(&candidate, ce->name, base - ce->name); + strbuf_addstr(&candidate, GITATTRIBUTES_FILE); + item = string_list_lookup(candidates, candidate.buf); + if (!item) + BUG("tracked attribute source lacks manifest candidate"); + state = item->util; + if ((S_ISREG(ce->ce_mode) || S_ISLNK(ce->ce_mode)) && + odb_has_object(istate->repo->objects, &ce->oid, 0)) { + state->index_present = 1; + memcpy(state->index_hash, ce->oid.hash, + istate->repo->hash_algo->rawsz); + } else if (S_ISREG(ce->ce_mode) || S_ISLNK(ce->ce_mode)) { + ret = -1; + break; + } + } + strbuf_release(&candidate); + return ret; +} + +static void *probe_attr_manifest_candidates(void *cb_data) +{ + struct attr_manifest_probe_data *data = cb_data; + struct semantic_verify_path *path = + semantic_verify_path_new(data->root); + size_t i; + + for (i = data->start; i < data->end; i++) { + struct string_list_item *item = &data->candidates->items[i]; + struct attr_manifest_candidate *candidate = item->util; + int found; + + if (worktree_attr_source_read(path, item->string, i, data->algo, + candidate->worktree_hash, &found)) + candidate->error = 1; + else + candidate->worktree_present = found; + } + semantic_verify_path_free(path, &data->namespace_unstable, NULL); + return NULL; +} + +static size_t select_thread_count(size_t candidates) +{ + size_t cpus, test_threads, threads; + + if (!HAVE_THREADS) + return 1; + threads = DIV_ROUND_UP(candidates, ATTR_MANIFEST_FILES_PER_THREAD); + cpus = online_cpus(); + if (threads > cpus * 2) + threads = cpus * 2; + test_threads = git_env_ulong("GIT_TEST_ATTR_MANIFEST_THREADS", 0); + if (test_threads) + threads = test_threads; + if (threads > ATTR_MANIFEST_MAX_THREADS) + threads = ATTR_MANIFEST_MAX_THREADS; + if (threads > candidates) + threads = candidates; + return threads ? threads : 1; +} + +static int create_probe_thread(struct attr_manifest_thread *worker, + size_t thread_id) +{ + if (git_env_ulong("GIT_TEST_ATTR_MANIFEST_THREAD_FAIL_AT", + ULONG_MAX) == thread_id) + return EAGAIN; + return pthread_create(&worker->pthread, NULL, + probe_attr_manifest_candidates, &worker->probe); +} + +static int probe_candidates(struct string_list *candidates, + struct semantic_verify_root *root, + const struct git_hash_algo *algo, + struct worktree_attr_manifest_stats *stats) +{ + struct attr_manifest_thread *workers; + size_t thread_id, threads = select_thread_count(candidates->nr); + int create_threads = HAVE_THREADS; + int ret = 0; + + CALLOC_ARRAY(workers, threads); + for (thread_id = 0; thread_id < threads; thread_id++) { + struct attr_manifest_thread *worker = &workers[thread_id]; + struct attr_manifest_probe_data *data = &worker->probe; + int err; + + data->candidates = candidates; + data->root = root; + data->algo = algo; + data->start = st_mult(candidates->nr, thread_id) / threads; + data->end = st_mult(candidates->nr, thread_id + 1) / threads; + if (threads == 1 || !create_threads) { + probe_attr_manifest_candidates(data); + continue; + } + err = create_probe_thread(worker, thread_id); + if (!err) { + worker->started = 1; + continue; + } + stats->thread_failures++; + create_threads = 0; + probe_attr_manifest_candidates(data); + } + for (thread_id = 0; thread_id < threads; thread_id++) { + struct attr_manifest_thread *worker = &workers[thread_id]; + + if (worker->started && pthread_join(worker->pthread, NULL)) + die("unable to join attribute manifest thread"); + ret |= worker->probe.namespace_unstable; + } + stats->threads = threads; + free(workers); + return ret ? -1 : 0; +} + +int worktree_attr_manifest_build( + struct index_state *istate, + struct strbuf *manifest, + unsigned char *manifest_hash, + struct worktree_attr_manifest_stats *stats) +{ + struct string_list candidates = STRING_LIST_INIT_DUP; + struct semantic_verify_root *root = NULL; + struct attr_manifest_writer writer; + const struct git_hash_algo *algo = istate->repo->hash_algo; + unsigned int i; + int ret = -1; + + memset(stats, 0, sizeof(*stats)); + if (istate->sparse_index != INDEX_EXPANDED || + semantic_verify_root_init(istate->repo, &root) || + collect_candidates(istate, &candidates) || + collect_index_sources(istate, &candidates)) + goto done; + stats->candidates = candidates.nr; + if (probe_candidates(&candidates, root, algo, stats)) + goto done; + attr_manifest_writer_init(&writer, manifest, algo); + for (i = 0; i < candidates.nr; i++) { + const char *name = candidates.items[i].string; + struct attr_manifest_candidate *state = candidates.items[i].util; + enum attr_manifest_source source; + const unsigned char *hash; + + if (state->error) + goto done; + if (state->worktree_present) { + source = ATTR_MANIFEST_WORKTREE; + hash = state->worktree_hash; + stats->worktree_sources++; + } else if (state->index_present) { + source = ATTR_MANIFEST_INDEX; + hash = state->index_hash; + stats->index_sources++; + } else { + continue; + } + if (attr_manifest_writer_add(&writer, name, source, hash)) + goto done; + } + if (!semantic_verify_root_stable(root)) + goto done; + if (!attr_manifest_valid(manifest->buf, manifest->len, algo)) + BUG("newly built attribute manifest is invalid"); + hash_buffer_digest(algo, manifest->buf, manifest->len, manifest_hash); + ret = 0; +done: + semantic_verify_root_clear(root); + string_list_clear(&candidates, 1); + if (ret) + strbuf_reset(manifest); + return ret; +} diff --git a/worktree-attr-manifest.h b/worktree-attr-manifest.h new file mode 100644 index 00000000000000..4b3e17c26abd35 --- /dev/null +++ b/worktree-attr-manifest.h @@ -0,0 +1,21 @@ +#ifndef WORKTREE_ATTR_MANIFEST_H +#define WORKTREE_ATTR_MANIFEST_H + +struct index_state; +struct strbuf; + +struct worktree_attr_manifest_stats { + size_t candidates; + size_t threads; + size_t worktree_sources; + size_t index_sources; + size_t thread_failures; +}; + +int worktree_attr_manifest_build( + struct index_state *istate, + struct strbuf *manifest, + unsigned char *manifest_hash, + struct worktree_attr_manifest_stats *stats); + +#endif /* WORKTREE_ATTR_MANIFEST_H */ diff --git a/worktree-attr-source.c b/worktree-attr-source.c new file mode 100644 index 00000000000000..d56043eba7a576 --- /dev/null +++ b/worktree-attr-source.c @@ -0,0 +1,93 @@ +#include "git-compat-util.h" +#include "attr.h" +#include "hash.h" +#include "path-namespace.h" +#include "semantic-verify-internal.h" +#include "worktree-attr-source.h" + +#define WORKTREE_ATTR_HASH_BUFFER_SIZE (64 * 1024) + +#if !SEMANTIC_VERIFY_HAS_ANCHORED_OPEN + +int worktree_attr_source_read( + struct semantic_verify_path *path UNUSED, + const char *name UNUSED, size_t position UNUSED, + const struct git_hash_algo *algo UNUSED, + unsigned char *hash UNUSED, int *found) +{ + *found = 0; + return -1; +} + +#else + +int worktree_attr_source_read(struct semantic_verify_path *path, + const char *name, size_t position, + const struct git_hash_algo *algo, + unsigned char *hash, int *found) +{ + struct git_hash_ctx ctx = { 0 }; + struct stat before, after, named; + unsigned char buffer[WORKTREE_ATTR_HASH_BUFFER_SIZE]; + const char *basename; + ssize_t got; + size_t remaining, size; + int parent_fd, fd = -1, ret = -1; + char extra; + + *found = 0; + if (semantic_verify_resolve_parent(path, name, position, + &parent_fd, &basename)) { + if (errno == ENOENT || errno == ENOTDIR || errno == ELOOP || + errno == EXDEV) + return 0; + return -1; + } + if (fstatat(parent_fd, basename, &before, AT_SYMLINK_NOFOLLOW)) + return errno == ENOENT || errno == ENOTDIR ? 0 : -1; + if (!S_ISREG(before.st_mode) || + before.st_size < 0 || before.st_size >= ATTR_MAX_FILE_SIZE) + return 0; + if (before.st_nlink != 1) + return -1; + + fd = semantic_verify_openat(parent_fd, basename, + O_RDONLY | O_NONBLOCK | O_NOFOLLOW); + if (fd < 0) + return -1; + if (fstat(fd, &after) || + !path_namespace_stat_equal(&before, &after)) + goto done; + size = xsize_t(before.st_size); + remaining = size; + git_hash_init(&ctx, algo); + while (remaining) { + size_t want = remaining < sizeof(buffer) ? + remaining : sizeof(buffer); + + got = xread(fd, buffer, want); + if (got <= 0) + goto done; + git_hash_update(&ctx, buffer, got); + remaining -= got; + } + got = xread(fd, &extra, 1); + if (got != 0 || + fstat(fd, &after) || + fstatat(parent_fd, basename, &named, AT_SYMLINK_NOFOLLOW) || + !path_namespace_stat_equal(&before, &after) || + !path_namespace_stat_equal(&after, &named) || + path_namespace_reopen_component( + parent_fd, basename, O_RDONLY | O_NONBLOCK | O_NOFOLLOW, + semantic_verify_openat, &after)) + goto done; + git_hash_final(hash, &ctx); + *found = 1; + ret = 0; +done: + git_hash_discard(&ctx); + close(fd); + return ret; +} + +#endif /* SEMANTIC_VERIFY_HAS_ANCHORED_OPEN */ diff --git a/worktree-attr-source.h b/worktree-attr-source.h new file mode 100644 index 00000000000000..2db26c68a71119 --- /dev/null +++ b/worktree-attr-source.h @@ -0,0 +1,12 @@ +#ifndef WORKTREE_ATTR_SOURCE_H +#define WORKTREE_ATTR_SOURCE_H + +struct git_hash_algo; +struct semantic_verify_path; + +int worktree_attr_source_read(struct semantic_verify_path *path, + const char *name, size_t position, + const struct git_hash_algo *algo, + unsigned char *hash, int *found); + +#endif /* WORKTREE_ATTR_SOURCE_H */ diff --git a/wt-status.c b/wt-status.c index 57772c7501fdba..095dfa13f661d5 100644 --- a/wt-status.c +++ b/wt-status.c @@ -2,18 +2,27 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "git-compat-util.h" +#include "abspath.h" #include "advice.h" +#include "attr.h" +#include "attr-fingerprint.h" #include "wt-status.h" +#include "cache-tree.h" #include "object.h" #include "dir.h" #include "commit.h" +#include "clean-status.h" +#include "clean-status-index.h" #include "diff.h" #include "environment.h" +#include "exclude-source-proof.h" #include "gettext.h" #include "hash.h" #include "hex.h" #include "object-name.h" #include "path.h" +#include "preload-index.h" +#include "replace-object.h" #include "revision.h" #include "diffcore.h" #include "quote.h" @@ -26,6 +35,7 @@ #include "column.h" #include "read-cache.h" #include "setup.h" +#include "semantic-verify.h" #include "strbuf.h" #include "trace.h" #include "trace2.h" @@ -34,7 +44,9 @@ #include "worktree.h" #include "lockfile.h" #include "sequencer.h" +#include "fsmonitor.h" #include "fsmonitor-settings.h" +#include "wrapper.h" #define AB_DELAY_WARNING_IN_MS (2 * 1000) #define UF_DELAY_WARNING_IN_MS (2 * 1000) @@ -457,6 +469,19 @@ static char short_submodule_status(struct wt_status_change_data *d) return d->worktree_status; } +static struct wt_status_change_data *wt_status_get_change( + struct wt_status *s, const char *path) +{ + struct string_list_item *it = string_list_insert(&s->change, path); + struct wt_status_change_data *d = it->util; + + if (!d) { + CALLOC_ARRAY(d, 1); + it->util = d; + } + return d; +} + static void wt_status_collect_changed_cb(struct diff_queue_struct *q, struct diff_options *options UNUSED, void *data) @@ -469,16 +494,10 @@ static void wt_status_collect_changed_cb(struct diff_queue_struct *q, s->workdir_dirty = 1; for (i = 0; i < q->nr; i++) { struct diff_filepair *p; - struct string_list_item *it; struct wt_status_change_data *d; p = q->queue[i]; - it = string_list_insert(&s->change, p->two->path); - d = it->util; - if (!d) { - CALLOC_ARRAY(d, 1); - it->util = d; - } + d = wt_status_get_change(s, p->two->path); if (!d->worktree_status) d->worktree_status = p->status; if (S_ISGITLINK(p->two->mode)) { @@ -524,6 +543,68 @@ static void wt_status_collect_changed_cb(struct diff_queue_struct *q, } } +static struct cache_entry **wt_status_collect_preload_changes( + struct wt_status *s, size_t *direct_nr) +{ + struct index_state *istate = s->repo->index; + struct cache_entry **direct = NULL; + size_t direct_alloc = 0; + uint64_t modified = 0, deleted = 0; + + *direct_nr = 0; + if (istate->preload_bulk_tracked_nr != istate->cache_nr) + goto clear; + for (size_t i = 0; i < istate->cache_nr; i++) { + struct cache_entry *ce = istate->cache[i]; + struct wt_status_change_data *d; + unsigned char state = + istate->preload_bulk_tracked_state[i]; + int status; + + if (state == PRELOAD_BULK_TRACKED_DEFINITIVE_MODIFIED) { + status = DIFF_STATUS_MODIFIED; + modified++; + } else if (state == PRELOAD_BULK_TRACKED_DEFINITIVE_DELETED) { + status = DIFF_STATUS_DELETED; + deleted++; + } else { + continue; + } + + d = wt_status_get_change(s, ce->name); + if (!d->worktree_status) + d->worktree_status = status; + d->mode_index = ce->ce_mode; + d->mode_worktree = status == DIFF_STATUS_MODIFIED ? + ce->ce_mode : 0; + oidcpy(&d->oid_index, &ce->oid); + ce_mark_uptodate(ce); + ALLOC_GROW(direct, *direct_nr + 1, direct_alloc); + direct[(*direct_nr)++] = ce; + s->workdir_dirty = 1; + } + trace2_data_intmax("status", s->repo, "preload/direct_modified", + modified); + trace2_data_intmax("status", s->repo, "preload/direct_deleted", + deleted); + +clear: + /* + * The tracked result is single-use. Keep a closed excludes digest + * available until the index writer has consumed it. + */ + preload_index_bulk_result_consume(istate); + return direct; +} + +static void wt_status_release_preload_changes( + struct cache_entry **direct, size_t direct_nr) +{ + for (size_t i = 0; i < direct_nr; i++) + direct[i]->ce_flags &= ~CE_UPTODATE; + free(direct); +} + static int unmerged_mask(struct index_state *istate, const char *path) { int pos, mask; @@ -553,16 +634,10 @@ static void wt_status_collect_updated_cb(struct diff_queue_struct *q, for (i = 0; i < q->nr; i++) { struct diff_filepair *p; - struct string_list_item *it; struct wt_status_change_data *d; p = q->queue[i]; - it = string_list_insert(&s->change, p->two->path); - d = it->util; - if (!d) { - CALLOC_ARRAY(d, 1); - it->util = d; - } + d = wt_status_get_change(s, p->two->path); if (!d->index_status) d->index_status = p->status; switch (p->status) { @@ -638,8 +713,16 @@ void wt_status_collect_changes_trees(struct wt_status *s, static void wt_status_collect_changes_worktree(struct wt_status *s) { + struct cache_entry **direct; + size_t direct_nr; struct rev_info rev; + if (s->tracked_from_fsmonitor) { + preload_index_bulk_result_consume(s->repo->index); + return; + } + + direct = wt_status_collect_preload_changes(s, &direct_nr); repo_init_revisions(s->repo, &rev, NULL); setup_revisions(0, NULL, &rev, NULL); rev.diffopt.output_format |= DIFF_FORMAT_CALLBACK; @@ -659,15 +742,57 @@ static void wt_status_collect_changes_worktree(struct wt_status *s) rev.diffopt.rename_limit = s->rename_limit >= 0 ? s->rename_limit : rev.diffopt.rename_limit; rev.diffopt.rename_score = s->rename_score >= 0 ? s->rename_score : rev.diffopt.rename_score; copy_pathspec(&rev.prune_data, &s->pathspec); - run_diff_files(&rev, 0); + run_diff_files(&rev, s->bulk_update_index_stat ? + DIFF_UPDATE_INDEX_STAT : 0); + wt_status_release_preload_changes(direct, direct_nr); release_revisions(&rev); } +static int wt_status_cache_tree_matches_reference(struct wt_status *s) +{ + struct index_state *istate = s->repo->index; + struct object_id reference_tree; + struct strbuf reference = STRBUF_INIT; + int matches = 0; + + if (!s->allow_clean_status_shortcuts || s->is_initial || + getenv(INDEX_ENVIRONMENT) || istate->split_index || + istate->sparse_index != INDEX_EXPANDED || + !istate->cache_tree || + istate->cache_tree->entry_count < 0 || + (unsigned int)istate->cache_tree->entry_count != + istate->cache_nr) + return 0; + + /* + * A replacement below the root can change the effective tree without + * changing the root object name stored in the commit. + */ + if (replace_refs_enabled(s->repo)) { + prepare_replace_object(s->repo); + if (oidmap_get_size(&s->repo->objects->replace_map)) + return 0; + } + + strbuf_addf(&reference, "%s^{tree}", s->reference); + if (!repo_get_oid_tree(s->repo, reference.buf, &reference_tree) && + oideq(&istate->cache_tree->oid, &reference_tree)) + matches = 1; + strbuf_release(&reference); + return matches; +} + static void wt_status_collect_changes_index(struct wt_status *s) { struct rev_info rev; struct setup_revision_opt opt; + if (wt_status_cache_tree_matches_reference(s)) { + trace2_data_intmax("status", s->repo, + "index/cache-tree-match", 1); + return; + } + repo_init_revisions(s->repo, &rev, NULL); memset(&opt, 0, sizeof(opt)); opt.def = s->is_initial ? empty_tree_oid_hex(s->repo->hash_algo) : s->reference; @@ -803,19 +928,245 @@ static void wt_status_collect_changes_initial(struct wt_status *s) strbuf_release(&base); } -static void wt_status_collect_untracked(struct wt_status *s) +static unsigned int wt_status_untracked_dir_flags(const struct wt_status *s) +{ + if (s->show_untracked_files == SHOW_ALL_UNTRACKED_FILES) + return 0; + return DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES; +} + +struct wt_status_exclude_context { + int root_fd; +}; + +static void wt_status_release_exclude_proof(struct wt_status *s) +{ + exclude_source_proof_release(s->certify_exclude_proof); + s->certify_exclude_proof = NULL; + if (s->certify_exclude_context) { + if (s->certify_exclude_context->root_fd >= 0) + close(s->certify_exclude_context->root_fd); + FREE_AND_NULL(s->certify_exclude_context); + } + oidclr(&s->certify_exclude_digest, s->repo->hash_algo); + s->certify_exclude_digest_valid = 0; + s->certify_untracked_scan_failed = 0; +} + +#if EXCLUDE_SOURCE_PROOF_HAS_ANCHORED_OPEN + +static int wt_status_open_exclude_parent(void *data, const char *path) +{ + struct wt_status_exclude_context *context = data; + int flags = O_RDONLY | O_NONBLOCK | O_DIRECTORY | O_CLOEXEC | + O_NOFOLLOW; + + if (is_absolute_path(path)) + return open(path, flags); + return openat(context->root_fd, path, flags); +} + +static void wt_status_prepare_exclude_proof( + struct wt_status *s, struct dir_struct *dir) +{ + struct wt_status_exclude_context *context; + const char *worktree; + + if (!s->certify_clean_status) + return; + if (!s->certify_exclude_proof) { + worktree = repo_get_work_tree(s->repo); + if (!worktree) + return; + CALLOC_ARRAY(context, 1); + context->root_fd = open_nofollow( + worktree, + O_RDONLY | O_NONBLOCK | O_DIRECTORY | O_CLOEXEC); + if (context->root_fd < 0) { + free(context); + return; + } + s->certify_exclude_context = context; + s->certify_exclude_proof = exclude_source_proof_create( + s->repo->index, context, + wt_status_open_exclude_parent, + EXCLUDE_SOURCE_PROOF_NONBLOCKING); + } + dir->internal.exclude_source_proof = + s->certify_exclude_proof; +} + +static void wt_status_record_exclude_digest(struct wt_status *s) +{ + if (s->certify_exclude_digest_valid || + !s->certify_exclude_proof) + return; + s->certify_exclude_digest_valid = + !exclude_source_proof_digest( + s->certify_exclude_proof, + s->repo->hash_algo, + &s->certify_exclude_digest); +} + +#else + +static void wt_status_prepare_exclude_proof( + struct wt_status *s UNUSED, struct dir_struct *dir UNUSED) +{ +} + +static void wt_status_record_exclude_digest(struct wt_status *s UNUSED) +{ +} + +#endif + +int wt_status_certified_excludes_digest( + struct wt_status *s, struct object_id *digest, + struct stat *scanned_worktree) +{ + if (s->certify_untracked_scan_failed || + !s->certify_exclude_digest_valid || + !s->certify_exclude_proof || + !s->certify_exclude_context || + s->certify_exclude_context->root_fd < 0 || + !exclude_source_proof_validate( + s->certify_exclude_proof) || + fstat(s->certify_exclude_context->root_fd, + scanned_worktree)) + return -1; + oidcpy(digest, &s->certify_exclude_digest); + return 0; +} + +static int wt_status_begin_attr_snapshot(struct wt_status *s) +{ + int ret; + int hook_provider = + fsm_settings__get_mode(s->repo) == FSMONITOR_MODE_HOOK; + + if (s->attr_snapshot_failed) + return -1; + if (s->attr_source_snapshot) + return 0; + ret = clean_status_capture_attr_snapshot( + s->repo->index, &s->attr_source_snapshot); + if (ret < 0) { + s->attr_snapshot_failed = 1; + untracked_cache_invalidate_all(s->repo->index); + fsmonitor_invalidate_semantics(s->repo->index); + return -1; + } + if (s->attr_source_snapshot) + git_attr_source_snapshot_begin(s->attr_source_snapshot); + if ((ret > 0 && + (!hook_provider || + (ret & CLEAN_STATUS_ATTR_CONTENT_CHANGED))) || + (clean_status_fsmonitor_strong_mismatch(s->repo->index) && + !hook_provider)) { + /* + * Hook providers have no closing query with which to adopt + * missing semantic history, so absence alone must preserve + * their established path-reporting contract. Namespace-only + * churn can arise from an index rewrite and is likewise not + * evidence that the hook missed a semantic change. Changed + * attribute contents are current evidence and invalidate + * cached semantics regardless of provider. + */ + untracked_cache_invalidate_all(s->repo->index); + fsmonitor_invalidate_semantics(s->repo->index); + } + return ret; +} + +void wt_status_start_untracked_cache_preload(struct wt_status *s) +{ + struct index_state *istate = s->repo->index; + unsigned int dir_flags; + int has_fsmonitor = + fsm_settings__get_mode(s->repo) > FSMONITOR_MODE_DISABLED; + + if (s->untracked_cache_preload) + BUG("untracked-cache preload already started"); + wt_status_begin_attr_snapshot(s); + /* Record the provider token before either filesystem traversal. */ + refresh_fsmonitor(istate); + if (s->certify_clean_status && + !fsmonitor_has_pending_token(istate)) + fsmonitor_reopen_token(istate); + if (s->pathspec.nr || + s->show_untracked_files == SHOW_NO_UNTRACKED_FILES || + s->show_ignored_mode) + return; + + dir_flags = wt_status_untracked_dir_flags(s); + if (has_fsmonitor && + (!fsmonitor_has_pending_token(istate) || + !fstat_is_reliable())) { + s->untracked_cache_preload = + untracked_cache_preload_start_fsmonitor_excludes( + istate, dir_flags); + return; + } + + if (s->show_untracked_files == SHOW_NORMAL_UNTRACKED_FILES && + !istate->untracked && + istate->sparse_index == INDEX_EXPANDED) + istate->preload_untracked = &s->untracked; + /* Restore verified stats before cached excludes inspect them. */ + if (fstat_is_reliable() && !istate->split_index && + fsm_settings__get_mode(s->repo) == FSMONITOR_MODE_IPC && + fsmonitor_pending_token_from_provider(istate) && + clean_status_fsmonitor_semantic_adoption_needed(istate) && + istate->untracked && istate->untracked->root) { + trace2_data_intmax("status", s->repo, + "fsmonitor_token/untracked-deferred", 1); + return; + } + if (has_fsmonitor) + return; + + s->untracked_cache_preload = + untracked_cache_preload_start_ordinary(istate, dir_flags); +} + +static void wt_status_finish_untracked_cache_preload(struct wt_status *s) +{ + struct index_state *istate = s->repo->index; + size_t index_invalidated = 0; + + if (!s->untracked_cache_preload) + return; + s->untracked_cache_preloaded = untracked_cache_preload_finish( + s->untracked_cache_preload, istate, + wt_status_untracked_dir_flags(s), &index_invalidated); + s->untracked_cache_preload = NULL; + if (!index_invalidated) + return; + + s->tracked_from_fsmonitor = 0; + preload_index_bulk_result_clear(istate); + trace2_data_intmax("status", s->repo, + "fsmonitor/exclude-index-invalidated", + index_invalidated); +} + +static int wt_status_collect_untracked_1( + struct wt_status *s, + struct string_list *untracked, + struct string_list *ignored) { int i; + int used_untracked_cache; struct dir_struct dir = DIR_INIT; uint64_t t_begin = getnanotime(); struct index_state *istate = s->repo->index; if (!s->show_untracked_files) - return; + return 0; if (s->show_untracked_files != SHOW_ALL_UNTRACKED_FILES) - dir.flags |= - DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES; + dir.flags |= wt_status_untracked_dir_flags(s); if (s->show_ignored_mode) { dir.flags |= DIR_SHOW_IGNORED_TOO; @@ -825,28 +1176,630 @@ static void wt_status_collect_untracked(struct wt_status *s) dir.untracked = istate->untracked; } + wt_status_prepare_exclude_proof(s, &dir); setup_standard_excludes(&dir); + wt_status_record_exclude_digest(s); + wt_status_finish_untracked_cache_preload(s); + dir.internal.untracked_cache_preloaded = + s->untracked_cache_preloaded; fill_directory(&dir, istate, &s->pathspec); + if (s->certify_clean_status && dir.internal.traversal_failed) + s->certify_untracked_scan_failed = 1; + used_untracked_cache = dir.untracked && + dir.untracked == istate->untracked; for (i = 0; i < dir.nr; i++) { struct dir_entry *ent = dir.entries[i]; if (index_name_is_other(istate, ent->name, ent->len)) - string_list_append(&s->untracked, ent->name); + string_list_append(untracked, ent->name); } - string_list_sort_u(&s->untracked, 0); + string_list_sort_u(untracked, 0); for (i = 0; i < dir.ignored_nr; i++) { struct dir_entry *ent = dir.ignored[i]; if (index_name_is_other(istate, ent->name, ent->len)) - string_list_append(&s->ignored, ent->name); + string_list_append(ignored, ent->name); } - string_list_sort_u(&s->ignored, 0); + string_list_sort_u(ignored, 0); dir_clear(&dir); if (advice_enabled(ADVICE_STATUS_U_OPTION)) s->untracked_in_ms = (getnanotime() - t_begin) / 1000000; + if (used_untracked_cache) + fsmonitor_mark_untracked_cache_valid(istate); + return used_untracked_cache; +} + +static int wt_status_can_use_bulk_provider( + struct wt_status *s, unsigned int refresh_flags) +{ + return !s->show_ignored_mode && !s->pathspec.nr && + (refresh_flags & REFRESH_DEFER_BULK_DIRTY) && + preload_index_bulk_can_close_provider(s->repo->index); +} + +static struct semantic_verify_proof *wt_status_prepare_semantic_verify( + struct wt_status *s, unsigned int refresh_flags) +{ + struct index_state *istate = s->repo->index; + struct semantic_verify_options options = SEMANTIC_VERIFY_OPTIONS_INIT; + struct semantic_verify_proof *proof = NULL; + int ret; + + if (!fstat_is_reliable() || istate->split_index || + s->show_ignored_mode || s->pathspec.nr || + fsm_settings__get_mode(s->repo) != FSMONITOR_MODE_IPC || + istate->sparse_index != INDEX_EXPANDED || + !fsmonitor_has_pending_token(istate) || + !fsmonitor_pending_token_from_provider(istate) || + !clean_status_fsmonitor_semantic_adoption_needed(istate)) + return NULL; + if (wt_status_can_use_bulk_provider(s, refresh_flags)) { + trace2_data_intmax("status", s->repo, + "semantic_verify/bulk_scan", 1); + return NULL; + } + + options.require_proof_epoch = 1; + options.attr_snapshot = s->attr_source_snapshot; + trace2_region_enter("status", "semantic_verify", s->repo); + ret = semantic_verify_prepare(istate, &options, &proof); + trace2_data_intmax("status", s->repo, + "semantic_verify/prepared", !ret); + trace2_region_leave("status", "semantic_verify", s->repo); + if (ret) { + semantic_verify_proof_clear(proof); + return NULL; + } + return proof; +} + +static int wt_status_collect_untracked(struct wt_status *s) +{ + if (s->untracked_from_token_closure && !s->show_ignored_mode) + return 1; + if (s->untracked_from_preload && !s->show_ignored_mode) + return 0; + return wt_status_collect_untracked_1( + s, &s->untracked, &s->ignored); +} + +#define FSMONITOR_TOKEN_MAX_QUERIES 3 + +struct wt_status_token_closure { + struct wt_status *status; + unsigned int refresh_flags; + int require_untracked; + int can_prime; + int use_bulk_provider; + int untracked_ready; + int untracked_proof_complete; + struct string_list staged_untracked; + struct string_list staged_ignored; + int staged_untracked_ready; + int refresh_result; + int queries; +}; + +static void wt_status_discard_staged_untracked( + struct wt_status_token_closure *closure) +{ + string_list_clear(&closure->staged_untracked, 0); + string_list_clear(&closure->staged_ignored, 0); + closure->staged_untracked_ready = 0; +} + +static int wt_status_stage_untracked( + struct wt_status_token_closure *closure) +{ + wt_status_discard_staged_untracked(closure); + closure->staged_untracked_ready = + wt_status_collect_untracked_1( + closure->status, + &closure->staged_untracked, + &closure->staged_ignored); + if (!closure->staged_untracked_ready) + wt_status_discard_staged_untracked(closure); + return closure->staged_untracked_ready; +} + +static void wt_status_publish_staged_untracked( + struct wt_status_token_closure *closure) +{ + struct wt_status *s = closure->status; + + if (!closure->staged_untracked_ready) + return; + if (s->untracked.nr || s->ignored.nr) + BUG("publishing untracked results over collected status"); + SWAP(s->untracked, closure->staged_untracked); + SWAP(s->ignored, closure->staged_ignored); + s->untracked_from_token_closure = 1; + closure->staged_untracked_ready = 0; +} + +static int wt_status_untracked_cache_valid( + const struct wt_status_token_closure *closure) +{ + const struct index_state *istate = closure->status->repo->index; + + return closure->untracked_ready && + istate->untracked && istate->untracked->root; +} + +static void wt_status_record_bulk_untracked( + struct wt_status_token_closure *closure) +{ + if (closure->status->repo->index->preload_untracked_complete) + closure->untracked_proof_complete = 1; +} + +static int fsmonitor_token_requires_rescan(enum fsmonitor_token_result result) +{ + return result == FSMONITOR_TOKEN_CHANGED || + result == FSMONITOR_TOKEN_TRIVIAL; +} + +static void wt_status_release_attr_snapshot(struct wt_status *s); + +static int wt_status_attr_snapshot_matches(struct wt_status *s) +{ + if (s->attr_snapshot_failed) + return 0; + return !s->attr_source_snapshot || + attr_source_snapshot_matches_repository( + s->repo, s->attr_source_snapshot); +} + +static int wt_status_refresh_invalidated_manifest(struct wt_status *s) +{ + if (!clean_status_worktree_manifest_needs_refresh(s->repo->index)) + return 0; + return clean_status_refresh_worktree_manifest(s->repo->index) < 0 ? + -1 : 0; +} + +static void wt_status_reset_attr_snapshot_if_changed(struct wt_status *s) +{ + if (wt_status_attr_snapshot_matches(s)) + return; + wt_status_release_attr_snapshot(s); + wt_status_begin_attr_snapshot(s); + trace2_data_intmax("status", s->repo, + "semantic/attribute-epoch-rejected", 1); +} + +static void wt_status_discard_semantic_verify( + struct wt_status *s, struct semantic_verify_proof **proof, + const char *reason) +{ + if (!*proof) + return; + trace2_data_string("status", s->repo, "semantic_verify/discard", + reason); + semantic_verify_proof_clear(*proof); + *proof = NULL; + git_attr_invalidate_all(); +} + +static void wt_status_refresh_for_token( + struct wt_status *s, unsigned int refresh_flags, + struct clean_status_proof_epoch **epoch, int use_bulk_provider, + int *refresh_result) +{ + struct index_state *istate = s->repo->index; + + clean_status_release_proof_epoch(*epoch); + *epoch = clean_status_capture_proof_epoch( + istate, s->attr_source_snapshot); + if (*epoch && use_bulk_provider) + istate->preload_bulk_proof_epoch = *epoch; + if (*epoch) { + *refresh_result |= refresh_index( + istate, refresh_flags, &s->pathspec, NULL, NULL); + } + istate->preload_bulk_proof_epoch = NULL; +} + +static int wt_status_close_ordinary_fsmonitor_token( + struct wt_status_token_closure *closure, + int refreshed_before_closure) +{ + struct wt_status *s = closure->status; + struct index_state *istate = s->repo->index; + struct clean_status_proof_epoch *scan_epoch = NULL; + int reliable_stat = fstat_is_reliable(); + + /* + * A pending token must close a refresh begun after its epoch was + * captured. A refresh performed before entering token closure cannot + * be validated by capturing its inputs afterward. + */ + if (reliable_stat) { + wt_status_refresh_for_token( + s, closure->refresh_flags, &scan_epoch, + closure->use_bulk_provider, + &closure->refresh_result); + if (!scan_epoch) + return 0; + } else if (!refreshed_before_closure) { + closure->refresh_result |= refresh_index( + istate, closure->refresh_flags, &s->pathspec, + NULL, NULL); + } + wt_status_record_bulk_untracked(closure); + if (!closure->untracked_proof_complete && closure->can_prime) { + closure->untracked_ready = + wt_status_stage_untracked(closure); + closure->untracked_proof_complete = + closure->untracked_ready; + if (closure->queries) + trace2_data_intmax( + "status", s->repo, + "fsmonitor_token/untracked-after-retry", + closure->untracked_ready); + } + + while (closure->queries < FSMONITOR_TOKEN_MAX_QUERIES) { + enum fsmonitor_token_result result; + + if (reliable_stat && + !clean_status_proof_epoch_start_token_matches( + istate, scan_epoch)) + break; + closure->queries++; + result = fsmonitor_query_pending_token( + istate, + wt_status_untracked_cache_valid(closure)); + if (result == FSMONITOR_TOKEN_CLEAN) { + if (reliable_stat && + !clean_status_proof_epoch_matches( + istate, scan_epoch)) { + wt_status_reset_attr_snapshot_if_changed(s); + break; + } + if (closure->untracked_proof_complete || + !closure->require_untracked) { + if (preload_index_bulk_result_accept(istate) < 0) + break; + if (reliable_stat) + clean_status_mark_fsmonitor_config_valid( + istate, + istate->fsmonitor_last_update_pending); + clean_status_release_proof_epoch(scan_epoch); + fsmonitor_accept_pending_token( + istate, + closure->untracked_proof_complete, + wt_status_untracked_cache_valid( + closure)); + return 1; + } + break; + } + wt_status_discard_staged_untracked(closure); + closure->untracked_proof_complete = + !closure->require_untracked || !istate->untracked; + clean_status_release_proof_epoch(scan_epoch); + scan_epoch = NULL; + if (!fsmonitor_token_requires_rescan(result)) + break; + + /* Rescan invalidations returned by the closure query. */ + wt_status_reset_attr_snapshot_if_changed(s); + if (wt_status_refresh_invalidated_manifest(s)) + break; + if (reliable_stat) { + wt_status_refresh_for_token( + s, closure->refresh_flags, &scan_epoch, + closure->use_bulk_provider, + &closure->refresh_result); + if (!scan_epoch) + break; + } else { + closure->refresh_result |= refresh_index( + istate, closure->refresh_flags, + &s->pathspec, NULL, NULL); + } + wt_status_record_bulk_untracked(closure); + if (!closure->untracked_proof_complete && + closure->can_prime) { + closure->untracked_ready = + wt_status_stage_untracked(closure); + closure->untracked_proof_complete = + closure->untracked_ready; + } + } + clean_status_release_proof_epoch(scan_epoch); + return 0; +} + +enum wt_status_token_closure_result { + WT_STATUS_TOKEN_CLOSURE_FALLBACK = -1, + WT_STATUS_TOKEN_CLOSURE_RETRY, + WT_STATUS_TOKEN_CLOSURE_ACCEPTED, +}; + +static enum wt_status_token_closure_result +wt_status_close_semantic_fsmonitor_token( + struct wt_status_token_closure *closure, + struct semantic_verify_proof **proof) +{ + struct wt_status *s = closure->status; + struct index_state *istate = s->repo->index; + enum fsmonitor_token_result result; + int defer_untracked = + closure->can_prime && + !closure->untracked_proof_complete; + int applied; + + if (!semantic_verify_start_token_is_current(istate, *proof)) { + wt_status_discard_semantic_verify( + s, proof, "start-token-drift"); + return WT_STATUS_TOKEN_CLOSURE_FALLBACK; + } + + /* The first query closes the tracked scan and its semantic proof. */ + closure->queries++; + result = fsmonitor_query_pending_token( + istate, defer_untracked ? 0 : + wt_status_untracked_cache_valid(closure)); + if (result != FSMONITOR_TOKEN_CLEAN) { + wt_status_discard_semantic_verify( + s, proof, "token-reset"); + if (fsmonitor_token_requires_rescan(result)) + return WT_STATUS_TOKEN_CLOSURE_RETRY; + return WT_STATUS_TOKEN_CLOSURE_FALLBACK; + } + + applied = semantic_verify_apply_after_closure(istate, *proof); + if (applied < 0) { + wt_status_reset_attr_snapshot_if_changed(s); + wt_status_discard_semantic_verify( + s, proof, "closure-drift"); + return WT_STATUS_TOKEN_CLOSURE_FALLBACK; + } + closure->refresh_result |= refresh_index( + istate, closure->refresh_flags, &s->pathspec, NULL, NULL); + trace2_data_intmax("status", s->repo, + "fsmonitor_token/semantic-closed", 1); + if (!semantic_verify_proof_is_current(istate, *proof)) { + wt_status_reset_attr_snapshot_if_changed(s); + wt_status_discard_semantic_verify( + s, proof, "closure-drift"); + return WT_STATUS_TOKEN_CLOSURE_FALLBACK; + } + + if (defer_untracked) { + closure->untracked_ready = + wt_status_stage_untracked(closure); + closure->untracked_proof_complete = + closure->untracked_ready; + trace2_data_intmax( + "status", s->repo, + "fsmonitor_token/untracked-after-semantic", + closure->untracked_ready); + if (!closure->untracked_ready || + closure->queries >= FSMONITOR_TOKEN_MAX_QUERIES) + return WT_STATUS_TOKEN_CLOSURE_FALLBACK; + + /* A second query closes the subsequent untracked scan. */ + closure->queries++; + result = fsmonitor_query_pending_token( + istate, + wt_status_untracked_cache_valid(closure)); + if (result != FSMONITOR_TOKEN_CLEAN) { + wt_status_discard_staged_untracked(closure); + untracked_cache_invalidate_all(istate); + fsmonitor_invalidate_semantics(istate); + closure->untracked_ready = 0; + closure->untracked_proof_complete = 0; + wt_status_discard_semantic_verify( + s, proof, "token-reset"); + if (fsmonitor_token_requires_rescan(result)) + return WT_STATUS_TOKEN_CLOSURE_RETRY; + return WT_STATUS_TOKEN_CLOSURE_FALLBACK; + } + if (!semantic_verify_proof_is_current(istate, *proof)) { + wt_status_reset_attr_snapshot_if_changed(s); + wt_status_discard_semantic_verify( + s, proof, "closure-drift"); + return WT_STATUS_TOKEN_CLOSURE_FALLBACK; + } + } + + clean_status_mark_fsmonitor_config_valid( + istate, istate->fsmonitor_last_update_pending); + semantic_verify_proof_clear(*proof); + *proof = NULL; + fsmonitor_accept_pending_token( + istate, closure->untracked_proof_complete, + wt_status_untracked_cache_valid(closure)); + return WT_STATUS_TOKEN_CLOSURE_ACCEPTED; +} + +static int wt_status_tracked_fsmonitor_state_is_current( + struct wt_status *s) +{ + struct index_state *istate = s->repo->index; + + return s->allow_clean_status_shortcuts && + !s->certify_clean_status && !s->pathspec.nr && + !getenv(INDEX_ENVIRONMENT) && !istate->split_index && + istate->sparse_index == INDEX_EXPANDED && + fsm_settings__get_mode(s->repo) == FSMONITOR_MODE_IPC && + is_fsmonitor_refreshed(istate) && + !fsmonitor_has_pending_token(istate) && + istate->fsmonitor_token_valid && + istate->fsmonitor_last_update && + *istate->fsmonitor_last_update && + clean_status_revalidated_token_matches(istate) && + !clean_status_manifest_global_fallback(istate) && + !clean_status_worktree_manifest_needs_refresh(istate); +} + +static int wt_status_close_fsmonitor_token( + struct wt_status *s, struct semantic_verify_proof *proof, + unsigned int refresh_flags, int require_untracked, + int refreshed_before_closure) +{ + struct index_state *istate = s->repo->index; + struct wt_status_token_closure closure = { + .status = s, + .refresh_flags = refresh_flags, + .require_untracked = require_untracked, + .staged_untracked = STRING_LIST_INIT_DUP, + .staged_ignored = STRING_LIST_INIT_DUP, + }; + enum wt_status_token_closure_result result; + + refresh_fsmonitor(istate); + if (!fsmonitor_has_pending_token(istate) || s->pathspec.nr || + fsm_settings__get_mode(s->repo) != FSMONITOR_MODE_IPC) { + int attr_inputs_match = + wt_status_attr_snapshot_matches(s) && + !clean_status_worktree_manifest_needs_refresh(istate); + + wt_status_discard_semantic_verify( + s, &proof, "provider-unavailable"); + if (!refreshed_before_closure && attr_inputs_match && + wt_status_tracked_fsmonitor_state_is_current(s) && + clean_status_index_entries_are_certifiable(istate)) { + s->tracked_from_fsmonitor = 1; + trace2_data_intmax( + "status", s->repo, + "fsmonitor/tracked-clean", 1); + return 0; + } + if (refreshed_before_closure && attr_inputs_match && + s->tracked_from_fsmonitor && + wt_status_tracked_fsmonitor_state_is_current(s)) + return closure.refresh_result; + s->tracked_from_fsmonitor = 0; + if (!refreshed_before_closure && attr_inputs_match) + return refresh_index( + istate, refresh_flags, &s->pathspec, + NULL, NULL); + if (refreshed_before_closure && attr_inputs_match) + return closure.refresh_result; + + wt_status_reset_attr_snapshot_if_changed(s); + if (fsmonitor_has_pending_token(istate)) + fsmonitor_reject_pending_token(istate); + untracked_cache_invalidate_all(istate); + fsmonitor_invalidate_semantics(istate); + closure.refresh_result |= refresh_index( + istate, refresh_flags, &s->pathspec, NULL, NULL); + return closure.refresh_result; + } + + s->tracked_from_fsmonitor = 0; + closure.can_prime = require_untracked && + istate->untracked && + s->show_untracked_files != SHOW_NO_UNTRACKED_FILES && + !s->show_ignored_mode; + closure.use_bulk_provider = + wt_status_can_use_bulk_provider(s, refresh_flags); + closure.untracked_ready = !istate->untracked || + !istate->untracked->root; + closure.untracked_proof_complete = + !require_untracked || !istate->untracked; + if (require_untracked && !closure.can_prime && + !closure.untracked_ready) + BUG("cannot close required untracked scan"); + trace2_region_enter("status", "fsmonitor_token_closure", s->repo); + wt_status_reset_attr_snapshot_if_changed(s); + if (wt_status_refresh_invalidated_manifest(s)) + goto fallback; + + if (proof) { + result = wt_status_close_semantic_fsmonitor_token( + &closure, &proof); + if (result == WT_STATUS_TOKEN_CLOSURE_ACCEPTED) + goto accepted; + if (result == WT_STATUS_TOKEN_CLOSURE_FALLBACK) + goto fallback; + wt_status_reset_attr_snapshot_if_changed(s); + if (wt_status_refresh_invalidated_manifest(s)) + goto fallback; + } + + if (wt_status_close_ordinary_fsmonitor_token( + &closure, refreshed_before_closure)) + goto accepted; + + /* Keep the last valid token and fall back to complete scans. */ +fallback: + wt_status_discard_semantic_verify(s, &proof, "fallback"); + wt_status_discard_staged_untracked(&closure); + preload_index_bulk_result_clear(istate); + fsmonitor_reject_pending_token(istate); + if (fstat_is_reliable()) { + if (closure.can_prime) + untracked_cache_invalidate_all(istate); + fsmonitor_invalidate_semantics(istate); + } + closure.refresh_result |= refresh_index( + istate, refresh_flags, &s->pathspec, NULL, NULL); +accepted: + wt_status_publish_staged_untracked(&closure); + wt_status_discard_staged_untracked(&closure); + trace2_region_leave("status", "fsmonitor_token_closure", s->repo); + return closure.refresh_result; +} + +int wt_status_refresh_index(struct wt_status *s, + unsigned int refresh_flags, + int require_untracked) +{ + struct index_state *istate = s->repo->index; + struct semantic_verify_proof *proof; + int ret; + + wt_status_begin_attr_snapshot(s); + refresh_fsmonitor(istate); + proof = wt_status_prepare_semantic_verify(s, refresh_flags); + ret = wt_status_close_fsmonitor_token( + s, proof, refresh_flags, require_untracked, 0); + if (istate->preload_untracked == &s->untracked) { + s->untracked_from_preload = + istate->preload_untracked_complete; + istate->preload_untracked = NULL; + istate->preload_untracked_complete = 0; + } + return ret; +} + +static void wt_status_release_attr_snapshot(struct wt_status *s) +{ + if (s->attr_source_snapshot) + git_attr_source_snapshot_end(s->attr_source_snapshot); + attr_source_snapshot_free(s->attr_source_snapshot); + s->attr_source_snapshot = NULL; + s->attr_snapshot_failed = 0; +} + +void wt_status_invalidate_refresh(struct wt_status *s) +{ + struct index_state *istate = s->repo->index; + + s->tracked_from_fsmonitor = 0; + if (s->untracked_from_token_closure) { + string_list_clear(&s->untracked, 0); + string_list_clear(&s->ignored, 0); + s->untracked_from_token_closure = 0; + } + wt_status_release_exclude_proof(s); + wt_status_release_attr_snapshot(s); + if (!s->pathspec.nr && !istate->split_index && + fsmonitor_reopen_token(istate)) + return; + if (fsmonitor_has_pending_token(istate)) + fsmonitor_reject_pending_token(istate); + clean_status_invalidate_current_manifest(istate); + untracked_cache_invalidate_all(istate); + fsmonitor_invalidate_semantics(istate); } static int has_unmerged(struct wt_status *s) @@ -864,6 +1817,16 @@ static int has_unmerged(struct wt_status *s) void wt_status_collect(struct wt_status *s) { + int used_untracked_cache; + + if (fsm_settings__get_mode(s->repo) > FSMONITOR_MODE_DISABLED) + wt_status_finish_untracked_cache_preload(s); + wt_status_begin_attr_snapshot(s); + wt_status_close_fsmonitor_token( + s, NULL, REFRESH_QUIET | REFRESH_UNMERGED, + s->show_untracked_files != SHOW_NO_UNTRACKED_FILES && + !s->show_ignored_mode, 1); + trace2_region_enter("status", "worktrees", s->repo); wt_status_collect_changes_worktree(s); trace2_region_leave("status", "worktrees", s->repo); @@ -879,9 +1842,21 @@ void wt_status_collect(struct wt_status *s) } trace2_region_enter("status", "untracked", s->repo); - wt_status_collect_untracked(s); + used_untracked_cache = wt_status_collect_untracked(s); trace2_region_leave("status", "untracked", s->repo); + /* Hook providers have no second query with which to close the scan. */ + if (fsmonitor_has_pending_token(s->repo->index) && !s->pathspec.nr && + fsm_settings__get_mode(s->repo) == FSMONITOR_MODE_HOOK && + (used_untracked_cache || !s->repo->index->untracked || + !s->repo->index->untracked->root)) { + if (fsmonitor_pending_token_from_provider(s->repo->index)) + fsmonitor_accept_pending_token( + s->repo->index, 1, used_untracked_cache); + else + fsmonitor_reject_pending_token(s->repo->index); + } + wt_status_get_state(s->repo, &s->state, s->branch && !strcmp(s->branch, "HEAD")); if (s->state.merge_in_progress && !has_unmerged(s)) s->committable = 1; @@ -889,6 +1864,10 @@ void wt_status_collect(struct wt_status *s) void wt_status_collect_free_buffers(struct wt_status *s) { + untracked_cache_preload_release(s->untracked_cache_preload); + s->untracked_cache_preload = NULL; + wt_status_release_exclude_proof(s); + wt_status_release_attr_snapshot(s); wt_status_state_free_buffers(&s->state); } diff --git a/wt-status.h b/wt-status.h index e9fe32e98cc18c..6f5300fe8e5481 100644 --- a/wt-status.h +++ b/wt-status.h @@ -7,7 +7,12 @@ #include "remote.h" struct repository; +struct stat; +struct attr_source_snapshot; +struct exclude_source_proof; +struct wt_status_exclude_context; struct worktree; +struct untracked_cache_preload; enum color_wt_status { WT_STATUS_HEADER = 0, @@ -138,6 +143,12 @@ struct wt_status { /* These are computed during processing of the individual sections */ int committable; int workdir_dirty; + unsigned allow_clean_status_shortcuts : 1; + unsigned certify_clean_status : 1; + unsigned tracked_from_fsmonitor : 1; + unsigned untracked_from_token_closure : 1; + unsigned untracked_from_preload : 1; + unsigned bulk_update_index_stat : 1; const char *index_file; FILE *fp; const char *prefix; @@ -145,6 +156,15 @@ struct wt_status { struct string_list untracked; struct string_list ignored; uint32_t untracked_in_ms; + struct untracked_cache_preload *untracked_cache_preload; + struct attr_source_snapshot *attr_source_snapshot; + struct exclude_source_proof *certify_exclude_proof; + struct wt_status_exclude_context *certify_exclude_context; + struct object_id certify_exclude_digest; + unsigned untracked_cache_preloaded : 1; + unsigned attr_snapshot_failed : 1; + unsigned certify_exclude_digest_valid : 1; + unsigned certify_untracked_scan_failed : 1; }; size_t wt_status_locate_end(const char *s, size_t len); @@ -153,6 +173,18 @@ void wt_status_add_cut_line(struct wt_status *s); void wt_status_prepare(struct repository *r, struct wt_status *s); void wt_status_print(struct wt_status *s); void wt_status_collect(struct wt_status *s); +void wt_status_start_untracked_cache_preload(struct wt_status *s); +/* + * Refresh tracked entries and close any provider token. When requested, + * also close a complete untracked-cache scan before accepting that token. + */ +int wt_status_refresh_index(struct wt_status *s, + unsigned int refresh_flags, + int require_untracked); +void wt_status_invalidate_refresh(struct wt_status *s); +int wt_status_certified_excludes_digest( + struct wt_status *s, struct object_id *digest, + struct stat *scanned_worktree); /* * Collect all changes between the two trees. Changes will be displayed as if