diff --git a/.github/workflows/cmake-binaries.yaml b/.github/workflows/cmake-binaries.yaml index 0285e67..f9155cc 100644 --- a/.github/workflows/cmake-binaries.yaml +++ b/.github/workflows/cmake-binaries.yaml @@ -6,25 +6,15 @@ on: branches: ["develop"] paths: ["**.c", "**.h"] -# Make sure to build each platform binary one-at-a-time. concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true env: - REPO: "dangduc/fzf-native" - SOURCE_BRANCH: ${{ github.ref }} - TARGET_BRANCH: ${{ github.ref }} - # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) BUILD_TYPE: Release jobs: build: - # The CMake configure and build commands are platform agnostic and - # should work equally well on Windows or Mac. You can convert - # this to a matrix build if you need cross-platform coverage. - # See: - # https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -32,26 +22,81 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] steps: - - uses: actions/checkout@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@v4 - - name: Pull branch changes - run: | - git pull + - name: Clean stale binaries + shell: bash + run: rm -rf bin - name: Configure CMake - # Configure CMake in a 'build' subdirectory. run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} - name: Build - # Build your program with the given configuration run: cmake --build ${{github.workspace}}/build --verbose --config ${{env.BUILD_TYPE}} + - name: Upload binaries + uses: actions/upload-artifact@v4 + with: + name: bin-${{ matrix.os }} + path: bin/ + if-no-files-found: error + retention-days: 1 + + build-freebsd: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Clean stale binaries + run: rm -rf bin + + - name: Build in FreeBSD VM + uses: vmactions/freebsd-vm@v1 + with: + usesh: true + prepare: | + pkg install -y cmake + run: | + cmake -B build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} + cmake --build build --verbose --config ${{env.BUILD_TYPE}} + + - name: Upload binaries + uses: actions/upload-artifact@v4 + with: + name: bin-freebsd + path: bin/ + if-no-files-found: error + retention-days: 1 + + commit: + needs: [build, build-freebsd] + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Download all binaries + uses: actions/download-artifact@v4 + with: + path: bin-artifacts + pattern: bin-* + merge-multiple: true + + - name: Stage binaries + run: | + mkdir -p bin + cp -R bin-artifacts/. bin/ + rm -rf bin-artifacts + - name: Commit changes uses: EndBug/add-and-commit@v9 with: author_name: github-actions author_email: github-actions@github.com - message: Update binary ${{ matrix.os }} - add: "bin/**" \ No newline at end of file + message: Update binaries for all platforms + add: "bin/**" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d93f2da..4584f39 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -40,4 +40,13 @@ jobs: - run: make lint - run: make ctest if: runner.os != 'Windows' + + - name: Clean stale binaries + shell: bash + run: rm -rf bin + - name: Configure CMake + run: cmake -B build -DCMAKE_BUILD_TYPE=Release + - name: Build native module + run: cmake --build build --config Release + - run: make test \ No newline at end of file diff --git a/.gitignore b/.gitignore index 55e1ecf..298acbd 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ build /.eask/ /dist/ fzf-native.log +fzf-native-pkg.el diff --git a/Makefile b/Makefile index 77f7ddf..6819377 100644 --- a/Makefile +++ b/Makefile @@ -78,7 +78,7 @@ emacs-asan: .PHONY: ctest ctest: mkdir -p $(BUILD_DIR) - $(CC) -std=c11 -Wall -Wextra -O2 -I. -pthread \ + $(CC) -std=gnu11 -Wall -Wextra -O2 -I. -pthread \ -o $(BUILD_DIR)/fzf-native-ctest fzf-native-ctest.c fzf.c $(BUILD_DIR)/fzf-native-ctest diff --git a/README.org b/README.org index 8fd7416..141b7f1 100644 --- a/README.org +++ b/README.org @@ -59,6 +59,16 @@ Clone / download this repository and modify your ~load-path~: (add-to-list 'load-path (expand-file-name "/path/to/fzf-native/" user-emacs-directory)) #+end_src +*** use-package with :vc + +#+begin_src emacs-lisp +; Configuration that uses pre-built dynamic module. +(use-package fzf-native + :vc (:url "https://github.com/dangduc/fzf-native" :rev :newest) + :config + (fzf-native-load-dyn)) +#+end_src + *** Straight Examples #+begin_src emacs-lisp @@ -85,17 +95,6 @@ Clone / download this repository and modify your ~load-path~: (fzf-native-load-dyn)) #+end_src -*** Multibyte character support - -Work around the lib's lack of support for multibyte chars. Add this -advice if you want accurate indices for multibyte chars. Don't add -this advice if you want better run time performance or you don't need -accurate indices for multibyte chars. - -#+begin_src emacs-lisp -(advice-add 'fzf-native-score :around #'fzf-native--fix-score-indices) -#+end_src - ** Use Cases [[https://github.com/jojojames/fussy][Fussy]]: ~fzf-native~ is used as @@ -111,6 +110,31 @@ stream candidates from long-running shell commands (~find~, ~rg~, [[https://github.com/jojojames/fzf-async/blob/HEAD/architecture.org][fzf-async's architecture overview]] for the Elisp-side pipeline. +** Customization + +The C module reads the following defcustoms via ~symbol-value~ at call +time. Higher-level packages (~fussy~, ~fzf-async~) keep their own +user-facing defcustoms and bridge their values onto these canonical +names — ~fussy~ via ~setq-local~ (synchronous, same-buffer call +pattern), ~fzf-async~ via ~:around~ advice on the C entry points +(timer-driven, cross-buffer). Set these directly only if you call +~fzf-native-score~ / ~fzf-native-score-all~ / ~fzf-native-async-*~ +yourself; otherwise prefer the package-level knob. + +| Variable | Default | Read by | Behavior | +|---------------------------------+---------+------------------------------------+---------------------------------------------| +| ~fzf-native-case-mode~ | ~smart~ | every scoring call (sync + async) | ~smart~ (lowercase = ignore, mixed = respect) / ~ignore~ / ~respect~ | +| ~fzf-native-batch-highlight~ | 25 | ~fzf-native-score~ / ~fzf-native-score-all~ | nil = off; positive N = apply ~completions-common-part~ face to top N candidates via ~fzf_get_positions~ in C. | +| ~fzf-native-async-highlight~ | 200 | ~fzf-native-async-candidates~ | nil = off; t = all returned candidates; positive N = top N. | +| ~fzf-native-max-line-length~ | 256 | ~fzf-native-async-start~ (once at session start) | nil = no limit; +N = drop lines longer than N; -N = truncate lines to N characters. | +| ~fzf-native-async-cache-size~ | 40 | ~fzf-native-async-start~ (once at session start) | LRU result-cache entries. Each entry stores top-K results plus the full matched-candidate index for one query, enabling exact-fresh hits and prefix-refinement hits. | + +Highlighting runs entirely inside the C module: after scoring, +~fzf_get_positions~ produces matched byte offsets, which are merged +into runs and applied to the candidate string with ~put-text-property~ +before strings are handed back to Emacs. No Elisp regex pass is +involved. + ** Building the Native Libraries #+begin_src bash diff --git a/architecture.org b/architecture.org index c6958c0..34e693e 100644 --- a/architecture.org +++ b/architecture.org @@ -47,7 +47,7 @@ process and must never block the main thread. | Function | Args | Returns | Path | |--------------------------------+--------------------------------------+------------------------+-------------| -| =fzf-native-score= | =(str query &optional slab)= | =(score idx idx ...)= | sync single | +| =fzf-native-score= | =(str query &optional slab)= | =(score)= | sync single | | =fzf-native-score-all= | =(collection query &optional slab)= | sorted candidate vector | sync batch | | =fzf-native-make-default-slab= | =()= | slab | (helper) | | =fzf-native-make-slab= | =(size16 size32)= | slab | (helper) | @@ -133,17 +133,22 @@ the fzf-native backend. Elisp call: (fzf-native-score "foobar" "fb" slab) │ ▼ - ┌──────────────────────────────────┐ - │ C: fzf_native_score │ - │ - copy_emacs_string("foobar") │ - │ - fzf_parse_pattern("fb") │ - │ - fzf_get_score(text, pat, slab)│ - │ - free pattern │ - │ - build (score idx idx ...) cons│ - └──────────────────────────────────┘ + ┌──────────────────────────────────────────┐ + │ C: fzf_native_score │ + │ - copy_emacs_string("foobar") │ + │ - fzf_parse_pattern("fb") │ + │ - fzf_get_score(text, pat, slab) │ + │ - if fussy-fzf-native-highlight non-nil │ + │ and score > 0: │ + │ - fzf_get_positions │ + │ - put-text-property on args[0] with │ + │ face=completions-common-part │ + │ - free pattern │ + │ - return (score) │ + └──────────────────────────────────────────┘ │ ▼ - Elisp value: (score idx0 idx1 ...) + Elisp value: (score) ; match indices not surfaced #+end_src ** Notes @@ -154,6 +159,12 @@ the fzf-native backend. avoid that cost. - Non-UTF-8 Emacs strings are coerced via =copy_emacs_string=, falling back to =encode-coding-string= if =make_string= would signal. +- Highlighting is applied in C, not Elisp. The C module reads + =fussy-fzf-native-highlight= via =symbol-value= on every call; any + non-nil value enables highlighting (the cap concept used by + =fzf_native_score_all= does not apply to a single candidate). The + return value is =(score)= — match indices are no longer surfaced + to Elisp callers. * Path 2 — sync batch (=fzf_native_score_all=) @@ -287,6 +298,10 @@ ready." │ score_count │ │ last_filtered │ │ last_total │ + ├─────────────────┤ + │ cache │ ← LRU result cache, own mutex + │ head, tail │ + │ count, max │ └─────────────────┘ #+end_src @@ -391,6 +406,84 @@ are still pulled. which serves as the "has any data arrived yet" signal but is not the cross-thread publish signal.) +** Result cache + +A per-session LRU cache sits between the dispatch path and the scoring +thread. Configured via the =fzf-async-cache-size= defcustom (read +once at session start; default 40 entries). + +*Data model*: + +#+begin_src + CacheEntry { + query : strdup'd filter string ← owned + pool_gen : s->count at time of scoring + top[K] : copy of top-K results published ← owned + m_idx : SharedIdx*, refcounted ← shared + parsed : fzf_pattern_t* (parsed query) ← owned + } + + Cache { + head, tail : doubly-linked list (MRU at head) + count, max + mu : mutex + } +#+end_src + +=SharedIdx= is a refcounted flexible-array struct of =uint32_t= +holding the *full set* of matched candidate indices for one scoring +run (not just the top-K). Lookup consumers retain in O(1) under the +cache mutex (atomic refcount bump — no memcpy); the last consumer +frees it. + +*Dispatch flow* (=fzf_native_async_candidates=, main thread): + +| Lookup result | Display | Scoring scheduled | +|----------------------+----------------------+-----------------------------------| +| Exact, pool_gen==now | Cached top-K | None (no work) | +| Exact, pool_gen1 term) and every term-set in Q' has an equivalent term-set + in Q. Captures: adding an AND term in non-prefix position + (=fo= → =x fo=), term reordering (=foo bar= → =bar foo=), + non-prefix negation (=fo= → =!x fo=). + +OR queries (containing =|=) are excluded from prefix lookups in both +directions — adding an OR alternate widens results unpredictably, so +prefix refinement is unsound. + +When multiple cached entries subsume Q, the one with the most +term-sets wins (= most constraints = smallest match set = fastest +refinement scan), with byte-length as a tiebreaker. + +*Refinement scoring*: When the scoring thread receives a refine +request, it builds the snap from =m_idx= ∪ =s->cands[delta_from..count]= +instead of the full =s->cands[]=. Typical match sets after 2-3 chars +are <1% of pool size, so refine scans drop by 100-1000× vs full +scans on large pools. + +*Critical-section discipline*: =cache_insert= pre-allocates everything +(strdup, malloc, parse_query) *before* taking the cache mutex. The +critical section is just pointer swaps + LRU list manipulation. +Evicted entries are freed after unlock. Lookups bump the SharedIdx +refcount under the mutex, then release it; the consumer holds the +=SharedIdx*= with no further locking. + +*fzf semantics gotcha*: within a term-set = OR (terms are +alternatives); across term-sets = AND. So =foo bar= parses as 2 sets +× 1 term (AND-of-two), and =foo | bar= parses as 1 set × 2 terms +(OR-of-two). The cache rejects any term-set with >1 term as an OR +query. + ** Memory and string ownership | Object | Owner | Lifetime | diff --git a/bin/Darwin/arm64/fzf-native-module.so b/bin/Darwin/arm64/fzf-native-module.so index 4d22e03..532f725 100755 Binary files a/bin/Darwin/arm64/fzf-native-module.so and b/bin/Darwin/arm64/fzf-native-module.so differ diff --git a/bin/FreeBSD/fzf-native-module.so b/bin/FreeBSD/fzf-native-module.so index 4a37ae0..a4dae77 100755 Binary files a/bin/FreeBSD/fzf-native-module.so and b/bin/FreeBSD/fzf-native-module.so differ diff --git a/bin/Linux/fzf-native-module.so b/bin/Linux/fzf-native-module.so index 1343630..7584d52 100755 Binary files a/bin/Linux/fzf-native-module.so and b/bin/Linux/fzf-native-module.so differ diff --git a/bin/Windows/Release/fzf-native-module.dll b/bin/Windows/Release/fzf-native-module.dll index 16a4486..42e91b5 100644 Binary files a/bin/Windows/Release/fzf-native-module.dll and b/bin/Windows/Release/fzf-native-module.dll differ diff --git a/bin/Windows/Release/fzf-native-module.exp b/bin/Windows/Release/fzf-native-module.exp index 2bdd6c9..ebec96b 100644 Binary files a/bin/Windows/Release/fzf-native-module.exp and b/bin/Windows/Release/fzf-native-module.exp differ diff --git a/bin/Windows/Release/fzf-native-module.lib b/bin/Windows/Release/fzf-native-module.lib index 8e1913b..5533696 100644 Binary files a/bin/Windows/Release/fzf-native-module.lib and b/bin/Windows/Release/fzf-native-module.lib differ diff --git a/fzf-native-ctest.c b/fzf-native-ctest.c index 3983b8c..6c1bb26 100644 --- a/fzf-native-ctest.c +++ b/fzf-native-ctest.c @@ -262,24 +262,33 @@ static void test_strip_ansi_bare_esc(void) { * async_reader (pipe-based; no Emacs runtime needed) * ===================================================================== */ +/* The `cap` parameter is ignored under the chunked-storage design — the + reader allocates blocks lazily. Kept in the signature so existing test + sites remain readable without rewrites. */ static AsyncSession *make_async_session(FILE *fp, size_t cap) { + (void)cap; AsyncSession *s = calloc(1, sizeof *s); if (!s) return NULL; - s->fp = fp; - s->cap = cap; - s->cands = calloc(cap, sizeof *s->cands); - if (!s->cands) { free(s); return NULL; } + s->fp = fp; pthread_mutex_init(&s->mu, NULL); return s; } static void free_async_session(AsyncSession *s) { arena_free(&s->arena); - free(s->cands); + for (size_t k = 0; k < CANDS_TOP_CAP; k++) + if (s->cands_top[k]) free(s->cands_top[k]); pthread_mutex_destroy(&s->mu); free(s); } +/* Convenience accessor: read s->cands_top[i >> SHIFT][i & MASK]. + Returns NULL if the block isn't allocated (which would be a bug). */ +static const char *cands_at(AsyncSession *s, size_t i) { + char **block = s->cands_top[i >> CANDS_BLOCK_SHIFT]; + return block ? block[i & CANDS_BLOCK_MASK] : NULL; +} + static void test_async_reader_basic(void) { int pfd[2]; CHECK(pipe(pfd) == 0); @@ -296,9 +305,9 @@ static void test_async_reader_basic(void) { async_reader((void *)s); CHECK(s->count == 3); - CHECK(strcmp(s->cands[0], "alpha") == 0); - CHECK(strcmp(s->cands[1], "beta") == 0); - CHECK(strcmp(s->cands[2], "gamma") == 0); + CHECK(strcmp(cands_at(s, 0), "alpha") == 0); + CHECK(strcmp(cands_at(s, 1), "beta") == 0); + CHECK(strcmp(cands_at(s, 2), "gamma") == 0); free_async_session(s); } @@ -319,13 +328,15 @@ static void test_async_reader_ansi_stripping(void) { async_reader((void *)s); CHECK(s->count == 2); - CHECK(strcmp(s->cands[0], "file.txt") == 0); - CHECK(strcmp(s->cands[1], "plain.c") == 0); + CHECK(strcmp(cands_at(s, 0), "file.txt") == 0); + CHECK(strcmp(cands_at(s, 1), "plain.c") == 0); free_async_session(s); } -static void test_async_reader_buffer_growth(void) { - /* Initial cap=4, write 32 lines — exercises the realloc doubling path. */ +static void test_async_reader_many_lines(void) { + /* Write 32 lines. Under the chunked-storage design no realloc is + involved; this just verifies sequential round-trip through the + accessor. */ enum { NLINES = 32 }; int pfd[2]; CHECK(pipe(pfd) == 0); @@ -345,11 +356,447 @@ static void test_async_reader_buffer_growth(void) { char expected[32]; for (int i = 0; i < NLINES; i++) { snprintf(expected, sizeof expected, "line%d", i); - CHECK(strcmp(s->cands[i], expected) == 0); + CHECK(strcmp(cands_at(s, i), expected) == 0); } + /* All 32 fit in block 0; later blocks must be untouched. */ + CHECK(s->cands_top[0] != NULL); + CHECK(s->cands_top[1] == NULL); free_async_session(s); } +/* ===================================================================== + * Chunked candidate storage — index split formula and accessor + * ===================================================================== */ + +static void test_cands_top_index_split(void) { + /* Verify hi = i >> SHIFT and lo = i & MASK match the documented + "i = hi * BLOCK_SIZE + lo" decomposition. */ + size_t cases[][3] = { + /* { i, expected_hi, expected_lo } */ + { 0, 0, 0 }, + { 1, 0, 1 }, + { CANDS_BLOCK_SIZE - 1, 0, CANDS_BLOCK_SIZE - 1 }, + { CANDS_BLOCK_SIZE , 1, 0 }, + { CANDS_BLOCK_SIZE + 5, 1, 5 }, + { CANDS_BLOCK_SIZE * 2 , 2, 0 }, + { CANDS_BLOCK_SIZE * 2 + 7, 2, 7 }, + { CANDS_BLOCK_SIZE * 100 , 100, 0 }, + }; + for (size_t k = 0; k < sizeof cases / sizeof *cases; k++) { + size_t i = cases[k][0]; + CHECK((i >> CANDS_BLOCK_SHIFT) == cases[k][1]); + CHECK((i & CANDS_BLOCK_MASK) == cases[k][2]); + /* Inverse: reconstruct i from (hi, lo). */ + CHECK((cases[k][1] << CANDS_BLOCK_SHIFT) + cases[k][2] == i); + } +} + +static void test_cands_top_accessor_reads_block_pointer(void) { + /* Manually populate a single slot via the accessor formula and + verify the read path returns the same pointer. */ + AsyncSession *s = calloc(1, sizeof *s); + CHECK(s != NULL); + pthread_mutex_init(&s->mu, NULL); + + /* Allocate block 3 and write a sentinel pointer at slot 42. */ + size_t hi = 3, lo = 42; + s->cands_top[hi] = calloc(CANDS_BLOCK_SIZE, sizeof *s->cands_top[hi]); + CHECK(s->cands_top[hi] != NULL); + char *sentinel = "sentinel"; + s->cands_top[hi][lo] = sentinel; + + /* Read it back via the accessor formula at the equivalent global index. */ + size_t i = (hi << CANDS_BLOCK_SHIFT) + lo; + CHECK(s->cands_top[i >> CANDS_BLOCK_SHIFT][i & CANDS_BLOCK_MASK] == sentinel); + + free(s->cands_top[hi]); + pthread_mutex_destroy(&s->mu); + free(s); +} + +/* ===================================================================== + * Result cache — phase 1: exact-match lookup, LRU eviction, MRU touch + * ===================================================================== */ + +static ScoredStr make_top(const char *str, int score) { + ScoredStr s = {0}; + s.str = (char *)str; /* not freed by the cache (cache strdups internally) */ + s.score = score; + return s; +} + +static void test_cache_lookup_miss_on_empty(void) { + Cache c; + cache_init(&c, 20); + ScoredStr *out_top = NULL; + SharedIdx *out_sidx = NULL; + size_t out_count = 0, out_gen = 0; + CHECK(cache_lookup_exact(&c, "foo", &out_top, &out_count, + &out_sidx, &out_gen) == false); + CHECK(out_top == NULL); + cache_free(&c); +} + +static void test_cache_insert_then_lookup_hit(void) { + Cache c; + cache_init(&c, 20); + ScoredStr top[2] = { make_top("alpha", 42), make_top("beta", 17) }; + + cache_insert(&c, "fo", 1000, CaseSmart, top, 2, NULL, 0); + + ScoredStr *out = NULL; + SharedIdx *out_sidx = NULL; + size_t out_count = 0, out_gen = 0; + CHECK(cache_lookup_exact(&c, "fo", &out, &out_count, &out_sidx, &out_gen) == true); + CHECK(out_count == 2); + CHECK(out_gen == 1000); + CHECK(out != NULL); + CHECK(out[0].score == 42); + CHECK(strcmp(out[0].str, "alpha") == 0); + CHECK(out[1].score == 17); + CHECK(strcmp(out[1].str, "beta") == 0); + CHECK(out_sidx == NULL); /* no matched_idx supplied */ + free(out); + cache_free(&c); +} + +static void test_cache_lookup_miss_distinct_query(void) { + Cache c; + cache_init(&c, 20); + ScoredStr top[1] = { make_top("alpha", 42) }; + cache_insert(&c, "fo", 100, CaseSmart, top, 1, NULL, 0); + + ScoredStr *out = NULL; + SharedIdx *out_sidx = NULL; + size_t out_count = 0, out_gen = 0; + CHECK(cache_lookup_exact(&c, "bar", &out, &out_count, &out_sidx, &out_gen) == false); + CHECK(out == NULL); + cache_free(&c); +} + +static void test_cache_insert_updates_in_place(void) { + /* Re-inserting the same query overwrites the existing entry rather + than creating a duplicate. Verify count stays at 1 and the new + data wins. */ + Cache c; + cache_init(&c, 20); + ScoredStr v1[1] = { make_top("alpha", 10) }; + ScoredStr v2[2] = { make_top("alpha", 99), make_top("beta", 50) }; + + cache_insert(&c, "fo", 100, CaseSmart, v1, 1, NULL, 0); + cache_insert(&c, "fo", 200, CaseSmart, v2, 2, NULL, 0); + CHECK(c.count == 1); + + ScoredStr *out = NULL; + SharedIdx *out_sidx = NULL; + size_t out_count = 0, out_gen = 0; + CHECK(cache_lookup_exact(&c, "fo", &out, &out_count, &out_sidx, &out_gen) == true); + CHECK(out_count == 2); + CHECK(out_gen == 200); + CHECK(out[0].score == 99); + CHECK(out[1].score == 50); + free(out); + cache_free(&c); +} + +static void test_cache_lru_eviction_at_capacity(void) { + /* Fill the cache, insert one more, verify the oldest entry is gone + and all others remain. */ + const size_t MAX = 8; + Cache c; + cache_init(&c, MAX); + ScoredStr one[1] = { make_top("x", 1) }; + + char qbuf[16]; + for (size_t i = 0; i < MAX; i++) { + snprintf(qbuf, sizeof qbuf, "q%zu", i); + cache_insert(&c, qbuf, (size_t)i, CaseSmart, one, 1, NULL, 0); + } + CHECK(c.count == MAX); + + /* Insert one more — should evict q0 (the LRU tail). */ + cache_insert(&c, "extra", 999, CaseSmart, one, 1, NULL, 0); + CHECK(c.count == MAX); + + ScoredStr *out = NULL; + SharedIdx *out_sidx = NULL; + size_t out_count = 0, out_gen = 0; + + /* q0 is gone. */ + CHECK(cache_lookup_exact(&c, "q0", &out, &out_count, &out_sidx, &out_gen) == false); + + /* q1 .. q(MAX-1) are still present. */ + for (size_t i = 1; i < MAX; i++) { + snprintf(qbuf, sizeof qbuf, "q%zu", i); + out = NULL; out_sidx = NULL; out_count = 0; out_gen = 0; + CHECK(cache_lookup_exact(&c, qbuf, &out, &out_count, &out_sidx, &out_gen) == true); + free(out); + } + + /* And the freshly inserted "extra" is present. */ + out = NULL; out_sidx = NULL; out_count = 0; out_gen = 0; + CHECK(cache_lookup_exact(&c, "extra", &out, &out_count, &out_sidx, &out_gen) == true); + CHECK(out_gen == 999); + free(out); + + cache_free(&c); +} + +static void test_cache_touch_on_hit(void) { + /* Fill the cache; touch q0 (the oldest) so it becomes MRU; insert + one more; verify q0 survived and q1 (now the LRU) was evicted. */ + const size_t MAX = 4; + Cache c; + cache_init(&c, MAX); + ScoredStr one[1] = { make_top("x", 1) }; + + char qbuf[16]; + for (size_t i = 0; i < MAX; i++) { + snprintf(qbuf, sizeof qbuf, "q%zu", i); + cache_insert(&c, qbuf, (size_t)i, CaseSmart, one, 1, NULL, 0); + } + + /* Touch q0 — moves it to head (MRU). */ + ScoredStr *out = NULL; + SharedIdx *out_sidx = NULL; + size_t out_count = 0, out_gen = 0; + CHECK(cache_lookup_exact(&c, "q0", &out, &out_count, &out_sidx, &out_gen) == true); + free(out); + + /* Now the LRU tail is q1. Insert one more; q1 should be evicted. */ + cache_insert(&c, "extra", 999, CaseSmart, one, 1, NULL, 0); + + out = NULL; out_sidx = NULL; out_count = 0; out_gen = 0; + CHECK(cache_lookup_exact(&c, "q0", &out, &out_count, &out_sidx, &out_gen) == true); + free(out); + + out = NULL; out_sidx = NULL; out_count = 0; out_gen = 0; + CHECK(cache_lookup_exact(&c, "q1", &out, &out_count, &out_sidx, &out_gen) == false); + + cache_free(&c); +} + +static void test_cache_insert_zero_count(void) { + /* Empty top[] is a legitimate "no matches" cache entry; verify it + stores and looks up cleanly. */ + Cache c; + cache_init(&c, 20); + cache_insert(&c, "nothing", 500, CaseSmart, NULL, 0, NULL, 0); + + ScoredStr *out = (ScoredStr *)0xdeadbeef; + SharedIdx *out_sidx = NULL; + size_t out_count = 99, out_gen = 0; + CHECK(cache_lookup_exact(&c, "nothing", &out, &out_count, &out_sidx, &out_gen) == true); + CHECK(out == NULL); + CHECK(out_count == 0); + CHECK(out_gen == 500); + cache_free(&c); +} + +static void test_cache_pool_gen_distinguishes_stale(void) { + /* The cache itself doesn't decide fresh-vs-stale — that's the dispatch + layer's job — but it must faithfully report pool_gen so the dispatch + can compare it against the current pool size. */ + Cache c; + cache_init(&c, 20); + ScoredStr top[1] = { make_top("alpha", 1) }; + cache_insert(&c, "fo", 100, CaseSmart, top, 1, NULL, 0); + + ScoredStr *out = NULL; + SharedIdx *out_sidx = NULL; + size_t out_count = 0, out_gen = 0; + CHECK(cache_lookup_exact(&c, "fo", &out, &out_count, &out_sidx, &out_gen) == true); + CHECK(out_gen == 100); + free(out); + + /* Re-insert at a new pool_gen; lookup should reflect the latest. */ + cache_insert(&c, "fo", 5000, CaseSmart, top, 1, NULL, 0); + out = NULL; out_count = 0; out_gen = 0; + CHECK(cache_lookup_exact(&c, "fo", &out, &out_count, &out_sidx, &out_gen) == true); + CHECK(out_gen == 5000); + free(out); + cache_free(&c); +} + +/* ===================================================================== + * Result cache — phase 2: term-set subsumption + prefix lookup + * ===================================================================== */ + +static void test_subsumes_pattern_extending_term_via_byte_prefix(void) { + /* "fo" → "foo": same single-term query getting longer. Term-set rule + alone says NO (terms "fo" and "foo" aren't equivalent), but + cache_lookup_prefix uses byte-prefix OR term-set, so this still + captures via the byte-prefix path. Verify the byte-prefix subsumes() + directly. */ + CHECK(subsumes("fo", "foo") == true); +} + +static void test_subsumes_pattern_adding_term_at_end(void) { + /* "fo" → "fo bar": both rules agree. Verify term-set path. */ + fzf_pattern_t *p1 = parse_query_for_cache("fo", CaseSmart); + fzf_pattern_t *p2 = parse_query_for_cache("fo bar", CaseSmart); + CHECK(p1 && p2); + CHECK(subsumes_pattern(p1, p2) == true); + CHECK(subsumes_pattern(p2, p1) == false); + fzf_free_pattern(p1); + fzf_free_pattern(p2); +} + +static void test_subsumes_pattern_adding_term_at_start(void) { + /* "fo" → "x fo": v2-only case. Byte-prefix says NO (fo not prefix of + x fo), term-set says YES (fo's terms ⊆ x fo's terms). */ + fzf_pattern_t *p1 = parse_query_for_cache("fo", CaseSmart); + fzf_pattern_t *p2 = parse_query_for_cache("x fo", CaseSmart); + CHECK(p1 && p2); + CHECK(subsumes("fo", "x fo") == false); /* v1 misses */ + CHECK(subsumes_pattern(p1, p2) == true); /* v2 catches */ + CHECK(subsumes_pattern(p2, p1) == false); + fzf_free_pattern(p1); + fzf_free_pattern(p2); +} + +static void test_subsumes_pattern_term_reorder(void) { + /* "foo bar" and "bar foo" are semantically equivalent in fzf — same + term set, different textual order. Term-set rule sees mutual + subsumption; byte-prefix rule sees neither. */ + fzf_pattern_t *p1 = parse_query_for_cache("foo bar", CaseSmart); + fzf_pattern_t *p2 = parse_query_for_cache("bar foo", CaseSmart); + CHECK(p1 && p2); + CHECK(subsumes("foo bar", "bar foo") == false); + CHECK(subsumes("bar foo", "foo bar") == false); + CHECK(subsumes_pattern(p1, p2) == true); + CHECK(subsumes_pattern(p2, p1) == true); + fzf_free_pattern(p1); + fzf_free_pattern(p2); +} + +static void test_subsumes_pattern_negation_at_start(void) { + /* "fo" → "!x fo": adding a negation term in non-prefix position. + Term-set rule catches it; byte-prefix doesn't. */ + fzf_pattern_t *p1 = parse_query_for_cache("fo", CaseSmart); + fzf_pattern_t *p2 = parse_query_for_cache("!x fo", CaseSmart); + CHECK(p1 && p2); + CHECK(subsumes_pattern(p1, p2) == true); + fzf_free_pattern(p1); + fzf_free_pattern(p2); +} + +static void test_subsumes_pattern_or_query_rejected(void) { + /* "fo | bar" parses as ONE term-set with TWO terms (within a set = + OR; across sets = AND). subsumes_pattern rejects any term-set with + >1 term — it can never serve as a refinement source. */ + fzf_pattern_t *p1 = parse_query_for_cache("fo", CaseSmart); + fzf_pattern_t *p2 = parse_query_for_cache("fo | bar", CaseSmart); + CHECK(p1 && p2); + CHECK(p1->size == 1 && p1->ptr[0]->size == 1); /* "fo": 1 set, 1 term */ + CHECK(p2->size == 1 && p2->ptr[0]->size == 2); /* "fo|bar": 1 set, 2 terms */ + CHECK(subsumes_pattern(p1, p2) == false); + CHECK(subsumes_pattern(p2, p1) == false); + fzf_free_pattern(p1); + fzf_free_pattern(p2); +} + +static void test_subsumes_pattern_distinct_terms(void) { + /* "foo" and "bar" share no terms; neither subsumes the other. */ + fzf_pattern_t *p1 = parse_query_for_cache("foo", CaseSmart); + fzf_pattern_t *p2 = parse_query_for_cache("bar", CaseSmart); + CHECK(p1 && p2); + CHECK(subsumes_pattern(p1, p2) == false); + CHECK(subsumes_pattern(p2, p1) == false); + fzf_free_pattern(p1); + fzf_free_pattern(p2); +} + +/* Helper: insert a cache entry that has a non-NULL m_idx (so it's eligible + as a prefix-refinement source) using a single dummy match index. Tests + the lookup logic without caring about the actual indices. */ +static void cache_insert_eligible(Cache *c, const char *query, size_t pool_gen) { + uint32_t idx[1] = { 0 }; + cache_insert(c, query, pool_gen, CaseSmart, NULL, 0, idx, 1); +} + +static void test_cache_lookup_prefix_v2_finds_term_subset(void) { + /* Cache has "fo". New query "x fo" should hit via term-set rule. */ + Cache c; + cache_init(&c, 20); + cache_insert_eligible(&c, "fo", 100); + + ScoredStr *out = NULL; + SharedIdx *out_sidx = NULL; + size_t out_count = 0, out_gen = 0; + CHECK(cache_lookup_prefix(&c, "x fo", CaseSmart, &out, &out_count, &out_sidx, &out_gen) == true); + CHECK(out_gen == 100); + shared_idx_release(out_sidx); + free(out); + cache_free(&c); +} + +static void test_cache_lookup_prefix_v2_finds_reordered(void) { + /* Cache has "foo bar". New query "bar foo" should hit via term-set + mutual subsumption. We exclude exact-match entries from prefix + lookup, but "bar foo" != "foo bar" textually so it counts as + non-exact and the term-set rule picks it up. */ + Cache c; + cache_init(&c, 20); + cache_insert_eligible(&c, "foo bar", 100); + + ScoredStr *out = NULL; + SharedIdx *out_sidx = NULL; + size_t out_count = 0, out_gen = 0; + CHECK(cache_lookup_prefix(&c, "bar foo", CaseSmart, &out, &out_count, &out_sidx, &out_gen) == true); + CHECK(out_gen == 100); + shared_idx_release(out_sidx); + free(out); + cache_free(&c); +} + +static void test_cache_lookup_prefix_picks_most_terms(void) { + /* Cache has "fo" (1 term) and "fo bar" (2 terms). New query + "fo bar baz" subsumes both. cache_lookup_prefix should prefer the + most-restricted entry — "fo bar" with 2 terms — over "fo". */ + Cache c; + cache_init(&c, 20); + cache_insert_eligible(&c, "fo", 100); + cache_insert_eligible(&c, "fo bar", 200); + + ScoredStr *out = NULL; + SharedIdx *out_sidx = NULL; + size_t out_count = 0, out_gen = 0; + CHECK(cache_lookup_prefix(&c, "fo bar baz", CaseSmart, &out, &out_count, &out_sidx, &out_gen) == true); + CHECK(out_gen == 200); /* "fo bar" entry wins */ + shared_idx_release(out_sidx); + free(out); + cache_free(&c); +} + +static void test_cache_lookup_prefix_skips_or_in_query(void) { + /* If the new query contains '|', prefix lookup short-circuits to false + (we never refine into an OR query). */ + Cache c; + cache_init(&c, 20); + cache_insert_eligible(&c, "fo", 100); + + ScoredStr *out = NULL; + SharedIdx *out_sidx = NULL; + size_t out_count = 0, out_gen = 0; + CHECK(cache_lookup_prefix(&c, "fo | bar", CaseSmart, &out, &out_count, &out_sidx, &out_gen) == false); + cache_free(&c); +} + +static void test_cache_lookup_prefix_skips_exact_match(void) { + /* Even if an entry's parsed pattern equals the new query's, we exclude + it from prefix lookup (that's what cache_lookup_exact is for). */ + Cache c; + cache_init(&c, 20); + cache_insert_eligible(&c, "fo bar", 100); + + ScoredStr *out = NULL; + SharedIdx *out_sidx = NULL; + size_t out_count = 0, out_gen = 0; + CHECK(cache_lookup_prefix(&c, "fo bar", CaseSmart, &out, &out_count, &out_sidx, &out_gen) == false); + cache_free(&c); +} + /* ================================================================= */ int main(void) { @@ -379,7 +826,35 @@ int main(void) { printf("--- async_reader ---\n"); RUN(test_async_reader_basic); RUN(test_async_reader_ansi_stripping); - RUN(test_async_reader_buffer_growth); + RUN(test_async_reader_many_lines); + + printf("--- chunked cands_top ---\n"); + RUN(test_cands_top_index_split); + RUN(test_cands_top_accessor_reads_block_pointer); + + printf("--- cache (phase 1: exact-match) ---\n"); + RUN(test_cache_lookup_miss_on_empty); + RUN(test_cache_insert_then_lookup_hit); + RUN(test_cache_lookup_miss_distinct_query); + RUN(test_cache_insert_updates_in_place); + RUN(test_cache_lru_eviction_at_capacity); + RUN(test_cache_touch_on_hit); + RUN(test_cache_insert_zero_count); + RUN(test_cache_pool_gen_distinguishes_stale); + + printf("--- cache (phase 2: term-set subsumption) ---\n"); + RUN(test_subsumes_pattern_extending_term_via_byte_prefix); + RUN(test_subsumes_pattern_adding_term_at_end); + RUN(test_subsumes_pattern_adding_term_at_start); + RUN(test_subsumes_pattern_term_reorder); + RUN(test_subsumes_pattern_negation_at_start); + RUN(test_subsumes_pattern_or_query_rejected); + RUN(test_subsumes_pattern_distinct_terms); + RUN(test_cache_lookup_prefix_v2_finds_term_subset); + RUN(test_cache_lookup_prefix_v2_finds_reordered); + RUN(test_cache_lookup_prefix_picks_most_terms); + RUN(test_cache_lookup_prefix_skips_or_in_query); + RUN(test_cache_lookup_prefix_skips_exact_match); if (failed == 0) { printf("\nAll tests passed.\n"); diff --git a/fzf-native-module.c b/fzf-native-module.c index b1aee3b..5a6fa53 100644 --- a/fzf-native-module.c +++ b/fzf-native-module.c @@ -92,6 +92,8 @@ emacs_value Fhashtablep, Fmessage, Fvectorp, Fconsp, Fcdr, Fcar, Fvconcat; emacs_value Ffunctionp, Fsymbolp, Fsymbolname, Flength, Fnth, Fprinc, Freverse; emacs_value Qcompletion_score, Fput_text_property, Qzero, Qone; emacs_value Fencode_coding_string, Qutf_8; +emacs_value Qface, Qcompletions_common_part; +emacs_value Fremove_text_properties, Qface_nil_plist; /** An Emacs string made accessible by copying. */ @@ -296,6 +298,112 @@ static void *worker_routine(void *ptr) { return NULL; } +/* Apply `completions-common-part' face to STR_VAL on positions matched by + PATTERN against CSTR. Computes positions via fzf_get_positions and groups + them into contiguous runs to minimize put-text-property calls. + + Byte offsets from fzf are used directly as character positions: accurate + for ASCII, may be slightly misaligned for multi-byte UTF-8 (same caveat + as the async highlight path). */ +static void apply_highlight_positions(emacs_env *env, + const char *cstr, + fzf_pattern_t *pattern, + fzf_slab_t *slab, + emacs_value str_val) { + /* Empty candidates can't carry text properties or matched positions; skip + the funcalls and the get_positions slab work entirely. */ + if (cstr[0] == '\0') return; + /* Strip any `completions-common-part' face left over from a prior highlight + pass on the same Emacs string (fussy reuses caller-owned candidate strings + across keystrokes; without this, e.g. "ab" highlight [0,2] persists when + the user backspaces to "a" because put-text-property only writes the new + [0,1] range and leaves byte [1,2] highlighted). One funcall per + highlighted candidate; the (face nil) plist is interned at init. */ + emacs_value len_v = env->funcall(env, Flength, 1, &str_val); + if (env->non_local_exit_check(env) == emacs_funcall_exit_return) { + emacs_value rargs[4] = { Qzero, len_v, Qface_nil_plist, str_val }; + env->funcall(env, Fremove_text_properties, 4, rargs); + env->non_local_exit_clear(env); + } else { + env->non_local_exit_clear(env); + } + fzf_position_t *pos = fzf_get_positions(cstr, pattern, slab); + if (pos && pos->size > 0) { + /* pos->data[] is in descending order: pos->data[0] = highest position. + Iterate ascending (j from size-1 to 0) to find contiguous runs. */ + size_t plen = pos->size; + size_t run_start = pos->data[plen - 1]; + size_t run_end = run_start; + for (ptrdiff_t j = (ptrdiff_t)plen - 2; j >= 0; j--) { + size_t p = pos->data[j]; + if (p == run_end + 1) { + run_end = p; + } else { + emacs_value a[5] = { + env->make_integer(env, (intmax_t)run_start), + env->make_integer(env, (intmax_t)(run_end + 1)), + Qface, Qcompletions_common_part, str_val }; + env->funcall(env, Fput_text_property, 5, a); + env->non_local_exit_clear(env); + run_start = run_end = p; + } + } + emacs_value a[5] = { + env->make_integer(env, (intmax_t)run_start), + env->make_integer(env, (intmax_t)(run_end + 1)), + Qface, Qcompletions_common_part, str_val }; + env->funcall(env, Fput_text_property, 5, a); + env->non_local_exit_clear(env); + } + fzf_free_positions(pos); +} + +/* Read `fzf-native-case-mode' via symbol-value and resolve to fzf_case_types. + Recognized symbol values: smart (default), ignore, respect. + Falls back to CaseSmart on any read or comparison failure. */ +static fzf_case_types resolve_fzf_native_case_mode(emacs_env *env) { + emacs_value sym = env->intern(env, "fzf-native-case-mode"); + emacs_value v = env->funcall(env, env->intern(env, "symbol-value"), 1, &sym); + if (env->non_local_exit_check(env) != emacs_funcall_exit_return) { + env->non_local_exit_clear(env); + return CaseSmart; + } + if (env->eq(env, v, env->intern(env, "ignore"))) return CaseIgnore; + if (env->eq(env, v, env->intern(env, "respect"))) return CaseRespect; + return CaseSmart; +} + +/* Read fussy-fzf-native-highlight via symbol-value and resolve to a cap. + Returns: + 0 — no highlighting (nil, negative, unreadable, or zero). + LEN — highlight all (t). + N — highlight top N (clamped to LEN). */ +static size_t resolve_fussy_highlight_cap(emacs_env *env, size_t len) { + /* Canonical name; fussy bridges its `fussy-fzf-native-highlight' + onto this via `setq-local' inside its all-completions entry. */ + emacs_value sym = env->intern(env, "fzf-native-batch-highlight"); + emacs_value v = env->funcall(env, env->intern(env, "symbol-value"), 1, &sym); + if (env->non_local_exit_check(env) != emacs_funcall_exit_return) { + env->non_local_exit_clear(env); + return 0; + } + if (env->eq(env, v, Qnil)) return 0; + if (env->eq(env, v, Qt)) return len; + intmax_t n = env->extract_integer(env, v); + if (env->non_local_exit_check(env) != emacs_funcall_exit_return) { + env->non_local_exit_clear(env); + return 0; + } + if (n <= 0) return 0; + return (size_t)n > len ? len : (size_t)n; +} + +// Forward declare. +emacs_value fzf_native_highlight_all(emacs_env *env, + ptrdiff_t nargs, + emacs_value args[], + void *data_ptr); + // fzf-native-score-all COLLECTION QUERY &optional SLAB emacs_value fzf_native_score_all(emacs_env *env, ptrdiff_t nargs, @@ -312,9 +420,13 @@ emacs_value fzf_native_score_all(emacs_env *env, fzf_log("fzf_native_score_all START: query='%.*s'\n", (int)query.len, query.b); - // Return all candidates if query is empty with doing anything else. + /* Empty query: don't score, but still strip stale `completions-common-part' + face from the top-N candidates so backspacing to "" clears highlights left + behind by a prior query. Delegate to highlight-all, which respects + `fussy-fzf-native-highlight' for the cap. */ if (query.len == 0) { - result = args[0]; + emacs_value hargs[2] = { args[0], args[1] }; + result = fzf_native_highlight_all(env, 2, hargs, NULL); success = true; goto err; } @@ -363,7 +475,8 @@ emacs_value fzf_native_score_all(emacs_env *env, return Qnil; } - fzf_pattern_t *pattern = fzf_parse_pattern(CaseIgnore, false, query.b, true); + fzf_case_types case_mode = resolve_fzf_native_case_mode(env); + fzf_pattern_t *pattern = fzf_parse_pattern(case_mode, false, query.b, true); struct Shared shared = { .pattern = pattern, .batches = batches, @@ -422,11 +535,23 @@ emacs_value fzf_native_score_all(emacs_env *env, counting_sort_candidates(xs, len); + /* Resolve C-side highlight cap from fussy-fzf-native-highlight. After the + sort, xs[0] is the highest-scoring candidate, so the top-N candidates + are xs[0..hl_cap-1]. The original parsing pattern was already freed; + re-parse for highlighting using the same case mode the scoring used. */ + size_t hl_cap = resolve_fussy_highlight_cap(env, len); + fzf_pattern_t *hl_pattern = NULL; + fzf_slab_t *hl_slab = NULL; + if (hl_cap > 0) { + hl_pattern = fzf_parse_pattern(case_mode, false, query.b, true); + if (hl_pattern) hl_slab = fzf_make_default_slab(); + if (!hl_slab) { + if (hl_pattern) { fzf_free_pattern(hl_pattern); hl_pattern = NULL; } + hl_cap = 0; + } + } + for (size_t i = len; i-- > 0;) { - /* printf("zero: %jd one: %jd score: %d", */ - /* env->extract_integer(env, Qzero), */ - /* env->extract_integer(env, Qone), */ - /* xs[i].score); */ /* e.g. (put-text-property 0 1 'completion-score score x) */ if (xs[i].s.len > 0) { env->funcall(env, Fput_text_property, 5, @@ -437,9 +562,17 @@ emacs_value fzf_native_score_all(emacs_env *env, }); } + if (hl_pattern && i < hl_cap) { + apply_highlight_positions(env, xs[i].s.b, hl_pattern, hl_slab, + xs[i].value); + } + result = env->funcall(env, Fcons, 2, (emacs_value[]) { xs[i].value, result }); } + if (hl_pattern) fzf_free_pattern(hl_pattern); + if (hl_slab) fzf_free_slab(hl_slab); + fzf_log("fzf_native_score_all DONE: query='%.*s' count=%zu\n", (int)query.len, query.b, n); free(xs); @@ -458,6 +591,104 @@ emacs_value fzf_native_score_all(emacs_env *env, return result; } +/* Strip `completions-common-part' face from STR_VAL without applying any new + positions. Used by the empty-query path of `fzf-native-highlight-all', and + shares the (face nil) plist with `apply_highlight_positions'. */ +static void clear_highlight_face(emacs_env *env, emacs_value str_val) { + emacs_value len_v = env->funcall(env, Flength, 1, &str_val); + if (env->non_local_exit_check(env) != emacs_funcall_exit_return) { + env->non_local_exit_clear(env); + return; + } + emacs_value rargs[4] = { Qzero, len_v, Qface_nil_plist, str_val }; + env->funcall(env, Fremove_text_properties, 4, rargs); + env->non_local_exit_clear(env); +} + +// fzf-native-highlight-all COLLECTION QUERY +// +// Apply `completions-common-part' face to each candidate in COLLECTION +// against QUERY without scoring or sorting. Intended for callers that +// already have a sorted result set but need to refresh stale highlights +// (e.g. fussy cache hits or the empty-query branch, where the C scoring +// path is skipped entirely and previously-applied face properties from +// a different query persist on the same Emacs string objects). +// +// When QUERY is empty, performs a clear-only pass: removes the face +// from the top-N candidates without computing new positions. This is +// the path that fixes the "type m, backspace, highlight stays" case. +// +// Honors `fussy-fzf-native-highlight' the same way `fzf-native-score-all' +// does: nil → no-op, t → process all, N → process top N. COLLECTION +// is assumed to be in display order (highest-scoring first). +// +// Returns COLLECTION unchanged. Mutates the candidate strings in-place. +emacs_value fzf_native_highlight_all(emacs_env *env, + ptrdiff_t UNUSED(nargs), + emacs_value args[], + void UNUSED(*data_ptr)) { + struct Bump *bump = NULL; + fzf_pattern_t *pattern = NULL; + fzf_slab_t *slab = NULL; + + /* Treat an empty *or* undecodable query as clear-only. The stale face + properties live on the COLLECTION strings, not on the query, so we still + need to walk the collection and strip face even if the query couldn't be + coerced through `encode-coding-string'. */ + struct Str query = copy_emacs_string(env, &bump, args[1]); + bool clear_only = (!query.b || query.len == 0); + + /* Accept both lists and vectors; mirror score-all's normalization so + callers don't have to care which one they have on hand. */ + emacs_value collection = args[0]; + if (!env->eq(env, env->type_of(env, collection), env->intern(env, "vector"))) { + collection = env->funcall(env, Fvconcat, 1, (emacs_value[]) { args[0] }); + if (env->non_local_exit_check(env) != emacs_funcall_exit_return) { + env->non_local_exit_clear(env); + goto done; + } + } + + ptrdiff_t n = env->vec_size(env, collection); + if (env->non_local_exit_check(env) != emacs_funcall_exit_return) { + env->non_local_exit_clear(env); + goto done; + } + + /* Cap the highlight/clear pass to the user's `fussy-fzf-native-highlight' + setting. Returns 0 when highlighting is disabled — at that point the + candidates can't have stale face from this module either, so skip. */ + size_t hl_cap = resolve_fussy_highlight_cap(env, (size_t)n); + if (hl_cap == 0) goto done; + + if (!clear_only) { + fzf_case_types case_mode = resolve_fzf_native_case_mode(env); + pattern = fzf_parse_pattern(case_mode, false, query.b, true); + if (!pattern) goto done; + slab = fzf_make_default_slab(); + if (!slab) goto done; + } + + for (ptrdiff_t i = 0; i < (ptrdiff_t)hl_cap; i++) { + emacs_value value = env->vec_get(env, collection, i); + if (clear_only) { + clear_highlight_face(env, value); + } else { + struct Str s = copy_emacs_string(env, &bump, value); + if (!s.b) continue; + apply_highlight_positions(env, s.b, pattern, slab, value); + } + } + +done: + if (slab) fzf_free_slab(slab); + if (pattern) fzf_free_pattern(pattern); + bump_free(bump); + /* Always return the original COLLECTION (not the vector-coerced copy) + so list callers see their list back. */ + return args[0]; +} + /* Signal `(wrong-type-argument stringp VALUE)' if VALUE is not a string. Returns true on failure (caller should return immediately). */ static bool signal_if_not_string(emacs_env *env, emacs_value value) { @@ -522,7 +753,8 @@ emacs_value fzf_native_score(emacs_env *env, ptrdiff_t nargs, emacs_value args[] * pattern char* : Pattern you want to match. e.g. "src | lua !.c$ * fuzzy bool : Enable or disable fuzzy matching */ - fzf_pattern_t *pattern = fzf_parse_pattern(CaseSmart, false, query.b, true); + fzf_case_types case_mode = resolve_fzf_native_case_mode(env); + fzf_pattern_t *pattern = fzf_parse_pattern(case_mode, false, query.b, true); if (!pattern) { goto err; } fzf_slab_t *slab; @@ -534,26 +766,19 @@ emacs_value fzf_native_score(emacs_env *env, ptrdiff_t nargs, emacs_value args[] slab = fzf_make_default_slab(); } - /* You can get the score/position for as many items as you want */ int score = fzf_get_score(str.b, pattern, slab); - fzf_position_t *pos = fzf_get_positions(str.b, pattern, slab); - - size_t offset = 1; - size_t len = 0; - if (pos) { - len = pos->size; - } - emacs_value *result_array = malloc(sizeof(emacs_value) * (offset + len)); - - result_array[0] = env->make_integer(env, score); - - for (size_t i = 0; i < len; i++) { - result_array[offset + i] = env->make_integer(env, pos->data[len - (i + 1)]); + /* Apply C-layer highlighting when fussy-fzf-native-highlight is non-nil + and the candidate matched. The cap concept does not apply to a single + candidate — any non-nil value enables highlighting for this call. */ + if (score > 0 && resolve_fussy_highlight_cap(env, 1) > 0) { + apply_highlight_positions(env, str.b, pattern, slab, args[0]); } - result = env->funcall(env, Flist, offset + len, result_array); - fzf_free_positions(pos); + /* Return (SCORE) — a single-element list. Match indices are no longer + surfaced to Elisp; highlighting is handled in C. */ + emacs_value score_val = env->make_integer(env, score); + result = env->funcall(env, Flist, 1, &score_val); fzf_free_pattern(pattern); if (nargs > 2) { @@ -604,10 +829,31 @@ emacs_value fzf_native_make_slab(emacs_env *env, #if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__) -#define ASYNC_INIT_CAP 4096 #define ASYNC_LINE_MAX 8192 #define ARENA_CHUNK_SIZE (4 * 1024 * 1024) /* 4 MB per chunk */ +/* Chunked candidate-pointer storage. + * + * The candidate-pointer table is split into fixed-size blocks owned by a + * top-level pointer table. The reader appends to the current block; when a + * block fills, it allocates the next one. No realloc ever moves pointer + * data, so the worst-case allocation the reader performs is a single + * block — predictable cost regardless of pool size. + * + * cands_top[] : CANDS_TOP_CAP slots × 8 B (~32 KB, fixed inline) + * cands_top[i] : CANDS_BLOCK_SIZE × 8 B (2 MB, on demand) + * + * Defaults: 256K pointers per block, 4096 blocks → 1 G candidates max. + * + * Index split: hi = i >> SHIFT (which block) + * lo = i & MASK (which slot in that block) + * Both ops are single CPU instructions because BLOCK_SIZE is a power of 2. + */ +#define CANDS_BLOCK_SHIFT 18 +#define CANDS_BLOCK_SIZE ((size_t)1 << CANDS_BLOCK_SHIFT) +#define CANDS_BLOCK_MASK (CANDS_BLOCK_SIZE - 1) +#define CANDS_TOP_CAP 4096 + /* Arena allocator: strings are packed into large chunks so freeing the entire candidate set is O(chunks) instead of O(candidates). */ typedef struct ArenaChunk { struct ArenaChunk *next; size_t used; char data[]; } ArenaChunk; @@ -649,7 +895,366 @@ static size_t async_strip_ansi(char *s, size_t len) { return w; } -typedef struct { char *str; int score; } ScoredStr; +typedef struct { char *str; int score; uint32_t idx; } ScoredStr; + +/* Reference-counted immutable index array. Allocated once by the scoring + thread on cache_insert, retained in O(1) (atomic refcount bump under the + cache mutex — no memcpy) by lookup consumers, and freed when the last + consumer releases it. Used to record the full set of matched candidate + indices for a query, so a later subsuming query can refine-score that + set + only the candidates that arrived since (delta scoring) instead of + re-scanning the whole pool. */ +typedef struct { + _Atomic uint32_t refcount; + size_t count; + uint32_t idx[]; /* flexible array */ +} SharedIdx; + +static SharedIdx *shared_idx_alloc(const uint32_t *src, size_t n) { + if (!n || !src) return NULL; + SharedIdx *p = malloc(sizeof *p + n * sizeof *p->idx); + if (!p) return NULL; + atomic_init(&p->refcount, 1); + p->count = n; + memcpy(p->idx, src, n * sizeof *p->idx); + return p; +} +static SharedIdx *shared_idx_retain(SharedIdx *p) { + if (p) atomic_fetch_add_explicit(&p->refcount, 1, memory_order_relaxed); + return p; +} +static void shared_idx_release(SharedIdx *p) { + if (p && atomic_fetch_sub_explicit(&p->refcount, 1, memory_order_acq_rel) == 1) + free(p); +} + +/* LRU result cache. Per-session, mutex-protected doubly-linked list with + MRU at head and LRU at tail. Each entry records: + query — the literal filter string (owned, strdup'd) + pool_gen — s->count at the moment the entry was scored + top — copy of the top-K ScoredStr published to Elisp + m_idx — SharedIdx of all matched candidate indices (NULL for OR + queries, which can never serve as refinement sources because + adding an OR alternate widens the result set) + + Lookups happen on the Emacs main thread (dispatch); inserts happen on + the scoring thread (publish). Both serialize through cache->mu. */ +typedef struct CacheEntry { + struct CacheEntry *prev, *next; + char *query; + size_t pool_gen; + ScoredStr *top; + size_t top_count; + SharedIdx *m_idx; + /* Parsed form of `query`, populated on insert. NULL when parsing failed + or for OR queries (which are excluded from prefix-refinement anyway). + Owned by the entry; freed in cache_entry_free. */ + fzf_pattern_t *parsed; +} CacheEntry; + +typedef struct { + pthread_mutex_t mu; + CacheEntry *head; /* MRU */ + CacheEntry *tail; /* LRU */ + size_t count; + size_t max_entries; +} Cache; + +static void cache_init(Cache *c, size_t max_entries) { + pthread_mutex_init(&c->mu, NULL); + c->head = c->tail = NULL; + c->count = 0; + c->max_entries = max_entries ? max_entries : 40; +} + +static void cache_entry_free(CacheEntry *e) { + if (!e) return; + free(e->query); + free(e->top); + shared_idx_release(e->m_idx); + if (e->parsed) fzf_free_pattern(e->parsed); + free(e); +} + +static void cache_unlink_locked(Cache *c, CacheEntry *e) { + if (e->prev) e->prev->next = e->next; else c->head = e->next; + if (e->next) e->next->prev = e->prev; else c->tail = e->prev; + e->prev = e->next = NULL; + c->count--; +} + +static void cache_push_head_locked(Cache *c, CacheEntry *e) { + e->prev = NULL; + e->next = c->head; + if (c->head) c->head->prev = e; + c->head = e; + if (!c->tail) c->tail = e; + c->count++; +} + +static void cache_free(Cache *c) { + pthread_mutex_lock(&c->mu); + CacheEntry *e = c->head; + while (e) { CacheEntry *n = e->next; cache_entry_free(e); e = n; } + c->head = c->tail = NULL; + c->count = 0; + pthread_mutex_unlock(&c->mu); + pthread_mutex_destroy(&c->mu); +} + +/* Q' subsumes Q iff neither contains '|' AND Q' is a byte-prefix of Q. + Captures: extending a term (fo → foo), adding AND terms at end (fo → fo + bar), adding negations/anchors at end (fo → fo !x). Conservatively + rejects every query containing '|' — adding an OR alternate widens + results unpredictably. This is the v1 rule, kept as a fast-path. */ +static bool subsumes(const char *q_prime, const char *q) { + if (strchr(q_prime, '|') || strchr(q, '|')) return false; + size_t lp = strlen(q_prime); + if (lp == 0) return true; + size_t lq = strlen(q); + if (lq < lp) return false; + return memcmp(q, q_prime, lp) == 0; +} + +/* Two parsed terms are equivalent iff they would match exactly the same + strings: same algorithm, same negation flag, same case sensitivity, same + text after fzf's prefix stripping. Both terms must have been parsed with + the same case mode (CaseIgnore in our usage), so `ptr` (the lowercased + token) is directly comparable. */ +static bool term_equiv(const fzf_term_t *a, const fzf_term_t *b) { + if (a->fn != b->fn) return false; + if (a->inv != b->inv) return false; + if (a->case_sensitive != b->case_sensitive) return false; + return strcmp(a->ptr, b->ptr) == 0; +} + +/* P' subsumes P (term-set rule) iff every term-set in P' has an equivalent + term-set in P. In fzf's model, term-sets are AND'd together; adding + more term-sets monotonically narrows the match set, so P (with all of + P''s term-sets plus possibly more) matches a subset of P''s candidates. + + Restricted to non-OR queries: any term-set with >1 term is an OR (e.g. + "fo | bar" parses as one set with two terms), and OR queries can never + serve as refinement sources because adding an OR alternate widens the + match set unpredictably. + + Catches v2-only cases: adding an AND term in non-prefix position (fo → + x fo), term reordering (fo bar → bar fo), non-prefix negation (fo → + !x fo). Empty P' (zero term-sets) trivially subsumes anything. */ +static bool subsumes_pattern(const fzf_pattern_t *p_prime, + const fzf_pattern_t *p) { + if (!p_prime || !p) return false; + /* Reject if either side has any OR-containing term-set. */ + for (size_t i = 0; i < p_prime->size; i++) + if (p_prime->ptr[i]->size != 1) return false; + for (size_t i = 0; i < p->size; i++) + if (p->ptr[i]->size != 1) return false; + /* Every single-term set in p_prime must equal some single-term set in p. */ + for (size_t i = 0; i < p_prime->size; i++) { + fzf_term_t *t_prime = &p_prime->ptr[i]->ptr[0]; + bool found = false; + for (size_t j = 0; j < p->size; j++) { + if (term_equiv(t_prime, &p->ptr[j]->ptr[0])) { found = true; break; } + } + if (!found) return false; + } + return true; +} + +/* Parse a query string into an fzf_pattern_t. Returns NULL if the query + is empty or parsing fails. fzf_parse_pattern mutates its input, so we + strdup first and free after — the returned pattern is self-contained. */ +static fzf_pattern_t *parse_query_for_cache(const char *query, + fzf_case_types case_mode) { + if (!query || !*query) return NULL; + char *dup = strdup(query); + if (!dup) return NULL; + fzf_pattern_t *p = fzf_parse_pattern(case_mode, false, dup, true); + free(dup); + return p; +} + +/* Find an entry by exact query match. Caller holds c->mu. */ +static CacheEntry *cache_find_locked(Cache *c, const char *query) { + for (CacheEntry *e = c->head; e; e = e->next) + if (strcmp(e->query, query) == 0) return e; + return NULL; +} + +/* Exact lookup. On hit, bumps entry to MRU and returns: + *out_top, *out_top_count — caller-owned copy of the cached top-K + *out_m_idx — SharedIdx with refcount bumped (caller releases) + *out_pool_gen — pool size at the time this entry was scored + Returns true on hit, false on miss. */ +static bool cache_lookup_exact(Cache *c, const char *query, + ScoredStr **out_top, size_t *out_top_count, + SharedIdx **out_m_idx, size_t *out_pool_gen) { + pthread_mutex_lock(&c->mu); + CacheEntry *e = cache_find_locked(c, query); + if (!e) { pthread_mutex_unlock(&c->mu); return false; } + + ScoredStr *top_copy = NULL; + if (e->top_count) { + top_copy = malloc(e->top_count * sizeof *top_copy); + if (top_copy) memcpy(top_copy, e->top, e->top_count * sizeof *top_copy); + } + *out_top = top_copy; + *out_top_count = top_copy ? e->top_count : 0; + *out_m_idx = shared_idx_retain(e->m_idx); + *out_pool_gen = e->pool_gen; + + /* Bump to MRU. */ + if (e != c->head) { cache_unlink_locked(c, e); cache_push_head_locked(c, e); } + pthread_mutex_unlock(&c->mu); + return true; +} + +/* Prefix lookup: most-constrained Q' that subsumes Q (and is not Q itself). + Uses byte-prefix OR term-set subsumption. Skips entries with NULL m_idx + (OR queries / empty match sets — can't serve as refinement sources). + + Best = the entry whose parsed pattern has the most terms. More terms = + more constraints = smaller match set = faster refinement scan. Falls + back to byte-prefix-length tiebreak when both have equal term counts + (or for entries whose parsed pattern is unavailable). */ +static bool cache_lookup_prefix(Cache *c, const char *query, + fzf_case_types case_mode, + ScoredStr **out_top, size_t *out_top_count, + SharedIdx **out_m_idx, size_t *out_pool_gen) { + if (strchr(query, '|')) return false; /* fast reject */ + + fzf_pattern_t *p_query = parse_query_for_cache(query, case_mode); + /* If parse failed and query isn't empty, fall back to byte-prefix only. + Empty query has p_query == NULL but byte-prefix subsumes("", anything) + also returns true so the loop still works. */ + + pthread_mutex_lock(&c->mu); + CacheEntry *best = NULL; + size_t best_terms = 0; + size_t best_len = 0; + for (CacheEntry *e = c->head; e; e = e->next) { + if (!e->m_idx) continue; + if (strcmp(e->query, query) == 0) continue; + + bool match = subsumes(e->query, query) + || (p_query && subsumes_pattern(e->parsed, p_query)); + if (!match) continue; + + /* Term count = number of AND term-sets in the parsed pattern. More + sets = more constraints = smaller match set = better refinement + source. OR-containing entries have m_idx==NULL and were skipped. */ + size_t terms = e->parsed ? e->parsed->size : 0; + size_t len = strlen(e->query); + if (terms > best_terms || + (terms == best_terms && len > best_len)) { + best = e; + best_terms = terms; + best_len = len; + } + } + if (!best) { + pthread_mutex_unlock(&c->mu); + if (p_query) fzf_free_pattern(p_query); + return false; + } + + ScoredStr *top_copy = NULL; + if (best->top_count) { + top_copy = malloc(best->top_count * sizeof *top_copy); + if (top_copy) memcpy(top_copy, best->top, best->top_count * sizeof *top_copy); + } + *out_top = top_copy; + *out_top_count = top_copy ? best->top_count : 0; + *out_m_idx = shared_idx_retain(best->m_idx); + *out_pool_gen = best->pool_gen; + + if (best != c->head) { cache_unlink_locked(c, best); cache_push_head_locked(c, best); } + pthread_mutex_unlock(&c->mu); + if (p_query) fzf_free_pattern(p_query); + return true; +} + +/* Insert or update an entry. Performs all allocations BEFORE taking + c->mu, so the critical section is just pointer swaps + LRU manipulation. + Evicted entries are freed after the unlock. m_idx may be NULL (OR queries + or empty match sets); the entry is still inserted, but is then ineligible + as a prefix-refinement source. */ +static void cache_insert(Cache *c, const char *query, size_t pool_gen, + fzf_case_types case_mode, + const ScoredStr *top, size_t top_count, + const uint32_t *m_idx_src, size_t m_idx_count) { + /* Pre-allocate everything outside the mutex. */ + char *q_dup = strdup(query); + ScoredStr *top_dup = NULL; + if (top_count && top) { + top_dup = malloc(top_count * sizeof *top_dup); + if (top_dup) memcpy(top_dup, top, top_count * sizeof *top_dup); + else top_count = 0; + } + SharedIdx *sidx = (m_idx_src && m_idx_count && !strchr(query, '|')) + ? shared_idx_alloc(m_idx_src, m_idx_count) : NULL; + /* Parse once on insert so cache_lookup_prefix doesn't pay parse cost on + every iteration of its scan loop. NULL is fine — entries with NULL + parsed only participate via the byte-prefix subsumption fallback. */ + fzf_pattern_t *parsed = parse_query_for_cache(query, case_mode); + + if (!q_dup) { + free(top_dup); + shared_idx_release(sidx); + if (parsed) fzf_free_pattern(parsed); + return; + } + + pthread_mutex_lock(&c->mu); + CacheEntry *e = cache_find_locked(c, query); + if (e) { + /* Update existing entry: swap fields, release old refs after unlock. */ + char *old_q = e->query; + ScoredStr *old_top = e->top; + SharedIdx *old_idx = e->m_idx; + fzf_pattern_t *old_parsed = e->parsed; + e->query = q_dup; + e->top = top_dup; + e->top_count = top_dup ? top_count : 0; + e->m_idx = sidx; + e->parsed = parsed; + e->pool_gen = pool_gen; + if (e != c->head) { cache_unlink_locked(c, e); cache_push_head_locked(c, e); } + pthread_mutex_unlock(&c->mu); + free(old_q); + free(old_top); + shared_idx_release(old_idx); + if (old_parsed) fzf_free_pattern(old_parsed); + return; + } + + /* New entry. */ + CacheEntry *ne = calloc(1, sizeof *ne); + if (!ne) { + pthread_mutex_unlock(&c->mu); + free(q_dup); + free(top_dup); + shared_idx_release(sidx); + if (parsed) fzf_free_pattern(parsed); + return; + } + ne->query = q_dup; + ne->top = top_dup; + ne->top_count = top_dup ? top_count : 0; + ne->m_idx = sidx; + ne->parsed = parsed; + ne->pool_gen = pool_gen; + cache_push_head_locked(c, ne); + + /* Evict LRU if over capacity. */ + CacheEntry *evicted = NULL; + if (c->count > c->max_entries && c->tail) { + evicted = c->tail; + cache_unlink_locked(c, evicted); + } + pthread_mutex_unlock(&c->mu); + cache_entry_free(evicted); +} typedef struct { pthread_t reader; @@ -659,9 +1264,12 @@ typedef struct { pthread_mutex_t mu; Arena arena; /* backing storage for all candidate strings */ - char **cands; + /* Two-level pointer table; see CANDS_BLOCK_SHIFT comments above. + Top level is fixed-size and zero-initialized at session start; + blocks are allocated on demand by the reader. Access pattern: + cands_top[i >> CANDS_BLOCK_SHIFT][i & CANDS_BLOCK_MASK]. */ + char **cands_top[CANDS_TOP_CAP]; size_t count; - size_t cap; _Atomic int gen; size_t last_filtered; /* candidates matching last filter */ @@ -673,6 +1281,12 @@ typedef struct { pthread_cond_t score_req_cond; char *score_req_filter; /* owned; NULL = nothing pending */ size_t score_req_limit; + fzf_case_types score_req_case_mode; + /* Refinement request: when score_req_refine_idx is non-NULL the next scoring + run scores only those candidate indices plus s->cands[refine_delta_from..count]. + Ownership transfers to the scoring thread along with score_req_filter. */ + SharedIdx *score_req_refine_idx; + size_t score_req_refine_delta_from; bool score_req_stop; _Atomic bool score_abort; /* set to cancel in-flight workers */ @@ -682,6 +1296,15 @@ typedef struct { pthread_mutex_t score_res_mu; ScoredStr *score_results; /* latest scored+sorted results */ size_t score_count; /* number of entries in score_results */ + + /* Result cache (LRU keyed by query, values include matched_idx for + prefix refinement). Read on dispatch (main thread); written on + scoring publish (scoring thread). */ + Cache cache; + + /* Read-only after session start; set from fzf-async-max-line-length defcustom. + 0 = no limit. >0 = exclude lines longer than N chars. <0 = truncate to |N|. */ + ptrdiff_t max_line_length; } AsyncSession; static void *async_reader(void *arg) { @@ -695,19 +1318,57 @@ static void *async_reader(void *arg) { len = async_strip_ansi(line, len); if (!len) continue; + ptrdiff_t mll = s->max_line_length; + if (mll != 0) { + ptrdiff_t cap = mll > 0 ? mll : -mll; + if ((ptrdiff_t)len > cap) { + if (mll > 0) continue; /* exclude */ + len = (size_t)cap; /* truncate */ + line[len] = '\0'; + } + } + + /* Compute chunked-table coordinates BEFORE allocating in the arena. + The reader is the sole writer to s->count, so reading without the + lock is safe. Cap check first so a hit doesn't leak arena bytes + on every dropped line — at 1G candidates we keep draining the pipe + (so the child doesn't block on write) but allocate nothing. */ + size_t i = s->count; + size_t hi = i >> CANDS_BLOCK_SHIFT; + size_t lo = i & CANDS_BLOCK_MASK; + if (hi >= CANDS_TOP_CAP) { + /* Verbose by design: a single fzf-async session producing >1G + candidates is so far outside expected behavior that hitting this + almost certainly means a broken upstream command (infinite loop, + runaway find on a cyclic FS, etc.). Log every dropped line so + the cause is obvious in the log. */ + size_t preview = len > 80 ? 80 : len; + fzf_log("async_reader: TOP TABLE FULL count=%zu cap=%zu line='%.*s%s'\n", + s->count, (size_t)CANDS_TOP_CAP * CANDS_BLOCK_SIZE, + (int)preview, line, len > preview ? "..." : ""); + continue; + } + char *dup = arena_strdup(&s->arena, line, len); if (!dup) continue; - pthread_mutex_lock(&s->mu); - if (s->count >= s->cap) { - size_t ncap = s->cap * 2; - fzf_log("async_reader: reallocating candidates %zu -> %zu\n", s->cap, ncap); - char **nc = realloc(s->cands, ncap * sizeof *nc); - if (!nc) { free(dup); pthread_mutex_unlock(&s->mu); continue; } - s->cands = nc; - s->cap = ncap; + /* Pre-allocate the new block (if needed) OUTSIDE s->mu — that's the + largest allocation the reader ever does (2 MB). Doing it under the + lock would let a slow malloc (e.g. macOS compressor pressure) stall + the scoring thread's snapshot path. */ + char **block = s->cands_top[hi]; /* sole writer ⇒ safe to read unlocked */ + bool need_publish = (block == NULL); + if (need_publish) { + block = malloc(CANDS_BLOCK_SIZE * sizeof *block); + if (!block) continue; /* dup stays in arena, freed at session stop */ + fzf_log("async_reader: allocated block %zu (count=%zu, %zu MB)\n", + hi, s->count, (CANDS_BLOCK_SIZE * sizeof *block) >> 20); } - s->cands[s->count++] = dup; + + pthread_mutex_lock(&s->mu); + if (need_publish) s->cands_top[hi] = block; /* publish under lock */ + s->cands_top[hi][lo] = dup; + s->count++; pthread_mutex_unlock(&s->mu); atomic_fetch_add_explicit(&s->gen, 1, memory_order_relaxed); } @@ -732,6 +1393,8 @@ static void async_session_destroy(void *ptr) { pthread_mutex_lock(&s->score_req_mu); free(s->score_req_filter); s->score_req_filter = NULL; + shared_idx_release(s->score_req_refine_idx); + s->score_req_refine_idx = NULL; s->score_req_stop = true; pthread_cond_signal(&s->score_req_cond); pthread_mutex_unlock(&s->score_req_mu); @@ -739,6 +1402,7 @@ static void async_session_destroy(void *ptr) { free(s->score_results); free(s->score_current_filter); + cache_free(&s->cache); pthread_mutex_destroy(&s->score_res_mu); pthread_mutex_destroy(&s->score_req_mu); pthread_cond_destroy(&s->score_req_cond); @@ -749,7 +1413,8 @@ static void async_session_destroy(void *ptr) { if (s->pid > 0) { waitpid(s->pid, NULL, 0); s->pid = -1; } pthread_mutex_lock(&s->mu); arena_free(&s->arena); - free(s->cands); + for (size_t k = 0; k < CANDS_TOP_CAP; k++) + if (s->cands_top[k]) { free(s->cands_top[k]); s->cands_top[k] = NULL; } pthread_mutex_unlock(&s->mu); pthread_mutex_destroy(&s->mu); free(s); @@ -773,11 +1438,80 @@ fzf_native_async_start(emacs_env *env, ptrdiff_t nargs, if (dir) env->copy_string_contents(env, args[1], dir, &dlen); } + /* Use shell-file-name / shell-command-switch so behaviour matches + shell-command (M-!) rather than hardcoding /bin/sh -c. */ + char *shell_prog = NULL, *shell_switch = NULL; + { + emacs_value sym = env->intern(env, "shell-file-name"); + emacs_value v = env->funcall(env, env->intern(env, "symbol-value"), 1, &sym); + if (env->non_local_exit_check(env) == emacs_funcall_exit_return && + !env->eq(env, v, Qnil)) { + ptrdiff_t slen = 0; + env->copy_string_contents(env, v, NULL, &slen); + if (slen > 1) { + shell_prog = malloc((size_t)slen); + if (shell_prog) env->copy_string_contents(env, v, shell_prog, &slen); + } + } else { + env->non_local_exit_clear(env); + } + if (!shell_prog) shell_prog = strdup("/bin/sh"); + } + { + emacs_value sym = env->intern(env, "shell-command-switch"); + emacs_value v = env->funcall(env, env->intern(env, "symbol-value"), 1, &sym); + if (env->non_local_exit_check(env) == emacs_funcall_exit_return && + !env->eq(env, v, Qnil)) { + ptrdiff_t slen = 0; + env->copy_string_contents(env, v, NULL, &slen); + if (slen > 1) { + shell_switch = malloc((size_t)slen); + if (shell_switch) env->copy_string_contents(env, v, shell_switch, &slen); + } + } else { + env->non_local_exit_clear(env); + } + if (!shell_switch) shell_switch = strdup("-c"); + } + + /* Build PATH from exec-path so the child shell can find binaries that + Emacs can find, even on macOS GUI launches with a minimal inherited PATH. */ + char *exec_path_str = NULL; + { + emacs_value sym = env->intern(env, "exec-path"); + emacs_value v = env->funcall(env, env->intern(env, "symbol-value"), 1, &sym); + emacs_value sep = env->make_string(env, ":", 1); + emacs_value id = env->intern(env, "identity"); + emacs_value mc_fn = env->intern(env, "mapconcat"); + emacs_value mc_args[3] = {id, v, sep}; + if (env->non_local_exit_check(env) == emacs_funcall_exit_return) { + emacs_value joined = env->funcall(env, mc_fn, 3, mc_args); + if (env->non_local_exit_check(env) == emacs_funcall_exit_return) { + ptrdiff_t plen = 0; + env->copy_string_contents(env, joined, NULL, &plen); + if (plen > 1) { + exec_path_str = malloc((size_t)plen); + if (exec_path_str) + env->copy_string_contents(env, joined, exec_path_str, &plen); + } + } + } + if (env->non_local_exit_check(env) != emacs_funcall_exit_return) + env->non_local_exit_clear(env); + } + + fzf_log("async_start: shell='%s' switch='%s' cmd='%s' dir='%s' PATH='%s'\n", + shell_prog, shell_switch, cmd, dir ? dir : "(nil)", + exec_path_str ? exec_path_str : "(inherited)"); + int pfd[2]; if (pipe(pfd) != 0) { fzf_log("async_start: pipe failed\n"); free(cmd); free(dir); + free(shell_prog); + free(shell_switch); + free(exec_path_str); return Qnil; } @@ -788,6 +1522,9 @@ fzf_native_async_start(emacs_env *env, ptrdiff_t nargs, close(pfd[1]); free(cmd); free(dir); + free(shell_prog); + free(shell_switch); + free(exec_path_str); return Qnil; } @@ -797,11 +1534,28 @@ fzf_native_async_start(emacs_env *env, ptrdiff_t nargs, close(pfd[1]); int dn = open("/dev/null", O_WRONLY); if (dn >= 0) { dup2(dn, STDERR_FILENO); close(dn); } + if (exec_path_str) { + const char *old = getenv("PATH"); + if (old && *old) { + size_t nlen = strlen(exec_path_str) + 1 + strlen(old) + 1; + char *new_path = malloc(nlen); + if (new_path) { + snprintf(new_path, nlen, "%s:%s", exec_path_str, old); + setenv("PATH", new_path, 1); + free(new_path); + } + } else { + setenv("PATH", exec_path_str, 1); + } + } if (dir) chdir(dir); - execl("/bin/sh", "sh", "-c", cmd, (char *)NULL); + execl(shell_prog, shell_prog, shell_switch, cmd, (char *)NULL); _exit(127); } close(pfd[1]); + free(shell_prog); + free(shell_switch); + free(exec_path_str); AsyncSession *s = calloc(1, sizeof *s); if (!s) { @@ -822,8 +1576,8 @@ fzf_native_async_start(emacs_env *env, ptrdiff_t nargs, s->pid = pid; s->fp = fdopen(pfd[0], "r"); - s->cap = ASYNC_INIT_CAP; - s->cands = malloc(s->cap * sizeof *s->cands); + /* cands_top is zero-initialized by the calloc above; blocks are + allocated lazily by the reader on first write into each block. */ pthread_mutex_init(&s->mu, NULL); pthread_mutex_init(&s->score_req_mu, NULL); pthread_cond_init(&s->score_req_cond, NULL); @@ -831,7 +1585,44 @@ fzf_native_async_start(emacs_env *env, ptrdiff_t nargs, atomic_store(&s->gen, 0); atomic_store(&s->score_abort, false); - if (!s->fp || !s->cands || + { + /* Canonical name; fzf-async bridges `fzf-async-max-line-length' + onto this via :around advice on `fzf-native-async-start'. + Type is integer (positive = exclude, negative = truncate) or nil + (no limit). The defcustom default lives in fzf-native.el — no + hardcoded fallback here. */ + emacs_value sym = env->intern(env, "fzf-native-max-line-length"); + emacs_value val = env->funcall(env, env->intern(env, "symbol-value"), 1, &sym); + if (env->non_local_exit_check(env) != emacs_funcall_exit_return) + env->non_local_exit_clear(env); + else if (!env->eq(env, val, Qnil)) { + s->max_line_length = (ptrdiff_t)env->extract_integer(env, val); + if (env->non_local_exit_check(env) != emacs_funcall_exit_return) { + env->non_local_exit_clear(env); + s->max_line_length = 0; + } + } + } + + { + size_t cache_max = 40; + /* Canonical name; fzf-async bridges `fzf-async-cache-size' + onto this via :around advice on `fzf-native-async-start'. */ + emacs_value sym = env->intern(env, "fzf-native-async-cache-size"); + emacs_value val = env->funcall(env, env->intern(env, "symbol-value"), 1, &sym); + if (env->non_local_exit_check(env) != emacs_funcall_exit_return) + env->non_local_exit_clear(env); + else if (!env->eq(env, val, Qnil)) { + intmax_t n = env->extract_integer(env, val); + if (env->non_local_exit_check(env) != emacs_funcall_exit_return) + env->non_local_exit_clear(env); + else if (n > 0) + cache_max = (size_t)n; + } + cache_init(&s->cache, cache_max); + } + + if (!s->fp || pthread_create(&s->reader, NULL, async_reader, s) != 0 || pthread_create(&s->score_thread, NULL, scoring_thread_fn, s) != 0) { async_session_destroy(s); @@ -951,9 +1742,13 @@ static void *scoring_thread_fn(void *arg) { pthread_mutex_unlock(&s->score_req_mu); break; } - char *filter = s->score_req_filter; /* steal ownership */ - size_t limit = s->score_req_limit; - s->score_req_filter = NULL; + char *filter = s->score_req_filter; /* steal ownership */ + size_t limit = s->score_req_limit; + fzf_case_types case_mode = s->score_req_case_mode; + SharedIdx *refine_idx = s->score_req_refine_idx; /* steal */ + size_t refine_delta_from = s->score_req_refine_delta_from; + s->score_req_filter = NULL; + s->score_req_refine_idx = NULL; /* Record what we're about to score so main thread can skip abort for same filter */ free(s->score_current_filter); s->score_current_filter = strdup(filter); @@ -971,24 +1766,72 @@ static void *scoring_thread_fn(void *arg) { size_t count = s->count; pthread_mutex_unlock(&s->mu); - char **snap = count ? malloc(count * sizeof *snap) : NULL; - if (!snap && count) { + /* Determine snap layout. In full mode, snap[i] = s->cands[i] for + i in [0, count); snap_count = count. In refine mode, snap is the + union of refine_idx (the previous match set) and the delta range + s->cands[refine_delta_from .. count) — always a strict subset of + the full pool, hence the speedup. */ + bool refine = (refine_idx != NULL && refine_delta_from <= count); + size_t delta_len = refine ? (count - refine_delta_from) : 0; + size_t snap_count = refine ? (refine_idx->count + delta_len) : count; + + char **snap = snap_count ? malloc(snap_count * sizeof *snap) : NULL; + uint32_t *snap_idx = snap_count ? malloc(snap_count * sizeof *snap_idx) : NULL; + if (snap_count && (!snap || !snap_idx)) { pthread_mutex_lock(&s->score_req_mu); free(s->score_current_filter); s->score_current_filter = NULL; pthread_mutex_unlock(&s->score_req_mu); - free(filter); continue; + free(snap); free(snap_idx); free(filter); shared_idx_release(refine_idx); continue; } pthread_mutex_lock(&s->mu); if (s->count < count) count = s->count; /* cap if reader shrank (shouldn't happen) */ - if (snap) memcpy(snap, s->cands, count * sizeof *snap); + if (refine) { + /* Cap delta range too in case reader shrank. */ + if (refine_delta_from > count) refine_delta_from = count; + delta_len = count - refine_delta_from; + /* Refine entries first: validate each refine_idx[i] < count. + Random-access path — pay shift+mask per entry. */ + size_t w = 0; + for (size_t i = 0; i < refine_idx->count; i++) { + uint32_t gi = refine_idx->idx[i]; + if (gi < count) { + snap[w] = s->cands_top[gi >> CANDS_BLOCK_SHIFT][gi & CANDS_BLOCK_MASK]; + snap_idx[w] = gi; + w++; + } + } + /* Delta entries — sequential range, but a small one (delta size). + Walk via accessor too; the cost is negligible at delta-scale. */ + for (size_t k = 0; k < delta_len; k++) { + size_t gi = refine_delta_from + k; + snap[w] = s->cands_top[gi >> CANDS_BLOCK_SHIFT][gi & CANDS_BLOCK_MASK]; + snap_idx[w] = (uint32_t)gi; + w++; + } + snap_count = w; + } else if (snap) { + /* Full-pool snapshot — walk block-by-block. Inside a block this is + a flat memcpy over contiguous memory, same speed as the old flat + array. Boundary-crossing cost is paid once per block (every + CANDS_BLOCK_SIZE entries — ~250 crossings for a 60M pool). */ + size_t copied = 0; + for (size_t hi = 0; copied < count; hi++) { + size_t in_block = CANDS_BLOCK_SIZE; + if (count - copied < in_block) in_block = count - copied; + memcpy(snap + copied, s->cands_top[hi], in_block * sizeof *snap); + copied += in_block; + } + for (size_t i = 0; i < count; i++) snap_idx[i] = (uint32_t)i; + snap_count = count; + } pthread_mutex_unlock(&s->mu); /* Batch; check abort every 64 K items so a filter change is noticed quickly. */ struct AsyncScoringBatch *batches = NULL; size_t bi = 0, bcap = 0; bool batch_ok = true; - for (size_t i = 0; i < count; i++) { + for (size_t i = 0; i < snap_count; i++) { if ((i & 0xFFFF) == 0 && atomic_load_explicit(&s->score_abort, memory_order_relaxed)) { batch_ok = false; break; @@ -1002,20 +1845,22 @@ static void *scoring_thread_fn(void *arg) { } batches[bi].xs[batches[bi].len].str = snap[i]; batches[bi].xs[batches[bi].len].score = 0; + batches[bi].xs[batches[bi].len].idx = snap_idx[i]; batches[bi].len++; } if (!batch_ok) { pthread_mutex_lock(&s->score_req_mu); free(s->score_current_filter); s->score_current_filter = NULL; pthread_mutex_unlock(&s->score_req_mu); - free(snap); free(filter); free(batches); continue; + free(snap); free(snap_idx); free(filter); free(batches); + shared_idx_release(refine_idx); continue; } - unsigned num_batches = count ? (unsigned)(bi + 1) : 0; + unsigned num_batches = snap_count ? (unsigned)(bi + 1) : 0; unsigned max_workers = (unsigned)sysconf(_SC_NPROCESSORS_ONLN); size_t flen = strlen(filter); - fzf_pattern_t *pattern = flen ? fzf_parse_pattern(CaseIgnore, false, filter, true) : NULL; + fzf_pattern_t *pattern = flen ? fzf_parse_pattern(case_mode, false, filter, true) : NULL; bool has_pattern = (pattern != NULL); struct AsyncScoringShared shared = { @@ -1041,8 +1886,8 @@ static void *scoring_thread_fn(void *arg) { pthread_mutex_lock(&s->score_req_mu); free(s->score_current_filter); s->score_current_filter = NULL; pthread_mutex_unlock(&s->score_req_mu); - free(snap); free(batches); free(filter); - continue; + free(snap); free(snap_idx); free(batches); free(filter); + shared_idx_release(refine_idx); continue; } /* Compact into flat array */ @@ -1063,6 +1908,17 @@ static void *scoring_thread_fn(void *arg) { size_t emit = (limit && limit < pos) ? limit : pos; + /* Build matched_idx array (all pos matches, not just top-K) for the + cache so a future subsuming query can refine-score only this set. */ + uint32_t *m_idx_buf = (pos && flat) ? malloc(pos * sizeof *m_idx_buf) : NULL; + if (m_idx_buf) for (size_t k = 0; k < pos; k++) m_idx_buf[k] = flat[k].idx; + + /* Cache the result. pool_gen = count (the pool size we actually scored). + For refine runs, count may be > refine_delta_from, so the new entry + supersedes the old one as a refinement source for the same query. */ + cache_insert(&s->cache, filter, count, case_mode, flat, emit, m_idx_buf, pos); + free(m_idx_buf); + /* Clear active-filter marker before publishing results */ pthread_mutex_lock(&s->score_req_mu); free(s->score_current_filter); s->score_current_filter = NULL; @@ -1079,10 +1935,11 @@ static void *scoring_thread_fn(void *arg) { /* Increment gen so Elisp knows new results are available */ atomic_fetch_add_explicit(&s->gen, 1, memory_order_relaxed); - fzf_log("scoring_thread: filter='%s' filtered=%zu total=%zu emit=%zu\n", - filter, pos, count, emit); + fzf_log("scoring_thread: filter='%s' filtered=%zu total=%zu emit=%zu refine=%d scanned=%zu\n", + filter, pos, count, emit, refine ? 1 : 0, snap_count); - free(snap); free(batches); free(filter); + free(snap); free(snap_idx); free(batches); free(filter); + shared_idx_release(refine_idx); } fzf_log("scoring_thread EXIT\n"); @@ -1103,41 +1960,143 @@ fzf_native_async_candidates(emacs_env *env, ptrdiff_t nargs, char *filter = malloc((size_t)flen); if (!filter) return Qnil; env->copy_string_contents(env, args[1], filter, &flen); + /* Keep a copy for C-side highlighting; filter ownership may transfer below. */ + char *filter_for_hilit = (flen > 1) ? strdup(filter) : NULL; size_t limit = 0; if (nargs > 2 && !env->eq(env, args[2], Qnil)) limit = (size_t)env->extract_integer(env, args[2]); - fzf_log("async_candidates: filter='%s' limit=%zu — dispatching to bg thread\n", - filter, limit); + fzf_case_types case_mode = resolve_fzf_native_case_mode(env); + + /* Cache lookup. Exact-fresh hits skip scoring entirely; exact-stale + and prefix hits dispatch a refinement scoring run that scans only + the prior match set + candidates that arrived since. Misses fall + through to a normal full-pool scoring run. */ + ScoredStr *cached_top = NULL; + size_t cached_count = 0; + SharedIdx *cached_m_idx = NULL; + size_t cached_pool_gen = 0; + + bool exact_hit = cache_lookup_exact(&s->cache, filter, + &cached_top, &cached_count, + &cached_m_idx, &cached_pool_gen); + bool prefix_hit = false; + if (!exact_hit) + prefix_hit = cache_lookup_prefix(&s->cache, filter, case_mode, + &cached_top, &cached_count, + &cached_m_idx, &cached_pool_gen); + + pthread_mutex_lock(&s->mu); + size_t current_pool = s->count; + pthread_mutex_unlock(&s->mu); + + bool exact_fresh = exact_hit && (cached_pool_gen == current_pool); + + fzf_log("async_candidates: filter='%s' limit=%zu pool=%zu hit=%s%s%s\n", + filter, limit, current_pool, + exact_fresh ? "exact-fresh" : "", + (exact_hit && !exact_fresh) ? "exact-stale" : "", + prefix_hit ? "prefix" : (!exact_hit ? "miss" : "")); /* Enqueue the new request. Only abort in-flight scoring if the filter actually changed — same-filter timer re-triggers must not interrupt a scoring run that is still working on the same query, which would cause - a livelock where scoring never completes on large candidate sets. */ + a livelock where scoring never completes on large candidate sets. + + On exact-fresh hit we skip enqueueing entirely — no work needed. */ pthread_mutex_lock(&s->score_req_mu); bool filter_changed = !(s->score_current_filter && strcmp(s->score_current_filter, filter) == 0 && s->score_current_limit == limit); if (filter_changed) atomic_store_explicit(&s->score_abort, true, memory_order_seq_cst); - free(s->score_req_filter); - s->score_req_filter = filter; /* scoring thread owns this now */ - s->score_req_limit = limit; - pthread_cond_signal(&s->score_req_cond); + + if (exact_fresh) { + /* No work to dispatch; release locally-owned filter + retained idx. */ + free(filter); + shared_idx_release(cached_m_idx); + } else { + free(s->score_req_filter); + shared_idx_release(s->score_req_refine_idx); + s->score_req_filter = filter; /* transfer */ + s->score_req_limit = limit; + s->score_req_case_mode = case_mode; + s->score_req_refine_idx = cached_m_idx; /* transfer (NULL on miss) */ + s->score_req_refine_delta_from = cached_pool_gen; + pthread_cond_signal(&s->score_req_cond); + } pthread_mutex_unlock(&s->score_req_mu); - /* Copy latest scored results under lock so we can release quickly */ - pthread_mutex_lock(&s->score_res_mu); - size_t rcount = s->score_count; - ScoredStr *snap = rcount ? malloc(rcount * sizeof *snap) : NULL; - if (snap && s->score_results) - memcpy(snap, s->score_results, rcount * sizeof *snap); - else - rcount = 0; - pthread_mutex_unlock(&s->score_res_mu); + /* Update displayed stats so the [FILTERED](TOTAL) overlay matches what + we're about to return: + - TOTAL always reflects the *current* pool, not the pool at the + last scoring time. Without this, the total in the prompt lags + behind the streaming counter visible elsewhere — and on cache + hits (which skip scoring) it can stay stale for many seconds. + - FILTERED on a cache hit is the cached entry's full match-set + count (m_idx->count), which describes the candidate set the + user is currently looking at. On a miss we leave it alone: + scoring will publish a fresh value shortly, and meanwhile the + existing value (from the previous query) is at least + consistent with the candidates we're about to fall back to. */ + ScoredStr *snap = NULL; + size_t rcount = 0; + if (exact_hit || prefix_hit) { + snap = cached_top; /* ownership transferred from cache_lookup_*. */ + rcount = cached_count; + size_t cached_filtered = cached_m_idx ? cached_m_idx->count : rcount; + pthread_mutex_lock(&s->score_res_mu); + s->last_filtered = cached_filtered; + s->last_total = current_pool; + pthread_mutex_unlock(&s->score_res_mu); + } else { + pthread_mutex_lock(&s->score_res_mu); + rcount = s->score_count; + snap = rcount ? malloc(rcount * sizeof *snap) : NULL; + if (snap && s->score_results) + memcpy(snap, s->score_results, rcount * sizeof *snap); + else + rcount = 0; + s->last_total = current_pool; /* keep TOTAL live even on miss */ + pthread_mutex_unlock(&s->score_res_mu); + } - /* Build Emacs list from stale results — strings are stable until session destroy */ + /* Resolve C-side highlight cap from the canonical defcustom. Read + via symbol-value so the user can change it without reloading the + module. fzf-async bridges `fzf-async-highlight' onto this via + :around advice on `fzf-native-async-candidates'. */ + size_t hl_cap = 0; + fzf_pattern_t *hl_pattern = NULL; + fzf_slab_t *hl_slab = NULL; + + if (filter_for_hilit) { + emacs_value sym_hi = env->intern(env, "fzf-native-async-highlight"); + emacs_value hi = env->funcall(env, env->intern(env, "symbol-value"), 1, &sym_hi); + if (env->non_local_exit_check(env) != emacs_funcall_exit_return) + env->non_local_exit_clear(env); + else if (env->eq(env, hi, Qt)) + hl_cap = rcount; /* t → highlight everything */ + else if (!env->eq(env, hi, Qnil)) { /* integer → highlight top N */ + intmax_t n = env->extract_integer(env, hi); + if (env->non_local_exit_check(env) != emacs_funcall_exit_return) + env->non_local_exit_clear(env); + else if (n > 0) + hl_cap = (size_t)n; + } + /* nil or negative integer → hl_cap stays 0, no highlighting */ + } + if (hl_cap > 0) { + hl_pattern = fzf_parse_pattern(case_mode, false, filter_for_hilit, true); + hl_slab = fzf_make_default_slab(); + } + + /* Build Emacs list from stale results — strings are stable until session destroy. + snap[0] is the highest-scoring candidate (prepend loop puts it at list head). + Apply fzf_get_positions highlighting for snap[0..hl_cap-1] (i < hl_cap). + NOTE: fzf positions are byte offsets; put-text-property uses character + positions. For ASCII candidates these are identical. Multi-byte UTF-8 + candidates may have slightly misaligned highlights — acceptable for now. */ emacs_value result = Qnil; for (size_t i = rcount; i-- > 0;) { emacs_value str = env->make_string(env, snap[i].str, @@ -1149,11 +2108,47 @@ fzf_native_async_candidates(emacs_env *env, ptrdiff_t nargs, } else if (status != emacs_funcall_exit_return) { break; } + + if (hl_pattern && i < hl_cap) { + fzf_position_t *pos = fzf_get_positions(snap[i].str, hl_pattern, hl_slab); + if (pos && pos->size > 0) { + /* pos->data[] is in descending order: pos->data[0] = highest position. + Iterate ascending (j from size-1 to 0) to find contiguous runs. */ + size_t plen = pos->size; + size_t run_start = pos->data[plen - 1]; + size_t run_end = run_start; + for (ptrdiff_t j = (ptrdiff_t)plen - 2; j >= 0; j--) { + size_t p = pos->data[j]; + if (p == run_end + 1) { + run_end = p; + } else { + emacs_value a[5] = { + env->make_integer(env, (intmax_t)run_start), + env->make_integer(env, (intmax_t)(run_end + 1)), + Qface, Qcompletions_common_part, str }; + env->funcall(env, Fput_text_property, 5, a); + env->non_local_exit_clear(env); + run_start = run_end = p; + } + } + emacs_value a[5] = { + env->make_integer(env, (intmax_t)run_start), + env->make_integer(env, (intmax_t)(run_end + 1)), + Qface, Qcompletions_common_part, str }; + env->funcall(env, Fput_text_property, 5, a); + env->non_local_exit_clear(env); + } + fzf_free_positions(pos); + } + result = env->funcall(env, Fcons, 2, (emacs_value[]){ str, result }); if (env->non_local_exit_check(env) != emacs_funcall_exit_return) break; } + if (hl_pattern) fzf_free_pattern(hl_pattern); + if (hl_slab) fzf_free_slab(hl_slab); + free(filter_for_hilit); free(snap); return result; } @@ -1217,6 +2212,20 @@ int emacs_module_init(struct emacs_runtime *rt) { &data), }); + // fzf-native-highlight-all COLLECTION QUERY + env->funcall(env, env->intern(env, "defalias"), 2, (emacs_value[]) { + env->intern(env, "fzf-native-highlight-all"), + env->make_function(env, 2, 2, fzf_native_highlight_all, + "Apply fzf match highlights to COLLECTION against QUERY.\n" + "Mutates each candidate string's text properties in place;\n" + "stale `completions-common-part' face from a prior query is\n" + "stripped before new positions are applied. No scoring or\n" + "sorting is performed.\n" + "\n" + "\\(fn COLLECTION QUERY)", + &data), + }); + // fzf-native-score STR QUERY &optional SLAB env->funcall(env, env->intern(env, "defalias"), 2, (emacs_value[]) { env->intern(env, "fzf-native-score"), @@ -1314,6 +2323,14 @@ int emacs_module_init(struct emacs_runtime *rt) { Qcompletion_score = env->make_global_ref(env, env->intern(env, "completion-score")); Fput_text_property = env->make_global_ref(env, env->intern(env, "put-text-property")); Fencode_coding_string = env->make_global_ref(env, env->intern(env, "encode-coding-string")); + Qface = env->make_global_ref(env, env->intern(env, "face")); + Qcompletions_common_part = env->make_global_ref(env, env->intern(env, "completions-common-part")); + Fremove_text_properties = env->make_global_ref(env, env->intern(env, "remove-text-properties")); + /* Pre-built (face nil) plist passed to remove-text-properties to strip the + `face' property regardless of value. Built once to avoid allocating a + fresh cons cell on every highlight call. */ + Qface_nil_plist = env->make_global_ref( + env, env->funcall(env, Flist, 2, (emacs_value[]){Qface, Qnil})); Qutf_8 = env->make_global_ref(env, env->intern(env, "utf-8")); Qlistofzero = env->make_global_ref( env, env->funcall(env, Fcons, 2, diff --git a/fzf-native-test.el b/fzf-native-test.el index 5388bb4..d667479 100644 --- a/fzf-native-test.el +++ b/fzf-native-test.el @@ -4,20 +4,7 @@ (fzf-native-load-dyn) -(ert-deftest fzf-native-score-indices-order-test () - (let ((result (fzf-native-score "abcdefghi" "acef"))) - (should (= (nth 1 result) 0)) - (should (= (nth 2 result) 2)) - (should (= (nth 3 result) 4)) - (should (= (nth 4 result) 5)))) - -(ert-deftest score-with-default-slab-indices-order-test () - (let* ((slab (fzf-native-make-default-slab)) - (result (fzf-native-score "abcdefghi" "acef" slab))) - (should (= (nth 1 result) 0)) - (should (= (nth 2 result) 2)) - (should (= (nth 3 result) 4)) - (should (= (nth 4 result) 5)))) + (ert-deftest fzf-native-score-with-default-slab-test () "Test slab can be reused." @@ -25,16 +12,16 @@ (_result (fzf-native-score "abcdefghi" "acef" slab))) (should (equal (fzf-native-score "abcdefghi" "acef" slab) - '(78 0 2 4 5))) + '(78))) (should (equal (fzf-native-score "abc" "acef" slab) '(0))) (should (equal (fzf-native-score "zzzzzabc" "z" slab) - '(32 0))) + '(32))) (should (equal (fzf-native-score "sfsjoc" "jo" slab) - '(36 3 4))))) + '(36))))) (ert-deftest fzf-native-score-with-slab-test () "Test slab can be reused." @@ -42,16 +29,16 @@ (_result (fzf-native-score "abcdefghi" "acef" slab))) (should (equal (fzf-native-score "abcdefghi" "acef" slab) - '(78 0 2 4 5))) + '(78))) (should (equal (fzf-native-score "abc" "acef" slab) '(0))) (should (equal (fzf-native-score "zzzzzabc" "z" slab) - '(32 0))) + '(32))) (should (equal (fzf-native-score "sfsjoc" "jo" slab) - '(36 3 4))))) + '(36))))) (ert-deftest fzf-native-score-empty-query-test () (let ((result (fzf-native-score "abcdefghi" ""))) @@ -81,13 +68,34 @@ (let* ((len 4096) (str (concat (make-string len ?s) "d")) (result (fzf-native-score str "d"))) - (should (equal result `(16 ,len))))) + (should (equal result '(16))))) (ert-deftest fzf-native-score-very-long-str-test () (let* ((len 65536) (str (concat (make-string len ?s) "d")) (result (fzf-native-score str "d"))) - (should (equal result `(16 ,len))))) + (should (equal result '(16))))) + +(ert-deftest fzf-native-score-case-mode-smart-test () + "Default `fzf-native-case-mode' is smart: lowercase query is +case-insensitive, query with any uppercase becomes case-sensitive." + (should (eq fzf-native-case-mode 'smart)) + ;; Lowercase query → insensitive: matches uppercase target. + (should (equal (fzf-native-score "Foo" "foo") '(80))) + ;; Uppercase query → sensitive: lowercase target no longer matches. + (should (equal (fzf-native-score "foo" "Foo") '(0)))) + +(ert-deftest fzf-native-score-case-mode-ignore-test () + "`fzf-native-case-mode' = ignore matches regardless of case." + (let ((fzf-native-case-mode 'ignore)) + (should (equal (fzf-native-score "foo" "Foo") '(80))) + (should (equal (fzf-native-score "Foo" "foo") '(80))))) + +(ert-deftest fzf-native-score-case-mode-respect-test () + "`fzf-native-case-mode' = respect requires exact case." + (let ((fzf-native-case-mode 'respect)) + (should (equal (fzf-native-score "Foo" "foo") '(0))) + (should (equal (fzf-native-score "foo" "foo") '(80))))) (ert-deftest fzf-native-score-with-default-slab-benchmark-test () "Test scoring with slab is faster." @@ -118,32 +126,7 @@ (benchmark-run 10000 (fzf-native-score str query large-slab))))))) -(ert-deftest fzf-native-score-indices-multibyte-not-supported-test () - ;; Force `str' to be unambiguously multibyte regardless of the coding - ;; system used to load this file (eask may differ from an interactive - ;; session). Without the advice, the C module returns BYTE positions. - (let ((str (decode-coding-string - (encode-coding-string "ポケモン.txt" 'utf-8) 'utf-8))) - (should (multibyte-string-p str)) - (should - (equal (cdr (fzf-native-score str "txt")) - '(13 14 15))))) - -(ert-deftest fzf-native-score-indices-multibyte-support-through-advice-test () - ;; Force `str' to be multibyte (see `...not-supported-test' above). - ;; With the advice, byte positions are mapped back to character - ;; positions, so we expect (5 6 7) instead of (13 14 15). - (advice-add 'fzf-native-score :around #'fzf-native--fix-score-indices) - (unwind-protect - (let ((str (decode-coding-string - (encode-coding-string "ポケモン.txt" 'utf-8) 'utf-8))) - (should (multibyte-string-p str)) - (should - (equal (cdr (fzf-native-score str "txt")) - '(5 6 7)))) - ;; Always remove the advice, even if the assertions above failed. - ;; Otherwise the advice leaks into subsequent tests. - (advice-remove 'fzf-native-score #'fzf-native--fix-score-indices))) + (defun fzf-native-generate-random-string (length) "Generate a random string of LENGTH using alphanumeric characters." @@ -215,7 +198,7 @@ ;; the same as "did not match" and the candidate is silently dropped. (defconst fzf-native-test--bad-bytes - (string-as-multibyte ";; Copyright 2022 Jo Be") + (string-as-multibyte ";; Copyright 2022 Jo Be�����") "Raw-byte string used as a reproducer for the `unicode-string-p' bug. Note: on Emacs 30+ this WILL coerce successfully through `encode-coding-string', so it scores like any other string. The tests @@ -402,7 +385,64 @@ currently being scored. Stats are only written on completion, so (should done))) (fzf-native-async-stop handle)))) -(ert-deftest fzf-native-async-start-wrong-type-test () - "`fzf-native-async-start' signals on a non-string command." - (should-error (fzf-native-async-start 42) - :type 'wrong-type-argument)) +(ert-deftest fzf-native-async-cache-prefix-refinement-test () + "Cache returns consistent results across a typing progression and on +backspace. Setup: a small corpus where 'fo'/'foo'/'food' produce +predictably-different result sets. We type the progression, verify +each query's results, then backspace back to 'fo' and verify it +returns the same set as the original 'fo' call. + +This exercises: +- Phase-1 exact lookup (each first call inserts; second call hits) +- Phase-2 prefix refinement (typing extends matched_idx) +- Backspace coverage (LRU keeps prior queries)" + (skip-unless (fboundp 'fzf-native-async-start)) + (let ((handle (fzf-native-async-start + "printf '%s\\n' food foo foobar fool bar baz"))) + (unwind-protect + (progn + (fzf-native-test--wait-for-data handle) + (let ((r-fo-1 (sort (copy-sequence + (fzf-native-test--wait-for-scoring handle "fo")) + #'string<))) + (should (member "foo" r-fo-1)) + (should (member "food" r-fo-1)) + (should (member "foobar" r-fo-1)) + (should (member "fool" r-fo-1)) + (should-not (member "bar" r-fo-1)) + ;; Type "foo": narrower than "fo" — refinement scenario + (let ((r-foo (fzf-native-test--wait-for-scoring handle "foo"))) + (should (member "foo" r-foo)) + (should (member "food" r-foo)) + (should (member "foobar" r-foo)) + ;; "fool" doesn't fuzzy-match "foo" cleanly; just check non-foo + ;; candidates are absent + (should-not (member "bar" r-foo)) + (should-not (member "baz" r-foo))) + ;; Backspace to "fo" — should hit cached entry, return same set + (let ((r-fo-2 (sort (copy-sequence + (fzf-native-async-candidates handle "fo")) + #'string<))) + (should (equal r-fo-2 r-fo-1))))) + (fzf-native-async-stop handle)))) + +(ert-deftest fzf-native-async-cache-term-reorder-test () + "Term reordering: \"foo bar\" and \"bar foo\" are semantically equal +in fzf and the cache should treat them so via term-set subsumption +(v2). Both queries should return the same candidates." + (skip-unless (fboundp 'fzf-native-async-start)) + (let ((handle (fzf-native-async-start + "printf '%s\\n' foobar fooXbar bar foo barfoo barXfoo"))) + (unwind-protect + (progn + (fzf-native-test--wait-for-data handle) + (let ((r1 (sort (copy-sequence + (fzf-native-test--wait-for-scoring handle "foo bar")) + #'string<)) + (r2 (sort (copy-sequence + (fzf-native-test--wait-for-scoring handle "bar foo")) + #'string<))) + (should r1) + (should (equal r1 r2)))) + (fzf-native-async-stop handle)))) + diff --git a/fzf-native.el b/fzf-native.el index 95b59ed..2595157 100644 --- a/fzf-native.el +++ b/fzf-native.el @@ -3,7 +3,7 @@ ;; Copyright 2021 Duc Dang ;; Author: Duc Dang ;; Version: 0.3 -;; Package-Requires: ((emacs "27.1")) +;; Package-Requires: ((emacs "29.1")) ;; Keywords: matching ;; Homepage: https://github.com/dangduc/fzf-native ;; SPDX-License-Identifier: GPL-3.0-or-later @@ -21,6 +21,7 @@ :link '(url-link :tag "GitHub" "https://github.com/dangduc/fzf-native")) (declare-function fzf-native-score-all "fzf-native-module" (collection query &optional slab)) +(declare-function fzf-native-highlight-all "fzf-native-module" (collection query)) (declare-function fzf-native-score "fzf-native-module" (str query &optional slab)) (declare-function fzf-native-make-default-slab "fzf-native-module" ()) (declare-function fzf-native-make-slab "fzf-native-module" (size16 size32)) @@ -54,6 +55,76 @@ confirmation before compiling." :type 'boolean :group 'fzf-native) +;; Canonical knobs the C module reads via `symbol-value' at call time. +;; Higher-level packages (fzf-async, fussy) keep their own user-facing +;; defcustoms and bridge their values onto these names — fussy via +;; `setq-local' (synchronous, same-buffer call pattern), fzf-async via +;; `:around' advice on the C entry points (timer-driven, cross-buffer). + +(defcustom fzf-native-case-mode 'smart + "How fzf-native treats letter case when matching queries. +smart Case-insensitive when the query is all lowercase; case-sensitive + once it contains any uppercase character (fzf's default). +ignore Always case-insensitive. +respect Always case-sensitive. + +Read on every scoring call; changes take effect immediately." + :type '(choice (const :tag "Smart case (default)" smart) + (const :tag "Ignore case" ignore) + (const :tag "Respect case" respect)) + :group 'fzf-native) + +(defcustom fzf-native-batch-highlight 25 + "Highlight cap for the synchronous (batch) scoring path. +Read by `fzf-native-score' / `fzf-native-score-all' on every call. +nil disables highlighting; a positive integer caps the number of +top-scoring candidates that get `completions-common-part' face +applied via `fzf_get_positions' inside the C module. + +Bridged by fussy from `fussy-fzf-native-highlight' via `setq-local'." + :type '(choice (const :tag "Disabled" nil) + (integer :tag "Top N candidates")) + :group 'fzf-native) + +(defcustom fzf-native-async-highlight 200 + "Highlight cap for the streaming (async) candidate path. +Read by `fzf-native-async-candidates' on every call. Same semantics +as `fzf-native-batch-highlight' (nil / positive integer). + +Bridged by fzf-async from `fzf-async-highlight' via `:around' advice." + :type '(choice (const :tag "Disabled" nil) + (const :tag "All candidates" t) + (integer :tag "Top N candidates")) + :group 'fzf-native) + +(defcustom fzf-native-max-line-length 256 + "Per-line character cap applied by the async reader thread. +nil — no limit. +positive N — exclude lines longer than N characters. +negative -N — include but truncate lines to N characters. + +Read once at session start by `fzf-native-async-start'. + +Bridged by fzf-async from `fzf-async-max-line-length' via `:around' +advice; the read happens inside `fzf-native-async-start' so the +advice is in scope for the symbol-value lookup." + :type '(choice (const :tag "No limit" nil) + (integer :tag "N (positive = exclude, negative = truncate)")) + :group 'fzf-native) + +(defcustom fzf-native-async-cache-size 40 + "Per-session LRU result cache capacity for the async path. +Each entry stores top-K results and the full matched-candidate index +for one query — enables exact-fresh hits (skip scoring) and prefix- +refinement hits (rescore only previously-matched candidates plus +deltas) without re-scanning the full pool. + +Read once at session start by `fzf-native-async-start'. + +Bridged by fzf-async from `fzf-async-cache-size' via `:around' advice." + :type 'integer + :group 'fzf-native) + (defun fzf-native-module--cmake-is-available () "Return t if cmake is available. CMake is needed to build fzf-native, here we check that we can find @@ -143,23 +214,5 @@ module load." (require 'fzf-native-module)) (error "Fzf-Native will not work until `fzf-native-module' is compiled!")))) -(defun fzf-native--fix-score-indices (fn str &rest args) - "An around advice to fix score indices if STR is multibyte. -FN should be `fzf-native-score'." - (let ((score (apply fn str args))) - (if (or (null score) (not (multibyte-string-p str))) - score - ;; fzf-native makes score indices as byte position. - ;; But we want it as character position. - (let ((idx (cl-loop for i from 0 to (1- (length str)) - vconcat (make-vector (string-bytes (char-to-string (aref str i))) i)))) - (cons (car score) (mapcar (lambda (x) (aref idx x)) (cdr score))))))) - -;; Work around the lib's lack of support for multibyte chars. Add this -;; advice if you want accurate indices for multibyte chars. Don't add -;; this advice if you want better run time performance or you don't -;; need accurate indices for multibyte chars. -; e.g. (advice-add 'fzf-native-score :around #'fzf-native--fix-score-indices) - (provide 'fzf-native) ;;; fzf-native.el ends here