From d1044942521c225cc279c82653b2f50c286cb3a0 Mon Sep 17 00:00:00 2001 From: Oranje AI <293843428+oranjeai@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:36:49 +1200 Subject: [PATCH] tests: pin the pre-recall prompt-cache freeze AgentEngine._get_or_create_client freezes a session's first successful pre-recall in session metadata (engine.py:1149) so every later rebuild renders a byte-identical system prompt and hits the prompt cache. Nothing tested it, in either direction: `git grep -c recalled_memories origin/main -- tests/` and the same for meta_updates both return 0 hits in any test file, and tests/test_prompts.py never passes recalled_memories at all. Deleting engine.py:1149 leaves the whole suite green, and nothing else re-derives the value, so the regression would be silent - every rebuild re-recalls live, a session's priors drift, and the cache hit the freeze exists to buy is lost with no test failing. The gap survived because the payoff is a cross-rebuild property. Dropping the write leaves an observably identical first turn; the loss shows only on a later rebuild of the same session, and no existing test spanned two rebuilds. Test-only and insert-only. TestPreRecallFreeze goes in the file that already drives _get_or_create_client end to end against the real db fixture with a local stub backend, so no production line changes and no new seam is needed - the stub's create_client keeps the SessionSpec it is already handed. Phase A asserts the metadata now holds the priors AND that the first rendered system prompt carries them (prompts.py:166-168, previously unpinned in tests/ and in test_prompts.py). Phase B evicts the client via sessions.remove_client, rearms recall with different priors, calls again, and asserts a second create_client really happened before checking that the second prompt carries the ORIGINAL priors and none of the rearmed ones. The precondition is not optional: a live client returns at engine.py:1077 before session_meta is parsed, so without the eviction the second call never re-enters the freeze block and both assertions would be satisfied by the first call's side effects. Coverage-only, so discrimination is the deliverable. Eight mutants, all killed, each arm asserting its mutation applied and restoring the tree afterwards, with the unmutated control green at both ends of the matrix: deleting engine.py:1149 (phase A metadata); frozen_recall = None so every rebuild re-recalls (phase B, second prompt carried the rearmed priors); freezing the wrong value, = [] (phase A); the render call dropping the priors (phase A prompt only); the frozen-read branch reading then discarding (phase B prompt only); reversing the frozen list and doubling it, each of which changes the prompt bytes while leaving membership, exclusion, await_count and the store all satisfied (phase B byte-identity only); and removing the eviction from the test itself, which fails loudly at the precondition (assert 1 == 2) rather than passing silently. Weakening phase A to the key-presence-plus-await_count shape makes the wrong-value mutant survive phase A - the second-prompt assertion is what kills it - so the store check, the first-prompt check, the second-prompt check and the byte-identity check each uniquely kill a different mutant and none is decoration. "Reused verbatim" is byte-identity, and membership is strictly weaker: prompts.py:167 renders "\n".join(f"- {m}" for m in recalled_memories), so order and multiplicity are prompt bytes, and byte-identity is exactly the property the prompt cache needs. A reordered or duplicated frozen list therefore satisfies every membership, exclusion, await_count and store assertion while destroying the payoff; the reversal and duplication mutants both survive without the byte-identity check and both die on it, at that assertion specifically. It is ADDITIVE rather than a replacement: under the mutant where the render call drops the priors entirely, both prompts are equally wrong, so comparing them to each other passes and the first-prompt membership check is that mutant's only observer. Substituting would have traded one kill for another. Target file 20 -> 21 passed. Full suite compared by failure NAME, not count: 7 failed / 2933 passed before, 7 failed / 2934 passed after, the two sorted name sets byte-identical, all 7 pre-existing and none in a file this touches. The new test is 20/20 stable and clean in one process with test_memorize_background, test_session_context_tool and test_prompts. ruff reports one F401 at tests/test_engine_backend_selection.py:196; it is pre-existing (the identical error is reported on the untouched origin/main blob at the same line) and deliberately left alone as an unrelated concern. --- tests/test_engine_backend_selection.py | 98 ++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/tests/test_engine_backend_selection.py b/tests/test_engine_backend_selection.py index 87b5c601..cda37e27 100644 --- a/tests/test_engine_backend_selection.py +++ b/tests/test_engine_backend_selection.py @@ -252,6 +252,104 @@ async def create_client(self, spec): assert session.get("backend") == "codex" +class TestPreRecallFreeze: + """The first successful pre-recall is persisted to session metadata and + reused verbatim on every later rebuild instead of re-recalled.""" + + @pytest.mark.asyncio + async def test_first_recall_is_frozen_and_reused_on_rebuild( + self, tmp_path, db, + ): + """Phase A pins the metadata write; phase B pins that a rebuild + renders the stored priors, not a fresh recall.""" + import json + from unittest.mock import AsyncMock, MagicMock + + from nerve.agent.backends.base import BackendCapabilities + + first = ["prior-alpha-9f1", "prior-beta-9f2"] + second = ["rearmed-gamma-7c1", "rearmed-delta-7c2"] + + engine = _engine(tmp_path, db) + await db.create_session("s-freeze", source="web", backend="codex") + + bridge = MagicMock(available=True) + bridge.recall = AsyncMock( + return_value=[{"summary": m} for m in first], + ) + engine._memory_bridge = bridge + + prompts: list[str] = [] + + class StubClient: + model = "gpt-5.6-sol" + + def is_alive(self): + return True + + async def disconnect(self): + pass + + class StubBackend: + name = "codex" + capabilities = BackendCapabilities( + cost_is_cumulative=False, + supports_idle_stream=False, + supports_cache_ttl=False, + interactive_builtins=False, + reports_context_window=True, + ) + + def default_model(self, source): + return "gpt-5.6-sol" + + def excluded_tools(self): + return set() + + def validate_resume_target(self, native_id, cwd): + return True + + async def create_client(self, spec): + prompts.append(spec.system_prompt) + return StubClient() + + engine._backends["codex"] = StubBackend() + + # Phase A: the write under test. + await engine._get_or_create_client("s-freeze", "web", None) + session = await db.get_session("s-freeze") + meta = json.loads(session.get("metadata") or "{}") + assert meta.get("recalled_memories") == first, meta + assert len(prompts) == 1 + for m in first: + assert f"- {m}" in prompts[0] + + # Phase B: evict the cached client so the rebuild re-enters the + # freeze block (a live client returns before metadata is parsed), + # and rearm recall so a live re-recall would be visible. + engine.sessions.remove_client("s-freeze") + bridge.recall.return_value = [{"summary": m} for m in second] + + await engine._get_or_create_client("s-freeze", "web", None) + + # Precondition: the rebuild really happened. + assert len(prompts) == 2, prompts + # The frozen priors are what the second prompt carries. + for m in first: + assert f"- {m}" in prompts[1] + for m in second: + assert m not in prompts[1] + # Verbatim means byte-identical: order and multiplicity are prompt + # bytes (prompts.py:167), and byte-identity is the cache property. + assert prompts[1] == prompts[0] + # Corroborating: nothing re-recalled, nothing overwrote the store. + assert bridge.recall.await_count == 1 + after = await db.get_session("s-freeze") + assert json.loads(after.get("metadata") or "{}").get( + "recalled_memories", + ) == first + + class TestCreateSessionRoute: """POST /api/sessions with the new-chat backend selector's param."""