diff --git a/src/context_system/prompt_assembly.py b/src/context_system/prompt_assembly.py index 63f19138..5ef6b4e9 100644 --- a/src/context_system/prompt_assembly.py +++ b/src/context_system/prompt_assembly.py @@ -911,12 +911,20 @@ def _emit_group( "investigation only when results rule them out or reveal new evidence; " "do not replace focused diagnosis with an exhaustive search of the " "environment.\n" + # The trailing sentence here used to be: "When the user asks for all, + # every, multiple, or an exhaustive set of results, actively search for + # additional valid results instead of stopping after the first one." + # It keyed off bare quantifiers, the same over-broad signal as the + # (now removed) exhaustive-audit nudge, and pushed extra verification + # rounds on any request that merely contained "each" or "all". What + # remains is the part that holds regardless of phrasing: check the + # requirements you were actually given. If a request genuinely asks for + # every qualifying result, that IS an explicit requirement and the + # first sentence already covers it. "- Before finishing, audit the result against every explicit requirement " "in the user's request and verify the requirements that can be checked. " "Do not treat producing a plausible result as proof that the task is " - "complete. When the user asks for all, every, multiple, or an exhaustive " - "set of results, actively search for additional valid results instead of " - "stopping after the first one.\n" + "complete.\n" "- Use the narrowest sufficient verification for the change. Prefer an " "existing focused check, add a regression test when it provides lasting " "value, and run broader checks when the scope or risk warrants them. " diff --git a/src/query/continuation_nudge.py b/src/query/continuation_nudge.py index 2a20069b..dd302980 100644 --- a/src/query/continuation_nudge.py +++ b/src/query/continuation_nudge.py @@ -26,27 +26,6 @@ "explicitly and state what you did." ) -EXHAUSTIVE_AUDIT_NUDGE = ( - "Before finishing, audit the result against the user's exhaustive-result " - "requirement. Actively search for additional valid results; do not assume " - "the first valid result is the only one. For an enumerable or mechanically " - "checkable state space, you MUST use a tool to enumerate or independently " - "validate candidates rather than declaring a prose-only audit exhaustive. " - "Evaluate the state after each candidate action, including candidates that " - "do not have the relevant property before they move. Update the deliverable " - "if the audit finds more." -) - -_EXHAUSTIVE_REQUIREMENT = re.compile( - r"\b(all|every|multiple|each|exhaustive|complete set|any other|others)\b", - re.IGNORECASE, -) - - -def requests_exhaustive_results(text: str) -> bool: - """Whether a user request explicitly asks for a result set audit.""" - return bool(_EXHAUSTIVE_REQUIREMENT.search(text)) - # Don't nudge when the model signaled completion (query.ts:1487). _COMPLETION_MARKERS = re.compile( r"\b(done|finished|completed|complete|summary|that's all|that is all" diff --git a/src/query/query.py b/src/query/query.py index 0ec9b3a6..30971a9c 100644 --- a/src/query/query.py +++ b/src/query/query.py @@ -2171,49 +2171,6 @@ def _marking_chunk_cb(text: str) -> None: for b in content if getattr(b, "type", None) == "text" ) - # A completion claim is not sufficient when the user asked - # for an exhaustive result set. Require one bounded audit - # pass before accepting the clean exit. This closes a common - # harness failure mode where the model finds one plausible - # answer and stops despite an explicit "all"/"multiple" - # requirement. A dedicated latch survives tool rounds so the - # guard cannot create an unbounded self-review loop. - from .continuation_nudge import ( - EXHAUSTIVE_AUDIT_NUDGE, - requests_exhaustive_results, - ) - - exhaustive_request = any( - not getattr(msg, "isMeta", False) - and getattr(msg, "role", None) == "user" - and requests_exhaustive_results( - getattr(msg, "content", "") - if isinstance(getattr(msg, "content", ""), str) - else "" - ) - for msg in messages - ) - if exhaustive_request and not state.exhaustive_audit_performed: - logger.debug("Exhaustive-result audit nudge triggered") - state = QueryState( - messages=[ - *messages, - *assistant_messages, - UserMessage(content=EXHAUSTIVE_AUDIT_NUDGE, isMeta=True), - ], - tool_use_context=tool_use_context, - auto_compact_tracking=state.auto_compact_tracking, - max_output_tokens_recovery_count=0, - has_attempted_reactive_compact=False, - max_output_tokens_override=None, - stop_hook_active=None, - turn_count=turn_count, - pending_tool_use_summary=None, - continuation_nudge_count=state.continuation_nudge_count, - exhaustive_audit_performed=True, - transition=Transition(reason="continuation_nudge"), - ) - continue # An assistant turn with NO tool calls and NO text is not a # completion — it is a degenerate response, and accepting it # ends the run with an empty answer while every caller @@ -2512,7 +2469,6 @@ def _marking_chunk_cb(text: str) -> None: stop_hook_active=state.stop_hook_active, # Phase A: reset per-turn counter; carry pending summary forward. continuation_nudge_count=0, - exhaustive_audit_performed=state.exhaustive_audit_performed, pending_tool_use_summary=state.pending_tool_use_summary, transition=Transition(reason="next_turn"), ) diff --git a/src/query/transitions.py b/src/query/transitions.py index 0b3dec14..5c24ff2d 100644 --- a/src/query/transitions.py +++ b/src/query/transitions.py @@ -110,7 +110,4 @@ class QueryState: # when the model keeps matching continuation signals without tool calls. # Mirrors TS State.continuationNudgeCount at query.ts:218. continuation_nudge_count: int = 0 - # Session-chain latch for the one-time exhaustive-result audit. Unlike - # continuation_nudge_count, this survives successful tool rounds. - exhaustive_audit_performed: bool = False transition: Transition | None = None diff --git a/tests/test_query_loop_wires_round3.py b/tests/test_query_loop_wires_round3.py index 3f4c3952..06f2d097 100644 --- a/tests/test_query_loop_wires_round3.py +++ b/tests/test_query_loop_wires_round3.py @@ -30,10 +30,8 @@ def get_pending_post_compaction() -> bool: return _bs._STATE.pending_post_compaction from src.providers.base import ChatResponse from src.query.continuation_nudge import ( - EXHAUSTIVE_AUDIT_NUDGE, MAX_CONTINUATION_NUDGES, detect_continuation_signal, - requests_exhaustive_results, ) from src.query.query import QueryParams, run_query from src.query.stop_hooks import StopHookResult @@ -623,10 +621,30 @@ def test_completion_text_no_nudge(self): sum(1 for m in msgs if isinstance(m, AssistantMessage)), 1 ) - def test_exhaustive_request_forces_one_audit_pass(self): + def test_no_forced_audit_pass_on_a_completion_claim(self): + """A completion claim ends the turn — no extra audit round-trip. + + #740 added a one-shot "exhaustive audit" nudge: when the request + matched a bare quantifier (all|every|multiple|each|...), the loop + appended a MUST-enumerate instruction and forced another model + turn. Removed, on evidence from the latest Claude Code rather than + from the older vendored fork: + + * CC scores 1.0 on the two tasks the mechanism was built for + (chess-best-move, regex-chess) with no such machinery; + * only 1 of its 89 trials writes more than two verification files, + against clawcodex's 57 temporary verification scripts overall; + * the trigger was a plain-English quantifier match, so it fired on + 49% of terminal-bench 2.1 — "each line of the file", "where each + task is", "in all years" — and in regex-log it demanded + exhaustive enumeration of a task that asks to match only ONE. + + Verification guidance now lives entirely in the system prompt, + which is where the model can weigh it against the request. + """ provider = _provider([ _completion("I found one result and finished."), - _completion("I audited all candidates and updated the result."), + _completion("second turn should never happen"), ]) params = _params(self.workspace, provider) params.messages = [ @@ -637,16 +655,10 @@ def test_exhaustive_request_forces_one_audit_pass(self): self.assertEqual(terminal.reason, "completed") self.assertEqual( - sum(1 for m in msgs if isinstance(m, AssistantMessage)), 2 + sum(1 for m in msgs if isinstance(m, AssistantMessage)), 1, + "an exhaustive-sounding request must not buy an extra turn", ) - def test_exhaustive_request_detector(self): - self.assertTrue(requests_exhaustive_results("print them all")) - self.assertTrue(requests_exhaustive_results("multiple winning moves")) - self.assertFalse(requests_exhaustive_results("find the best move")) - self.assertIn("MUST use a tool", EXHAUSTIVE_AUDIT_NUDGE) - self.assertIn("state after each candidate action", EXHAUSTIVE_AUDIT_NUDGE) - # --------------------------------------------------------------------------- # G3 + G4 — /clear resets; compaction marks diff --git a/tests/test_system_prompt_full.py b/tests/test_system_prompt_full.py index 92df7492..be5843d9 100644 --- a/tests/test_system_prompt_full.py +++ b/tests/test_system_prompt_full.py @@ -59,11 +59,25 @@ def test_basic_prompt_has_intro(self): assert "interactive agent" in prompt assert "software engineering tasks" in prompt - def test_task_prompt_requires_requirement_audit_and_exhaustive_results(self): + def test_task_prompt_audits_requirements_without_a_quantifier_trigger(self): + """Keep the requirement audit; drop the quantifier-keyed clause. + + The removed sentence told the model to search for additional + results "when the user asks for all, every, multiple…" — keying off + bare quantifiers, which appear in ordinary task descriptions + ("each line of the file", "in all years"). Paired with the + exhaustive-audit nudge it drove verification rounds the latest + Claude Code does not spend: 1 of its 89 terminal-bench 2.1 trials + writes more than two verification files. + + A genuine "give me every match" request is an explicit requirement, + so the surviving sentence already covers it. + """ prompt = build_full_system_prompt(use_cache=False) assert "audit the result against every explicit requirement" in prompt - assert "all, every, multiple, or an exhaustive set" in prompt - assert "stopping after the first one" in prompt + assert "plausible result as proof" in prompt + assert "all, every, multiple, or an exhaustive set" not in prompt + assert "stopping after the first one" not in prompt def test_task_prompt_prioritizes_evidence_and_cheap_discriminating_checks(self): prompt = build_full_system_prompt(use_cache=False)