diff --git a/src/context_system/prompt_assembly.py b/src/context_system/prompt_assembly.py index 5c5420b0..e64adce0 100644 --- a/src/context_system/prompt_assembly.py +++ b/src/context_system/prompt_assembly.py @@ -883,6 +883,12 @@ def _emit_group( "the error, check your assumptions, try a focused fix. Don't retry " "the identical action blindly, but don't abandon a viable approach " "after a single failure either.\n" + "- Before finishing, audit the result against every explicit requirement " + "in the user's request and verify the requirements that can be checked. " + "Do not treat producing a plausible result as proof that the task is " + "complete. When the user asks for all, every, multiple, or an exhaustive " + "set of results, actively search for additional valid results instead of " + "stopping after the first one.\n" "- Be cautious not to introduce security vulnerabilities such as " "command injection, XSS, SQL injection, and other OWASP top 10 " "vulnerabilities.\n" diff --git a/src/query/continuation_nudge.py b/src/query/continuation_nudge.py index 56bd2208..b2e1fbcf 100644 --- a/src/query/continuation_nudge.py +++ b/src/query/continuation_nudge.py @@ -16,6 +16,27 @@ NUDGE_MESSAGE = "Continue with the task. Use the appropriate tools to proceed." +EXHAUSTIVE_AUDIT_NUDGE = ( + "Before finishing, audit the result against the user's exhaustive-result " + "requirement. Actively search for additional valid results; do not assume " + "the first valid result is the only one. For an enumerable or mechanically " + "checkable state space, you MUST use a tool to enumerate or independently " + "validate candidates rather than declaring a prose-only audit exhaustive. " + "Evaluate the state after each candidate action, including candidates that " + "do not have the relevant property before they move. Update the deliverable " + "if the audit finds more." +) + +_EXHAUSTIVE_REQUIREMENT = re.compile( + r"\b(all|every|multiple|each|exhaustive|complete set|any other|others)\b", + re.IGNORECASE, +) + + +def requests_exhaustive_results(text: str) -> bool: + """Whether a user request explicitly asks for a result set audit.""" + return bool(_EXHAUSTIVE_REQUIREMENT.search(text)) + # Don't nudge when the model signaled completion (query.ts:1487). _COMPLETION_MARKERS = re.compile( r"\b(done|finished|completed|complete|summary|that's all|that is all" diff --git a/src/query/query.py b/src/query/query.py index 70cd2777..c9216a9f 100644 --- a/src/query/query.py +++ b/src/query/query.py @@ -2101,6 +2101,49 @@ def _marking_chunk_cb(text: str) -> None: for b in content if getattr(b, "type", None) == "text" ) + # A completion claim is not sufficient when the user asked + # for an exhaustive result set. Require one bounded audit + # pass before accepting the clean exit. This closes a common + # harness failure mode where the model finds one plausible + # answer and stops despite an explicit "all"/"multiple" + # requirement. A dedicated latch survives tool rounds so the + # guard cannot create an unbounded self-review loop. + from .continuation_nudge import ( + EXHAUSTIVE_AUDIT_NUDGE, + requests_exhaustive_results, + ) + + exhaustive_request = any( + not getattr(msg, "isMeta", False) + and getattr(msg, "role", None) == "user" + and requests_exhaustive_results( + getattr(msg, "content", "") + if isinstance(getattr(msg, "content", ""), str) + else "" + ) + for msg in messages + ) + if exhaustive_request and not state.exhaustive_audit_performed: + logger.debug("Exhaustive-result audit nudge triggered") + state = QueryState( + messages=[ + *messages, + *assistant_messages, + UserMessage(content=EXHAUSTIVE_AUDIT_NUDGE, isMeta=True), + ], + tool_use_context=tool_use_context, + auto_compact_tracking=state.auto_compact_tracking, + max_output_tokens_recovery_count=0, + has_attempted_reactive_compact=False, + max_output_tokens_override=None, + stop_hook_active=None, + turn_count=turn_count, + pending_tool_use_summary=None, + continuation_nudge_count=state.continuation_nudge_count, + exhaustive_audit_performed=True, + transition=Transition(reason="continuation_nudge"), + ) + continue if last_text and detect_continuation_signal(last_text): logger.debug( "Continuation nudge triggered (%d/%d)", @@ -2349,6 +2392,7 @@ def _marking_chunk_cb(text: str) -> None: stop_hook_active=state.stop_hook_active, # Phase A: reset per-turn counter; carry pending summary forward. continuation_nudge_count=0, + exhaustive_audit_performed=state.exhaustive_audit_performed, pending_tool_use_summary=state.pending_tool_use_summary, transition=Transition(reason="next_turn"), ) diff --git a/src/query/transitions.py b/src/query/transitions.py index 5c24ff2d..0b3dec14 100644 --- a/src/query/transitions.py +++ b/src/query/transitions.py @@ -110,4 +110,7 @@ class QueryState: # when the model keeps matching continuation signals without tool calls. # Mirrors TS State.continuationNudgeCount at query.ts:218. continuation_nudge_count: int = 0 + # Session-chain latch for the one-time exhaustive-result audit. Unlike + # continuation_nudge_count, this survives successful tool rounds. + exhaustive_audit_performed: bool = False transition: Transition | None = None diff --git a/tests/test_query_loop_wires_round3.py b/tests/test_query_loop_wires_round3.py index b066bab2..fa4e26e6 100644 --- a/tests/test_query_loop_wires_round3.py +++ b/tests/test_query_loop_wires_round3.py @@ -30,8 +30,10 @@ def get_pending_post_compaction() -> bool: return _bs._STATE.pending_post_compaction from src.providers.base import ChatResponse from src.query.continuation_nudge import ( + EXHAUSTIVE_AUDIT_NUDGE, MAX_CONTINUATION_NUDGES, detect_continuation_signal, + requests_exhaustive_results, ) from src.query.query import QueryParams, run_query from src.query.stop_hooks import StopHookResult @@ -582,6 +584,30 @@ def test_completion_text_no_nudge(self): sum(1 for m in msgs if isinstance(m, AssistantMessage)), 1 ) + def test_exhaustive_request_forces_one_audit_pass(self): + provider = _provider([ + _completion("I found one result and finished."), + _completion("I audited all candidates and updated the result."), + ]) + params = _params(self.workspace, provider) + params.messages = [ + UserMessage(content="If there are multiple results, print them all.") + ] + msgs, terminal = _run(run_query(params)) + from src.types.messages import AssistantMessage + + self.assertEqual(terminal.reason, "completed") + self.assertEqual( + sum(1 for m in msgs if isinstance(m, AssistantMessage)), 2 + ) + + def test_exhaustive_request_detector(self): + self.assertTrue(requests_exhaustive_results("print them all")) + self.assertTrue(requests_exhaustive_results("multiple winning moves")) + self.assertFalse(requests_exhaustive_results("find the best move")) + self.assertIn("MUST use a tool", EXHAUSTIVE_AUDIT_NUDGE) + self.assertIn("state after each candidate action", EXHAUSTIVE_AUDIT_NUDGE) + # --------------------------------------------------------------------------- # G3 + G4 — /clear resets; compaction marks diff --git a/tests/test_system_prompt_full.py b/tests/test_system_prompt_full.py index fbebd9e4..8c083778 100644 --- a/tests/test_system_prompt_full.py +++ b/tests/test_system_prompt_full.py @@ -59,6 +59,12 @@ def test_basic_prompt_has_intro(self): assert "interactive agent" in prompt assert "software engineering tasks" in prompt + def test_task_prompt_requires_requirement_audit_and_exhaustive_results(self): + prompt = build_full_system_prompt(use_cache=False) + assert "audit the result against every explicit requirement" in prompt + assert "all, every, multiple, or an exhaustive set" in prompt + assert "stopping after the first one" in prompt + def test_identity_prompt_backward_compat(self): """_IDENTITY_PROMPT is an alias for _INTRO_SECTION.""" assert _IDENTITY_PROMPT is _INTRO_SECTION