Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions src/context_system/prompt_assembly.py
Original file line number Diff line number Diff line change
Expand Up @@ -883,6 +883,12 @@ def _emit_group(
"the error, check your assumptions, try a focused fix. Don't retry "
"the identical action blindly, but don't abandon a viable approach "
"after a single failure either.\n"
"- Before finishing, audit the result against every explicit requirement "
"in the user's request and verify the requirements that can be checked. "
"Do not treat producing a plausible result as proof that the task is "
"complete. When the user asks for all, every, multiple, or an exhaustive "
"set of results, actively search for additional valid results instead of "
"stopping after the first one.\n"
"- Be cautious not to introduce security vulnerabilities such as "
"command injection, XSS, SQL injection, and other OWASP top 10 "
"vulnerabilities.\n"
Expand Down
21 changes: 21 additions & 0 deletions src/query/continuation_nudge.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,27 @@

NUDGE_MESSAGE = "Continue with the task. Use the appropriate tools to proceed."

EXHAUSTIVE_AUDIT_NUDGE = (
"Before finishing, audit the result against the user's exhaustive-result "
"requirement. Actively search for additional valid results; do not assume "
"the first valid result is the only one. For an enumerable or mechanically "
"checkable state space, you MUST use a tool to enumerate or independently "
"validate candidates rather than declaring a prose-only audit exhaustive. "
"Evaluate the state after each candidate action, including candidates that "
"do not have the relevant property before they move. Update the deliverable "
"if the audit finds more."
)

_EXHAUSTIVE_REQUIREMENT = re.compile(
r"\b(all|every|multiple|each|exhaustive|complete set|any other|others)\b",
re.IGNORECASE,
)


def requests_exhaustive_results(text: str) -> bool:
"""Whether a user request explicitly asks for a result set audit."""
return bool(_EXHAUSTIVE_REQUIREMENT.search(text))

# Don't nudge when the model signaled completion (query.ts:1487).
_COMPLETION_MARKERS = re.compile(
r"\b(done|finished|completed|complete|summary|that's all|that is all"
Expand Down
44 changes: 44 additions & 0 deletions src/query/query.py
Original file line number Diff line number Diff line change
Expand Up @@ -2101,6 +2101,49 @@ def _marking_chunk_cb(text: str) -> None:
for b in content
if getattr(b, "type", None) == "text"
)
# A completion claim is not sufficient when the user asked
# for an exhaustive result set. Require one bounded audit
# pass before accepting the clean exit. This closes a common
# harness failure mode where the model finds one plausible
# answer and stops despite an explicit "all"/"multiple"
# requirement. A dedicated latch survives tool rounds so the
# guard cannot create an unbounded self-review loop.
from .continuation_nudge import (
EXHAUSTIVE_AUDIT_NUDGE,
requests_exhaustive_results,
)

exhaustive_request = any(
not getattr(msg, "isMeta", False)
and getattr(msg, "role", None) == "user"
and requests_exhaustive_results(
getattr(msg, "content", "")
if isinstance(getattr(msg, "content", ""), str)
else ""
)
for msg in messages
)
if exhaustive_request and not state.exhaustive_audit_performed:
logger.debug("Exhaustive-result audit nudge triggered")
state = QueryState(
messages=[
*messages,
*assistant_messages,
UserMessage(content=EXHAUSTIVE_AUDIT_NUDGE, isMeta=True),
],
tool_use_context=tool_use_context,
auto_compact_tracking=state.auto_compact_tracking,
max_output_tokens_recovery_count=0,
has_attempted_reactive_compact=False,
max_output_tokens_override=None,
stop_hook_active=None,
turn_count=turn_count,
pending_tool_use_summary=None,
continuation_nudge_count=state.continuation_nudge_count,
exhaustive_audit_performed=True,
transition=Transition(reason="continuation_nudge"),
)
continue
if last_text and detect_continuation_signal(last_text):
logger.debug(
"Continuation nudge triggered (%d/%d)",
Expand Down Expand Up @@ -2349,6 +2392,7 @@ def _marking_chunk_cb(text: str) -> None:
stop_hook_active=state.stop_hook_active,
# Phase A: reset per-turn counter; carry pending summary forward.
continuation_nudge_count=0,
exhaustive_audit_performed=state.exhaustive_audit_performed,
pending_tool_use_summary=state.pending_tool_use_summary,
transition=Transition(reason="next_turn"),
)
Expand Down
3 changes: 3 additions & 0 deletions src/query/transitions.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,4 +110,7 @@ class QueryState:
# when the model keeps matching continuation signals without tool calls.
# Mirrors TS State.continuationNudgeCount at query.ts:218.
continuation_nudge_count: int = 0
# Session-chain latch for the one-time exhaustive-result audit. Unlike
# continuation_nudge_count, this survives successful tool rounds.
exhaustive_audit_performed: bool = False
transition: Transition | None = None
26 changes: 26 additions & 0 deletions tests/test_query_loop_wires_round3.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,10 @@ def get_pending_post_compaction() -> bool:
return _bs._STATE.pending_post_compaction
from src.providers.base import ChatResponse
from src.query.continuation_nudge import (
EXHAUSTIVE_AUDIT_NUDGE,
MAX_CONTINUATION_NUDGES,
detect_continuation_signal,
requests_exhaustive_results,
)
from src.query.query import QueryParams, run_query
from src.query.stop_hooks import StopHookResult
Expand Down Expand Up @@ -582,6 +584,30 @@ def test_completion_text_no_nudge(self):
sum(1 for m in msgs if isinstance(m, AssistantMessage)), 1
)

def test_exhaustive_request_forces_one_audit_pass(self):
provider = _provider([
_completion("I found one result and finished."),
_completion("I audited all candidates and updated the result."),
])
params = _params(self.workspace, provider)
params.messages = [
UserMessage(content="If there are multiple results, print them all.")
]
msgs, terminal = _run(run_query(params))
from src.types.messages import AssistantMessage

self.assertEqual(terminal.reason, "completed")
self.assertEqual(
sum(1 for m in msgs if isinstance(m, AssistantMessage)), 2
)

def test_exhaustive_request_detector(self):
self.assertTrue(requests_exhaustive_results("print them all"))
self.assertTrue(requests_exhaustive_results("multiple winning moves"))
self.assertFalse(requests_exhaustive_results("find the best move"))
self.assertIn("MUST use a tool", EXHAUSTIVE_AUDIT_NUDGE)
self.assertIn("state after each candidate action", EXHAUSTIVE_AUDIT_NUDGE)


# ---------------------------------------------------------------------------
# G3 + G4 — /clear resets; compaction marks
Expand Down
6 changes: 6 additions & 0 deletions tests/test_system_prompt_full.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,12 @@ def test_basic_prompt_has_intro(self):
assert "interactive agent" in prompt
assert "software engineering tasks" in prompt

def test_task_prompt_requires_requirement_audit_and_exhaustive_results(self):
prompt = build_full_system_prompt(use_cache=False)
assert "audit the result against every explicit requirement" in prompt
assert "all, every, multiple, or an exhaustive set" in prompt
assert "stopping after the first one" in prompt

def test_identity_prompt_backward_compat(self):
"""_IDENTITY_PROMPT is an alias for _INTRO_SECTION."""
assert _IDENTITY_PROMPT is _INTRO_SECTION
Expand Down
Loading