From 13daf04bb47d76dec8a7275441449e005aa69bdb Mon Sep 17 00:00:00 2001 From: Eric Lee Date: Sun, 26 Jul 2026 00:18:45 -0700 Subject: [PATCH] fix(prompt): restore two dropped qualifiers from the reference Doing-tasks section MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both are faithful-port restorations of text present in typescript/src/constants/prompts.ts and lost in the Python port — not new guidance. 1. "Do not create files unless they're absolutely necessary" was missing the reference's "for achieving your goal" AND its rationale ("as this prevents file bloat and builds on existing work more effectively"). Stripped of both, a bloat-avoidance heuristic reads as a flat ban on creating files. Cost, measured on terminal-bench 2.1 (torch-tensor-parallelism, 2026-07-25): the instruction was "Create the file /app/parallel_linear.py", /app was empty, and the container had no Python to verify against. clawcodex spent all 8 of its steps hunting for an interpreter — two Bash probes, a ToolSearch, then four filesystem-wide Globs — and never wrote the file. Claude Code ran the same probes, concluded nothing was runnable, wrote the deliverable on its 5th action, and scored 1.0. 2. The "If an approach fails" bullet lost its closing clause scoping escalation to genuine blockage. Restored WITHOUT naming AskUserQuestion, which the reference interpolates there: that tool is unregistered on the headless surface, and a prompt naming an unavailable tool is its own failure mode (fix-git, 2026-07-25 — asked twice, then handed the task back to a user who could not exist). Note what is NOT here: no instruction telling the agent to produce the artifact when the environment cannot be verified. The reference has no such line — Claude Code's behavior there is emergent — and inventing one is what a previous A/B on this benchmark already falsified. Tests: 8843 passed; both restorations are revert-sensitive. Co-Authored-By: Claude Opus 5 --- src/context_system/prompt_assembly.py | 27 +++++++++++++++++--- tests/test_system_prompt_full.py | 36 +++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/src/context_system/prompt_assembly.py b/src/context_system/prompt_assembly.py index 13d67a7e..63f19138 100644 --- a/src/context_system/prompt_assembly.py +++ b/src/context_system/prompt_assembly.py @@ -877,12 +877,33 @@ def _emit_group( "- In general, do not propose changes to code you haven't read. " "If a user asks about or wants you to modify a file, read it first. " "Understand existing code before suggesting modifications.\n" - "- Do not create files unless they're absolutely necessary. " - "Generally prefer editing an existing file to creating a new one.\n" + # Both qualifiers below are load-bearing and were dropped in the port + # (reference: typescript/src/constants/prompts.ts "Doing tasks"). Without + # "for achieving your goal" the bullet reads as a flat prohibition on + # creating files, and without the rationale it loses the scope that makes + # it a bloat-avoidance rule rather than a ban. Cost of the truncation, + # observed on terminal-bench 2.1 (torch-tensor-parallelism, 2026-07-25): + # the task said "Create the file /app/parallel_linear.py", /app was empty, + # the container had no Python to verify against — and the agent spent all + # 8 of its steps hunting for an interpreter and never wrote the file. + # Claude Code wrote it on its 5th action and scored 1.0. + "- Do not create files unless they're absolutely necessary for achieving " + "your goal. Generally prefer editing an existing file to creating a new " + "one, as this prevents file bloat and builds on existing work more " + "effectively.\n" + # Final clause restored from the reference, which ends this bullet with + # "Escalate to the user with only when you're genuinely + # stuck after investigation, not as a first response to friction." + # Deliberately phrased without naming the tool: it is unregistered on the + # headless surface (nothing there can answer), and a prompt that names an + # unavailable tool is its own failure mode \u2014 fix-git, 2026-07-25, asked + # twice and then handed the task back to a user who did not exist. "- If an approach fails, diagnose why before switching tactics\u2014read " "the error, check your assumptions, try a focused fix. Don't retry " "the identical action blindly, but don't abandon a viable approach " - "after a single failure either.\n" + "after a single failure either. Escalate to the user only when you're " + "genuinely stuck after investigation, not as a first response to " + "friction.\n" "- For ambiguous bugs, failures, or unfamiliar data, form a small set " "of plausible hypotheses from the available evidence and rank them by " "likelihood and the cost of checking them. Start with the cheapest " diff --git a/tests/test_system_prompt_full.py b/tests/test_system_prompt_full.py index a180305e..92df7492 100644 --- a/tests/test_system_prompt_full.py +++ b/tests/test_system_prompt_full.py @@ -87,6 +87,42 @@ def test_task_prompt_groups_related_mechanical_edits(self): assert "long sequence of tiny edits" in prompt assert "Keep unrelated or semantically distinct changes separate" in prompt + def test_task_prompt_keeps_the_reference_file_creation_qualifiers(self): + """The file-creation bullet must carry BOTH reference qualifiers. + + Reference (typescript/src/constants/prompts.ts, "Doing tasks"): + "Do not create files unless they're absolutely necessary for + achieving your goal. Generally prefer editing an existing file to + creating a new one, as this prevents file bloat and builds on + existing work more effectively." + + The port dropped "for achieving your goal" and the rationale, + leaving what reads as a flat ban on new files. That cost a + terminal-bench 2.1 task outright (torch-tensor-parallelism, + 2026-07-25): the instruction was "Create the file + /app/parallel_linear.py", /app was empty, no interpreter existed to + verify against, and the agent spent every step hunting for one + instead of writing the deliverable it had been asked for. + """ + prompt = build_full_system_prompt(use_cache=False) + assert "absolutely necessary for achieving your goal" in prompt, ( + "dropping 'for achieving your goal' turns a bloat-avoidance rule " + "into a prohibition on producing a requested file" + ) + assert "prevents file bloat" in prompt + + def test_task_prompt_scopes_escalation_to_genuine_blockage(self): + """Restored from the reference's "If an approach fails" bullet. + + The named tool is omitted on purpose: AskUserQuestion is + unregistered on the headless surface, and a prompt that names an + unavailable tool is its own failure mode (fix-git, 2026-07-25 — + asked twice, then handed the task back to a user who did not exist). + """ + prompt = build_full_system_prompt(use_cache=False) + assert "genuinely stuck after investigation" in prompt + assert "not as a first response to friction" in prompt + def test_identity_prompt_backward_compat(self): """_IDENTITY_PROMPT is an alias for _INTRO_SECTION.""" assert _IDENTITY_PROMPT is _INTRO_SECTION