diff --git a/src/context_system/prompt_assembly.py b/src/context_system/prompt_assembly.py index 066cf6f8..295807b9 100644 --- a/src/context_system/prompt_assembly.py +++ b/src/context_system/prompt_assembly.py @@ -904,6 +904,32 @@ def _emit_group( "after a single failure either. Escalate to the user only when you're " "genuinely stuck after investigation, not as a first response to " "friction.\n" + # Measured on seven shared terminal-bench 2.1 tasks, same model and + # effort, classifying every step by purpose (2026-07-26, two clawcodex + # runs vs the latest Claude Code, per trial): + # + # clawcodex Claude Code + # explore:inspect 1.71 / 1.57 2.43 <- CC inspects MORE + # author:inline 2.71 / 2.29 1.14 + # run:exec 2.29 / 3.57 1.14 <- biggest gap + # + # Claude Code buys ~0.7 extra inspection steps and saves 1.2-2.4 + # execution steps: it looks at the data, then writes something that works. + # clawcodex writes from assumptions, runs, and pays for it in fix cycles. + # This is also why the reverse guidance failed: a bullet telling the model + # to act as soon as it had enough information (withdrawn 2026-07-26) made + # both step count and reward worse, because under-inspection was the + # deficit, not over-deliberation. + # + # Deliberately narrow: the assumptions the NEXT action depends on, not a + # survey of the environment. The exhaustive-audit machinery removed in + # #749 is what over-broad exploration guidance costs. + "- Before running something you just wrote, confirm the specific " + "assumptions it depends on — the shape of the input, which tools " + "actually exist, the exact output expected. One cheap check on the real " + "data usually costs less than a failed run plus the fix cycle that " + "follows it. This is about the assumptions your next action rests on, " + "not a survey of the environment.\n" "- For ambiguous bugs, failures, or unfamiliar data, form a small set " "of plausible hypotheses from the available evidence and rank them by " "likelihood and the cost of checking them. Start with the cheapest " @@ -1040,7 +1066,17 @@ def _emit_group( " - Reserve Bash for system commands and terminal operations that " "require shell execution. If you are unsure and there is a relevant " "dedicated tool, default to using the dedicated tool.\n" - "- Break down and manage your work with the TaskCreate tool.\n" + # The port dropped the reference's skip conditions, leaving an + # unconditional imperative. Measured over 14 clawcodex and 21 Claude Code + # trials on seven shared terminal-bench 2.1 tasks (2026-07-26), steps per + # trial spent on task-tool bookkeeping: clawcodex 0.21, Claude Code 0.00 — + # the reference never reaches for it on work this size. Together with + # memory writes (0.43 vs 0.00) that is 27% of the remaining step gap, + # spent on bookkeeping rather than the task. + "- Break down and manage multi-step work with the TaskCreate tool. Skip " + "it when the work is a single straightforward task, or is trivial enough " + "that tracking it adds nothing — a task list you are the only reader of " + "is overhead, not progress.\n" # Two clauses were dropped in the port, and together they are what turns # a permission into a practice: the imperative to maximize, and the # dependency carve-out that tells the model when NOT to — without which diff --git a/tests/test_system_prompt_full.py b/tests/test_system_prompt_full.py index c41b1beb..0cebf308 100644 --- a/tests/test_system_prompt_full.py +++ b/tests/test_system_prompt_full.py @@ -79,6 +79,33 @@ def test_task_prompt_audits_requirements_without_a_quantifier_trigger(self): assert "all, every, multiple, or an exhaustive set" not in prompt assert "stopping after the first one" not in prompt + def test_task_prompt_checks_assumptions_before_executing(self): + """Under-inspection, not over-deliberation, is the measured deficit. + + Classifying every trajectory step by purpose on seven shared + terminal-bench 2.1 tasks at the same model and effort (2026-07-26, + two clawcodex runs vs the latest Claude Code, per trial): + + explore:inspect 1.71 / 1.57 vs 2.43 <- CC inspects MORE + author:inline 2.71 / 2.29 vs 1.14 + run:exec 2.29 / 3.57 vs 1.14 <- biggest gap + + Claude Code spends ~0.7 more steps looking and 1.2-2.4 fewer steps + executing. The opposite bullet — act as soon as you have enough + information — was tried and withdrawn the same day for making both + step count and reward worse. + + The wording must stay scoped to the next action's assumptions; broad + exploration guidance is what #749 had to remove. + """ + prompt = build_full_system_prompt(use_cache=False) + assert "Before running something you just wrote" in prompt + assert "cheap check on the real data" in prompt + assert "not a survey of the environment" in prompt, ( + "the scope limiter is load-bearing — without it this becomes the " + "exhaustive-exploration guidance removed in #749" + ) + def test_task_prompt_prioritizes_evidence_and_cheap_discriminating_checks(self): prompt = build_full_system_prompt(use_cache=False) assert "small set of plausible hypotheses" in prompt @@ -350,3 +377,27 @@ def test_all_sections_together(self): assert "echo-arg" in prompt # skills assert "Non-Interactive" in prompt assert "Final note" in prompt + + +class TestTaskToolGating: + def setup_method(self): + get_system_prompt_cache().invalidate_all() + + def test_task_tool_carries_the_reference_skip_conditions(self): + """The port dropped them, leaving an unconditional imperative. + + Reference: "Skip using this tool when: There is only a single, + straightforward task; the task is trivial and tracking it provides no + organizational benefit." clawcodex kept only "Break down and manage + your work with the TaskCreate tool." + + Measured over 14 clawcodex and 21 Claude Code trials on seven shared + terminal-bench 2.1 tasks (2026-07-26): task-tool steps per trial were + 0.21 for clawcodex and 0.00 for Claude Code — it never reaches for it + on work this size. With memory writes (0.43 vs 0.00) that is 27% of + the remaining step gap spent on bookkeeping. + """ + prompt = build_full_system_prompt(use_cache=False) + assert "Break down and manage multi-step work" in prompt + assert "Skip it when" in prompt, "the skip condition is the whole fix" + assert "trivial enough" in prompt