From 4e1cb22c4067caf6675ba82c7cb5646b3a3bd8ef Mon Sep 17 00:00:00 2001 From: Eric Lee Date: Sun, 26 Jul 2026 05:57:26 -0700 Subject: [PATCH 1/2] perf(prompt): check the next action's assumptions before running it Iteration 5. First one derived from classifying trajectory steps by PURPOSE rather than by tool, which is what finally located the gap. Seven shared terminal-bench 2.1 tasks, same model (claude-opus-5), same effort (xhigh), per trial, two clawcodex runs against the latest Claude Code: clawcodex Claude Code explore:inspect 1.71 / 1.57 2.43 <- Claude Code inspects MORE explore:read 0.86 / 1.14 0.57 author:inline 2.71 / 2.29 1.14 author:deliverable 1.29 / 2.43 1.14 run:exec 2.29 / 3.57 1.14 <- the biggest single gap meta:memory 0.29 / 0.57 0.00 meta:tasks 0.14 / 0.29 0.00 TOTAL 12.14 / 15.43 8.43 Claude Code buys about 0.7 extra inspection steps and saves 1.2-2.4 execution steps. It looks at the actual data, then writes something that works first time. clawcodex writes from assumptions, runs, and pays in write-run-fix cycles -- run:exec and author:inline are each roughly double Claude Code's, while inspection is LOWER. Every tool-level metric already matches (Bash 78.8% vs 79.7%, Write 11.8% vs 11.9%, per-call density 5.2 vs 5.5 statements, heredoc 20.9% vs 23.4%, exact-repeat 2.4% vs 0.0%). Purpose-level classification is what separated "same actions, more of them" into "fewer looks, more retries". This also explains a failure. Iteration 3 added the opposite instruction -- act once you have enough information, do not re-check what the evidence supports -- and made both step count (+3.2 -> +7.2) and reward (1.000 -> 0.857) worse. It was pushing clawcodex further toward acting before looking, when under-inspection was the actual deficit. That bullet was withdrawn; this is the corrected direction, and the earlier failure is evidence for it. Scope is deliberately narrow: the assumptions the NEXT action rests on, not a survey of the environment. #749 had to remove exhaustive-audit machinery that fired on ordinary English, so the limiter is load-bearing and pinned by the test. Not claimed: a verified step reduction. Same-code replicates on this task set swing up to 11 steps per task and 3.43 in the mean, so at k=1 nothing under ~3 mean steps is resolvable. Claude Code replicates are in flight to establish a k=3 baseline; this change gets measured against it, not against a single run. --- src/context_system/prompt_assembly.py | 26 ++++++++++++++++++++++++++ tests/test_system_prompt_full.py | 27 +++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/src/context_system/prompt_assembly.py b/src/context_system/prompt_assembly.py index 066cf6f8..c0ed7992 100644 --- a/src/context_system/prompt_assembly.py +++ b/src/context_system/prompt_assembly.py @@ -904,6 +904,32 @@ def _emit_group( "after a single failure either. Escalate to the user only when you're " "genuinely stuck after investigation, not as a first response to " "friction.\n" + # Measured on seven shared terminal-bench 2.1 tasks, same model and + # effort, classifying every step by purpose (2026-07-26, two clawcodex + # runs vs the latest Claude Code, per trial): + # + # clawcodex Claude Code + # explore:inspect 1.71 / 1.57 2.43 <- CC inspects MORE + # author:inline 2.71 / 2.29 1.14 + # run:exec 2.29 / 3.57 1.14 <- biggest gap + # + # Claude Code buys ~0.7 extra inspection steps and saves 1.2-2.4 + # execution steps: it looks at the data, then writes something that works. + # clawcodex writes from assumptions, runs, and pays for it in fix cycles. + # This is also why the reverse guidance failed: a bullet telling the model + # to act as soon as it had enough information (withdrawn 2026-07-26) made + # both step count and reward worse, because under-inspection was the + # deficit, not over-deliberation. + # + # Deliberately narrow: the assumptions the NEXT action depends on, not a + # survey of the environment. The exhaustive-audit machinery removed in + # #749 is what over-broad exploration guidance costs. + "- Before running something you just wrote, confirm the specific " + "assumptions it depends on — the shape of the input, which tools " + "actually exist, the exact output expected. One cheap check on the real " + "data usually costs less than a failed run plus the fix cycle that " + "follows it. This is about the assumptions your next action rests on, " + "not a survey of the environment.\n" "- For ambiguous bugs, failures, or unfamiliar data, form a small set " "of plausible hypotheses from the available evidence and rank them by " "likelihood and the cost of checking them. Start with the cheapest " diff --git a/tests/test_system_prompt_full.py b/tests/test_system_prompt_full.py index c41b1beb..677175e4 100644 --- a/tests/test_system_prompt_full.py +++ b/tests/test_system_prompt_full.py @@ -79,6 +79,33 @@ def test_task_prompt_audits_requirements_without_a_quantifier_trigger(self): assert "all, every, multiple, or an exhaustive set" not in prompt assert "stopping after the first one" not in prompt + def test_task_prompt_checks_assumptions_before_executing(self): + """Under-inspection, not over-deliberation, is the measured deficit. + + Classifying every trajectory step by purpose on seven shared + terminal-bench 2.1 tasks at the same model and effort (2026-07-26, + two clawcodex runs vs the latest Claude Code, per trial): + + explore:inspect 1.71 / 1.57 vs 2.43 <- CC inspects MORE + author:inline 2.71 / 2.29 vs 1.14 + run:exec 2.29 / 3.57 vs 1.14 <- biggest gap + + Claude Code spends ~0.7 more steps looking and 1.2-2.4 fewer steps + executing. The opposite bullet — act as soon as you have enough + information — was tried and withdrawn the same day for making both + step count and reward worse. + + The wording must stay scoped to the next action's assumptions; broad + exploration guidance is what #749 had to remove. + """ + prompt = build_full_system_prompt(use_cache=False) + assert "Before running something you just wrote" in prompt + assert "cheap check on the real data" in prompt + assert "not a survey of the environment" in prompt, ( + "the scope limiter is load-bearing — without it this becomes the " + "exhaustive-exploration guidance removed in #749" + ) + def test_task_prompt_prioritizes_evidence_and_cheap_discriminating_checks(self): prompt = build_full_system_prompt(use_cache=False) assert "small set of plausible hypotheses" in prompt From 47ec936d914847cd448465c4b4f95a6c6b12eb80 Mon Sep 17 00:00:00 2001 From: Eric Lee Date: Sun, 26 Jul 2026 06:28:03 -0700 Subject: [PATCH 2/2] perf(prompt): restore the reference's skip conditions on the task tool The port kept "Break down and manage your work with the TaskCreate tool" and dropped the reference's gating, which reads: "Skip using this tool when: There is only a single, straightforward task; the task is trivial and tracking it provides no organizational benefit." What survived is an unconditional imperative. Measured over 14 clawcodex and 21 Claude Code trials on seven shared terminal-bench 2.1 tasks, same model (claude-opus-5) and effort (xhigh), steps per trial by purpose: clawcodex Claude Code delta run:exec 3.36 2.52 +0.83 other 1.43 0.67 +0.76 author:inline 2.50 1.90 +0.60 meta:memory 0.43 0.00 +0.43 explore:env 1.07 0.67 +0.40 meta:tasks 0.21 0.00 +0.21 explore:inspect 1.64 2.29 -0.64 author:file 2.00 2.19 -0.19 TOTAL 13.79 11.38 +2.40 Claude Code spends ZERO steps on the task tool across 21 trials and zero on memory writes. clawcodex spends 0.64 per trial on the two together -- 27% of the whole remaining gap, on bookkeeping rather than the task. This commit addresses the task-tool half, where the fix is a dropped reference qualifier rather than a judgment call. The memory half is deliberately untouched. Its "build up this memory system over time" imperative IS in the reference, and memory is a shipped feature whose quality this eval cannot see -- changing it on step-count evidence is what made iteration 3 regress reward. Also of note in the table: "Claude Code inspects more" (-0.64) survives Claude Code's replication, so the iteration-5 premise was sound even though the change built on it measured null and was dropped. Not claimed: a verified step reduction. 0.21 steps/trial is far inside the same-code noise floor (median 2, max 11 per task). This lands because the dropped qualifier is a real port defect, not because the number is measurable. --- src/context_system/prompt_assembly.py | 12 +++++++++++- tests/test_system_prompt_full.py | 24 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/context_system/prompt_assembly.py b/src/context_system/prompt_assembly.py index c0ed7992..295807b9 100644 --- a/src/context_system/prompt_assembly.py +++ b/src/context_system/prompt_assembly.py @@ -1066,7 +1066,17 @@ def _emit_group( " - Reserve Bash for system commands and terminal operations that " "require shell execution. If you are unsure and there is a relevant " "dedicated tool, default to using the dedicated tool.\n" - "- Break down and manage your work with the TaskCreate tool.\n" + # The port dropped the reference's skip conditions, leaving an + # unconditional imperative. Measured over 14 clawcodex and 21 Claude Code + # trials on seven shared terminal-bench 2.1 tasks (2026-07-26), steps per + # trial spent on task-tool bookkeeping: clawcodex 0.21, Claude Code 0.00 — + # the reference never reaches for it on work this size. Together with + # memory writes (0.43 vs 0.00) that is 27% of the remaining step gap, + # spent on bookkeeping rather than the task. + "- Break down and manage multi-step work with the TaskCreate tool. Skip " + "it when the work is a single straightforward task, or is trivial enough " + "that tracking it adds nothing — a task list you are the only reader of " + "is overhead, not progress.\n" # Two clauses were dropped in the port, and together they are what turns # a permission into a practice: the imperative to maximize, and the # dependency carve-out that tells the model when NOT to — without which diff --git a/tests/test_system_prompt_full.py b/tests/test_system_prompt_full.py index 677175e4..0cebf308 100644 --- a/tests/test_system_prompt_full.py +++ b/tests/test_system_prompt_full.py @@ -377,3 +377,27 @@ def test_all_sections_together(self): assert "echo-arg" in prompt # skills assert "Non-Interactive" in prompt assert "Final note" in prompt + + +class TestTaskToolGating: + def setup_method(self): + get_system_prompt_cache().invalidate_all() + + def test_task_tool_carries_the_reference_skip_conditions(self): + """The port dropped them, leaving an unconditional imperative. + + Reference: "Skip using this tool when: There is only a single, + straightforward task; the task is trivial and tracking it provides no + organizational benefit." clawcodex kept only "Break down and manage + your work with the TaskCreate tool." + + Measured over 14 clawcodex and 21 Claude Code trials on seven shared + terminal-bench 2.1 tasks (2026-07-26): task-tool steps per trial were + 0.21 for clawcodex and 0.00 for Claude Code — it never reaches for it + on work this size. With memory writes (0.43 vs 0.00) that is 27% of + the remaining step gap spent on bookkeeping. + """ + prompt = build_full_system_prompt(use_cache=False) + assert "Break down and manage multi-step work" in prompt + assert "Skip it when" in prompt, "the skip condition is the whole fix" + assert "trivial enough" in prompt