From dd0b705ba8b8c0702de68980e4ce946e74c17b03 Mon Sep 17 00:00:00 2001 From: Eric Lee Date: Sun, 26 Jul 2026 06:36:16 -0700 Subject: [PATCH] revert: drop the iteration-5 bullet that #754 merged by accident #754 was branched while HEAD still sat on the iteration-5 commit, so the squash carried in a change I had measured as null and said I would not merge. This removes it. The task-tool skip conditions that #754 was actually for are untouched. The bullet ("Before running something you just wrote, confirm the specific assumptions it depends on...") rested on a purpose-level classification showing Claude Code inspecting more than clawcodex (2.43 vs 1.57-1.71 steps per trial). That result was a CLASSIFIER ARTIFACT: the regex anchored the verb at the start of the command, so clawcodex's `cd /app && grep ...` inspection calls fell into an "other" bucket instead. Stripping `cd`/`&&` prefixes before classifying reverses it: clawcodex Claude Code delta explore:inspect 4.93 3.95 +0.98 <- clawcodex inspects MORE meta 0.64 0.00 +0.64 author:inline 2.93 2.52 +0.40 run:exec 1.86 1.57 +0.29 So the premise was false, which explains the null measurement (4 of 5 tasks worse, mean 9.9 -> 10.2). Anchor a command classifier on the first real verb, not the first token. What survives from that analysis is the `meta` row: memory writes plus task-tool calls cost clawcodex 0.64 steps per trial where Claude Code spends ZERO across 21 trials -- 26% of the remaining gap. #754's skip-condition restoration addresses the task-tool half and stands on the dropped-qualifier argument, independent of the misclassified rows. --- src/context_system/prompt_assembly.py | 26 -------------------------- tests/test_system_prompt_full.py | 27 --------------------------- 2 files changed, 53 deletions(-) diff --git a/src/context_system/prompt_assembly.py b/src/context_system/prompt_assembly.py index 295807b9..8d37874d 100644 --- a/src/context_system/prompt_assembly.py +++ b/src/context_system/prompt_assembly.py @@ -904,32 +904,6 @@ def _emit_group( "after a single failure either. Escalate to the user only when you're " "genuinely stuck after investigation, not as a first response to " "friction.\n" - # Measured on seven shared terminal-bench 2.1 tasks, same model and - # effort, classifying every step by purpose (2026-07-26, two clawcodex - # runs vs the latest Claude Code, per trial): - # - # clawcodex Claude Code - # explore:inspect 1.71 / 1.57 2.43 <- CC inspects MORE - # author:inline 2.71 / 2.29 1.14 - # run:exec 2.29 / 3.57 1.14 <- biggest gap - # - # Claude Code buys ~0.7 extra inspection steps and saves 1.2-2.4 - # execution steps: it looks at the data, then writes something that works. - # clawcodex writes from assumptions, runs, and pays for it in fix cycles. - # This is also why the reverse guidance failed: a bullet telling the model - # to act as soon as it had enough information (withdrawn 2026-07-26) made - # both step count and reward worse, because under-inspection was the - # deficit, not over-deliberation. - # - # Deliberately narrow: the assumptions the NEXT action depends on, not a - # survey of the environment. The exhaustive-audit machinery removed in - # #749 is what over-broad exploration guidance costs. - "- Before running something you just wrote, confirm the specific " - "assumptions it depends on — the shape of the input, which tools " - "actually exist, the exact output expected. One cheap check on the real " - "data usually costs less than a failed run plus the fix cycle that " - "follows it. This is about the assumptions your next action rests on, " - "not a survey of the environment.\n" "- For ambiguous bugs, failures, or unfamiliar data, form a small set " "of plausible hypotheses from the available evidence and rank them by " "likelihood and the cost of checking them. Start with the cheapest " diff --git a/tests/test_system_prompt_full.py b/tests/test_system_prompt_full.py index 0cebf308..2d52c63e 100644 --- a/tests/test_system_prompt_full.py +++ b/tests/test_system_prompt_full.py @@ -79,33 +79,6 @@ def test_task_prompt_audits_requirements_without_a_quantifier_trigger(self): assert "all, every, multiple, or an exhaustive set" not in prompt assert "stopping after the first one" not in prompt - def test_task_prompt_checks_assumptions_before_executing(self): - """Under-inspection, not over-deliberation, is the measured deficit. - - Classifying every trajectory step by purpose on seven shared - terminal-bench 2.1 tasks at the same model and effort (2026-07-26, - two clawcodex runs vs the latest Claude Code, per trial): - - explore:inspect 1.71 / 1.57 vs 2.43 <- CC inspects MORE - author:inline 2.71 / 2.29 vs 1.14 - run:exec 2.29 / 3.57 vs 1.14 <- biggest gap - - Claude Code spends ~0.7 more steps looking and 1.2-2.4 fewer steps - executing. The opposite bullet — act as soon as you have enough - information — was tried and withdrawn the same day for making both - step count and reward worse. - - The wording must stay scoped to the next action's assumptions; broad - exploration guidance is what #749 had to remove. - """ - prompt = build_full_system_prompt(use_cache=False) - assert "Before running something you just wrote" in prompt - assert "cheap check on the real data" in prompt - assert "not a survey of the environment" in prompt, ( - "the scope limiter is load-bearing — without it this becomes the " - "exhaustive-exploration guidance removed in #749" - ) - def test_task_prompt_prioritizes_evidence_and_cheap_discriminating_checks(self): prompt = build_full_system_prompt(use_cache=False) assert "small set of plausible hypotheses" in prompt