diff --git a/eval/harbor/clawcodex_agent.py b/eval/harbor/clawcodex_agent.py index ee25f1b6..00dcb9b5 100644 --- a/eval/harbor/clawcodex_agent.py +++ b/eval/harbor/clawcodex_agent.py @@ -103,7 +103,6 @@ ToolCall, Trajectory, ) -from time_budget import build_deadline_prompt, resolve_agent_timeout_seconds # Host env vars forwardable into the container (clawcodex's builtin provider # key candidates — see src/providers/__init__.py in the clawcodex repo), keyed @@ -448,24 +447,29 @@ async def _inject_subscription_credentials( async def _seed_container_settings( self, environment: BaseEnvironment, - *, - deadline_prompt: str | None = None, ) -> None: - """Seed session-wide effort and deadline guidance in global config. + """Seed session-wide effort in global config. clawcodex's ``--effort`` flag governs the MAIN loop only; subagents (Agent tool) resolve effort from ``settings.effort``. Seeding the container's global config (home-anchored ``~/.clawcodex/config.json`` — the global-config path deliberately does not follow - CLAWCODEX_CONFIG_DIR) makes the requested effort session-wide and - appends Harbor's real wall-clock deadline to the model instructions. + CLAWCODEX_CONFIG_DIR) makes the requested effort session-wide. + + This seeds NOTHING but effort. An earlier version also appended + Harbor's wall-clock deadline to the system prompt ("stop broad + exploration, switch to the narrowest checks"). That is harness-shaped + guidance no other agent receives: the built-in claude-code adapter + appends nothing, so it made clawcodex-vs-Claude-Code trajectories + non-comparable, and it would not exist on a submitted run scored by + the official tooling. Any behavior worth having under a deadline + belongs in the product's own prompt, applied to every task, not + injected by an eval adapter. """ settings: dict[str, Any] = {} effort = self._resolved_flags.get("effort") if effort: settings["effort"] = effort - if deadline_prompt: - settings["append_system_prompt"] = deadline_prompt if not settings: return payload = json.dumps({"settings": settings}) @@ -491,19 +495,7 @@ async def run( self._captured_instruction = instruction if self._subscription: await self._inject_subscription_credentials(environment) - timeout_seconds = resolve_agent_timeout_seconds( - environment.environment_dir.parent / "task.toml", - environment.trial_paths.lock_path, - ) - deadline_prompt = ( - build_deadline_prompt(timeout_seconds) - if timeout_seconds is not None - else None - ) - await self._seed_container_settings( - environment, - deadline_prompt=deadline_prompt, - ) + await self._seed_container_settings(environment) parts: list[str] = [ "clawcodex", diff --git a/eval/harbor/compare_trajectories.py b/eval/harbor/compare_trajectories.py new file mode 100644 index 00000000..984716c0 --- /dev/null +++ b/eval/harbor/compare_trajectories.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Compare two Harbor jobs' ATIF trajectories on the tasks they share. + +Every metric is computed over the INTERSECTION of task names. Mixing task sets +silently inverts conclusions: comparing a 7-task clawcodex job against an +89-task Claude Code job made Claude Code look like it repeated tool calls more +often (6.8% vs 5.3%) and used heredocs less often (7.4% vs 12.1%). Restricted +to the seven shared tasks, both directions flip (0.0% vs 5.3%; 23.4% vs 12.1%). + +Trials killed by the harness are reported separately and excluded from the +step-count means: a right-censored trajectory's step count is a lower bound on +what the agent would have spent, so averaging it against completed runs is not +a valid comparison. Rewards are always reported -- a step-count delta on tasks +where both sides already score 1.0 is a cost difference, not a quality one. + +Usage: + python3 eval/harbor/compare_trajectories.py +""" + +from __future__ import annotations + +import json +import re +import statistics as st +import sys +from pathlib import Path + +HEREDOC = re.compile(r"(cat|tee)\s*>+\s*\S+\s*<<") +PROBE = re.compile( + r"\b(command -v|which |type -p|ls /usr/bin|ls /usr/local|find / |find /usr" + r"|apt-get|apt |pip install|pip3 install|--version)" +) +MEMORY = re.compile(r"/memory/|MEMORY\.md|CLAWCODEX\.md|CLAUDE\.md") + + +def _norm(value: object) -> str: + return " ".join(str(value or "").split()) + + +def load_trials(job: Path) -> dict[str, dict]: + """Map task name -> per-trial metrics for one job directory.""" + out: dict[str, dict] = {} + for traj in sorted(job.glob("*__*/agent/trajectory.json")): + trial_dir = traj.parent.parent + task = trial_dir.name.split("__")[0] + steps = json.loads(traj.read_text()).get("steps") or [] + + calls: list[tuple[str, str, str]] = [] + for step in steps: + for call in step.get("tool_calls") or []: + args = call.get("arguments") or {} + calls.append(( + call.get("function_name") or "?", + _norm(args.get("command") or args.get("content")), + _norm(args.get("file_path")), + )) + + bash = [c for c in calls if c[0] == "Bash"] + stmts = [ + len([x for x in re.split(r"&&|\|\||;|\n", c[1]) if x.strip()]) + for c in bash + ] + seen: set[tuple[str, str, str]] = set() + repeats = 0 + for key in calls: + if key in seen: + repeats += 1 + seen.add(key) + + reward_file = trial_dir / "verifier" / "reward.txt" + out.setdefault(task, {"trials": []})["trials"].append({ + "steps": len([s for s in steps if s.get("tool_calls")]), + "raw_steps": len(steps), + "calls": len(calls), + "bash": len(bash), + "heredoc": sum(1 for c in bash if HEREDOC.search(c[1])), + "probe": sum(1 for c in bash if PROBE.search(c[1])), + "memory": sum(1 for c in calls if MEMORY.search(c[2])), + "repeats": repeats, + "stmts": stmts, + "tools": [c[0] for c in calls], + "reward": ( + reward_file.read_text().strip() + if reward_file.exists() else None + ), + "killed": (trial_dir / "exception.txt").exists(), + }) + return out + + +def agg(trials: list[dict], key: str) -> float: + return st.mean([t[key] for t in trials]) if trials else float("nan") + + +def report(job_a: Path, job_b: Path) -> None: + A, B = load_trials(job_a), load_trials(job_b) + shared = sorted(set(A) & set(B)) + if not shared: + sys.exit("no shared tasks between the two jobs") + + print(f"A = {job_a.name}\nB = {job_b.name}") + print(f"shared tasks: {len(shared)} (A has {len(A)}, B has {len(B)})\n") + + killed = [ + (lbl, t) + for lbl, src in (("A", A), ("B", B)) + for t in shared + for tr in src[t]["trials"] if tr["killed"] + ] + if killed: + print("RIGHT-CENSORED (killed by harness; excluded from step means):") + for lbl, t in killed: + print(f" {lbl}: {t}") + print() + + print(f"{'task':28}{'A stp':>7}{'B stp':>7}{'A rew':>7}{'B rew':>7}") + a_ok: list[float] = [] + b_ok: list[float] = [] + for t in shared: + at, bt = A[t]["trials"], B[t]["trials"] + alive_a = [x for x in at if not x["killed"]] + alive_b = [x for x in bt if not x["killed"]] + if alive_a: + a_ok.append(agg(alive_a, "steps")) + if alive_b: + b_ok.append(agg(alive_b, "steps")) + mark = "*" if len(alive_a) < len(at) or len(alive_b) < len(bt) else " " + rw = lambda tr: ",".join(str(x["reward"]) for x in tr) # noqa: E731 + print(f"{t:28}{agg(at,'steps'):>7.1f}{agg(bt,'steps'):>7.1f}" + f"{rw(at):>7}{rw(bt):>7}{mark}") + + print(f"\nmean steps (completed only): A={st.mean(a_ok):.2f} " + f"B={st.mean(b_ok):.2f} gap={st.mean(a_ok)-st.mean(b_ok):+.2f}") + print(' "steps" = steps carrying >=1 tool call. Raw len(steps): ' + f"A={st.mean([agg(A[t]['trials'],'raw_steps') for t in shared]):.1f} " + f"B={st.mean([agg(B[t]['trials'],'raw_steps') for t in shared]):.1f}") + + print("\nstructural metrics over shared tasks (all trials):") + fa = [x for t in shared for x in A[t]["trials"]] + fb = [x for t in shared for x in B[t]["trials"]] + for lbl, num, den in ( + ("heredoc-authored", "heredoc", "bash"), + ("env-probe", "probe", "bash"), + ("exact-repeat", "repeats", "calls"), + ): + na, da = sum(x[num] for x in fa), sum(x[den] for x in fa) + nb, db = sum(x[num] for x in fb), sum(x[den] for x in fb) + print(f" {lbl:18} A={na:4}/{da:4} ({na/max(da,1):5.1%}) " + f"B={nb:4}/{db:4} ({nb/max(db,1):5.1%})") + + for lbl, f in (("A", fa), ("B", fb)): + s = [v for x in f for v in x["stmts"]] + tools: dict[str, int] = {} + for x in f: + for tool in x["tools"]: + tools[tool] = tools.get(tool, 0) + 1 + tot = sum(tools.values()) + mix = " ".join( + f"{k} {v/tot:.1%}" + for k, v in sorted(tools.items(), key=lambda kv: -kv[1])[:5] + ) + print(f" {lbl}: bash stmts median={st.median(s):.1f} " + f"mean={st.mean(s):.1f} | memory-writes=" + f"{sum(x['memory'] for x in f)} | {mix}") + + +if __name__ == "__main__": + if len(sys.argv) != 3: + sys.exit(__doc__) + report(Path(sys.argv[1]), Path(sys.argv[2])) diff --git a/eval/harbor/time_budget.py b/eval/harbor/time_budget.py deleted file mode 100644 index 50d86975..00000000 --- a/eval/harbor/time_budget.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Deadline helpers shared by Harbor adapters. - -Harbor owns the agent-phase timeout but does not pass it to ``Agent.run``. -The resolved inputs are nevertheless available beside the environment and -trial, so adapters can make the deadline visible to an otherwise unaware -agent without encoding dataset or task names. -""" - -from __future__ import annotations - -import json -import time -import tomllib -from datetime import datetime, timezone -from pathlib import Path - - -def resolve_agent_timeout_seconds(task_toml: Path, lock_json: Path) -> float | None: - """Return Harbor's effective agent timeout from resolved trial inputs.""" - try: - task = tomllib.loads(task_toml.read_text(encoding="utf-8")) - lock = json.loads(lock_json.read_text(encoding="utf-8")) - base = float(task["agent"]["timeout_sec"]) - multiplier = lock.get("agent_timeout_multiplier") - if multiplier is None: - multiplier = lock.get("timeout_multiplier", 1.0) - effective = base * float(multiplier) - except (KeyError, OSError, TypeError, ValueError, json.JSONDecodeError): - return None - return effective if effective > 0 else None - - -def build_deadline_prompt( - timeout_seconds: float, - *, - started_at: float | None = None, -) -> str: - """Build model-neutral guidance for a real external execution deadline.""" - start = time.time() if started_at is None else started_at - deadline = start + timeout_seconds - # Reserve 15%, bounded so short tasks still get two minutes and very long - # tasks do not abandon productive work excessively early. - reserve = min(10 * 60, max(2 * 60, timeout_seconds * 0.15)) - finalize_at = deadline - reserve - - def stamp(value: float) -> str: - return datetime.fromtimestamp(value, tz=timezone.utc).isoformat( - timespec="seconds" - ) - - return ( - "This run has a hard external execution deadline at " - f"{stamp(deadline)} ({timeout_seconds / 60:.1f} minutes from start). " - f"By {stamp(finalize_at)}, preserve the best valid deliverable, stop " - "broad exploration, and switch to the narrowest checks needed for the " - "explicit requirements. Do not start optional audits, repeated passing " - "checks, or long refinements that cannot finish before the deadline. " - "If the core result already works, finish and return control rather " - "than consuming the remaining budget. You can use `date -u` to compare " - "the current time with these timestamps." - ) diff --git a/tests/test_harbor_time_budget.py b/tests/test_harbor_time_budget.py deleted file mode 100644 index 36b7b058..00000000 --- a/tests/test_harbor_time_budget.py +++ /dev/null @@ -1,57 +0,0 @@ -from __future__ import annotations - -import json -import sys -from pathlib import Path - - -sys.path.insert(0, str(Path(__file__).parents[1] / "eval" / "harbor")) - -from time_budget import build_deadline_prompt, resolve_agent_timeout_seconds - - -def test_resolve_agent_timeout_uses_agent_multiplier(tmp_path: Path) -> None: - task = tmp_path / "task.toml" - lock = tmp_path / "lock.json" - task.write_text("[agent]\ntimeout_sec = 900\n", encoding="utf-8") - lock.write_text( - json.dumps( - { - "timeout_multiplier": 3, - "agent_timeout_multiplier": 2, - } - ), - encoding="utf-8", - ) - - assert resolve_agent_timeout_seconds(task, lock) == 1800 - - -def test_resolve_agent_timeout_falls_back_to_global_multiplier( - tmp_path: Path, -) -> None: - task = tmp_path / "task.toml" - lock = tmp_path / "lock.json" - task.write_text("[agent]\ntimeout_sec = 1200\n", encoding="utf-8") - lock.write_text('{"timeout_multiplier": 1.5}', encoding="utf-8") - - assert resolve_agent_timeout_seconds(task, lock) == 1800 - - -def test_deadline_prompt_reserves_finalization_time() -> None: - prompt = build_deadline_prompt(1800, started_at=0) - - assert "30.0 minutes from start" in prompt - assert "1970-01-01T00:25:30+00:00" in prompt - assert "preserve the best valid deliverable" in prompt - assert "repeated passing checks" in prompt - - -def test_invalid_timeout_inputs_disable_attachment(tmp_path: Path) -> None: - assert ( - resolve_agent_timeout_seconds( - tmp_path / "missing-task.toml", - tmp_path / "missing-lock.json", - ) - is None - )