From 5ca9ca7b4d9a4911a7c3e4a8ef5b1cf3c1361057 Mon Sep 17 00:00:00 2001 From: Eric Lee Date: Thu, 23 Jul 2026 23:45:09 -0700 Subject: [PATCH 1/5] Fix deferred tool discovery in API history --- src/tool_system/tool_search.py | 11 +++- src/tool_system/tools/tool_search.py | 17 ++++++- tests/test_tool_search.py | 75 +++++++++++++++++++++++++++- 3 files changed, 99 insertions(+), 4 deletions(-) diff --git a/src/tool_system/tool_search.py b/src/tool_system/tool_search.py index adabe8ee..1d7572e9 100644 --- a/src/tool_system/tool_search.py +++ b/src/tool_system/tool_search.py @@ -275,7 +275,16 @@ def extract_discovered_tool_names(messages: list[Any]) -> set[str]: discovered: set[str] = set() for msg in messages: - msg_type = getattr(msg, "type", None) or (msg.get("type") if isinstance(msg, dict) else None) + msg_type = getattr(msg, "type", None) + if isinstance(msg, dict): + # Internal Message objects use ``type`` while provider-ready API + # messages use ``role``. _call_model_sync passes the latter into + # filter_tools_for_request(), so ignoring ``role`` made every + # freshly returned tool_reference invisible on the next request. + # The reference remained in history but its schema stayed + # filtered out, causing providers to reject the dangling + # reference as an invalid request. + msg_type = msg_type or msg.get("type") or msg.get("role") # Compact boundary carries pre-compact discovered set if msg_type == "system": diff --git a/src/tool_system/tools/tool_search.py b/src/tool_system/tools/tool_search.py index 7df6e002..602abad3 100644 --- a/src/tool_system/tools/tool_search.py +++ b/src/tool_system/tools/tool_search.py @@ -31,11 +31,22 @@ def _map_result_to_api(output: Any, tool_use_id: str) -> dict[str, Any]: def make_tool_search_tool(registry: ToolRegistry) -> Tool: + def _is_available_tool(tool: Tool | None) -> bool: + """Only advertise tools that can be present on the next request.""" + if tool is None: + return False + try: + return bool(tool.is_enabled()) + except Exception: + # A broken runtime gate must not produce a reference whose schema + # the request builder will subsequently omit. + return False + def _deferred_count() -> int: return sum( 1 for tool in registry.list_tools() - if tool.should_defer or tool.is_mcp + if (tool.should_defer or tool.is_mcp) and _is_available_tool(tool) ) def _tool_search_call(tool_input: dict[str, Any], context: ToolContext) -> ToolResult: @@ -51,7 +62,7 @@ def _tool_search_call(tool_input: dict[str, Any], context: ToolContext) -> ToolR if lowered.startswith("select:"): name = q.split(":", 1)[1].strip() tool = registry.get(name) - matches = [tool.name] if tool else [] + matches = [tool.name] if _is_available_tool(tool) else [] return ToolResult( name="ToolSearch", output={ @@ -63,6 +74,8 @@ def _tool_search_call(tool_input: dict[str, Any], context: ToolContext) -> ToolR scored: list[tuple[int, str]] = [] for t in registry.list_tools(): + if not _is_available_tool(t): + continue hay = f"{t.name}\n{t.prompt()}".lower() if lowered in t.name.lower(): scored.append((0, t.name)) diff --git a/tests/test_tool_search.py b/tests/test_tool_search.py index eeb13ada..6dd17581 100644 --- a/tests/test_tool_search.py +++ b/tests/test_tool_search.py @@ -27,9 +27,17 @@ build_default_registry, ) from src.tool_system.protocol import ToolCall +from src.tool_system.registry import ToolRegistry +from src.tool_system.tools.tool_search import make_tool_search_tool -def _make_tool(name: str, *, should_defer: bool = False, is_mcp: bool = False): +def _make_tool( + name: str, + *, + should_defer: bool = False, + is_mcp: bool = False, + is_enabled=None, +): return build_tool( name=name, input_schema={"type": "object", "properties": {}}, @@ -37,6 +45,7 @@ def _make_tool(name: str, *, should_defer: bool = False, is_mcp: bool = False): prompt=f"Tool {name}", should_defer=should_defer, is_mcp=is_mcp, + is_enabled=is_enabled, ) @@ -181,6 +190,20 @@ def test_tool_reference_in_user_message(self): result = extract_discovered_tool_names(msgs) self.assertIn("mcp__tool_x", result) + def test_tool_reference_in_provider_role_message(self): + """Provider-ready history uses role=user, not type=user.""" + msgs = [{ + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": "123", + "content": [ + {"type": "tool_reference", "tool_name": "Grep"}, + ], + }], + }] + self.assertEqual(extract_discovered_tool_names(msgs), {"Grep"}) + def test_no_tool_reference(self): msgs = [{ "type": "user", @@ -244,6 +267,33 @@ def test_discovered_tools_kept(self): names = [t.name for t in result] self.assertIn("mcp_tool", names) + def test_provider_role_history_loads_discovered_tool(self): + """Regression: real API messages use role and previously lost refs.""" + with patch.dict(os.environ, { + "ENABLE_TOOL_SEARCH": "true", + "CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS": "", + }): + tools = [ + _make_tool("Read"), + _make_tool("ToolSearch"), + _make_tool("Grep", should_defer=True), + ] + messages = [{ + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": "search-1", + "content": [ + {"type": "tool_reference", "tool_name": "Grep"}, + ], + }], + }] + result = filter_tools_for_request( + tools, "claude-opus-4-8", messages=messages, + ) + + self.assertIn("Grep", {tool.name for tool in result}) + def test_missing_tool_search_keeps_deferred_tools_reachable(self): with patch.dict(os.environ, { "ENABLE_TOOL_SEARCH": "true", @@ -306,6 +356,29 @@ def test_tool_search_result_loads_deferred_schema_next_request(self): self.assertIn("Grep", {tool.name for tool in loaded}) + def test_tool_search_does_not_reference_disabled_tool(self): + registry = ToolRegistry([ + _make_tool( + "TodoWrite", + should_defer=True, + is_enabled=lambda: False, + ), + ]) + registry.register(make_tool_search_tool(registry)) + + result = registry.dispatch( + ToolCall( + name="ToolSearch", + input={"query": "select:TodoWrite"}, + tool_use_id="search-1", + ), + ToolContext(workspace_root=Path.cwd()), + ) + search_tool = registry.get("ToolSearch") + assert search_tool is not None + block = search_tool.map_result_to_api(result.output, "search-1") + + self.assertEqual(block["content"], "No matching deferred tools found") if __name__ == "__main__": unittest.main() From d6af702e0ca8e5b89466c03e3621e328883338e5 Mon Sep 17 00:00:00 2001 From: Eric Lee Date: Fri, 24 Jul 2026 00:32:41 -0700 Subject: [PATCH 2/5] Support long-running Harbor agent trials --- eval/harbor/README.md | 6 ++++++ eval/harbor/clawcodex_agent.py | 14 ++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/eval/harbor/README.md b/eval/harbor/README.md index 5c1da52b..8003cc4f 100644 --- a/eval/harbor/README.md +++ b/eval/harbor/README.md @@ -74,10 +74,16 @@ PYTHONPATH=$PWD/eval/harbor harbor run \ --ak subscription=true \ --ak effort=high \ --jobs-dir eval/harbor/jobs \ + --agent-timeout-multiplier 2 \ --n-concurrent 2 ``` Notes: +- The long-run profile doubles only the agent phase timeout. This gives + complex tasks room for compilation, training, simulation, and iterative + debugging without weakening verifier or environment-setup limits. Omit + `--agent-timeout-multiplier 2` when reproducing a benchmark's stock + wall-clock policy exactly. - `effort=high` maps to `clawcodex --effort high` → `output_config.effort` on effort-capable models (Opus 4.6/4.8, Sonnet 4.6, Fable 5). Requires clawcodex > 1.2.1 in the container — diff --git a/eval/harbor/clawcodex_agent.py b/eval/harbor/clawcodex_agent.py index 9db0b24a..026bca49 100644 --- a/eval/harbor/clawcodex_agent.py +++ b/eval/harbor/clawcodex_agent.py @@ -157,11 +157,13 @@ def _save_host_oauth(credentials: dict[str, Any]) -> None: os.replace(tmp, path) -# Refresh when under this much runway. Must exceed the longest agent -# timeout (terminal-bench tasks: 900s) with margin, because containers get -# an access token WITHOUT a refresh token (see the injection notes) and so -# cannot refresh mid-trial. -_MIN_TOKEN_RUNWAY_SEC = 1800 +# Refresh when under this much runway. Containers intentionally receive an +# access token WITHOUT a refresh token (see the injection notes), so the +# token must outlive the whole trial. Long problem-solving evaluations can +# legitimately run beyond the benchmark's original 15-60 minute budgets; +# keep two hours available so callers can raise Harbor's agent timeout +# without introducing an unrelated mid-trial authentication failure. +_MIN_TOKEN_RUNWAY_SEC = 2 * 60 * 60 def fresh_subscription_credentials() -> dict[str, Any]: @@ -169,7 +171,7 @@ def fresh_subscription_credentials() -> dict[str, Any]: Returns the credentials dict, refreshing (and persisting back to the host file) when the access token has under ``_MIN_TOKEN_RUNWAY_SEC`` of - runway, so every trial starts with more runway than its agent timeout. + runway, so long-running trials do not inherit a nearly-expired token. Raises RuntimeError with a remedial message when no usable credentials exist. """ From cf0f7b6ace66dc55370d0601de46bae53c0db9b4 Mon Sep 17 00:00:00 2001 From: Eric Lee Date: Fri, 24 Jul 2026 07:36:05 -0700 Subject: [PATCH 3/5] Make Harbor agents deadline aware --- eval/harbor/clawcodex_agent.py | 36 +++++++++++++++---- eval/harbor/time_budget.py | 62 ++++++++++++++++++++++++++++++++ tests/test_harbor_time_budget.py | 57 +++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 6 deletions(-) create mode 100644 eval/harbor/time_budget.py create mode 100644 tests/test_harbor_time_budget.py diff --git a/eval/harbor/clawcodex_agent.py b/eval/harbor/clawcodex_agent.py index 026bca49..3ec28787 100644 --- a/eval/harbor/clawcodex_agent.py +++ b/eval/harbor/clawcodex_agent.py @@ -86,6 +86,7 @@ ToolCall, Trajectory, ) +from time_budget import build_deadline_prompt, resolve_agent_timeout_seconds # Host env vars forwardable into the container (clawcodex's builtin provider # key candidates — see src/providers/__init__.py in the clawcodex repo), keyed @@ -427,19 +428,30 @@ async def _inject_subscription_credentials( env={"CLAWCODEX_OAUTH_JSON": json.dumps(credentials)}, ) - async def _seed_container_settings(self, environment: BaseEnvironment) -> None: - """Seed ``settings.effort`` in the container's global config. + async def _seed_container_settings( + self, + environment: BaseEnvironment, + *, + deadline_prompt: str | None = None, + ) -> None: + """Seed session-wide effort and deadline guidance in global config. clawcodex's ``--effort`` flag governs the MAIN loop only; subagents (Agent tool) resolve effort from ``settings.effort``. Seeding the container's global config (home-anchored ``~/.clawcodex/config.json`` — the global-config path deliberately does not follow - CLAWCODEX_CONFIG_DIR) makes the requested effort session-wide. + CLAWCODEX_CONFIG_DIR) makes the requested effort session-wide and + appends Harbor's real wall-clock deadline to the model instructions. """ + settings: dict[str, Any] = {} effort = self._resolved_flags.get("effort") - if not effort: + if effort: + settings["effort"] = effort + if deadline_prompt: + settings["append_system_prompt"] = deadline_prompt + if not settings: return - payload = json.dumps({"settings": {"effort": effort}}) + payload = json.dumps({"settings": settings}) await self.exec_as_agent( environment, command=( @@ -462,7 +474,19 @@ async def run( self._captured_instruction = instruction if self._subscription: await self._inject_subscription_credentials(environment) - await self._seed_container_settings(environment) + timeout_seconds = resolve_agent_timeout_seconds( + environment.environment_dir.parent / "task.toml", + environment.trial_paths.lock_path, + ) + deadline_prompt = ( + build_deadline_prompt(timeout_seconds) + if timeout_seconds is not None + else None + ) + await self._seed_container_settings( + environment, + deadline_prompt=deadline_prompt, + ) parts: list[str] = [ "clawcodex", diff --git a/eval/harbor/time_budget.py b/eval/harbor/time_budget.py new file mode 100644 index 00000000..3a49fa51 --- /dev/null +++ b/eval/harbor/time_budget.py @@ -0,0 +1,62 @@ +"""Deadline helpers shared by Harbor adapters. + +Harbor owns the agent-phase timeout but does not pass it to ``Agent.run``. +The resolved inputs are nevertheless available beside the environment and +trial, so adapters can make the deadline visible to an otherwise unaware +agent without encoding dataset or task names. +""" + +from __future__ import annotations + +import json +import time +import tomllib +from datetime import datetime, timezone +from pathlib import Path + + +def resolve_agent_timeout_seconds(task_toml: Path, lock_json: Path) -> float | None: + """Return Harbor's effective agent timeout from resolved trial inputs.""" + try: + task = tomllib.loads(task_toml.read_text(encoding="utf-8")) + lock = json.loads(lock_json.read_text(encoding="utf-8")) + base = float(task["agent"]["timeout_sec"]) + multiplier = lock.get("agent_timeout_multiplier") + if multiplier is None: + multiplier = lock.get("timeout_multiplier", 1.0) + effective = base * float(multiplier) + except (KeyError, OSError, TypeError, ValueError, json.JSONDecodeError): + return None + return effective if effective > 0 else None + + +def build_deadline_prompt( + timeout_seconds: float, + *, + started_at: float | None = None, +) -> str: + """Build model-neutral guidance for a real external execution deadline.""" + start = time.time() if started_at is None else started_at + deadline = start + timeout_seconds + # Reserve 15%, bounded so short tasks still get two minutes and very long + # tasks do not abandon productive work excessively early. + reserve = min(10 * 60, max(2 * 60, timeout_seconds * 0.15)) + finalize_at = deadline - reserve + + def stamp(value: float) -> str: + return datetime.fromtimestamp(value, tz=timezone.utc).isoformat( + timespec="seconds" + ) + + return ( + "This run has a hard external execution deadline at " + f"{stamp(deadline)} ({timeout_seconds / 60:.1f} minutes from start). " + f"By {stamp(finalize_at)}, preserve the best valid deliverable, stop " + "broad exploration, and switch to the narrowest checks needed for the " + "explicit requirements. Do not start optional audits, repeated passing " + "checks, or long refinements that cannot finish before the deadline. " + "If the core result already works, finish and return control rather " + "than consuming the remaining budget. You can use `date -u` to compare " + "the current time with these timestamps." + ) + diff --git a/tests/test_harbor_time_budget.py b/tests/test_harbor_time_budget.py new file mode 100644 index 00000000..36b7b058 --- /dev/null +++ b/tests/test_harbor_time_budget.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path + + +sys.path.insert(0, str(Path(__file__).parents[1] / "eval" / "harbor")) + +from time_budget import build_deadline_prompt, resolve_agent_timeout_seconds + + +def test_resolve_agent_timeout_uses_agent_multiplier(tmp_path: Path) -> None: + task = tmp_path / "task.toml" + lock = tmp_path / "lock.json" + task.write_text("[agent]\ntimeout_sec = 900\n", encoding="utf-8") + lock.write_text( + json.dumps( + { + "timeout_multiplier": 3, + "agent_timeout_multiplier": 2, + } + ), + encoding="utf-8", + ) + + assert resolve_agent_timeout_seconds(task, lock) == 1800 + + +def test_resolve_agent_timeout_falls_back_to_global_multiplier( + tmp_path: Path, +) -> None: + task = tmp_path / "task.toml" + lock = tmp_path / "lock.json" + task.write_text("[agent]\ntimeout_sec = 1200\n", encoding="utf-8") + lock.write_text('{"timeout_multiplier": 1.5}', encoding="utf-8") + + assert resolve_agent_timeout_seconds(task, lock) == 1800 + + +def test_deadline_prompt_reserves_finalization_time() -> None: + prompt = build_deadline_prompt(1800, started_at=0) + + assert "30.0 minutes from start" in prompt + assert "1970-01-01T00:25:30+00:00" in prompt + assert "preserve the best valid deliverable" in prompt + assert "repeated passing checks" in prompt + + +def test_invalid_timeout_inputs_disable_attachment(tmp_path: Path) -> None: + assert ( + resolve_agent_timeout_seconds( + tmp_path / "missing-task.toml", + tmp_path / "missing-lock.json", + ) + is None + ) From 2581ef86eb5d774bbc8fc5c6bc7b6956c6c86ee7 Mon Sep 17 00:00:00 2001 From: Eric Lee Date: Fri, 24 Jul 2026 09:30:19 -0700 Subject: [PATCH 4/5] Keep leaderboard runs on stock timeouts --- eval/harbor/README.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/eval/harbor/README.md b/eval/harbor/README.md index 8003cc4f..841f57f0 100644 --- a/eval/harbor/README.md +++ b/eval/harbor/README.md @@ -74,16 +74,13 @@ PYTHONPATH=$PWD/eval/harbor harbor run \ --ak subscription=true \ --ak effort=high \ --jobs-dir eval/harbor/jobs \ - --agent-timeout-multiplier 2 \ --n-concurrent 2 ``` Notes: -- The long-run profile doubles only the agent phase timeout. This gives - complex tasks room for compilation, training, simulation, and iterative - debugging without weakening verifier or environment-setup limits. Omit - `--agent-timeout-multiplier 2` when reproducing a benchmark's stock - wall-clock policy exactly. +- Official Terminal-Bench submissions may not modify task timeouts or + resources. Keep Harbor's default timeout policy for leaderboard-comparable + runs; use timeout multipliers only for explicitly labeled diagnostics. - `effort=high` maps to `clawcodex --effort high` → `output_config.effort` on effort-capable models (Opus 4.6/4.8, Sonnet 4.6, Fable 5). Requires clawcodex > 1.2.1 in the container — From 49d7fe5b48790578f26e245a11c175b81d9e65cb Mon Sep 17 00:00:00 2001 From: Eric Lee Date: Fri, 24 Jul 2026 10:31:29 -0700 Subject: [PATCH 5/5] Fix Harbor deadline helper formatting --- eval/harbor/time_budget.py | 1 - 1 file changed, 1 deletion(-) diff --git a/eval/harbor/time_budget.py b/eval/harbor/time_budget.py index 3a49fa51..50d86975 100644 --- a/eval/harbor/time_budget.py +++ b/eval/harbor/time_budget.py @@ -59,4 +59,3 @@ def stamp(value: float) -> str: "than consuming the remaining budget. You can use `date -u` to compare " "the current time with these timestamps." ) -