Total wall-clock limit; checked between runs. Must be ≥ the per-run timeout.
+Runs
@@ -301,6 +321,7 @@ const defaultForm = () => ({ stop_signal: '', delay_seconds: 0, timeout_per_run: null, + max_duration_seconds: null, model: '', allowed_tools: null, }) @@ -354,6 +375,7 @@ async function submit() { if (form.stop_signal && form.stop_signal.trim()) payload.stop_signal = form.stop_signal.trim() if (form.delay_seconds) payload.delay_seconds = form.delay_seconds if (form.timeout_per_run) payload.timeout_per_run = form.timeout_per_run + if (form.max_duration_seconds) payload.max_duration_seconds = form.max_duration_seconds if (form.model) payload.model = form.model if (form.allowed_tools !== null) payload.allowed_tools = form.allowed_tools @@ -397,12 +419,22 @@ function formatStopReason(reason) { max_runs_reached: 'reached max runs', stop_signal_matched: 'stop signal matched', user_stopped: 'stopped by user', + deadline_exceeded: 'deadline exceeded', error: 'error', interrupted: 'interrupted', } return map[reason] || reason } +function formatSeconds(secs) { + if (secs === null || secs === undefined) return '—' + if (secs < 60) return `${secs}s` + if (secs < 3600) return `${Math.floor(secs / 60)}m ${secs % 60}s` + const h = Math.floor(secs / 3600) + const m = Math.floor((secs % 3600) / 60) + return `${h}h ${m}m` +} + function formatCost(cost) { if (cost === null || cost === undefined) return '—' return `$${cost.toFixed(4)}` diff --git a/src/mcp-server/src/client.ts b/src/mcp-server/src/client.ts index 25b9f77f0..2ec583837 100644 --- a/src/mcp-server/src/client.ts +++ b/src/mcp-server/src/client.ts @@ -1821,6 +1821,7 @@ export class TrinityClient { stop_signal?: string; delay_seconds?: number; timeout_per_run?: number; + max_duration_seconds?: number; model?: string; allowed_tools?: string[]; } diff --git a/src/mcp-server/src/tools/loops.ts b/src/mcp-server/src/tools/loops.ts index 8b5c8cae4..a331f9278 100644 --- a/src/mcp-server/src/tools/loops.ts +++ b/src/mcp-server/src/tools/loops.ts @@ -108,6 +108,19 @@ export function createLoopTools( .describe( "Per-iteration timeout in seconds (defaults to agent's configured execution_timeout_seconds)." ), + max_duration_seconds: z + .number() + .int() + .min(1) + .max(604_800) + .optional() + .describe( + "Optional loop-level wall-clock deadline in seconds (1–604800 = up to 7 days). " + + "Checked at each iteration boundary; an in-flight run is never killed mid-turn. " + + "When the deadline passes the loop stops with stop_reason='deadline_exceeded'. " + + "Must be >= timeout_per_run (or the agent's execution timeout when unset). " + + "Omit for no time bound (max_runs still applies)." + ), model: z .string() .optional() @@ -125,6 +138,7 @@ export function createLoopTools( stop_signal?: string; delay_seconds?: number; timeout_per_run?: number; + max_duration_seconds?: number; model?: string; allowed_tools?: string[]; }, @@ -146,6 +160,7 @@ export function createLoopTools( stop_signal: params.stop_signal, delay_seconds: params.delay_seconds, timeout_per_run: params.timeout_per_run, + max_duration_seconds: params.max_duration_seconds, model: params.model, allowed_tools: params.allowed_tools, }); diff --git a/tests/unit/test_loop_service.py b/tests/unit/test_loop_service.py index 29d2a0d59..0b9a09d75 100644 --- a/tests/unit/test_loop_service.py +++ b/tests/unit/test_loop_service.py @@ -16,6 +16,7 @@ import asyncio import sys from dataclasses import dataclass, field +from datetime import datetime, timedelta from pathlib import Path from typing import Any, Optional from unittest.mock import AsyncMock, MagicMock @@ -544,6 +545,160 @@ def test_orphan_sweep_idempotent(self, loop_module): assert db.mark_orphan_loops_interrupted() == 0 +# --------------------------------------------------------------------------- +# Runner — wall-clock deadline (#1156) +# --------------------------------------------------------------------------- + +class _FakeClock: + """Controllable stand-in for ``datetime`` inside loop_service. + + Only ``utcnow()`` is exercised by the runner; it returns the current + fake instant. Tests advance ``now`` (directly or via a task that bumps + it each run) to drive the deadline deterministically — no real sleeping. + """ + now = datetime(2026, 1, 1, 0, 0, 0) + + @classmethod + def utcnow(cls): + return cls.now + + +class TestDeadline: + def _install_clock(self, ls, monkeypatch, *, advance_per_run: float): + """Swap in the fake clock; each execute_task advances it.""" + _FakeClock.now = datetime(2026, 1, 1, 0, 0, 0) + monkeypatch.setattr(ls, "datetime", _FakeClock) + + async def _exec(**kwargs): + ts = self._ts + ts.calls.append(kwargs) + result = ts.results[ts._idx] if ts._idx < len(ts.results) else _Result() + ts._idx += 1 + _FakeClock.now = _FakeClock.now + timedelta(seconds=advance_per_run) + return result + + return _exec + + def test_deadline_stops_loop_at_boundary(self, loop_module, monkeypatch): + ls, db, ts = loop_module + self._ts = ts + ts.results = [_Result(response=f"r{i}") for i in range(1, 6)] + ts.execute_task = self._install_clock(ls, monkeypatch, advance_per_run=6) + + async def go(): + service = ls.LoopService() + row = await service.start_loop( + agent_name="a1", + message_template="m", + max_runs=5, + max_duration_seconds=10, # ~1.6 runs fit before the deadline + ) + handle = service._handles.get(row["id"]) + if handle is not None: + await handle.task + return row["id"] + + loop_id = _run(go()) + loop = db.get_loop(loop_id) + assert loop["status"] == "stopped" + assert loop["stop_reason"] == "deadline_exceeded" + # Run 1 (t0→6) and run 2 (t6→12) both started before the deadline; the + # boundary check before run 3 (t12 ≥ 10) trips. max_runs never reached. + assert loop["runs_completed"] == 2 + assert len(ts.calls) == 2 + + def test_in_flight_run_is_not_killed_mid_turn(self, loop_module, monkeypatch): + ls, db, ts = loop_module + self._ts = ts + ts.results = [_Result(response="done-run")] + # One run pushes the clock well past the deadline; that run must still + # finalize as completed (deadline is enforced only at the boundary). + ts.execute_task = self._install_clock(ls, monkeypatch, advance_per_run=999) + + async def go(): + service = ls.LoopService() + row = await service.start_loop( + agent_name="a1", + message_template="m", + max_runs=5, + max_duration_seconds=10, + ) + handle = service._handles.get(row["id"]) + if handle is not None: + await handle.task + return row["id"] + + loop_id = _run(go()) + loop = db.get_loop(loop_id) + assert loop["stop_reason"] == "deadline_exceeded" + assert loop["runs_completed"] == 1 # the in-flight run completed + runs = db.list_loop_runs(loop_id) + assert runs[0]["status"] == "completed" + assert runs[0]["response"] == "done-run" + + def test_delay_does_not_sleep_past_deadline(self, loop_module, monkeypatch): + ls, db, ts = loop_module + self._ts = ts + ts.results = [_Result(response="r1"), _Result(response="r2")] + ts.execute_task = self._install_clock(ls, monkeypatch, advance_per_run=3) + + # Capture sleep durations and advance the fake clock by them instead + # of really sleeping. + slept: list[float] = [] + + async def _fake_sleep(secs): + slept.append(secs) + _FakeClock.now = _FakeClock.now + timedelta(seconds=secs) + + monkeypatch.setattr(ls.asyncio, "sleep", _fake_sleep) + + async def go(): + service = ls.LoopService() + row = await service.start_loop( + agent_name="a1", + message_template="m", + max_runs=5, + delay_seconds=100, # would blow way past the deadline + max_duration_seconds=10, + ) + handle = service._handles.get(row["id"]) + if handle is not None: + await handle.task + return row["id"] + + loop_id = _run(go()) + loop = db.get_loop(loop_id) + assert loop["stop_reason"] == "deadline_exceeded" + # run1 t0→3, then delay capped to remaining (10−3=7), not the full 100. + assert slept == [7] + + def test_no_deadline_runs_all_when_unset(self, loop_module, monkeypatch): + ls, db, ts = loop_module + self._ts = ts + ts.results = [_Result(response=f"r{i}") for i in range(1, 4)] + # Clock jumps far each run; with no deadline it must be ignored. + ts.execute_task = self._install_clock(ls, monkeypatch, advance_per_run=10_000) + + async def go(): + service = ls.LoopService() + row = await service.start_loop( + agent_name="a1", + message_template="m", + max_runs=3, + max_duration_seconds=None, + ) + handle = service._handles.get(row["id"]) + if handle is not None: + await handle.task + return row["id"] + + loop_id = _run(go()) + loop = db.get_loop(loop_id) + assert loop["status"] == "completed" + assert loop["stop_reason"] == "max_runs_reached" + assert loop["runs_completed"] == 3 + + # --------------------------------------------------------------------------- # get_status # --------------------------------------------------------------------------- diff --git a/tests/unit/test_loops_router_validation.py b/tests/unit/test_loops_router_validation.py new file mode 100644 index 000000000..b148e81b1 --- /dev/null +++ b/tests/unit/test_loops_router_validation.py @@ -0,0 +1,98 @@ +"""Validation tests for the loops router — max_duration_seconds deadline (#1156). + +Calls the ``start_loop`` endpoint coroutine directly with mocked auth/db/service +so the cross-field validation (deadline must be >= the effective per-run +timeout) can be exercised without a live FastAPI app or database. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +import pytest + +_BACKEND = Path(__file__).resolve().parent.parent.parent / "src" / "backend" +_BACKEND_STR = str(_BACKEND) +while _BACKEND_STR in sys.path: + sys.path.remove(_BACKEND_STR) +sys.path.insert(0, _BACKEND_STR) + +pytestmark = pytest.mark.unit + + +def _load_router(monkeypatch): + from routers import loops as loops_router + + fake_db = MagicMock() + fake_db.get_execution_timeout.return_value = 900 # agent default for tests + monkeypatch.setattr(loops_router, "db", fake_db) + + fake_service = MagicMock() + fake_service.start_loop = AsyncMock( + return_value={"id": "loop_x", "status": "queued"} + ) + monkeypatch.setattr(loops_router, "get_loop_service", lambda: fake_service) + + return loops_router, fake_db, fake_service + + +def _user(): + u = MagicMock() + u.id = 1 + u.email = "u@example.com" + return u + + +async def _call(loops_router, payload): + return await loops_router.start_loop( + payload=payload, + name="a1", + current_user=_user(), + x_source_agent=None, + x_mcp_key_id=None, + x_mcp_key_name=None, + ) + + +class TestMaxDurationValidation: + def test_rejects_deadline_below_explicit_timeout_per_run(self, monkeypatch): + loops_router, _, service = _load_router(monkeypatch) + payload = loops_router.StartLoopRequest( + message="m", max_runs=5, timeout_per_run=600, max_duration_seconds=300, + ) + with pytest.raises(loops_router.HTTPException) as exc: + __import__("asyncio").run(_call(loops_router, payload)) + assert exc.value.status_code == 400 + assert "max_duration_seconds" in exc.value.detail + service.start_loop.assert_not_called() + + def test_rejects_deadline_below_agent_timeout_when_per_run_unset(self, monkeypatch): + loops_router, fake_db, service = _load_router(monkeypatch) + payload = loops_router.StartLoopRequest( + message="m", max_runs=5, max_duration_seconds=100, # < 900 agent default + ) + with pytest.raises(loops_router.HTTPException) as exc: + __import__("asyncio").run(_call(loops_router, payload)) + assert exc.value.status_code == 400 + fake_db.get_execution_timeout.assert_called_once_with("a1") + service.start_loop.assert_not_called() + + def test_accepts_deadline_at_or_above_timeout(self, monkeypatch): + loops_router, _, service = _load_router(monkeypatch) + payload = loops_router.StartLoopRequest( + message="m", max_runs=5, timeout_per_run=600, max_duration_seconds=600, + ) + resp = __import__("asyncio").run(_call(loops_router, payload)) + assert resp.loop_id == "loop_x" + # deadline threaded through to the service + assert service.start_loop.await_args.kwargs["max_duration_seconds"] == 600 + + def test_no_deadline_skips_validation(self, monkeypatch): + loops_router, fake_db, service = _load_router(monkeypatch) + payload = loops_router.StartLoopRequest(message="m", max_runs=5) + resp = __import__("asyncio").run(_call(loops_router, payload)) + assert resp.loop_id == "loop_x" + fake_db.get_execution_timeout.assert_not_called() + assert service.start_loop.await_args.kwargs["max_duration_seconds"] is None