From 66c9fed9c093437b0ddcdee0e8152025ccd1fd74 Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Thu, 6 Aug 2026 16:46:49 +0900 Subject: [PATCH] fix: bind tool approvals to concrete invocations --- src/agents/_tool_invocation.py | 302 ++ src/agents/agent.py | 1 + src/agents/items.py | 5 + src/agents/models/interface.py | 9 +- src/agents/realtime/session.py | 298 +- src/agents/run_context.py | 623 +++- src/agents/run_internal/items.py | 20 +- src/agents/run_internal/run_loop.py | 288 +- src/agents/run_internal/tool_actions.py | 177 +- src/agents/run_internal/tool_execution.py | 299 +- src/agents/run_internal/tool_planning.py | 455 ++- src/agents/run_internal/turn_resolution.py | 479 ++- src/agents/run_state.py | 289 +- src/agents/tool_context.py | 5 +- tests/mcp/test_mcp_tracing.py | 9 +- tests/realtime/test_session.py | 666 +++- .../capabilities/test_apply_patch_tool.py | 6 +- tests/sandbox/test_runtime.py | 10 +- tests/test_agent_as_tool.py | 3 + tests/test_agent_hooks.py | 18 +- tests/test_agent_runner.py | 120 +- tests/test_agent_runner_streamed.py | 34 +- tests/test_apply_patch_tool.py | 7 + tests/test_example_workflows.py | 22 +- tests/test_global_hooks.py | 16 +- tests/test_hitl_error_scenarios.py | 789 ++++- tests/test_max_turns.py | 36 +- tests/test_responses.py | 8 +- tests/test_run_context_approvals.py | 41 +- tests/test_run_context_wrapper.py | 60 +- tests/test_run_hooks.py | 9 +- tests/test_run_state.py | 1034 ++++++- tests/test_run_step_execution.py | 222 +- tests/test_soft_cancel.py | 8 +- tests/test_stream_events.py | 133 +- tests/test_tool_approval_call_id_reuse.py | 2683 +++++++++++++++++ tests/test_tool_guardrails.py | 8 +- tests/test_tool_name_collision_policy.py | 150 +- tests/test_tracing_errors.py | 18 +- tests/test_tracing_errors_streamed.py | 18 +- 40 files changed, 8489 insertions(+), 889 deletions(-) create mode 100644 src/agents/_tool_invocation.py create mode 100644 tests/test_tool_approval_call_id_reuse.py diff --git a/src/agents/_tool_invocation.py b/src/agents/_tool_invocation.py new file mode 100644 index 0000000000..a18d6dd94e --- /dev/null +++ b/src/agents/_tool_invocation.py @@ -0,0 +1,302 @@ +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping, Sequence +from typing import Any, TypeGuard + +from ._tool_identity import ( + FunctionToolLookupKey, + get_function_tool_lookup_key_for_call, + get_hosted_mcp_approval_request_identity, +) + +_TOOL_INVOCATION_TYPES = frozenset( + { + "apply_patch_call", + "computer_call", + "custom_tool_call", + "function_call", + "local_shell_call", + "mcp_approval_request", + "shell_call", + } +) +_TOOL_OUTPUT_TYPES = { + "apply_patch_call_output": "apply_patch_call", + "computer_call_output": "computer_call", + "custom_tool_call_output": "custom_tool_call", + "function_call_output": "function_call", + "local_shell_call_output": "local_shell_call", + "mcp_approval_response": "mcp_approval_request", + "shell_call_output": "shell_call", +} +_SEMANTIC_FIELDS = ( + "type", + "name", + "namespace", + "server_label", + "arguments", + "input", + "action", + "actions", + "pending_safety_checks", + "operation", + "operations", + "environment", + "caller", +) + + +def is_tool_invocation_type(value: Any) -> TypeGuard[str]: + """Return whether a value names a canonical tool invocation type.""" + return isinstance(value, str) and value in _TOOL_INVOCATION_TYPES + + +def is_tool_invocation_digest(value: Any) -> TypeGuard[str]: + """Return whether a value is a canonical lowercase SHA-256 digest.""" + return ( + isinstance(value, str) + and len(value) == 64 + and all(character in "0123456789abcdef" for character in value) + ) + + +def _as_mapping(value: Any) -> Mapping[str, Any] | None: + if isinstance(value, Mapping): + return value + model_dump = getattr(value, "model_dump", None) + if callable(model_dump): + dumped = model_dump(exclude_none=True, exclude_unset=True) + return dumped if isinstance(dumped, Mapping) else None + return None + + +def _normalize_value(value: Any) -> Any: + mapping = _as_mapping(value) + if mapping is not None: + return { + str(key): _normalize_value(item) + for key, item in sorted(mapping.items(), key=lambda pair: str(pair[0])) + } + if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray): + return [_normalize_value(item) for item in value] + if value is None or isinstance(value, str | int | float | bool): + return value + return str(value) + + +def _normalize_arguments(value: Any) -> Any: + if not isinstance(value, str): + return _normalize_value(value) + try: + parsed = json.loads( + value, + parse_constant=lambda constant: (_ for _ in ()).throw( + ValueError(f"Invalid JSON constant: {constant}") + ), + ) + except (TypeError, ValueError, json.JSONDecodeError): + return value + return _normalize_value(parsed) + + +def _unwrap_hosted_mcp_approval(raw_item: Any) -> Mapping[str, Any] | None: + mapping = _as_mapping(raw_item) + if mapping is None: + return None + provider_data = mapping.get("provider_data") + if ( + mapping.get("type") == "hosted_tool_call" + and isinstance(provider_data, Mapping) + and provider_data.get("type") == "mcp_approval_request" + ): + request_identity = get_hosted_mcp_approval_request_identity(mapping) + if request_identity is None: + return None + merged = dict(mapping) + merged.update(provider_data) + if request_identity.request_id is None: + merged.pop("id", None) + else: + merged["id"] = request_identity.request_id + if request_identity.tool_name is not None: + merged["name"] = request_identity.tool_name + return merged + return mapping + + +def tool_invocation_identity( + raw_item: Any, + *, + tool_lookup_key: FunctionToolLookupKey | None = None, + tool_name: str | None = None, + invocation_role: str | None = None, +) -> tuple[str, str, str] | None: + """Return invocation type, provider call ID, and a stable semantic fingerprint.""" + identity = tool_invocation_identity_and_scope( + raw_item, + tool_lookup_key=tool_lookup_key, + tool_name=tool_name, + invocation_role=invocation_role, + ) + if identity is None: + return None + invocation_type, call_id, _, fingerprint = identity + return invocation_type, call_id, fingerprint + + +def tool_invocation_identity_and_scope( + raw_item: Any, + *, + tool_lookup_key: FunctionToolLookupKey | None = None, + tool_name: str | None = None, + invocation_role: str | None = None, +) -> tuple[str, str, str, str] | None: + """Return invocation identity together with its stable approval scope.""" + call_identity = tool_invocation_call_id(raw_item) + approval_scope_identity = tool_invocation_approval_scope( + raw_item, + tool_lookup_key=tool_lookup_key, + tool_name=tool_name, + invocation_role=invocation_role, + ) + if call_identity is None or approval_scope_identity is None: + return None + invocation_type, call_id = call_identity + scope_invocation_type, approval_scope = approval_scope_identity + if call_id is None or scope_invocation_type != invocation_type: + return None + + mapping = _unwrap_hosted_mcp_approval(raw_item) + if mapping is None: + return None + + if invocation_type == "function_call": + if "arguments" not in mapping: + return None + elif invocation_type == "mcp_approval_request": + if "arguments" not in mapping: + return None + elif invocation_type == "custom_tool_call": + if not isinstance(mapping.get("name"), str) or not mapping["name"]: + return None + if "input" not in mapping: + return None + elif invocation_type in {"computer_call", "local_shell_call", "shell_call"}: + if "action" not in mapping: + return None + elif invocation_type == "apply_patch_call": + if "operation" not in mapping and "operations" not in mapping: + return None + + semantic_payload: dict[str, Any] = {"approval_scope": approval_scope} + for field_name in _SEMANTIC_FIELDS: + if invocation_type == "function_call" and field_name in {"name", "namespace"}: + continue + if field_name not in mapping: + continue + value = mapping[field_name] + semantic_payload[field_name] = ( + _normalize_arguments(value) if field_name == "arguments" else _normalize_value(value) + ) + + return ( + invocation_type, + call_id, + approval_scope, + _fingerprint(semantic_payload), + ) + + +def tool_invocation_call_id(raw_item: Any) -> tuple[str, str | None] | None: + """Return a recognized invocation type and its valid non-empty call ID, if present.""" + mapping = _unwrap_hosted_mcp_approval(raw_item) + if mapping is None: + return None + invocation_type = mapping.get("type") + if invocation_type not in _TOOL_INVOCATION_TYPES: + return None + candidate = ( + mapping.get("id") if invocation_type == "mcp_approval_request" else mapping.get("call_id") + ) + return invocation_type, candidate if isinstance(candidate, str) and candidate else None + + +def tool_invocation_approval_scope( + raw_item: Any, + *, + tool_lookup_key: FunctionToolLookupKey | None = None, + tool_name: str | None = None, + invocation_role: str | None = None, +) -> tuple[str, str] | None: + """Return the stable authorization scope for a recognized tool invocation.""" + mapping = _unwrap_hosted_mcp_approval(raw_item) + if mapping is None: + return None + invocation_type = mapping.get("type") + if invocation_type not in _TOOL_INVOCATION_TYPES: + return None + + payload: dict[str, Any] = {"type": invocation_type} + if invocation_role is not None: + payload["invocation_role"] = invocation_role + if invocation_type == "function_call": + resolved_lookup_key = tool_lookup_key or get_function_tool_lookup_key_for_call(mapping) + if resolved_lookup_key is None: + return None + payload["tool_lookup_key"] = _normalize_value(resolved_lookup_key) + elif invocation_type == "mcp_approval_request": + tool_name = mapping.get("name") + server_label = mapping.get("server_label") + if ( + not isinstance(tool_name, str) + or not tool_name + or not isinstance(server_label, str) + or not server_label + ): + return None + payload["name"] = tool_name + payload["server_label"] = server_label + else: + resolved_tool_name = tool_name or mapping.get("name") + if isinstance(resolved_tool_name, str) and resolved_tool_name: + payload["name"] = resolved_tool_name + return invocation_type, _fingerprint(payload) + + +def _fingerprint(payload: Mapping[str, Any]) -> str: + encoded = json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def is_mcp_approval_invocation(raw_item: Any) -> bool: + """Return whether an item represents a hosted MCP approval request.""" + mapping = _unwrap_hosted_mcp_approval(raw_item) + return mapping is not None and mapping.get("type") == "mcp_approval_request" + + +def tool_output_identity(raw_item: Any) -> tuple[str, str] | None: + """Return the invocation type and call ID completed by a tool output item.""" + mapping = _as_mapping(raw_item) + if mapping is None: + return None + output_type = mapping.get("type") + if not isinstance(output_type, str): + return None + invocation_type = _TOOL_OUTPUT_TYPES.get(output_type) + if invocation_type is None: + return None + candidate = ( + mapping.get("approval_request_id") + if output_type == "mcp_approval_response" + else mapping.get("call_id") + ) + if not isinstance(candidate, str) or not candidate: + return None + return invocation_type, candidate diff --git a/src/agents/agent.py b/src/agents/agent.py index 677e8cf868..bcd0a86f61 100644 --- a/src/agents/agent.py +++ b/src/agents/agent.py @@ -999,6 +999,7 @@ async def enqueue_stream_events() -> None: run_result, scope_id=tool_state_scope_id, ) + return run_result.final_output if custom_output_extractor is not None: return await custom_output_extractor(run_result) diff --git a/src/agents/items.py b/src/agents/items.py index 012d81b1dd..424b7f7e05 100644 --- a/src/agents/items.py +++ b/src/agents/items.py @@ -367,9 +367,14 @@ class ToolCallItem(RunItemBase[Any]): tool_origin: ToolOrigin | None = None """Optional metadata describing the source of a function-tool-backed item.""" + _resolved_tool_name: str | None = field(default=None, kw_only=True, repr=False) + """SDK-resolved tool name when the provider payload does not carry one.""" + @property def tool_name(self) -> str | None: """Return the tool name from the raw item, if available.""" + if self._resolved_tool_name is not None: + return self._resolved_tool_name if isinstance(self.raw_item, dict): return self.raw_item.get("name") return getattr(self.raw_item, "name", None) diff --git a/src/agents/models/interface.py b/src/agents/models/interface.py index 3be588c2a8..6c4bd2cc03 100644 --- a/src/agents/models/interface.py +++ b/src/agents/models/interface.py @@ -35,7 +35,14 @@ def include_data(self) -> bool: class Model(abc.ABC): - """The base interface for calling an LLM.""" + """The base interface for calling an LLM. + + Model implementations must assign a non-empty call ID to each tool invocation. A call ID must + identify one canonical invocation for the lifetime of the run and its serialized resume + lineage; it must not be reused for changed tool identity or payload. An exact completed replay + may be omitted by the runtime without re-executing the invocation. Tool outputs must retain the + call ID for correlation. + """ async def _cleanup_on_run_end(self, owner: object) -> None: """Release run-scoped resources after the runner finishes using this model.""" diff --git a/src/agents/realtime/session.py b/src/agents/realtime/session.py index 7d6265a96c..18d8d716f1 100644 --- a/src/agents/realtime/session.py +++ b/src/agents/realtime/session.py @@ -14,9 +14,11 @@ from .. import _debug from .._tool_identity import ( FunctionToolLookupKey, + get_function_tool_lookup_key, get_function_tool_lookup_key_for_tool, get_function_tool_namespace, ) +from .._tool_invocation import tool_invocation_identity from ..agent import Agent from ..exceptions import ( ModelBehaviorError, @@ -228,8 +230,11 @@ def __init__( self._cleanup_task: asyncio.Task[None] | None = None self._stored_exception: BaseException | None = None self._pending_tool_calls: dict[str, _PendingToolCall] = {} - self._active_tool_call_ids: set[str] = set() - self._completed_tool_call_ids: set[str] = set() + self._tool_invocation_routes: dict[ + str, + tuple[FunctionToolLookupKey | None, str | None], + ] = {} + self._active_tool_invocations: dict[str, tuple[str, str, str]] = {} self._pending_tool_outputs: dict[str, _PendingToolOutput] = {} self._current_dispatch_snapshot: _RealtimeDispatchSnapshot | None = None @@ -793,9 +798,8 @@ async def _run_tool_input_guardrails( if not guardrails: return None - tool_context = ToolContext( - context=self._context_wrapper.context, - usage=self._context_wrapper.usage, + tool_context = ToolContext.from_agent_context( + self._context_wrapper, tool_name=tool_call.name, tool_call_id=tool_call.call_id, tool_arguments=tool_call.arguments, @@ -843,6 +847,7 @@ async def _send_tool_rejection( rejection_message = await self._resolve_approval_rejection_message( tool=tool, call_id=event.call_id, + tool_call=self._build_tool_approval_item(tool, event, agent).raw_item, ) await self._send_tool_output_completion( _PendingToolOutput( @@ -866,21 +871,29 @@ async def _send_tool_output_completion(self, pending_output: _PendingToolOutput) call_id = pending_output.tool_call.call_id self._pending_tool_outputs[call_id] = pending_output try: - await self._send_pending_tool_output(pending_output) + output_sent = await self._send_pending_tool_output(pending_output) except Exception as exc: if self._closing or self._closed: self._pending_tool_outputs.pop(call_id, None) return raise _PendingToolOutputSendError(call_id, exc) from exc + if not output_sent: + self._pending_tool_outputs.pop(call_id, None) + return + self._context_wrapper._mark_tool_call_completed( + {"type": "function_call_output", "call_id": call_id}, + ) self._pending_tool_outputs.pop(call_id, None) + if pending_output.tool_end_event is not None: + self._put_event_nowait(pending_output.tool_end_event) - async def _send_pending_tool_output(self, pending_output: _PendingToolOutput) -> None: + async def _send_pending_tool_output(self, pending_output: _PendingToolOutput) -> bool: if self._closing or self._closed: - return + return False if pending_output.session_update is not None: await self._model.send_event(pending_output.session_update) if self._closing or self._closed: - return + return False await self._model.send_event( RealtimeModelSendToolOutput( tool_call=pending_output.tool_call, @@ -888,12 +901,15 @@ async def _send_pending_tool_output(self, pending_output: _PendingToolOutput) -> start_response=pending_output.start_response, ) ) - if self._closing or self._closed: - return - if pending_output.tool_end_event is not None: - await self._put_event(pending_output.tool_end_event) + return True - async def _resolve_approval_rejection_message(self, *, tool: FunctionTool, call_id: str) -> str: + async def _resolve_approval_rejection_message( + self, + *, + tool: FunctionTool, + call_id: str, + tool_call: Any | None = None, + ) -> str: """Resolve model-visible output text for approval rejections.""" explicit_message = self._context_wrapper.get_rejection_message( tool.name, @@ -907,6 +923,11 @@ async def _resolve_approval_rejection_message(self, *, tool: FunctionTool, call_ if formatter is None: return REJECTION_MESSAGE + if tool_call is not None: + self._context_wrapper._mark_tool_invocation_executed( + tool_call, + tool_lookup_key=get_function_tool_lookup_key_for_tool(tool), + ) try: maybe_message = formatter( ToolErrorFormatterArgs( @@ -948,7 +969,17 @@ async def approve_tool_call(self, call_id: str, *, always: bool = False) -> None if pending is None: return - if not self._begin_tool_call(call_id, from_pending_approval=True): + pending_identity = tool_invocation_identity( + pending.approval_item.raw_item, + tool_lookup_key=pending.approval_item.tool_lookup_key, + ) + if pending_identity is None: + raise ModelBehaviorError("Realtime tool calls require a canonical invocation identity.") + if not self._begin_tool_call( + call_id, + pending_identity, + from_pending_approval=True, + ): return try: @@ -971,7 +1002,7 @@ async def approve_tool_call(self, call_id: str, *, always: bool = False) -> None call_id_reserved=True, ) except Exception: - if call_id in self._active_tool_call_ids: + if call_id in self._active_tool_invocations: self._finish_tool_call(call_id, mark_completed=False) raise @@ -990,7 +1021,17 @@ async def reject_tool_call( if pending is None: return - if not self._begin_tool_call(call_id, from_pending_approval=True): + pending_identity = tool_invocation_identity( + pending.approval_item.raw_item, + tool_lookup_key=pending.approval_item.tool_lookup_key, + ) + if pending_identity is None: + raise ModelBehaviorError("Realtime tool calls require a canonical invocation identity.") + if not self._begin_tool_call( + call_id, + pending_identity, + from_pending_approval=True, + ): return mark_completed = False @@ -1020,15 +1061,91 @@ async def _handle_tool_call( ) -> None: """Handle a tool call event.""" mark_completed = False + agent = dispatch_snapshot.agent if dispatch_snapshot is not None else agent_snapshot + agent = agent or self._current_agent + recorded_route = self._tool_invocation_routes.get(event.call_id) + recorded_role = recorded_route[1] if recorded_route is not None else None + if ( + recorded_route is not None + and recorded_route[0] is not None + and recorded_route[0][-1] != event.name + ): + raise ModelBehaviorError( + "Model reused a Realtime tool call ID for a different invocation. " + "Use a unique call ID for each tool invocation." + ) + dispatch_role = self._resolve_tool_dispatch_role( + event.name, + agent=agent, + dispatch_snapshot=dispatch_snapshot, + ) + if ( + recorded_route is not None + and dispatch_role is not None + and recorded_role != dispatch_role + ): + raise ModelBehaviorError( + "Model reused a Realtime tool call ID for a different invocation. " + "Use a unique call ID for each tool invocation." + ) + identity_role = ( + recorded_role if dispatch_role is None and recorded_route is not None else dispatch_role + ) + current_raw_item = { + "type": "function_call", + "name": event.name, + "call_id": event.call_id, + "arguments": event.arguments, + } + current_lookup_key = ( + recorded_route[0] + if recorded_route is not None + else get_function_tool_lookup_key(event.name, None) + ) + current_identity = tool_invocation_identity( + current_raw_item, + tool_lookup_key=current_lookup_key, + invocation_role="handoff" if identity_role == "handoff" else None, + ) + if current_identity is None: + raise ModelBehaviorError("Realtime tool calls require a non-empty string call ID.") + active_identity = self._active_tool_invocations.get(event.call_id) + if active_identity is not None and active_identity != current_identity: + raise ModelBehaviorError( + "Model reused a Realtime tool call ID for a different invocation. " + "Use a unique call ID for each tool invocation." + ) + invocation_status = self._context_wrapper._tool_invocation_status( + current_raw_item, + tool_lookup_key=current_lookup_key, + invocation_role="handoff" if identity_role == "handoff" else None, + ) + if invocation_status is None: + raise ModelBehaviorError("Realtime tool calls require a non-empty string call ID.") + + pending_output = self._pending_tool_outputs.get(event.call_id) + has_pending_output = pending_output is not None + is_duplicate_call = ( + active_identity is not None + or event.call_id in self._pending_tool_calls + or invocation_status[1] + ) + if not call_id_reserved: + if is_duplicate_call: + return + if invocation_status[2] and not invocation_status[1] and not has_pending_output: + raise ModelBehaviorError( + "A Realtime tool call already executed, but its output was not committed. " + "Start a new call instead of retrying the invocation." + ) if not call_id_reserved and not self._begin_tool_call( - event.call_id, from_pending_approval=from_pending_approval + event.call_id, + current_identity, + from_pending_approval=from_pending_approval, ): return - agent = dispatch_snapshot.agent if dispatch_snapshot is not None else agent_snapshot - agent = agent or self._current_agent try: - pending_output = self._pending_tool_outputs.get(event.call_id) if pending_output is not None: await self._send_tool_output_completion(pending_output) mark_completed = True @@ -1046,6 +1163,14 @@ async def _handle_tool_call( if event.name in function_map: func_tool = function_map[event.name] + approval_item = self._build_tool_approval_item(func_tool, event, agent) + self._bind_resolved_tool_invocation( + event.call_id, + approval_item.raw_item, + preliminary_identity=current_identity, + tool_lookup_key=approval_item.tool_lookup_key, + route_role="function", + ) approval_status = await self._maybe_request_tool_approval( event, function_tool=func_tool, @@ -1065,6 +1190,10 @@ async def _handle_tool_call( if approval_status is None: return + self._context_wrapper._mark_tool_invocation_executed( + approval_item.raw_item, + tool_lookup_key=approval_item.tool_lookup_key, + ) rejected_message = await self._run_tool_input_guardrails( tool=func_tool, tool_call=event, @@ -1095,9 +1224,8 @@ async def _handle_tool_call( if self._closing or self._closed: return - tool_context = ToolContext( - context=self._context_wrapper.context, - usage=self._context_wrapper.usage, + tool_context = ToolContext.from_agent_context( + self._context_wrapper, tool_name=event.name, tool_call_id=event.call_id, tool_arguments=event.arguments, @@ -1128,9 +1256,15 @@ async def _handle_tool_call( mark_completed = True elif event.name in handoff_map: handoff = handoff_map[event.name] - tool_context = ToolContext( - context=self._context_wrapper.context, - usage=self._context_wrapper.usage, + self._bind_resolved_tool_invocation( + event.call_id, + current_raw_item, + preliminary_identity=current_identity, + tool_lookup_key=get_function_tool_lookup_key(event.name, None), + route_role="handoff", + ) + tool_context = ToolContext.from_agent_context( + self._context_wrapper, tool_name=event.name, tool_call_id=event.call_id, tool_arguments=event.arguments, @@ -1138,6 +1272,11 @@ async def _handle_tool_call( ) # Execute the handoff to get the new agent + self._context_wrapper._mark_tool_invocation_executed( + current_raw_item, + tool_lookup_key=get_function_tool_lookup_key(event.name, None), + invocation_role="handoff", + ) result = await handoff.on_invoke_handoff(self._context_wrapper, event.arguments) if self._closing or self._closed: return @@ -1185,6 +1324,14 @@ async def _handle_tool_call( ) mark_completed = True else: + fallback_role = "handoff" if identity_role == "handoff" else None + self._bind_resolved_tool_invocation( + event.call_id, + current_raw_item, + preliminary_identity=current_identity, + tool_lookup_key=get_function_tool_lookup_key(event.name, None), + route_role=fallback_role, + ) error_message = f"Tool {event.name} not found" await self._send_tool_output_completion( _PendingToolOutput( @@ -1203,20 +1350,101 @@ async def _handle_tool_call( finally: self._finish_tool_call(event.call_id, mark_completed=mark_completed) - def _begin_tool_call(self, call_id: str, *, from_pending_approval: bool) -> bool: + def _begin_tool_call( + self, + call_id: str, + identity: tuple[str, str, str], + *, + from_pending_approval: bool, + ) -> bool: if self._closing or self._closed: return False - if call_id in self._active_tool_call_ids or call_id in self._completed_tool_call_ids: + active_identity = self._active_tool_invocations.get(call_id) + if active_identity is not None: + if active_identity != identity: + raise ModelBehaviorError( + "Model reused a Realtime tool call ID for a different invocation. " + "Use a unique call ID for each tool invocation." + ) return False if not from_pending_approval and call_id in self._pending_tool_calls: return False - self._active_tool_call_ids.add(call_id) + self._active_tool_invocations[call_id] = identity return True + def _bind_resolved_tool_invocation( + self, + call_id: str, + raw_item: Any, + *, + preliminary_identity: tuple[str, str, str], + tool_lookup_key: FunctionToolLookupKey | None, + route_role: str | None, + ) -> None: + """Atomically replace a provisional Realtime identity with its resolved identity.""" + resolved_identity = tool_invocation_identity( + raw_item, + tool_lookup_key=tool_lookup_key, + invocation_role="handoff" if route_role == "handoff" else None, + ) + if resolved_identity is None: + raise ModelBehaviorError("Realtime tool calls require a canonical invocation identity.") + + active_identity = self._active_tool_invocations.get(call_id) + if active_identity not in {None, preliminary_identity, resolved_identity}: + raise ModelBehaviorError( + "Model reused a Realtime tool call ID for a different invocation. " + "Use a unique call ID for each tool invocation." + ) + self._context_wrapper._rebind_tool_invocation( + raw_item, + previous_identity=preliminary_identity, + tool_lookup_key=tool_lookup_key, + invocation_role="handoff" if route_role == "handoff" else None, + ) + if active_identity is not None: + self._active_tool_invocations[call_id] = resolved_identity + self._tool_invocation_routes[call_id] = (tool_lookup_key, route_role) + + def _resolve_tool_dispatch_role( + self, + tool_name: str, + *, + agent: RealtimeAgent[Any], + dispatch_snapshot: _RealtimeDispatchSnapshot | None, + ) -> str | None: + """Return the known dispatch role without invoking dynamic tool resolvers.""" + snapshot = dispatch_snapshot + if snapshot is None and self._current_dispatch_snapshot is not None: + if self._current_dispatch_snapshot.agent is agent: + snapshot = self._current_dispatch_snapshot + + tools: Sequence[Any] + handoffs: Sequence[Any] + if snapshot is not None: + tools = snapshot.tools + handoffs = snapshot.handoffs + else: + raw_tools = getattr(agent, "tools", ()) + raw_handoffs = getattr(agent, "handoffs", ()) + tools = raw_tools if isinstance(raw_tools, Sequence) else () + handoffs = raw_handoffs if isinstance(raw_handoffs, Sequence) else () + + if any( + (isinstance(handoff, Handoff) and handoff.tool_name == tool_name) + or ( + isinstance(handoff, RealtimeAgent) + and Handoff.default_tool_name(handoff) == tool_name + ) + for handoff in handoffs + ): + return "handoff" + if any(isinstance(tool, FunctionTool) and tool.name == tool_name for tool in tools): + return "function" + return None + def _finish_tool_call(self, call_id: str, *, mark_completed: bool) -> None: - self._active_tool_call_ids.discard(call_id) - if mark_completed and not self._closing and not self._closed: - self._completed_tool_call_ids.add(call_id) + self._active_tool_invocations.pop(call_id, None) @classmethod def _get_new_history( @@ -1770,9 +1998,9 @@ async def _cleanup(self) -> None: # Clear pending approval tracking self._pending_tool_calls.clear() + self._tool_invocation_routes.clear() self._pending_tool_outputs.clear() - self._active_tool_call_ids.clear() - self._completed_tool_call_ids.clear() + self._active_tool_invocations.clear() # Mark as closed self._closed = True diff --git a/src/agents/run_context.py b/src/agents/run_context.py index 946b3db879..ba0b1d4c17 100644 --- a/src/agents/run_context.py +++ b/src/agents/run_context.py @@ -16,7 +16,17 @@ is_reserved_synthetic_tool_namespace, tool_qualified_name, ) -from .exceptions import UserError +from ._tool_invocation import ( + is_mcp_approval_invocation, + is_tool_invocation_digest, + is_tool_invocation_type, + tool_invocation_approval_scope, + tool_invocation_call_id, + tool_invocation_identity, + tool_invocation_identity_and_scope, + tool_output_identity, +) +from .exceptions import ModelBehaviorError, UserError from .usage import Usage if TYPE_CHECKING: @@ -30,6 +40,17 @@ TContext = TypeVar("TContext", default=Any) +@dataclass(eq=False) +class _ToolInvocationRecord: + """Tracks the canonical identity and lifecycle of one provider tool call ID.""" + + invocation_type: str + approval_scope: str + fingerprint: str + executed: bool = False + completed: bool = False + + @dataclass(eq=False) class _ApprovalRecord: """Tracks approval/rejection state for a tool. @@ -42,6 +63,7 @@ class _ApprovalRecord: rejected: bool | list[str] = field(default_factory=list) rejection_messages: dict[str, str] = field(default_factory=dict) sticky_rejection_message: str | None = None + sticky_scope: str | None = None @dataclass(eq=False) @@ -63,8 +85,32 @@ class RunContextWrapper(Generic[TContext]): turn_input: list[TResponseInputItem] = field(default_factory=list) _approvals: dict[str | HostedMCPApprovalKey, _ApprovalRecord] = field(default_factory=dict) + _tool_invocations: dict[str, _ToolInvocationRecord] = field( + default_factory=dict, + init=False, + repr=False, + ) tool_input: Any | None = None """Structured input for the current agent tool run, when available.""" + _allow_legacy_approval_binding_reconstruction: bool = field( + default=False, + init=False, + repr=False, + ) + _restored_unbound_approval_call_ids: set[str] = field( + default_factory=set, + init=False, + repr=False, + ) + + def _share_tool_state_with(self, target: RunContextWrapper[Any]) -> None: + """Share tool approval and invocation state with a derived context wrapper.""" + target._approvals = self._approvals + target._tool_invocations = self._tool_invocations + target._allow_legacy_approval_binding_reconstruction = ( + self._allow_legacy_approval_binding_reconstruction + ) + target._restored_unbound_approval_call_ids = self._restored_unbound_approval_call_ids @staticmethod def _to_str_or_none(value: Any) -> str | None: @@ -149,8 +195,13 @@ def _resolve_tool_lookup_key(approval_item: ToolApprovalItem) -> FunctionToolLoo @staticmethod def _resolve_call_id(approval_item: ToolApprovalItem) -> str | None: + hosted_request = get_hosted_mcp_approval_request_identity(approval_item) + if hosted_request is not None: + return hosted_request.request_id + raw = approval_item.raw_item if isinstance(raw, dict): + raw_type = raw.get("type") provider_data = raw.get("provider_data") if ( isinstance(provider_data, dict) @@ -159,8 +210,11 @@ def _resolve_call_id(approval_item: ToolApprovalItem) -> str | None: candidate = provider_data.get("id") if isinstance(candidate, str): return candidate - candidate = raw.get("call_id") or raw.get("id") + candidate = raw.get("id") if raw_type == "mcp_approval_request" else raw.get("call_id") + if candidate is None and raw_type is None: + candidate = raw.get("id") else: + raw_type = getattr(raw, "type", None) provider_data = getattr(raw, "provider_data", None) if ( isinstance(provider_data, dict) @@ -169,7 +223,13 @@ def _resolve_call_id(approval_item: ToolApprovalItem) -> str | None: candidate = provider_data.get("id") if isinstance(candidate, str): return candidate - candidate = getattr(raw, "call_id", None) or getattr(raw, "id", None) + candidate = ( + getattr(raw, "id", None) + if raw_type == "mcp_approval_request" + else getattr(raw, "call_id", None) + ) + if candidate is None and raw_type is None: + candidate = getattr(raw, "id", None) return RunContextWrapper._to_str_or_none(candidate) def _get_or_create_approval_entry( @@ -182,6 +242,349 @@ def _get_or_create_approval_entry( self._approvals[approval_key] = approval_entry return approval_entry + def _approved_tool_invocation_status( + self, + raw_item: Any, + *, + tool_lookup_key: FunctionToolLookupKey | None = None, + tool_name: str | None = None, + invocation_role: str | None = None, + ) -> tuple[tuple[str, str], bool, bool] | None: + """Validate an invocation and return status when an approval decision applies.""" + status = self._tool_invocation_status( + raw_item, + tool_lookup_key=tool_lookup_key, + tool_name=tool_name, + invocation_role=invocation_role, + ) + if status is None: + return None + identity = tool_invocation_identity_and_scope( + raw_item, + tool_lookup_key=tool_lookup_key, + tool_name=tool_name, + invocation_role=invocation_role, + ) + if identity is None: + return None + _, call_id, approval_scope, _ = identity + sticky_approval_keys = self._matching_sticky_approval_keys( + raw_item, + tool_lookup_key=tool_lookup_key, + tool_name=tool_name, + approval_scope=approval_scope, + ) + has_per_call_decision = any( + (isinstance(record.approved, list) and call_id in record.approved) + or (isinstance(record.rejected, list) and call_id in record.rejected) + for record in self._approvals.values() + ) + if not has_per_call_decision and not sticky_approval_keys: + return None + return status + + def _tool_invocation_status( + self, + raw_item: Any, + *, + tool_lookup_key: FunctionToolLookupKey | None = None, + tool_name: str | None = None, + invocation_role: str | None = None, + ) -> tuple[tuple[str, str], bool, bool] | None: + """Validate and register one canonical invocation for a provider call ID.""" + call_identity = tool_invocation_call_id(raw_item) + call_id = call_identity[1] if call_identity is not None else None + is_restored_unbound = ( + call_id is not None and call_id in self._restored_unbound_approval_call_ids + ) + identity = tool_invocation_identity_and_scope( + raw_item, + tool_lookup_key=tool_lookup_key, + tool_name=tool_name, + invocation_role=invocation_role, + ) + if identity is None: + if is_mcp_approval_invocation(raw_item): + return None + if call_id is not None and call_id in self._tool_invocations: + raise ModelBehaviorError( + "Model reused a tool call ID for a different invocation. " + "Use a unique call ID for each tool invocation." + ) + return None + invocation_type, call_id, approval_scope, fingerprint = identity + record = self._tool_invocations.get(call_id) + if record is None: + if is_restored_unbound: + return None + record = _ToolInvocationRecord( + invocation_type=invocation_type, + approval_scope=approval_scope, + fingerprint=fingerprint, + ) + self._tool_invocations[call_id] = record + elif ( + record.invocation_type != invocation_type + or record.approval_scope != approval_scope + or record.fingerprint != fingerprint + ): + raise ModelBehaviorError( + "Model reused a tool call ID for a different invocation. " + "Use a unique call ID for each tool invocation." + ) + if is_restored_unbound: + return None + return ((invocation_type, call_id), record.completed, record.executed) + + def _rebind_tool_invocation( + self, + raw_item: Any, + *, + previous_identity: tuple[str, str, str], + tool_lookup_key: FunctionToolLookupKey | None = None, + tool_name: str | None = None, + invocation_role: str | None = None, + ) -> tuple[tuple[str, str], bool, bool] | None: + """Replace an unresolved invocation identity before execution begins.""" + identity = tool_invocation_identity_and_scope( + raw_item, + tool_lookup_key=tool_lookup_key, + tool_name=tool_name, + invocation_role=invocation_role, + ) + if identity is None: + return None + invocation_type, call_id, approval_scope, fingerprint = identity + record = self._tool_invocations.get(call_id) + resolved_identity = (invocation_type, approval_scope, fingerprint) + if record is None: + return self._tool_invocation_status( + raw_item, + tool_lookup_key=tool_lookup_key, + tool_name=tool_name, + invocation_role=invocation_role, + ) + current_identity = ( + record.invocation_type, + record.approval_scope, + record.fingerprint, + ) + if current_identity == resolved_identity: + return ((invocation_type, call_id), record.completed, record.executed) + matches_previous = ( + record.invocation_type == previous_identity[0] + and call_id == previous_identity[1] + and record.fingerprint == previous_identity[2] + ) + if not matches_previous or record.executed or record.completed: + raise ModelBehaviorError( + "Model reused a tool call ID for a different invocation. " + "Use a unique call ID for each tool invocation." + ) + record.invocation_type = invocation_type + record.approval_scope = approval_scope + record.fingerprint = fingerprint + return ((invocation_type, call_id), False, False) + + def _matching_sticky_approval_keys( + self, + raw_item: Any, + *, + tool_lookup_key: FunctionToolLookupKey | None, + tool_name: str | None = None, + approval_scope: str, + ) -> frozenset[str | HostedMCPApprovalKey]: + """Return sticky approval keys that independently authorize this tool identity.""" + if isinstance(raw_item, Mapping): + mapping = raw_item + else: + model_dump = getattr(raw_item, "model_dump", None) + dumped = ( + model_dump(exclude_none=True, exclude_unset=True) if callable(model_dump) else None + ) + mapping = dumped if isinstance(dumped, Mapping) else {} + provider_data = mapping.get("provider_data") + if ( + mapping.get("type") == "hosted_tool_call" + and isinstance(provider_data, Mapping) + and provider_data.get("type") == "mcp_approval_request" + ): + merged = dict(mapping) + merged.update(provider_data) + mapping = merged + + invocation_type = mapping.get("type") + if not isinstance(invocation_type, str): + return frozenset() + tool_name = tool_name or self._to_str_or_none(mapping.get("name")) + tool_namespace = self._to_str_or_none(mapping.get("namespace")) + if invocation_type == "function_call": + approval_keys: tuple[str | HostedMCPApprovalKey, ...] = get_function_tool_approval_keys( + tool_name=tool_name, + tool_namespace=tool_namespace, + tool_lookup_key=tool_lookup_key, + include_legacy_deferred_key=True, + ) + elif invocation_type == "mcp_approval_request": + server_label = self._to_str_or_none(mapping.get("server_label")) + approval_keys = ( + (("hosted_mcp", server_label, tool_name),) + if server_label is not None and tool_name is not None + else () + ) + else: + if tool_name is None: + tool_name = { + "apply_patch_call": "apply_patch", + "computer_call": "computer", + "local_shell_call": "local_shell", + "shell_call": "shell", + }.get(invocation_type) + approval_keys = (tool_name,) if tool_name else () + + matching_keys: set[str | HostedMCPApprovalKey] = set() + for approval_key in approval_keys: + record = self._approvals.get(approval_key) + if ( + record is not None + and (isinstance(record.approved, bool) or isinstance(record.rejected, bool)) + and record.sticky_scope == approval_scope + ): + matching_keys.add(approval_key) + return frozenset(matching_keys) + + def _mark_tool_call_completed( + self, + raw_item: Any, + ) -> None: + """Mark a canonical invocation completed when its output is committed.""" + identity = tool_output_identity(raw_item) + if identity is None: + return + invocation_type, call_id = identity + record = self._tool_invocations.get(call_id) + if record is None or record.invocation_type != invocation_type: + return + record.executed = True + record.completed = True + + def _mark_tool_invocation_executed( + self, + raw_item: Any, + *, + tool_lookup_key: FunctionToolLookupKey | None = None, + tool_name: str | None = None, + invocation_role: str | None = None, + ) -> None: + """Mark an invocation executed before the first user-code side effect.""" + status = self._tool_invocation_status( + raw_item, + tool_lookup_key=tool_lookup_key, + tool_name=tool_name, + invocation_role=invocation_role, + ) + if status is None: + return + _, call_id = status[0] + self._tool_invocations[call_id].executed = True + + def _restore_pending_approval_binding(self, approval_item: ToolApprovalItem) -> None: + """Rebuild a missing binding from a serialized pending approval item.""" + if not self._allow_legacy_approval_binding_reconstruction: + return + approval_keys: list[str | HostedMCPApprovalKey] = list( + self._resolve_approval_keys(approval_item) + ) + hosted_request = get_hosted_mcp_approval_request_identity(approval_item) + if hosted_request is not None and hosted_request.request_id is not None: + hosted_key: HostedMCPApprovalKey = ( + hosted_request.approval_identity + if hosted_request.approval_identity is not None + else ("hosted_mcp_call", hosted_request.request_id) + ) + approval_keys.append(hosted_key) + scope_identity = tool_invocation_approval_scope( + approval_item.raw_item, + tool_lookup_key=approval_item.tool_lookup_key, + tool_name=approval_item.tool_name, + ) + if scope_identity is not None: + _, approval_scope = scope_identity + for approval_key in approval_keys: + record = self._approvals.get(approval_key) + if record is not None and ( + isinstance(record.approved, bool) or isinstance(record.rejected, bool) + ): + record.sticky_scope = record.sticky_scope or approval_scope + call_id = self._resolve_call_id(approval_item) + if call_id is None: + return + identity = tool_invocation_identity_and_scope( + approval_item.raw_item, + tool_lookup_key=approval_item.tool_lookup_key, + tool_name=approval_item.tool_name, + ) + if identity is None: + self._restored_unbound_approval_call_ids.add(call_id) + return + has_matching_decision = False + for approval_key in approval_keys: + record = self._approvals.get(approval_key) + if record is None: + continue + has_per_call_decision = ( + isinstance(record.approved, list) and call_id in record.approved + ) or (isinstance(record.rejected, list) and call_id in record.rejected) + has_sticky_decision = ( + record.sticky_scope == approval_scope + and self._get_approval_status_for_record(record, call_id) is not None + ) + has_matching_decision = ( + has_matching_decision or has_per_call_decision or has_sticky_decision + ) + if has_matching_decision: + self._tool_invocation_status( + approval_item.raw_item, + tool_lookup_key=approval_item.tool_lookup_key, + tool_name=approval_item.tool_name, + ) + + def _mark_restored_unbound_pending_approval( + self, + approval_item: ToolApprovalItem, + ) -> None: + """Remember a current-schema pending call whose sticky binding was not restored.""" + if self._allow_legacy_approval_binding_reconstruction: + return + call_id = self._resolve_call_id(approval_item) + if call_id is None: + return + identity = tool_invocation_identity_and_scope( + approval_item.raw_item, + tool_lookup_key=approval_item.tool_lookup_key, + tool_name=approval_item.tool_name, + ) + if identity is None: + self._restored_unbound_approval_call_ids.add(call_id) + return + _, identity_call_id, approval_scope, _ = identity + if identity_call_id != call_id: + self._restored_unbound_approval_call_ids.add(call_id) + return + sticky_keys = self._matching_sticky_approval_keys( + approval_item.raw_item, + tool_lookup_key=approval_item.tool_lookup_key, + tool_name=approval_item.tool_name, + approval_scope=approval_scope, + ) + has_per_call_decision = any( + (isinstance(record.approved, list) and call_id in record.approved) + or (isinstance(record.rejected, list) and call_id in record.rejected) + for record in self._approvals.values() + ) + if (sticky_keys or has_per_call_decision) and call_id not in self._tool_invocations: + self._restored_unbound_approval_call_ids.add(call_id) + def is_tool_approved(self, tool_name: str, call_id: str) -> bool | None: """Return True/False/None for the given tool call.""" hosted_query_record = self._approvals.get(("hosted_mcp_query", tool_name, call_id)) @@ -350,6 +753,18 @@ def _resolve_hosted_mcp_approval_decision( status = self._get_per_call_approval_status_for_record(approval_record, request_id) else: status = self._get_approval_status_for_record(approval_record, request_id) + if ( + status is not None + and approval_record.sticky_scope is None + and self._allow_legacy_approval_binding_reconstruction + ): + scope_identity = tool_invocation_approval_scope( + approval_item.raw_item, + tool_lookup_key=approval_item.tool_lookup_key, + tool_name=approval_item.tool_name, + ) + if scope_identity is not None: + approval_record.sticky_scope = scope_identity[1] return status, self._get_rejection_message_for_key(approval_record, request_id) def get_rejection_message( @@ -463,8 +878,62 @@ def _apply_approval_decision( call_id = self._resolve_call_id(approval_item) hosted_identity = None + call_identity = tool_invocation_call_id(approval_item.raw_item) + if call_identity is not None and call_identity[1] is None: + raise ModelBehaviorError( + "Approval decisions require a non-empty call ID for recognized tool invocations." + ) + + raw_item = approval_item.raw_item + if isinstance(raw_item, Mapping): + raw_call_id = raw_item.get("call_id") if "call_id" in raw_item else raw_item.get("id") + else: + raw_call_id = ( + getattr(raw_item, "call_id", None) + if hasattr(raw_item, "call_id") + else getattr(raw_item, "id", None) + ) + if raw_call_id == "" and not always: + raise ModelBehaviorError("Per-call approval decisions require a non-empty call ID.") + invocation = ( + None + if call_id is None + else tool_invocation_identity_and_scope( + approval_item.raw_item, + tool_lookup_key=approval_item.tool_lookup_key, + tool_name=approval_item.tool_name, + ) + ) + if call_id is not None and invocation is None: + raise ModelBehaviorError("Approval decisions require a canonical invocation identity.") + if call_id is None and raw_call_id is not None: + raise ModelBehaviorError("Approval decisions require a canonical invocation identity.") + scope_identity = tool_invocation_approval_scope( + approval_item.raw_item, + tool_lookup_key=approval_item.tool_lookup_key, + tool_name=approval_item.tool_name, + ) + if invocation is not None: + assert call_id is not None + if invocation[1] != call_id: + raise ModelBehaviorError( + "Approval decision call ID does not match its canonical invocation ID." + ) + was_restored_unbound = call_id in self._restored_unbound_approval_call_ids + if was_restored_unbound: + self._restored_unbound_approval_call_ids.remove(call_id) + try: + self._tool_invocation_status( + approval_item.raw_item, + tool_lookup_key=approval_item.tool_lookup_key, + tool_name=approval_item.tool_name, + ) + finally: + if was_restored_unbound: + self._restored_unbound_approval_call_ids.add(call_id) approval_entries: tuple[tuple[_ApprovalRecord, bool], ...] if hosted_request is not None: + approval_keys: tuple[str, ...] = () assert call_id is not None hosted_key: HostedMCPApprovalKey if hosted_identity is None: @@ -497,6 +966,9 @@ def _apply_approval_decision( for approval_entry, entry_is_sticky in approval_entries: if entry_is_sticky or call_id is None: + approval_entry.sticky_scope = ( + scope_identity[1] if scope_identity is not None else None + ) approval_entry.approved = approve approval_entry.rejected = [] if approve else True if not approve: @@ -526,6 +998,10 @@ def _apply_approval_decision( else: self._clear_rejection_message(approval_entry, call_id) + if invocation is not None: + assert call_id is not None + self._restored_unbound_approval_call_ids.discard(call_id) + def approve_tool(self, approval_item: ToolApprovalItem, always_approve: bool = False) -> None: """Approve a tool call, optionally for all future calls.""" self._apply_approval_decision( @@ -556,13 +1032,38 @@ def get_approval_status( tool_namespace: str | None = None, existing_pending: ToolApprovalItem | None = None, tool_lookup_key: FunctionToolLookupKey | None = None, + current_invocation: ToolApprovalItem | None = None, ) -> bool | None: """Return approval status, retrying with pending item's tool name if necessary.""" + if not isinstance(call_id, str) or not call_id: + raise ModelBehaviorError("Approval-gated tool calls require a non-empty call ID.") if existing_pending is not None: + self._restore_pending_approval_binding(existing_pending) + pending_identity = tool_invocation_identity( + existing_pending.raw_item, + tool_lookup_key=existing_pending.tool_lookup_key, + tool_name=existing_pending.tool_name, + ) + if pending_identity is None: + pending_call_id = self._resolve_call_id(existing_pending) + if pending_call_id is not None and ( + current_invocation is None or pending_call_id not in self._tool_invocations + ): + self._restored_unbound_approval_call_ids.add(pending_call_id) + if current_invocation is None: + return None hosted_request = get_hosted_mcp_approval_request_identity(existing_pending) if hosted_request is not None: hosted_status, _ = self._resolve_hosted_mcp_approval_decision(existing_pending) - return hosted_status + if hosted_status is None: + return None + effective_invocation = current_invocation or existing_pending + binding_status = self._approved_tool_invocation_status( + effective_invocation.raw_item, + tool_lookup_key=effective_invocation.tool_lookup_key, + tool_name=effective_invocation.tool_name, + ) + return hosted_status if binding_status is not None else None candidates: list[str] = [] explicit_namespace = ( @@ -618,10 +1119,73 @@ def get_approval_status( candidates.append(pending_tool_name) status: bool | None = None + matched_record: _ApprovalRecord | None = None for candidate in candidates: status = self._get_approval_status_for_key(candidate, call_id) if status is not None: + matched_record = self._approvals.get(candidate) break + selected_invocation = current_invocation or existing_pending + if status is None or matched_record is None or selected_invocation is None: + return status + is_sticky = isinstance(matched_record.approved, bool) or isinstance( + matched_record.rejected, bool + ) + if is_sticky: + if ( + matched_record.sticky_scope is None + and self._allow_legacy_approval_binding_reconstruction + ): + scope_identity = tool_invocation_approval_scope( + selected_invocation.raw_item, + tool_lookup_key=selected_invocation.tool_lookup_key, + tool_name=selected_invocation.tool_name, + ) + if scope_identity is not None: + matched_record.sticky_scope = scope_identity[1] + binding_status = self._approved_tool_invocation_status( + selected_invocation.raw_item, + tool_lookup_key=selected_invocation.tool_lookup_key, + tool_name=selected_invocation.tool_name, + ) + return status if binding_status is not None else None + if current_invocation is not None: + current_identity = tool_invocation_identity( + current_invocation.raw_item, + tool_lookup_key=current_invocation.tool_lookup_key, + tool_name=current_invocation.tool_name, + ) + if current_identity is None: + self._approved_tool_invocation_status( + current_invocation.raw_item, + tool_lookup_key=current_invocation.tool_lookup_key, + tool_name=current_invocation.tool_name, + ) + return None + binding_status = self._approved_tool_invocation_status( + selected_invocation.raw_item, + tool_lookup_key=selected_invocation.tool_lookup_key, + tool_name=selected_invocation.tool_name, + ) + if binding_status is None: + current_identity = tool_invocation_identity( + selected_invocation.raw_item, + tool_lookup_key=selected_invocation.tool_lookup_key, + tool_name=selected_invocation.tool_name, + ) + if current_identity is not None: + return None + if existing_pending is not None: + pending_identity = tool_invocation_identity( + existing_pending.raw_item, + tool_lookup_key=existing_pending.tool_lookup_key, + tool_name=existing_pending.tool_name, + ) + if pending_identity is None and is_mcp_approval_invocation( + existing_pending.raw_item + ): + return None + return status return status def _rebuild_approvals(self, approvals: Any) -> None: @@ -649,8 +1213,55 @@ def _restore_approval_record(cls, record_dict: Mapping[str, Any]) -> _ApprovalRe sticky_rejection_message = record_dict.get("sticky_rejection_message") if isinstance(sticky_rejection_message, str): record.sticky_rejection_message = sticky_rejection_message + sticky_scope = record_dict.get("sticky_scope") + if isinstance(sticky_scope, str): + record.sticky_scope = sticky_scope return record + def _rebuild_tool_invocations(self, invocations: Any) -> None: + """Restore the current-schema canonical tool invocation ledger.""" + self._tool_invocations = {} + if not isinstance(invocations, Mapping): + raise UserError("RunState tool_invocations must be a mapping.") + for call_id, serialized_invocation in invocations.items(): + if not isinstance(call_id, str) or not call_id: + raise UserError("RunState tool_invocations contains an invalid call ID.") + if not isinstance(serialized_invocation, Mapping): + raise UserError(f"RunState tool invocation {call_id!r} must be a mapping.") + invocation_type = serialized_invocation.get("type") + approval_scope = serialized_invocation.get("approval_scope") + fingerprint = serialized_invocation.get("fingerprint") + executed = serialized_invocation.get("executed") + completed = serialized_invocation.get("completed") + if ( + not is_tool_invocation_type(invocation_type) + or not is_tool_invocation_digest(approval_scope) + or not is_tool_invocation_digest(fingerprint) + or not isinstance(executed, bool) + or not isinstance(completed, bool) + or (completed and not executed) + ): + raise UserError( + f"RunState tool invocation {call_id!r} contains invalid lifecycle data." + ) + self._tool_invocations[call_id] = _ToolInvocationRecord( + invocation_type=invocation_type, + approval_scope=approval_scope, + fingerprint=fingerprint, + executed=executed, + completed=completed, + ) + + def _mark_restored_unbound_approval_call_ids(self) -> None: + """Require reapproval for restored per-call decisions without a ledger binding.""" + for record in self._approvals.values(): + for decision in (record.approved, record.rejected): + if not isinstance(decision, list): + continue + self._restored_unbound_approval_call_ids.update( + call_id for call_id in decision if call_id not in self._tool_invocations + ) + def _rebuild_hosted_mcp_approvals(self, approvals: Any) -> None: """Restore typed hosted MCP approval records from serialized state.""" if not isinstance(approvals, list): @@ -692,7 +1303,7 @@ def _fork_with_tool_input(self, tool_input: Any) -> RunContextWrapper[TContext]: """Create a child context that shares approvals and usage with tool input set.""" fork = RunContextWrapper(context=self.context) fork.usage = self.usage - fork._approvals = self._approvals + self._share_tool_state_with(fork) fork.turn_input = self.turn_input fork.tool_input = tool_input return fork @@ -701,7 +1312,7 @@ def _fork_without_tool_input(self) -> RunContextWrapper[TContext]: """Create a child context that shares approvals and usage without tool input.""" fork = RunContextWrapper(context=self.context) fork.usage = self.usage - fork._approvals = self._approvals + self._share_tool_state_with(fork) fork.turn_input = self.turn_input return fork diff --git a/src/agents/run_internal/items.py b/src/agents/run_internal/items.py index 9a4c0ea6bf..9f15335879 100644 --- a/src/agents/run_internal/items.py +++ b/src/agents/run_internal/items.py @@ -16,6 +16,7 @@ from openai.types.responses import ResponseFunctionToolCall from pydantic import BaseModel +from .._tool_identity import get_hosted_mcp_approval_request_identity from ..agent_tool_state import drop_agent_tool_run_result from ..items import ItemHelpers, RunItem, ToolCallOutputItem, TResponseInputItem from ..models.fake_id import FAKE_RESPONSES_ID @@ -667,6 +668,12 @@ def _dedupe_key(item: TResponseInputItem) -> str | None: item_type = payload.get("type") or role if role is not None or item_type == "message": return None + call_id = payload.get("call_id") + if isinstance(call_id, str) and item_type in { + *_TOOL_CALL_TO_OUTPUT_TYPE, + *_TOOL_CALL_TO_OUTPUT_TYPE.values(), + }: + return f"call_id:{item_type}:{call_id}" item_id = payload.get("id") if item_id == FAKE_RESPONSES_ID: # Ignore placeholder IDs so call_id-based dedupe remains possible. @@ -674,7 +681,6 @@ def _dedupe_key(item: TResponseInputItem) -> str | None: if isinstance(item_id, str): return f"id:{item_type}:{item_id}" - call_id = payload.get("call_id") if isinstance(call_id, str): return f"call_id:{item_type}:{call_id}" @@ -865,6 +871,12 @@ def apply_patch_rejection_item( def extract_mcp_request_id(raw_item: Any) -> str | None: """Pull the request id from hosted MCP approval payloads.""" + try: + hosted_request = get_hosted_mcp_approval_request_identity(raw_item) + except Exception: + hosted_request = None + if hosted_request is not None: + return hosted_request.request_id if isinstance(raw_item, dict): provider_data = raw_item.get("provider_data") if isinstance(provider_data, dict): @@ -891,6 +903,12 @@ def extract_mcp_request_id(raw_item: Any) -> str | None: def extract_mcp_request_id_from_run(mcp_run: Any) -> str | None: """Extract the hosted MCP request id from a streaming run item.""" request_item = getattr(mcp_run, "request_item", None) or getattr(mcp_run, "requestItem", None) + try: + hosted_request = get_hosted_mcp_approval_request_identity(request_item) + except Exception: + hosted_request = None + if hosted_request is not None: + return hosted_request.request_id if isinstance(request_item, dict): provider_data = request_item.get("provider_data") if isinstance(provider_data, dict): diff --git a/src/agents/run_internal/run_loop.py b/src/agents/run_internal/run_loop.py index 1429270c2e..5b7efbefbe 100644 --- a/src/agents/run_internal/run_loop.py +++ b/src/agents/run_internal/run_loop.py @@ -7,26 +7,20 @@ import asyncio import dataclasses as _dc -import json -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable from functools import partial from typing import Any, TypeVar, cast +from uuid import uuid4 from openai.types.responses import ( Response, ResponseCompletedEvent, - ResponseFunctionToolCall, ResponseOutputItemDoneEvent, ) -from openai.types.responses.response_output_item import McpCall, McpListTools, ResponseOutputItem +from openai.types.responses.response_output_item import ResponseOutputItem from openai.types.responses.response_prompt_param import ResponsePromptParam -from openai.types.responses.response_reasoning_item import ResponseReasoningItem -from .._mcp_tool_metadata import collect_mcp_list_tools_metadata from .._tool_identity import ( - NamedToolLookupKey, - build_function_tool_lookup_map, - get_function_tool_lookup_key_for_call, get_tool_trace_name_for_tool, resolve_tool_name_collisions, ) @@ -46,19 +40,11 @@ ) from ..handoffs import Handoff from ..items import ( - HandoffCallItem, ItemHelpers, ModelResponse, - ReasoningItem, RunItem, ToolApprovalItem, - ToolCallItem, - ToolCallItemTypes, - ToolSearchCallItem, - ToolSearchOutputItem, TResponseInputItem, - coerce_tool_search_call_raw_item, - coerce_tool_search_output_raw_item, ) from ..lifecycle import RunHooks from ..logger import ( @@ -83,16 +69,11 @@ from ..stream_events import ( AgentUpdatedStreamEvent, RawResponsesStreamEvent, - RunItemStreamEvent, ) from ..tool import ( - FunctionTool, ProgrammaticToolCallingTool, Tool, - ToolOrigin, - ToolOriginType, dispose_resolved_computers, - get_function_tool_origin, ) from ..tracing import Span, SpanError, agent_span, get_current_trace, task_span, turn_span from ..tracing.config import include_task_and_turn_spans @@ -174,7 +155,6 @@ from .streaming import stream_step_items_to_queue, stream_step_result_to_queue from .tool_actions import ApplyPatchAction, ComputerAction, LocalShellAction, ShellAction from .tool_execution import ( - build_litellm_json_tool_call, coerce_shell_call, execute_apply_patch_calls, execute_computer_actions, @@ -189,7 +169,6 @@ ) from .tool_planning import execute_mcp_approval_requests from .tool_use_tracker import ( - TOOL_CALL_TYPES, AgentToolUseTracker, hydrate_tool_use_tracker, serialize_tool_use_tracker, @@ -209,7 +188,6 @@ execute_handoffs, execute_tools_and_side_effects, get_single_step_result_from_response, - is_handoff_tool_call, process_model_response, resolve_interrupted_turn, run_final_output_hooks, @@ -278,6 +256,21 @@ "input_guardrail_tripwire_triggered_for_stream", ] +_STREAM_EVENT_ITEM_OCCURRENCE_KEY = "_agents_stream_event_item_occurrence_key" + + +def _stream_event_item_occurrence_key(item: RunItem) -> str | None: + key = getattr(item, _STREAM_EVENT_ITEM_OCCURRENCE_KEY, None) + return key if isinstance(key, str) and key else None + + +def _ensure_stream_event_item_occurrence_key(item: RunItem) -> str: + key = _stream_event_item_occurrence_key(item) + if key is None: + key = uuid4().hex + setattr(item, _STREAM_EVENT_ITEM_OCCURRENCE_KEY, key) + return key + async def cleanup_models_after_run(tool_use_tracker: AgentToolUseTracker) -> None: """Notify every model resolved during the run that its owning run has ended.""" @@ -1510,22 +1503,6 @@ async def raise_if_input_guardrail_tripwire_known() -> None: if tripwire_result is not None: raise InputGuardrailTripwireTriggered(tripwire_result) - emitted_tool_call_ids: set[str] = set() - emitted_reasoning_item_ids: set[str] = set() - emitted_tool_search_fingerprints: set[str] = set() - - def _tool_search_fingerprint(raw_item: Any) -> str: - if isinstance(raw_item, Mapping): - payload: Any = dict(raw_item) - elif hasattr(raw_item, "model_dump"): - payload = cast(Any, raw_item).model_dump(exclude_unset=True) - else: - payload = { - "type": getattr(raw_item, "type", None), - "id": getattr(raw_item, "id", None), - } - return json.dumps(payload, sort_keys=True, default=str) - try: turn_input = ItemHelpers.input_to_new_input_list(streamed_result.input) except Exception: @@ -1536,9 +1513,9 @@ def _tool_search_fingerprint(raw_item: Any) -> str: agent_hook_context = AgentHookContext( context=context_wrapper.context, usage=context_wrapper.usage, - _approvals=context_wrapper._approvals, turn_input=turn_input, ) + context_wrapper._share_tool_state_with(agent_hook_context) await gather_with_cancel( hooks.on_agent_start(agent_hook_context, public_agent), ( @@ -1572,22 +1549,6 @@ def _tool_search_fingerprint(raw_item: Any) -> str: if (tool_name := get_tool_trace_name_for_tool(tool)) is not None ] - # Precompute the lookup map used for streaming descriptions. Function tools use the same - # collision-free lookup keys as runtime dispatch, including deferred top-level aliases. - tool_map: dict[NamedToolLookupKey, Any] = cast( - dict[NamedToolLookupKey, Any], - build_function_tool_lookup_map( - [tool for tool in all_tools if isinstance(tool, FunctionTool)] - ), - ) - for tool in all_tools: - tool_name = getattr(tool, "name", None) - if not isinstance(tool_name, str) or not tool_name: - continue - if isinstance(tool, FunctionTool): - continue - tool_map[tool_name] = tool - handoff_tool_names = {handoff.tool_name for handoff in handoffs} model = get_model(execution_agent, run_config) tool_use_tracker.record_model(model) model_settings = get_model_settings(execution_agent, run_config) @@ -1595,6 +1556,8 @@ def _tool_search_fingerprint(raw_item: Any) -> str: final_response: ModelResponse | None = None streamed_response_output: list[ResponseOutputItem] = [] + emitted_model_item_occurrence_keys: set[str] = set() + validated_model_items: list[RunItem] | None = None if server_conversation_tracker is not None: items_for_input = ( @@ -1624,9 +1587,6 @@ def _tool_search_fingerprint(raw_item: Any) -> str: ) if isinstance(filtered.input, list): filtered.input = deduplicate_input_items_preferring_latest(filtered.input) - hosted_mcp_tool_metadata = collect_mcp_list_tools_metadata(streamed_result._model_input_items) - if isinstance(filtered.input, list): - hosted_mcp_tool_metadata.update(collect_mcp_list_tools_metadata(filtered.input)) if server_conversation_tracker is not None: logger.debug( "filtered.input has %s items; ids=%s", @@ -1777,109 +1737,24 @@ async def rewind_model_request() -> None: ) if isinstance(event, ResponseOutputItemDoneEvent): - output_item = event.item - streamed_response_output.append(output_item) - output_item_type = getattr(output_item, "type", None) - - if output_item_type == "tool_search_call": - emitted_tool_search_fingerprints.add(_tool_search_fingerprint(output_item)) - streamed_result._event_queue.put_nowait( - RunItemStreamEvent( - item=ToolSearchCallItem( - raw_item=coerce_tool_search_call_raw_item(output_item), - agent=public_agent, - ), - name="tool_search_called", - ) - ) - - elif output_item_type == "tool_search_output": - emitted_tool_search_fingerprints.add(_tool_search_fingerprint(output_item)) - streamed_result._event_queue.put_nowait( - RunItemStreamEvent( - item=ToolSearchOutputItem( - raw_item=coerce_tool_search_output_raw_item(output_item), - agent=public_agent, - ), - name="tool_search_output_created", - ) - ) + streamed_response_output.append(event.item) - elif isinstance(output_item, McpListTools): - hosted_mcp_tool_metadata.update(collect_mcp_list_tools_metadata([output_item])) - - elif isinstance(output_item, TOOL_CALL_TYPES) and not is_handoff_tool_call( - output_item, handoff_tool_names - ): - # Handoff calls are streamed as `handoff_requested` once the turn is processed, - # so emitting them here too would duplicate the item under a second event name. - output_call_id: str | None = getattr( - output_item, "call_id", getattr(output_item, "id", None) - ) - - if ( - output_call_id - and isinstance(output_call_id, str) - and output_call_id not in emitted_tool_call_ids - ): - emitted_tool_call_ids.add(output_call_id) - - # Look up tool description from precomputed map ("last wins" matches - # execution behavior in process_model_response). - tool_lookup_key = get_function_tool_lookup_key_for_call(output_item) - matched_tool = ( - tool_map.get(tool_lookup_key) if tool_lookup_key is not None else None - ) - if ( - matched_tool is None - and output_schema is not None - and isinstance(output_item, ResponseFunctionToolCall) - and output_item.name == "json_tool_call" - ): - matched_tool = build_litellm_json_tool_call(output_item) - tool_description: str | None = None - tool_title: str | None = None - tool_origin = None - if isinstance(output_item, McpCall): - metadata = hosted_mcp_tool_metadata.get( - (output_item.server_label, output_item.name) - ) - if metadata is not None: - tool_description = metadata.description - tool_title = metadata.title - tool_origin = ToolOrigin( - type=ToolOriginType.MCP, - mcp_server_name=output_item.server_label, - ) - elif matched_tool is not None: - tool_description = getattr(matched_tool, "description", None) - tool_title = getattr(matched_tool, "_mcp_title", None) - tool_origin = get_function_tool_origin(matched_tool) - - tool_item = ToolCallItem( - raw_item=cast(ToolCallItemTypes, output_item), - agent=public_agent, - description=tool_description, - title=tool_title, - tool_origin=tool_origin, - ) - streamed_result._event_queue.put_nowait( - RunItemStreamEvent(item=tool_item, name="tool_called") - ) - - elif isinstance(output_item, ResponseReasoningItem): - reasoning_id: str | None = getattr(output_item, "id", None) + if not final_response: + raise ModelBehaviorError("Model did not produce a final response!") - if reasoning_id and reasoning_id not in emitted_reasoning_item_ids: - emitted_reasoning_item_ids.add(reasoning_id) + context_wrapper.usage.add(final_response.usage) - reasoning_item = ReasoningItem(raw_item=output_item, agent=public_agent) - streamed_result._event_queue.put_nowait( - RunItemStreamEvent(item=reasoning_item, name="reasoning_item_created") - ) + if server_conversation_tracker is not None: + # Streaming uses the same rewind helper, so a successful retry must restore delivered + # input tracking before the next turn computes server-managed deltas. + server_conversation_tracker.mark_input_as_sent(filtered.input) + server_conversation_tracker.track_server_items(final_response) - if final_response is not None: - context_wrapper.usage.add(final_response.usage) + async def after_invocation_validation( + model_items: list[RunItem] | None, + ) -> None: + nonlocal validated_model_items + validated_model_items = model_items await gather_with_cancel( ( public_agent.hooks.on_llm_end(context_wrapper, public_agent, final_response) @@ -1889,14 +1764,13 @@ async def rewind_model_request() -> None: hooks.on_llm_end(context_wrapper, public_agent, final_response), ) - if not final_response: - raise ModelBehaviorError("Model did not produce a final response!") - - if server_conversation_tracker is not None: - # Streaming uses the same rewind helper, so a successful retry must restore delivered - # input tracking before the next turn computes server-managed deltas. - server_conversation_tracker.mark_input_as_sent(filtered.input) - server_conversation_tracker.track_server_items(final_response) + async def emit_validated_model_items_before_side_effects() -> None: + await raise_if_input_guardrail_tripwire_known() + if validated_model_items is not None: + emitted_model_item_occurrence_keys.update( + _ensure_stream_event_item_occurrence_key(item) for item in validated_model_items + ) + stream_step_items_to_queue(validated_model_items, streamed_result._event_queue) single_step_result = await get_single_step_result_from_response( bindings=bindings, @@ -1912,47 +1786,17 @@ async def rewind_model_request() -> None: error_handlers=error_handlers, tool_use_tracker=tool_use_tracker, server_manages_conversation=server_conversation_tracker is not None, - event_queue=streamed_result._event_queue, - before_side_effects=raise_if_input_guardrail_tripwire_known, + after_invocation_validation=after_invocation_validation, + before_side_effects=emit_validated_model_items_before_side_effects, ) items_to_filter = session_items_for_turn(single_step_result) - if emitted_tool_call_ids: - items_to_filter = [ - item - for item in items_to_filter - if not ( - isinstance(item, ToolCallItem) - and ( - call_id := getattr(item.raw_item, "call_id", getattr(item.raw_item, "id", None)) - ) - and call_id in emitted_tool_call_ids - ) - ] - - if emitted_reasoning_item_ids: - items_to_filter = [ - item - for item in items_to_filter - if not ( - isinstance(item, ReasoningItem) - and (reasoning_id := getattr(item.raw_item, "id", None)) - and reasoning_id in emitted_reasoning_item_ids - ) - ] - - if emitted_tool_search_fingerprints: - items_to_filter = [ - item - for item in items_to_filter - if not ( - isinstance(item, ToolSearchCallItem | ToolSearchOutputItem) - and _tool_search_fingerprint(item.raw_item) in emitted_tool_search_fingerprints - ) - ] - - items_to_filter = [item for item in items_to_filter if not isinstance(item, HandoffCallItem)] + items_to_filter = [ + item + for item in items_to_filter + if _stream_event_item_occurrence_key(item) not in emitted_model_item_occurrence_keys + ] filtered_result = _dc.replace(single_step_result, new_step_items=items_to_filter) stream_step_result_to_queue(filtered_result, streamed_result._event_queue) @@ -1991,9 +1835,9 @@ async def run_single_turn( agent_hook_context = AgentHookContext( context=context_wrapper.context, usage=context_wrapper.usage, - _approvals=context_wrapper._approvals, turn_input=turn_input, ) + context_wrapper._share_tool_state_with(agent_hook_context) await gather_with_cancel( hooks.on_agent_start(agent_hook_context, public_agent), ( @@ -2044,8 +1888,21 @@ async def run_single_turn( session=session, session_items_to_rewind=session_items_to_rewind, prompt_cache_key_resolver=prompt_cache_key_resolver, + defer_llm_end_hooks=True, ) + async def after_invocation_validation( + _validated_model_items: list[RunItem] | None, + ) -> None: + await gather_with_cancel( + ( + public_agent.hooks.on_llm_end(context_wrapper, public_agent, new_response) + if public_agent.hooks + else _coro.noop_coroutine() + ), + hooks.on_llm_end(context_wrapper, public_agent, new_response), + ) + return await get_single_step_result_from_response( bindings=bindings, original_input=original_input, @@ -2060,6 +1917,7 @@ async def run_single_turn( error_handlers=error_handlers, tool_use_tracker=tool_use_tracker, server_manages_conversation=server_conversation_tracker is not None, + after_invocation_validation=after_invocation_validation, ) @@ -2079,6 +1937,7 @@ async def get_new_response( session: Session | None = None, session_items_to_rewind: list[TResponseInputItem] | None = None, prompt_cache_key_resolver: PromptCacheKeyResolver | None = None, + defer_llm_end_hooks: bool = False, ) -> ModelResponse: """Call the model and return the raw response, handling retries and hooks.""" public_agent = bindings.public_agent @@ -2186,13 +2045,14 @@ async def rewind_model_request() -> None: context_wrapper.usage.add(new_response.usage) - await gather_with_cancel( - ( - public_agent.hooks.on_llm_end(context_wrapper, public_agent, new_response) - if public_agent.hooks - else _coro.noop_coroutine() - ), - hooks.on_llm_end(context_wrapper, public_agent, new_response), - ) + if not defer_llm_end_hooks: + await gather_with_cancel( + ( + public_agent.hooks.on_llm_end(context_wrapper, public_agent, new_response) + if public_agent.hooks + else _coro.noop_coroutine() + ), + hooks.on_llm_end(context_wrapper, public_agent, new_response), + ) return new_response diff --git a/src/agents/run_internal/tool_actions.py b/src/agents/run_internal/tool_actions.py index 6b1d5cc97b..cea85b4645 100644 --- a/src/agents/run_internal/tool_actions.py +++ b/src/agents/run_internal/tool_actions.py @@ -9,6 +9,7 @@ import dataclasses import inspect import json +from collections.abc import Callable from typing import TYPE_CHECKING, Any, Literal, cast from openai.types.responses import ResponseComputerToolCall @@ -20,7 +21,7 @@ from .._tool_identity import get_mapping_or_attr, get_tool_trace_name_for_tool from ..agent import Agent from ..exceptions import ModelBehaviorError -from ..items import ItemHelpers, RunItem, ToolCallOutputItem +from ..items import ItemHelpers, RunItem, ToolApprovalItem, ToolCallOutputItem from ..logger import logger from ..run_config import RunConfig from ..run_context import RunContextWrapper @@ -112,6 +113,7 @@ async def execute( context_wrapper: RunContextWrapper[Any], config: RunConfig, acknowledged_safety_checks: list[ComputerCallOutputAcknowledgedSafetyCheck] | None = None, + tool_output_committer: Callable[[RunItem], None] | None = None, ) -> RunItem: """Run a computer action, capturing a screenshot and notifying hooks.""" trace_tool_name = get_tool_trace_name_for_tool(action.computer_tool) or cls.TRACE_TOOL_NAME @@ -166,6 +168,13 @@ async def _run_action(span: Any | None) -> RunItem: type="computer_call_output", acknowledged_safety_checks=acknowledged_safety_checks, ) + output_item = ToolCallOutputItem( + agent=agent, + output=image_url, + raw_item=raw_item, + ) + if tool_output_committer is not None: + tool_output_committer(output_item) custom_data = await maybe_extract_custom_data( action.computer_tool.custom_data_extractor, ComputerToolCustomDataContext( @@ -176,6 +185,7 @@ async def _run_action(span: Any | None) -> RunItem: raw_item=copy.deepcopy(raw_item), ), ) + output_item.custom_data = custom_data await gather_with_cancel( hooks.on_tool_end(context_wrapper, agent, action.computer_tool, output), @@ -189,12 +199,7 @@ async def _run_action(span: Any | None) -> RunItem: if span and config.trace_include_sensitive_data: span.span_data.output = image_url - return ToolCallOutputItem( - agent=agent, - output=image_url, - raw_item=raw_item, - custom_data=custom_data, - ) + return output_item return await with_tool_function_span( config=config, @@ -390,9 +395,14 @@ async def execute( hooks: RunHooks[Any], context_wrapper: RunContextWrapper[Any], config: RunConfig, + tool_output_committer: Callable[[RunItem], None] | None = None, ) -> RunItem: """Run a local shell tool call and wrap the result as a ToolCallOutputItem.""" agent_hooks = agent.hooks + context_wrapper._mark_tool_invocation_executed( + call.tool_call, + tool_name=call.local_shell_tool.name, + ) await gather_with_cancel( hooks.on_tool_start(context_wrapper, agent, call.local_shell_tool), ( @@ -409,25 +419,28 @@ async def execute( output = call.local_shell_tool.executor(request) result = await output if inspect.isawaitable(output) else output - await gather_with_cancel( - hooks.on_tool_end(context_wrapper, agent, call.local_shell_tool, result), - ( - agent_hooks.on_tool_end(context_wrapper, agent, call.local_shell_tool, result) - if agent_hooks - else _coro.noop_coroutine() - ), - ) - raw_payload: dict[str, Any] = { "type": "local_shell_call_output", "call_id": call.tool_call.call_id, "output": result, } - return ToolCallOutputItem( + output_item = ToolCallOutputItem( agent=agent, output=result, raw_item=raw_payload, ) + if tool_output_committer is not None: + tool_output_committer(output_item) + + await gather_with_cancel( + hooks.on_tool_end(context_wrapper, agent, call.local_shell_tool, result), + ( + agent_hooks.on_tool_end(context_wrapper, agent, call.local_shell_tool, result) + if agent_hooks + else _coro.noop_coroutine() + ), + ) + return output_item class ShellAction: @@ -442,11 +455,17 @@ async def execute( hooks: RunHooks[Any], context_wrapper: RunContextWrapper[Any], config: RunConfig, + tool_output_committer: Callable[[RunItem], None] | None = None, ) -> RunItem: """Run a shell tool call and return a normalized ToolCallOutputItem.""" shell_call = coerce_shell_call(call.tool_call) shell_tool = call.shell_tool agent_hooks = agent.hooks + current_item = ToolApprovalItem( + agent=agent, + raw_item=call.tool_call, + tool_name=shell_tool.name, + ) async def _run_call(span: Any | None) -> RunItem: if span and config.trace_include_sensitive_data: @@ -455,7 +474,9 @@ async def _run_call(span: Any | None) -> RunItem: ) approval_status = context_wrapper.get_approval_status( - shell_tool.name, shell_call.call_id + shell_tool.name, + shell_call.call_id, + current_invocation=current_item, ) if approval_status is None: needs_approval_result = await evaluate_needs_approval_setting( @@ -465,7 +486,9 @@ async def _run_call(span: Any | None) -> RunItem: shell_call.call_id, ) approval_status = context_wrapper.get_approval_status( - shell_tool.name, shell_call.call_id + shell_tool.name, + shell_call.call_id, + current_invocation=current_item, ) else: needs_approval_result = False @@ -487,6 +510,7 @@ async def _run_call(span: Any | None) -> RunItem: rejection_message = await resolve_approval_rejection_message( context_wrapper=context_wrapper, run_config=config, + tool_call=call.tool_call, tool_type="shell", tool_name=shell_tool.name, call_id=shell_call.call_id, @@ -498,6 +522,10 @@ async def _run_call(span: Any | None) -> RunItem: rejection_message=rejection_message, ) + context_wrapper._mark_tool_invocation_executed( + call.tool_call, + tool_name=shell_tool.name, + ) await gather_with_cancel( hooks.on_tool_start(context_wrapper, agent, shell_tool), ( @@ -572,15 +600,6 @@ async def _run_call(span: Any | None) -> RunItem: output_text = output_text[:max_output_length] log_tool_action_error("Shell executor failed", exc) - await gather_with_cancel( - hooks.on_tool_end(context_wrapper, agent, call.shell_tool, output_text), - ( - agent_hooks.on_tool_end(context_wrapper, agent, call.shell_tool, output_text) - if agent_hooks - else _coro.noop_coroutine() - ), - ) - raw_entries: list[dict[str, Any]] | None = None if shell_output_payload: raw_entries = shell_output_payload @@ -610,14 +629,27 @@ async def _run_call(span: Any | None) -> RunItem: if provider_meta: raw_item["provider_data"] = provider_meta - if span and config.trace_include_sensitive_data: - span.span_data.output = output_text - - return ToolCallOutputItem( + output_item = ToolCallOutputItem( agent=agent, output=output_text, raw_item=raw_item, ) + if tool_output_committer is not None: + tool_output_committer(output_item) + + await gather_with_cancel( + hooks.on_tool_end(context_wrapper, agent, call.shell_tool, output_text), + ( + agent_hooks.on_tool_end(context_wrapper, agent, call.shell_tool, output_text) + if agent_hooks + else _coro.noop_coroutine() + ), + ) + + if span and config.trace_include_sensitive_data: + span.span_data.output = output_text + + return output_item return await with_tool_function_span( config=config, @@ -638,13 +670,14 @@ async def execute( hooks: RunHooks[Any], context_wrapper: RunContextWrapper[Any], config: RunConfig, + tool_output_committer: Callable[[RunItem], None] | None = None, ) -> RunItem: custom_tool: CustomTool = call.custom_tool agent_hooks = agent.hooks call_id = get_mapping_or_attr(call.tool_call, "call_id") tool_input = get_mapping_or_attr(call.tool_call, "input") - if not isinstance(call_id, str): - raise ModelBehaviorError("Custom tool call is missing call_id.") + if not isinstance(call_id, str) or not call_id: + raise ModelBehaviorError("Custom tool call is missing a non-empty call_id.") if not isinstance(tool_input, str): raise ModelBehaviorError("Custom tool call is missing input.") @@ -656,17 +689,30 @@ async def execute( agent=agent, run_config=config, ) + current_item = ToolApprovalItem( + agent=agent, + raw_item=call.tool_call, + tool_name=custom_tool.name, + ) async def _run_call(span: Any | None) -> RunItem: if span and config.trace_include_sensitive_data: span.span_data.input = tool_input - approval_status = context_wrapper.get_approval_status(custom_tool.name, call_id) + approval_status = context_wrapper.get_approval_status( + custom_tool.name, + call_id, + current_invocation=current_item, + ) if approval_status is None: needs_approval_result = await evaluate_needs_approval_setting( custom_tool.runtime_needs_approval(), context_wrapper, tool_input, call_id ) - approval_status = context_wrapper.get_approval_status(custom_tool.name, call_id) + approval_status = context_wrapper.get_approval_status( + custom_tool.name, + call_id, + current_invocation=current_item, + ) else: needs_approval_result = False @@ -687,6 +733,7 @@ async def _run_call(span: Any | None) -> RunItem: rejection_message = await resolve_approval_rejection_message( context_wrapper=context_wrapper, run_config=config, + tool_call=call.tool_call, tool_type="custom", tool_name=custom_tool.name, call_id=call_id, @@ -702,6 +749,10 @@ async def _run_call(span: Any | None) -> RunItem: ), ) + context_wrapper._mark_tool_invocation_executed( + call.tool_call, + tool_name=custom_tool.name, + ) await gather_with_cancel( hooks.on_tool_start(tool_context, agent, custom_tool), ( @@ -738,6 +789,14 @@ async def _run_call(span: Any | None) -> RunItem: output_text, tool_call=call.tool_call, ) + output_item = cls._tool_output_item( + agent, + call_id, + output_text, + raw_item=raw_item, + ) + if tool_output_committer is not None: + tool_output_committer(output_item) custom_data = await maybe_extract_custom_data( custom_tool.custom_data_extractor, CustomToolCustomDataContext( @@ -748,6 +807,7 @@ async def _run_call(span: Any | None) -> RunItem: raw_item=copy.deepcopy(raw_item), ), ) + output_item.custom_data = custom_data await gather_with_cancel( hooks.on_tool_end(tool_context, agent, custom_tool, output_text), @@ -760,13 +820,7 @@ async def _run_call(span: Any | None) -> RunItem: if span and config.trace_include_sensitive_data: span.span_data.output = output_text - return cls._tool_output_item( - agent, - call_id, - output_text, - raw_item=raw_item, - custom_data=custom_data, - ) + return output_item return await with_tool_function_span( config=config, @@ -824,6 +878,7 @@ async def execute( hooks: RunHooks[Any], context_wrapper: RunContextWrapper[Any], config: RunConfig, + tool_output_committer: Callable[[RunItem], None] | None = None, ) -> RunItem: """Run an apply_patch call and serialize the editor result for the model.""" apply_patch_tool: ApplyPatchTool = call.apply_patch_tool @@ -833,6 +888,11 @@ async def execute( context_wrapper=context_wrapper, ) call_id = extract_apply_patch_call_id(call.tool_call) + current_item = ToolApprovalItem( + agent=agent, + raw_item=call.tool_call, + tool_name=apply_patch_tool.name, + ) async def _run_call(span: Any | None) -> RunItem: if span and config.trace_include_sensitive_data: @@ -847,7 +907,11 @@ async def _run_call(span: Any | None) -> RunItem: ] ) - approval_status = context_wrapper.get_approval_status(apply_patch_tool.name, call_id) + approval_status = context_wrapper.get_approval_status( + apply_patch_tool.name, + call_id, + current_invocation=current_item, + ) needs_approval_result = False if approval_status is None: for operation in operations: @@ -855,7 +919,9 @@ async def _run_call(span: Any | None) -> RunItem: apply_patch_tool.needs_approval, context_wrapper, operation, call_id ) approval_status = context_wrapper.get_approval_status( - apply_patch_tool.name, call_id + apply_patch_tool.name, + call_id, + current_invocation=current_item, ) if approval_status is not None or needs_approval_result: break @@ -877,6 +943,7 @@ async def _run_call(span: Any | None) -> RunItem: rejection_message = await resolve_approval_rejection_message( context_wrapper=context_wrapper, run_config=config, + tool_call=call.tool_call, tool_type="apply_patch", tool_name=apply_patch_tool.name, call_id=call_id, @@ -889,6 +956,10 @@ async def _run_call(span: Any | None) -> RunItem: rejection_message=rejection_message, ) + context_wrapper._mark_tool_invocation_executed( + call.tool_call, + tool_name=apply_patch_tool.name, + ) await gather_with_cancel( hooks.on_tool_start(context_wrapper, agent, apply_patch_tool), ( @@ -954,6 +1025,14 @@ async def _run_call(span: Any | None) -> RunItem: if output_text: raw_item["output"] = output_text + output_item = ToolCallOutputItem( + agent=agent, + output=output_text, + raw_item=raw_item, + ) + if tool_output_committer is not None: + tool_output_committer(output_item) + custom_data = await maybe_extract_custom_data( apply_patch_tool.custom_data_extractor, ApplyPatchToolCustomDataContext( @@ -965,6 +1044,7 @@ async def _run_call(span: Any | None) -> RunItem: raw_item=copy.deepcopy(raw_item), ), ) + output_item.custom_data = custom_data await gather_with_cancel( hooks.on_tool_end(context_wrapper, agent, apply_patch_tool, output_text), @@ -978,12 +1058,7 @@ async def _run_call(span: Any | None) -> RunItem: if span and config.trace_include_sensitive_data: span.span_data.output = output_text - return ToolCallOutputItem( - agent=agent, - output=output_text, - raw_item=raw_item, - custom_data=custom_data, - ) + return output_item return await with_tool_function_span( config=config, diff --git a/src/agents/run_internal/tool_execution.py b/src/agents/run_internal/tool_execution.py index 08f6d9fb3a..3bf3d7a940 100644 --- a/src/agents/run_internal/tool_execution.py +++ b/src/agents/run_internal/tool_execution.py @@ -27,6 +27,7 @@ build_function_tool_lookup_map, get_function_tool_lookup_key, get_function_tool_lookup_key_for_call, + get_function_tool_lookup_key_for_tool, get_function_tool_trace_name, get_hosted_mcp_approval_request_identity, get_tool_approval_item_call_id, @@ -103,6 +104,7 @@ from .approvals import append_approval_error_output from .items import ( REJECTION_MESSAGE, + extract_mcp_request_id, extract_mcp_request_id_from_run, function_rejection_item, function_tool_error_output, @@ -128,6 +130,7 @@ "coerce_shell_call", "parse_apply_patch_custom_input", "parse_apply_patch_function_args", + "normalize_apply_patch_fallback_call", "extract_apply_patch_call_id", "coerce_apply_patch_operation", "coerce_apply_patch_operations", @@ -634,7 +637,7 @@ def extract_tool_call_id(raw: Any) -> str | None: def extract_shell_call_id(tool_call: Any) -> str: """Ensure shell calls include a call_id before executing them.""" - value = extract_tool_call_id(tool_call) + value = get_mapping_or_attr(tool_call, "call_id") if not value: raise ModelBehaviorError("Shell call is missing call_id.") return str(value) @@ -733,9 +736,37 @@ def parse_apply_patch_function_args(arguments: str) -> dict[str, Any]: return _parse_apply_patch_json(arguments, label="arguments") +def normalize_apply_patch_fallback_call(tool_call: Any) -> dict[str, Any] | None: + """Normalize supported custom/function apply_patch fallbacks into one pseudo-call.""" + call_type = get_mapping_or_attr(tool_call, "type") + call_id = get_mapping_or_attr(tool_call, "call_id") + if call_type == "custom_tool_call": + parsed_operation = parse_apply_patch_custom_input( + str(get_mapping_or_attr(tool_call, "input") or "") + ) + pseudo_call = { + "type": "apply_patch_call", + "call_id": call_id, + **parsed_operation, + } + elif call_type == "function_call": + parsed_operation = parse_apply_patch_function_args( + str(get_mapping_or_attr(tool_call, "arguments") or "") + ) + pseudo_call = { + "type": "apply_patch_call", + "call_id": call_id, + "operation": parsed_operation, + } + else: + return None + ItemHelpers.copy_tool_call_caller(tool_call, pseudo_call) + return pseudo_call + + def extract_apply_patch_call_id(tool_call: Any) -> str: """Ensure apply_patch calls include a call_id for approvals and tracing.""" - value = extract_tool_call_id(tool_call) + value = get_mapping_or_attr(tool_call, "call_id") if not value: raise ModelBehaviorError("Apply patch call is missing call_id.") return str(value) @@ -754,7 +785,7 @@ def coerce_apply_patch_operation( def coerce_apply_patch_operations( - tool_call: Any, + tool_call: Any | None = None, *, context_wrapper: RunContextWrapper[Any], ) -> list[ApplyPatchOperation]: @@ -1155,6 +1186,7 @@ async def resolve_approval_status( tool_namespace=tool_namespace, existing_pending=approval_item, tool_lookup_key=tool_lookup_key, + current_invocation=approval_item, ) if approval_status is None and on_approval: decision_result = on_approval(context_wrapper, approval_item) @@ -1176,6 +1208,7 @@ async def resolve_approval_status( tool_namespace=tool_namespace, existing_pending=approval_item, tool_lookup_key=tool_lookup_key, + current_invocation=approval_item, ) return approval_status, approval_item @@ -1201,6 +1234,7 @@ async def resolve_approval_rejection_message( tool_type: Literal["function", "computer", "shell", "apply_patch", "custom"], tool_name: str, call_id: str, + tool_call: Any | None = None, tool_namespace: str | None = None, tool_lookup_key: FunctionToolLookupKey | None = None, existing_pending: ToolApprovalItem | None = None, @@ -1220,6 +1254,12 @@ async def resolve_approval_rejection_message( if formatter is None: return REJECTION_MESSAGE + if tool_call is not None: + context_wrapper._mark_tool_invocation_executed( + tool_call, + tool_lookup_key=tool_lookup_key, + tool_name=tool_name, + ) try: maybe_message = formatter( ToolErrorFormatterArgs( @@ -1320,6 +1360,49 @@ def _classify_hosted_mcp_pending_request( return "reuse_pending" +def process_hosted_mcp_approvals( + *, + original_pre_step_items: Sequence[RunItem], + mcp_approval_requests: Sequence[Any], + context_wrapper: RunContextWrapper[Any], + agent: Agent[Any], + append_item: Callable[[RunItem], None], +) -> tuple[list[ToolApprovalItem], set[str]]: + """Filter hosted MCP outputs and merge manual approvals so only coherent items remain.""" + hosted_mcp_approvals_by_id: dict[str, ToolApprovalItem] = {} + for item in original_pre_step_items: + if not isinstance(item, ToolApprovalItem): + continue + raw = item.raw_item + if get_hosted_mcp_approval_request_identity(item) is None: + continue + request_id = extract_mcp_request_id(raw) + if request_id: + hosted_mcp_approvals_by_id[request_id] = item + + resumed_requests = [ + request + for request in mcp_approval_requests + if extract_mcp_request_id_from_run(request) in hosted_mcp_approvals_by_id + ] + responses, pending = collect_manual_mcp_approvals( + agent=agent, + requests=resumed_requests, + context_wrapper=context_wrapper, + existing_pending_by_call_id=hosted_mcp_approvals_by_id, + ) + for item in responses: + append_item(item) + for item in pending: + append_item(item) + pending_ids = { + request_id + for item in pending + if (request_id := extract_mcp_request_id(item.raw_item)) is not None + } + return pending, pending_ids + + def collect_manual_mcp_approvals( *, agent: Agent[Any], @@ -1352,6 +1435,11 @@ def collect_manual_mcp_approvals( tool_name=tool_name, ) existing_pending = pending_lookup.get(request_id or "") + if existing_pending is not None: + context_wrapper._restore_pending_approval_binding(existing_pending) + binding_status = context_wrapper._approved_tool_invocation_status( + current_approval_item.raw_item + ) pending_resolution = ( _classify_hosted_mcp_pending_request(existing_pending, request_item) if existing_pending is not None @@ -1390,6 +1478,20 @@ def collect_manual_mcp_approvals( ) ) + if approval_status is not None and binding_status is None: + binding_status = context_wrapper._approved_tool_invocation_status( + current_approval_item.raw_item + ) + + if binding_status is None: + approval_item = current_approval_item + + if approval_status is not None and request_id: + if binding_status is None: + approval_status = None + elif binding_status[1]: + continue + if approval_status is not None and request_id: approval_response_raw: McpApprovalResponse = { "type": "mcp_approval_response", @@ -1422,6 +1524,23 @@ def index_approval_items_by_call_id(items: Sequence[RunItem]) -> dict[str, ToolA return approvals +def should_keep_hosted_mcp_item( + item: RunItem, + *, + pending_hosted_mcp_approvals: Sequence[ToolApprovalItem], + pending_hosted_mcp_approval_ids: set[str], +) -> bool: + """Keep only hosted MCP approvals that match pending requests from the provider.""" + if not isinstance(item, ToolApprovalItem): + return True + if get_hosted_mcp_approval_request_identity(item) is None: + return False + request_id = extract_mcp_request_id(item.raw_item) + return item in pending_hosted_mcp_approvals or ( + request_id is not None and request_id in pending_hosted_mcp_approval_ids + ) + + def _uses_programmatic_output_schema( function_tool: FunctionTool, tool_call: Any, @@ -1443,6 +1562,7 @@ def __init__( config: RunConfig, isolate_parallel_failures: bool | None, sibling_category_failure: asyncio.Event | None, + tool_output_committer: Callable[[RunItem], None] | None, ) -> None: self.execution_agent = bindings.execution_agent self.public_agent = bindings.public_agent @@ -1454,6 +1574,7 @@ def __init__( len(tool_runs) > 1 if isolate_parallel_failures is None else isolate_parallel_failures ) self.sibling_category_failure = sibling_category_failure + self.tool_output_committer = tool_output_committer self.tool_input_guardrail_results: list[ToolInputGuardrailResult] = [] self.tool_output_guardrail_results: list[ToolOutputGuardrailResult] = [] self.tool_state_scope_id = get_agent_tool_state_scope(context_wrapper) @@ -1462,6 +1583,7 @@ def __init__( self.results_by_tool_run: dict[int, Any] = {} self.schema_bypassed_tool_runs: set[int] = set() self.custom_data_by_tool_run: dict[int, dict[str, Any]] = {} + self.output_items_by_tool_run: dict[int, ToolCallOutputItem] = {} self.pending_tasks: set[asyncio.Task[Any]] = set() self.propagating_failure: BaseException | None = None self.available_function_tools: list[FunctionTool] = [] @@ -1745,11 +1867,24 @@ async def _maybe_execute_tool_approval( tool_lookup_key = get_function_tool_lookup_key_for_call(raw_tool_call) if is_deferred_top_level_function_tool(func_tool): tool_lookup_key = ("deferred_top_level", func_tool.name) + current_approval_item = ToolApprovalItem( + agent=self.public_agent, + raw_item=raw_tool_call, + tool_name=func_tool.name, + tool_namespace=tool_namespace, + tool_origin=get_function_tool_origin(func_tool), + tool_lookup_key=tool_lookup_key, + _allow_bare_name_alias=should_allow_bare_name_approval_alias( + func_tool, + self.available_function_tools, + ), + ) approval_status = self.context_wrapper.get_approval_status( func_tool.name, tool_call.call_id, tool_namespace=tool_namespace, tool_lookup_key=tool_lookup_key, + current_invocation=current_approval_item, ) if approval_status is None: needs_approval_result = await function_needs_approval( @@ -1762,6 +1897,7 @@ async def _maybe_execute_tool_approval( tool_call.call_id, tool_namespace=tool_namespace, tool_lookup_key=tool_lookup_key, + current_invocation=current_approval_item, ) if approval_status is None and not needs_approval_result: return None @@ -1790,6 +1926,7 @@ async def _maybe_execute_tool_approval( tool_call.call_id, tool_namespace=tool_namespace, tool_lookup_key=tool_lookup_key, + current_invocation=current_approval_item, ) if approval_status is None and rejected_message is not None: return FunctionToolResult( @@ -1806,19 +1943,11 @@ async def _maybe_execute_tool_approval( ) if approval_status is None: - approval_item = ToolApprovalItem( - agent=self.public_agent, - raw_item=raw_tool_call, - tool_name=func_tool.name, - tool_namespace=tool_namespace, - tool_origin=get_function_tool_origin(func_tool), - tool_lookup_key=tool_lookup_key, - _allow_bare_name_alias=should_allow_bare_name_approval_alias( - func_tool, - self.available_function_tools, - ), + return FunctionToolResult( + tool=func_tool, + output=None, + run_item=current_approval_item, ) - return FunctionToolResult(tool=func_tool, output=None, run_item=approval_item) if approval_status is not False: return None @@ -1826,6 +1955,7 @@ async def _maybe_execute_tool_approval( rejection_message = await resolve_approval_rejection_message( context_wrapper=self.context_wrapper, run_config=self.config, + tool_call=tool_call, tool_type="function", tool_name=tool_trace_name(func_tool.name, tool_namespace) or func_tool.name, call_id=tool_call.call_id, @@ -1867,24 +1997,34 @@ async def _execute_single_tool_body( tool_context: ToolContext[Any], agent_hooks: Any, ) -> Any: - rejected_message = await _execute_tool_input_guardrails( - func_tool=func_tool, - tool_context=tool_context, - agent=self.public_agent, - tool_input_guardrail_results=self.tool_input_guardrail_results, - ) - if rejected_message is not None: - self.schema_bypassed_tool_runs.add(id(task_state.tool_run)) - return rejected_message - - await gather_with_cancel( - self.hooks.on_tool_start(tool_context, self.public_agent, func_tool), - ( - agent_hooks.on_tool_start(tool_context, self.public_agent, func_tool) - if agent_hooks - else _coro.noop_coroutine() - ), + pending_nested_result = peek_agent_tool_run_result( + task_state.tool_run.tool_call, + scope_id=self.tool_state_scope_id, ) + is_nested_continuation = bool(self._get_nested_tool_interruptions(pending_nested_result)) + if not is_nested_continuation: + self.context_wrapper._mark_tool_invocation_executed( + tool_call, + tool_lookup_key=get_function_tool_lookup_key_for_tool(func_tool), + ) + rejected_message = await _execute_tool_input_guardrails( + func_tool=func_tool, + tool_context=tool_context, + agent=self.public_agent, + tool_input_guardrail_results=self.tool_input_guardrail_results, + ) + if rejected_message is not None: + self.schema_bypassed_tool_runs.add(id(task_state.tool_run)) + return rejected_message + + await gather_with_cancel( + self.hooks.on_tool_start(tool_context, self.public_agent, func_tool), + ( + agent_hooks.on_tool_start(tool_context, self.public_agent, func_tool) + if agent_hooks + else _coro.noop_coroutine() + ), + ) invoke_task = asyncio.create_task( self._invoke_tool_and_run_post_invoke( @@ -1952,8 +2092,15 @@ async def _invoke_tool_and_run_post_invoke( ) real_result = result - task_state.in_post_invoke_phase = True + nested_run_result = peek_agent_tool_run_result( + task_state.tool_run.tool_call, + scope_id=self.tool_state_scope_id, + ) + nested_interruptions = self._get_nested_tool_interruptions(nested_run_result) + if nested_interruptions: + return real_result + task_state.in_post_invoke_phase = True output_guardrail_result = await _execute_tool_output_guardrails( func_tool=func_tool, tool_context=tool_context, @@ -1980,6 +2127,17 @@ async def _invoke_tool_and_run_post_invoke( output_json_schema=None if bypass_output_schema else func_tool.output_json_schema, output_type_adapter=None if bypass_output_schema else func_tool._output_type_adapter, ) + output_item: ToolCallOutputItem | None = None + if not nested_interruptions: + output_item = ToolCallOutputItem( + output=final_result, + raw_item=raw_output_item, + agent=self.public_agent, + tool_origin=get_function_tool_origin(func_tool), + ) + self.output_items_by_tool_run[id(task_state.tool_run)] = output_item + if self.tool_output_committer is not None: + self.tool_output_committer(output_item) extracted_custom_data = await maybe_extract_custom_data( func_tool.custom_data_extractor, FunctionToolCustomDataContext( @@ -1992,6 +2150,8 @@ async def _invoke_tool_and_run_post_invoke( custom_data = merge_custom_data(tool_context._custom_data, extracted_custom_data) if custom_data: self.custom_data_by_tool_run[id(task_state.tool_run)] = custom_data + if output_item is not None: + output_item.custom_data = custom_data await gather_with_cancel( self.hooks.on_tool_end(tool_context, self.public_agent, func_tool, final_result), @@ -2095,35 +2255,37 @@ def _build_function_tool_results(self) -> list[FunctionToolResult]: run_item: RunItem | None if not nested_interruptions: - provider_result = ( - function_tool_error_output( - tool_run.tool_call, - result, - output_json_schema=tool_run.function_tool.output_json_schema, + run_item = self.output_items_by_tool_run.get(id(tool_run)) + if run_item is None: + provider_result = ( + function_tool_error_output( + tool_run.tool_call, + result, + output_json_schema=tool_run.function_tool.output_json_schema, + ) + if bypass_output_schema + else result ) - if bypass_output_schema - else result - ) - run_item = ToolCallOutputItem( - output=result, - raw_item=ItemHelpers.tool_call_output_item( - tool_run.tool_call, - provider_result, - output_json_schema=( - None - if bypass_output_schema - else tool_run.function_tool.output_json_schema - ), - output_type_adapter=( - None - if bypass_output_schema - else tool_run.function_tool._output_type_adapter + run_item = ToolCallOutputItem( + output=result, + raw_item=ItemHelpers.tool_call_output_item( + tool_run.tool_call, + provider_result, + output_json_schema=( + None + if bypass_output_schema + else tool_run.function_tool.output_json_schema + ), + output_type_adapter=( + None + if bypass_output_schema + else tool_run.function_tool._output_type_adapter + ), ), - ), - agent=self.public_agent, - tool_origin=get_function_tool_origin(tool_run.function_tool), - custom_data=self.custom_data_by_tool_run.get(id(tool_run)), - ) + agent=self.public_agent, + tool_origin=get_function_tool_origin(tool_run.function_tool), + custom_data=self.custom_data_by_tool_run.get(id(tool_run)), + ) else: # Skip tool output until nested interruptions are resolved. run_item = None @@ -2150,6 +2312,7 @@ async def execute_function_tool_calls( config: RunConfig, isolate_parallel_failures: bool | None = None, sibling_category_failure: asyncio.Event | None = None, + tool_output_committer: Callable[[RunItem], None] | None = None, ) -> tuple[ list[FunctionToolResult], list[ToolInputGuardrailResult], list[ToolOutputGuardrailResult] ]: @@ -2162,6 +2325,7 @@ async def execute_function_tool_calls( config=config, isolate_parallel_failures=isolate_parallel_failures, sibling_category_failure=sibling_category_failure, + tool_output_committer=tool_output_committer, ).execute() @@ -2172,6 +2336,7 @@ async def execute_custom_tool_calls( context_wrapper: RunContextWrapper[Any], hooks: RunHooks[Any], config: RunConfig, + tool_output_committer: Callable[[RunItem], None] | None = None, ) -> list[RunItem]: """Run Responses custom tool calls serially and wrap outputs.""" from .tool_actions import CustomToolAction @@ -2185,6 +2350,7 @@ async def execute_custom_tool_calls( hooks=hooks, context_wrapper=context_wrapper, config=config, + tool_output_committer=tool_output_committer, ) ) return results @@ -2197,6 +2363,7 @@ async def execute_local_shell_calls( context_wrapper: RunContextWrapper[Any], hooks: RunHooks[Any], config: RunConfig, + tool_output_committer: Callable[[RunItem], None] | None = None, ) -> list[RunItem]: """Run local shell tool calls serially and wrap outputs.""" from .tool_actions import LocalShellAction @@ -2210,6 +2377,7 @@ async def execute_local_shell_calls( hooks=hooks, context_wrapper=context_wrapper, config=config, + tool_output_committer=tool_output_committer, ) ) return results @@ -2222,6 +2390,7 @@ async def execute_shell_calls( context_wrapper: RunContextWrapper[Any], hooks: RunHooks[Any], config: RunConfig, + tool_output_committer: Callable[[RunItem], None] | None = None, ) -> list[RunItem]: """Run shell tool calls serially and wrap outputs.""" from .tool_actions import ShellAction @@ -2235,6 +2404,7 @@ async def execute_shell_calls( hooks=hooks, context_wrapper=context_wrapper, config=config, + tool_output_committer=tool_output_committer, ) ) return results @@ -2247,6 +2417,7 @@ async def execute_apply_patch_calls( context_wrapper: RunContextWrapper[Any], hooks: RunHooks[Any], config: RunConfig, + tool_output_committer: Callable[[RunItem], None] | None = None, ) -> list[RunItem]: """Run apply_patch tool calls serially and normalize outputs.""" from .tool_actions import ApplyPatchAction @@ -2260,6 +2431,7 @@ async def execute_apply_patch_calls( hooks=hooks, context_wrapper=context_wrapper, config=config, + tool_output_committer=tool_output_committer, ) ) return results @@ -2272,12 +2444,17 @@ async def execute_computer_actions( hooks: RunHooks[Any], context_wrapper: RunContextWrapper[Any], config: RunConfig, + tool_output_committer: Callable[[RunItem], None] | None = None, ) -> list[RunItem]: """Run computer actions serially and emit screenshot outputs.""" from .tool_actions import ComputerAction results: list[RunItem] = [] for action in actions: + context_wrapper._mark_tool_invocation_executed( + action.tool_call, + tool_name=action.computer_tool.name, + ) acknowledged: list[ComputerCallOutputAcknowledgedSafetyCheck] | None = None if action.tool_call.pending_safety_checks and action.computer_tool.on_safety_check: acknowledged = [] @@ -2309,6 +2486,7 @@ async def execute_computer_actions( context_wrapper=context_wrapper, config=config, acknowledged_safety_checks=acknowledged, + tool_output_committer=tool_output_committer, ) ) @@ -2416,6 +2594,7 @@ async def _resolve_tool_run( message = await resolve_approval_rejection_message( context_wrapper=context_wrapper, run_config=run_config, + tool_call=tool_call, tool_type="function", tool_name=display_tool_name, call_id=call_id, diff --git a/src/agents/run_internal/tool_planning.py b/src/agents/run_internal/tool_planning.py index 8647859d67..84d4246ed9 100644 --- a/src/agents/run_internal/tool_planning.py +++ b/src/agents/run_internal/tool_planning.py @@ -10,12 +10,26 @@ from openai.types.responses import ResponseFunctionToolCall from openai.types.responses.response_input_param import McpApprovalResponse -from .._tool_identity import get_function_tool_lookup_key_for_call, get_tool_call_namespace +from .._tool_identity import ( + FunctionToolLookupKey, + get_function_tool_lookup_key_for_call, + get_function_tool_lookup_key_for_tool, + get_tool_call_namespace, +) +from .._tool_invocation import ( + tool_invocation_call_id, + tool_invocation_identity, + tool_output_identity, +) from ..agent import Agent -from ..exceptions import UserError +from ..exceptions import ModelBehaviorError, UserError from ..items import ( + HandoffCallItem, + HandoffOutputItem, ItemHelpers, + MCPApprovalRequestItem, MCPApprovalResponseItem, + ReasoningItem, RunItem, RunItemBase, ToolApprovalItem, @@ -28,6 +42,7 @@ from ..util._asyncio_tasks import gather_with_cancel from .agent_bindings import AgentBindings from .run_steps import ( + ProcessedResponse, ToolRunApplyPatchCall, ToolRunComputerAction, ToolRunCustom, @@ -53,6 +68,9 @@ "execute_mcp_approval_requests", "_build_tool_output_index", "_dedupe_tool_call_items", + "_dedupe_processed_response_invocations", + "_register_tool_call_items", + "_validate_unresolved_function_calls", "ToolExecutionPlan", "_build_plan_for_fresh_turn", "_build_plan_for_resume_turn", @@ -106,29 +124,67 @@ async def execute_mcp_approval_requests( ) -> list[RunItem]: """Run hosted MCP approval callbacks and return approval response items.""" + approval_requests, _ = _preflight_mcp_approval_requests(approval_requests) + async def run_single_approval(approval_request: ToolRunMCPApprovalRequest) -> RunItem: - callback = approval_request.mcp_tool.on_approval_request - assert callback is not None, "Callback is required for MCP approval requests" - maybe_awaitable_result = callback( - MCPToolApprovalRequest(context_wrapper, approval_request.request_item) - ) - if inspect.isawaitable(maybe_awaitable_result): - result = await maybe_awaitable_result - else: - result = maybe_awaitable_result - reason = result.get("reason", None) request_item = approval_request.request_item request_id = ( request_item.id if hasattr(request_item, "id") else cast(dict[str, Any], request_item).get("id", "") ) + approval_item = ToolApprovalItem( + agent=agent, + raw_item=request_item, + tool_name=get_mapping_or_attr(request_item, "name"), + ) + approval_status = context_wrapper.get_approval_status( + approval_item.tool_name or "", + request_id, + existing_pending=approval_item, + current_invocation=approval_item, + ) + reason = context_wrapper.get_rejection_message( + approval_item.tool_name or "", + request_id, + existing_pending=approval_item, + ) + if approval_status is None: + invocation_status = context_wrapper._tool_invocation_status(request_item) + if invocation_status is None: + raise ModelBehaviorError( + "Hosted MCP approval requests require a canonical invocation identity." + ) + if invocation_status[2]: + raise ModelBehaviorError( + "A Hosted MCP approval callback already ran, but its response was not " + "committed. Start a new request instead of retrying the invocation." + ) + context_wrapper._mark_tool_invocation_executed(request_item) + callback = approval_request.mcp_tool.on_approval_request + assert callback is not None, "Callback is required for MCP approval requests" + maybe_awaitable_result = callback( + MCPToolApprovalRequest(context_wrapper, approval_request.request_item) + ) + if inspect.isawaitable(maybe_awaitable_result): + result = await maybe_awaitable_result + else: + result = maybe_awaitable_result + approval_status = result["approve"] + reason = result.get("reason", None) + if approval_status: + context_wrapper.approve_tool(approval_item) + else: + context_wrapper.reject_tool( + approval_item, + rejection_message=reason if isinstance(reason, str) else None, + ) raw_item: McpApprovalResponse = { "approval_request_id": request_id, - "approve": result["approve"], + "approve": approval_status, "type": "mcp_approval_response", } - if not result["approve"] and reason: + if not approval_status and reason: raw_item["reason"] = reason ItemHelpers.copy_tool_call_caller(request_item, raw_item) return MCPApprovalResponseItem( @@ -140,6 +196,34 @@ async def run_single_approval(approval_request: ToolRunMCPApprovalRequest) -> Ru return list(await gather_with_cancel(*tasks)) +def _preflight_mcp_approval_requests( + approval_requests: Sequence[ToolRunMCPApprovalRequest], +) -> tuple[list[ToolRunMCPApprovalRequest], set[int]]: + """Reject changed same-ID MCP siblings and coalesce exact duplicates.""" + seen_by_call: dict[tuple[str, str], tuple[str, str, str]] = {} + deduped: list[ToolRunMCPApprovalRequest] = [] + skipped_raw_item_ids: set[int] = set() + for approval_request in approval_requests: + raw_item = approval_request.request_item + identity = tool_invocation_identity(raw_item) + if identity is None: + deduped.append(approval_request) + continue + call_key = identity[:2] + existing_identity = seen_by_call.get(call_key) + if existing_identity is None: + seen_by_call[call_key] = identity + deduped.append(approval_request) + continue + if existing_identity != identity: + raise ModelBehaviorError( + "Model reused an approval-gated tool call ID for a different invocation. " + "Use a unique call ID for each approval-gated invocation." + ) + skipped_raw_item_ids.add(id(raw_item)) + return deduped, skipped_raw_item_ids + + def _build_tool_output_index(items: Sequence[RunItem]) -> set[tuple[str, str]]: """Index tool call output items by (type, call_id) for fast lookups.""" index: set[tuple[str, str]] = set() @@ -159,7 +243,10 @@ def _build_tool_output_index(items: Sequence[RunItem]) -> set[tuple[str, str]]: def _dedupe_tool_call_items( - *, existing_items: Sequence[RunItem], new_items: Sequence[RunItem] + *, + existing_items: Sequence[RunItem], + new_items: Sequence[RunItem], + skipped_raw_item_ids: set[int], ) -> list[RunItem]: """Return new items while skipping tool call duplicates already seen by identity.""" existing_call_keys: set[tuple[str | None, str | None, Hashable | None]] = set() @@ -168,7 +255,9 @@ def _dedupe_tool_call_items( existing_call_keys.add(_tool_call_identity(item.raw_item)) deduped: list[RunItem] = [] for item in new_items: - if isinstance(item, ToolCallItem): + if isinstance(item, ToolCallItem | HandoffCallItem | MCPApprovalRequestItem): + if id(item.raw_item) in skipped_raw_item_ids: + continue identity = _tool_call_identity(item.raw_item) if identity in existing_call_keys: continue @@ -177,6 +266,275 @@ def _dedupe_tool_call_items( return deduped +def _register_tool_call_items( + context_wrapper: RunContextWrapper[Any], + items: Sequence[RunItem], + *, + validate_invocations: bool = True, +) -> None: + """Validate approval-bound calls and record their committed outputs.""" + call_item_types = (ToolCallItem, HandoffCallItem, MCPApprovalRequestItem, ToolApprovalItem) + for item in items: + if isinstance(item, ToolApprovalItem): + context_wrapper._restore_pending_approval_binding(item) + for item in items: + if not isinstance(item, call_item_types): + continue + if isinstance(item, ToolApprovalItem): + continue + if not validate_invocations and isinstance(item, ToolCallItem | MCPApprovalRequestItem): + raw_type = get_mapping_or_attr(item.raw_item, "type") + tool_name = get_mapping_or_attr(item.raw_item, "name") + if not isinstance(tool_name, str): + tool_name = { + "apply_patch_call": "apply_patch", + "computer_call": "computer", + "local_shell_call": "local_shell", + "shell_call": "shell", + }.get(raw_type) + if isinstance(tool_name, str): + context_wrapper._restore_pending_approval_binding( + ToolApprovalItem( + agent=item.agent, + raw_item=cast(Any, item.raw_item), + tool_name=tool_name, + tool_namespace=get_tool_call_namespace(item.raw_item), + tool_lookup_key=( + get_function_tool_lookup_key_for_call(item.raw_item) + if raw_type == "function_call" + else None + ), + ) + ) + if not validate_invocations: + continue + if ( + isinstance(item, ToolCallItem) + and get_mapping_or_attr(item.raw_item, "type") == "function_call" + ): + # Resolved function calls are validated from the plan, where canonical routing identity + # is available. Raw calls can omit deferred-loading routing metadata. + continue + context_wrapper._tool_invocation_status( + item.raw_item, + tool_name=(item.tool_name if isinstance(item, ToolCallItem) else None), + invocation_role=("handoff" if isinstance(item, HandoffCallItem) else None), + ) + for item in items: + if isinstance(item, call_item_types): + continue + if isinstance(item, ToolCallOutputItem | HandoffOutputItem | MCPApprovalResponseItem): + context_wrapper._mark_tool_call_completed(item.raw_item) + + +def _validate_unresolved_function_calls( + context_wrapper: RunContextWrapper[Any], + runs: Sequence[Any], +) -> None: + """Validate unresolved function calls before any sibling tool starts.""" + for run in runs: + context_wrapper._tool_invocation_status(get_mapping_or_attr(run, "tool_call")) + + +def _dedupe_processed_response_invocations( + processed_response: ProcessedResponse, + *, + context_wrapper: RunContextWrapper[Any], + existing_items: Sequence[RunItem], + deferred_binding_validation_raw_item_ids: set[int] | None = None, + filter_completed: bool = True, +) -> set[int]: + """Validate and coalesce one response's tool invocations before user callbacks run.""" + deferred_binding_validation_raw_item_ids = deferred_binding_validation_raw_item_ids or set() + completed_output_keys = { + output_identity + for item in existing_items + if (output_identity := tool_output_identity(getattr(item, "raw_item", None))) is not None + } + completed_historical_invocations: dict[str, tuple[str, str, str]] = {} + for item in existing_items: + identity = tool_invocation_identity( + getattr(item, "raw_item", None), + tool_lookup_key=(item.tool_lookup_key if isinstance(item, ToolApprovalItem) else None), + tool_name=( + item.tool_name if isinstance(item, ToolCallItem | ToolApprovalItem) else None + ), + invocation_role=("handoff" if isinstance(item, HandoffCallItem) else None), + ) + if identity is None or identity[:2] not in completed_output_keys: + continue + previous_identity = completed_historical_invocations.get(identity[1]) + if previous_identity is not None and previous_identity != identity: + raise ModelBehaviorError( + "Run history reused a tool call ID for different completed invocations. " + "Use a unique call ID for each tool invocation." + ) + completed_historical_invocations[identity[1]] = identity + current_response_invocations: dict[str, tuple[str, str, str]] = {} + ( + processed_response.mcp_approval_requests, + skipped_raw_item_ids, + ) = _preflight_mcp_approval_requests(processed_response.mcp_approval_requests) + uncanonical_response_call_ids: set[str] = set() + + def should_keep( + raw_item: Any, + tool_lookup_key: FunctionToolLookupKey | None = None, + tool_name: str | None = None, + invocation_role: str | None = None, + ) -> bool: + call_identity = tool_invocation_call_id(raw_item) + if call_identity is not None and call_identity[1] is None: + raise ModelBehaviorError( + "Tool invocations require a non-empty string call ID before execution." + ) + identity = tool_invocation_identity( + raw_item, + tool_lookup_key=tool_lookup_key, + tool_name=tool_name, + invocation_role=invocation_role, + ) + if identity is None: + context_wrapper._tool_invocation_status( + raw_item, + tool_lookup_key=tool_lookup_key, + tool_name=tool_name, + invocation_role=invocation_role, + ) + if call_identity is not None and call_identity[1] is not None: + call_id = call_identity[1] + if ( + call_id in current_response_invocations + or call_id in uncanonical_response_call_ids + ): + raise ModelBehaviorError( + "Model reused a tool call ID for a different invocation in one response. " + "Use a unique call ID for each tool invocation." + ) + uncanonical_response_call_ids.add(call_id) + return True + + if identity[1] in uncanonical_response_call_ids: + raise ModelBehaviorError( + "Model reused a tool call ID for a different invocation in one response. " + "Use a unique call ID for each tool invocation." + ) + + historical_identity = completed_historical_invocations.get(identity[1]) + if historical_identity is not None: + if historical_identity != identity: + raise ModelBehaviorError( + "Model reused a completed tool call ID for a different invocation. " + "Use a unique call ID for each tool invocation." + ) + if filter_completed: + skipped_raw_item_ids.add(id(raw_item)) + return False + previous_identity = current_response_invocations.get(identity[1]) + if previous_identity is not None: + if previous_identity != identity: + raise ModelBehaviorError( + "Model reused a tool call ID for a different invocation in one response. " + "Use a unique call ID for each tool invocation." + ) + skipped_raw_item_ids.add(id(raw_item)) + return False + current_response_invocations[identity[1]] = identity + + if id(raw_item) not in deferred_binding_validation_raw_item_ids: + try: + binding_status = context_wrapper._tool_invocation_status( + raw_item, + tool_lookup_key=tool_lookup_key, + tool_name=tool_name, + invocation_role=invocation_role, + ) + except ModelBehaviorError: + # A completed exact sibling with the same provider ID can predate approval + # binding, so preserve that released cross-kind resume behavior. Changed content + # has no exact historical identity and still fails closed. + if historical_identity == identity: + return True + raise + if filter_completed and binding_status is not None and binding_status[1]: + skipped_raw_item_ids.add(id(raw_item)) + return False + if binding_status is not None and binding_status[2] and not binding_status[1]: + raise ModelBehaviorError( + "A tool call already executed, but its output was not committed. " + "Start a new run instead of retrying the invocation." + ) + return True + + processed_response.functions = [ + run + for run in processed_response.functions + if should_keep( + run.tool_call, + get_function_tool_lookup_key_for_tool(run.function_tool), + ) + ] + processed_response.handoffs = [ + run + for run in processed_response.handoffs + if should_keep(run.tool_call, invocation_role="handoff") + ] + processed_response.function_tools_not_found = [ + run for run in processed_response.function_tools_not_found if should_keep(run.tool_call) + ] + processed_response.computer_actions = [ + run + for run in processed_response.computer_actions + if should_keep(run.tool_call, tool_name=run.computer_tool.name) + ] + processed_response.custom_tool_calls = [ + run + for run in processed_response.custom_tool_calls + if should_keep(run.tool_call, tool_name=run.custom_tool.name) + ] + processed_response.local_shell_calls = [ + run + for run in processed_response.local_shell_calls + if should_keep(run.tool_call, tool_name=run.local_shell_tool.name) + ] + processed_response.shell_calls = [ + run + for run in processed_response.shell_calls + if should_keep(run.tool_call, tool_name=run.shell_tool.name) + ] + processed_response.apply_patch_calls = [ + run + for run in processed_response.apply_patch_calls + if should_keep(run.tool_call, tool_name=run.apply_patch_tool.name) + ] + processed_response.mcp_approval_requests = [ + run for run in processed_response.mcp_approval_requests if should_keep(run.request_item) + ] + dropped_item_indexes = { + index + for index, item in enumerate(processed_response.new_items) + if isinstance(item, ToolCallItem | HandoffCallItem | MCPApprovalRequestItem) + and id(item.raw_item) in skipped_raw_item_ids + } + dropped_reasoning_indexes: set[int] = set() + for index in range(len(processed_response.new_items) - 1, -1, -1): + if not isinstance(processed_response.new_items[index], ReasoningItem): + continue + for next_index in range(index + 1, len(processed_response.new_items)): + if isinstance(processed_response.new_items[next_index], ReasoningItem): + continue + if next_index in dropped_item_indexes: + dropped_reasoning_indexes.add(index) + break + excluded_item_indexes = dropped_item_indexes | dropped_reasoning_indexes + processed_response.new_items = [ + item + for index, item in enumerate(processed_response.new_items) + if index not in excluded_item_indexes + ] + return skipped_raw_item_ids + + @_dc.dataclass class ToolExecutionPlan: """Represents tool execution work to perform in a single turn.""" @@ -203,7 +561,10 @@ def _partition_mcp_approval_requests( with_callback: list[ToolRunMCPApprovalRequest] = [] manual: list[ToolRunMCPApprovalRequest] = [] for request in requests: - if request.mcp_tool.on_approval_request: + if ( + request.mcp_tool.on_approval_request + and tool_invocation_identity(request.request_item) is not None + ): with_callback.append(request) else: manual.append(request) @@ -394,17 +755,34 @@ async def _collect_runs_by_approval( rejection_items: list[RunItem] = [] for run in runs: call_id = call_id_extractor(run) + if output_exists_checker and output_exists_checker(call_id): + continue tool_name = tool_name_resolver(run) existing_pending = approval_items_by_call_id.get(call_id) + function_tool = get_mapping_or_attr(run, "function_tool") + current_item = ToolApprovalItem( + agent=agent, + raw_item=get_mapping_or_attr(run, "tool_call"), + tool_name=tool_name, + tool_namespace=get_tool_call_namespace(get_mapping_or_attr(run, "tool_call")), + tool_origin=( + get_function_tool_origin(function_tool) + if isinstance(function_tool, FunctionTool) + else None + ), + tool_lookup_key=( + get_function_tool_lookup_key_for_tool(function_tool) + if isinstance(function_tool, FunctionTool) + else None + ), + ) approval_status = context_wrapper.get_approval_status( tool_name, call_id, existing_pending=existing_pending, + current_invocation=current_item, ) - if output_exists_checker and output_exists_checker(call_id): - continue - needs_approval = True if approval_status is None and needs_approval_checker: try: @@ -417,6 +795,7 @@ async def _collect_runs_by_approval( tool_name, call_id, existing_pending=existing_pending, + current_invocation=current_item, ) if approval_status is False: @@ -436,21 +815,7 @@ async def _collect_runs_by_approval( approved_runs.append(run) continue - function_tool = get_mapping_or_attr(run, "function_tool") - pending_item = existing_pending or ToolApprovalItem( - agent=agent, - raw_item=get_mapping_or_attr(run, "tool_call"), - tool_name=tool_name, - tool_namespace=get_tool_call_namespace(get_mapping_or_attr(run, "tool_call")), - tool_origin=( - get_function_tool_origin(function_tool) - if isinstance(function_tool, FunctionTool) - else None - ), - tool_lookup_key=get_function_tool_lookup_key_for_call( - get_mapping_or_attr(run, "tool_call") - ), - ) + pending_item = existing_pending or current_item pending_interruption_adder(pending_item) return approved_runs, rejection_items @@ -516,11 +881,14 @@ async def _select_function_tool_runs_for_resume( if output_exists_checker(run): continue + current_item = pending_item_builder(run) approval_status = context_wrapper.get_approval_status( run.function_tool.name, call_id, tool_namespace=get_tool_call_namespace(run.tool_call), existing_pending=approval_items_by_call_id.get(call_id), + tool_lookup_key=current_item.tool_lookup_key, + current_invocation=current_item, ) requires_approval = True @@ -531,6 +899,8 @@ async def _select_function_tool_runs_for_resume( call_id, tool_namespace=get_tool_call_namespace(run.tool_call), existing_pending=approval_items_by_call_id.get(call_id), + tool_lookup_key=current_item.tool_lookup_key, + current_invocation=current_item, ) if approval_status is False: @@ -546,7 +916,7 @@ async def _select_function_tool_runs_for_resume( continue pending_interruption_adder( - approval_items_by_call_id.get(run.tool_call.call_id) or pending_item_builder(run) + approval_items_by_call_id.get(run.tool_call.call_id) or current_item ) return selected @@ -560,6 +930,7 @@ async def _execute_tool_plan( context_wrapper: RunContextWrapper[Any], run_config, parallel: bool = True, + tool_output_committer: Callable[[RunItem], None] | None = None, ) -> tuple[ list[Any], list[ToolInputGuardrailResult], @@ -600,6 +971,7 @@ async def _execute_tool_plan( config=run_config, isolate_parallel_failures=isolate_function_tool_failures, sibling_category_failure=sibling_category_failure, + tool_output_committer=tool_output_committer, ), execute_computer_actions( public_agent=public_agent, @@ -607,6 +979,7 @@ async def _execute_tool_plan( hooks=hooks, context_wrapper=context_wrapper, config=run_config, + tool_output_committer=tool_output_committer, ), execute_custom_tool_calls( public_agent=public_agent, @@ -614,6 +987,7 @@ async def _execute_tool_plan( hooks=hooks, context_wrapper=context_wrapper, config=run_config, + tool_output_committer=tool_output_committer, ), execute_shell_calls( public_agent=public_agent, @@ -621,6 +995,7 @@ async def _execute_tool_plan( hooks=hooks, context_wrapper=context_wrapper, config=run_config, + tool_output_committer=tool_output_committer, ), execute_apply_patch_calls( public_agent=public_agent, @@ -628,6 +1003,7 @@ async def _execute_tool_plan( hooks=hooks, context_wrapper=context_wrapper, config=run_config, + tool_output_committer=tool_output_committer, ), execute_local_shell_calls( public_agent=public_agent, @@ -635,6 +1011,7 @@ async def _execute_tool_plan( hooks=hooks, context_wrapper=context_wrapper, config=run_config, + tool_output_committer=tool_output_committer, ), on_child_failure=sibling_category_failure.set, ) @@ -650,6 +1027,7 @@ async def _execute_tool_plan( context_wrapper=context_wrapper, config=run_config, isolate_parallel_failures=isolate_function_tool_failures, + tool_output_committer=tool_output_committer, ) computer_results = await execute_computer_actions( public_agent=public_agent, @@ -657,6 +1035,7 @@ async def _execute_tool_plan( hooks=hooks, context_wrapper=context_wrapper, config=run_config, + tool_output_committer=tool_output_committer, ) custom_tool_results = await execute_custom_tool_calls( public_agent=public_agent, @@ -664,6 +1043,7 @@ async def _execute_tool_plan( hooks=hooks, context_wrapper=context_wrapper, config=run_config, + tool_output_committer=tool_output_committer, ) shell_results = await execute_shell_calls( public_agent=public_agent, @@ -671,6 +1051,7 @@ async def _execute_tool_plan( hooks=hooks, context_wrapper=context_wrapper, config=run_config, + tool_output_committer=tool_output_committer, ) apply_patch_results = await execute_apply_patch_calls( public_agent=public_agent, @@ -678,6 +1059,7 @@ async def _execute_tool_plan( hooks=hooks, context_wrapper=context_wrapper, config=run_config, + tool_output_committer=tool_output_committer, ) local_shell_results = await execute_local_shell_calls( public_agent=public_agent, @@ -685,6 +1067,7 @@ async def _execute_tool_plan( hooks=hooks, context_wrapper=context_wrapper, config=run_config, + tool_output_committer=tool_output_committer, ) return ( diff --git a/src/agents/run_internal/turn_resolution.py b/src/agents/run_internal/turn_resolution.py index a30372f74b..7cf164b86c 100644 --- a/src/agents/run_internal/turn_resolution.py +++ b/src/agents/run_internal/turn_resolution.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import inspect from collections.abc import Awaitable, Callable, Container, Mapping, Sequence from copy import deepcopy @@ -47,6 +46,7 @@ restore_tool_call_routing_identity, should_allow_bare_name_approval_alias, ) +from .._tool_invocation import tool_invocation_call_id, tool_invocation_identity_and_scope from ..agent import Agent, ToolsToFinalOutputResult from ..agent_output import AgentOutputSchemaBase from ..agent_tool_state import ( @@ -95,7 +95,6 @@ from ..run_context import AgentHookContext, RunContextWrapper, TContext from ..run_error_handlers import RunErrorHandlers from ..run_state import RunState -from ..stream_events import StreamEvent from ..tool import ( ApplyPatchTool, CodeInterpreterTool, @@ -140,7 +139,6 @@ NextStepInterruption, NextStepRunAgain, ProcessedResponse, - QueueCompleteSentinel, SingleStepResult, ToolRunApplyPatchCall, ToolRunComputerAction, @@ -152,7 +150,6 @@ ToolRunMCPApprovalRequest, ToolRunShellCall, ) -from .streaming import stream_step_items_to_queue from .tool_caller import ensure_programmatic_tool_call_parent, ensure_tool_caller_allowed from .tool_execution import ( build_litellm_json_tool_call, @@ -165,9 +162,10 @@ get_mapping_or_attr, index_approval_items_by_call_id, is_apply_patch_name, - parse_apply_patch_custom_input, - parse_apply_patch_function_args, + normalize_apply_patch_fallback_call, + process_hosted_mcp_approvals, resolve_approval_rejection_message, + should_keep_hosted_mcp_item, ) from .tool_planning import ( _append_mcp_callback_results, @@ -177,10 +175,13 @@ _build_tool_result_items, _collect_runs_by_approval, _collect_tool_interruptions, + _dedupe_processed_response_invocations, _dedupe_tool_call_items, _execute_tool_plan, _make_unique_item_appender, + _register_tool_call_items, _select_function_tool_runs_for_resume, + _validate_unresolved_function_calls, ) from .turn_preparation import get_handoffs, get_output_schema @@ -260,6 +261,7 @@ async def _resolve_tool_not_found_message( *, context_wrapper: RunContextWrapper[Any], run_config: RunConfig, + tool_call: ResponseFunctionToolCall, tool_name: str, call_id: str, ) -> str: @@ -268,6 +270,7 @@ async def _resolve_tool_not_found_message( if formatter is None: return default_message + context_wrapper._mark_tool_invocation_executed(tool_call) try: maybe_message = formatter( ToolErrorFormatterArgs( @@ -313,6 +316,7 @@ async def _build_tool_not_found_output_items( message = await _resolve_tool_not_found_message( context_wrapper=context_wrapper, run_config=run_config, + tool_call=call.tool_call, tool_name=call.tool_name, call_id=call.tool_call.call_id, ) @@ -335,9 +339,9 @@ async def run_final_output_hooks( agent_hook_context = AgentHookContext( context=context_wrapper.context, usage=context_wrapper.usage, - _approvals=context_wrapper._approvals, turn_input=context_wrapper.turn_input, ) + context_wrapper._share_tool_state_with(agent_hook_context) await gather_with_cancel( hooks.on_agent_end(agent_hook_context, agent, final_output), @@ -529,6 +533,7 @@ async def execute_handoffs( nest_handoff_history_fn: Callable[..., HandoffInputData] | None = None, tool_input_guardrail_results: list[ToolInputGuardrailResult] | None = None, tool_output_guardrail_results: list[ToolOutputGuardrailResult] | None = None, + handoff_output_committer: Callable[[HandoffOutputItem, Agent[Any]], None] | None = None, ) -> SingleStepResult: """Execute a handoff and prepare the next turn for the new agent.""" @@ -566,6 +571,10 @@ def nest_history( actual_handoff = run_handoffs[0] with handoff_span(from_agent=public_agent.name) as span_handoff: handoff = actual_handoff.handoff + context_wrapper._mark_tool_invocation_executed( + actual_handoff.tool_call, + invocation_role="handoff", + ) new_agent: Agent[Any] = await handoff.on_invoke_handoff( context_wrapper, actual_handoff.tool_call.arguments ) @@ -581,17 +590,19 @@ def nest_history( ) ) - new_step_items.append( - HandoffOutputItem( - agent=public_agent, - raw_item=ItemHelpers.tool_call_output_item( - actual_handoff.tool_call, - handoff.get_transfer_message(new_agent), - ), - source_agent=public_agent, - target_agent=new_agent, - ) + handoff_output = HandoffOutputItem( + agent=public_agent, + raw_item=ItemHelpers.tool_call_output_item( + actual_handoff.tool_call, + handoff.get_transfer_message(new_agent), + ), + source_agent=public_agent, + target_agent=new_agent, ) + new_step_items.append(handoff_output) + if handoff_output_committer is not None: + _register_tool_call_items(context_wrapper, [handoff_output]) + handoff_output_committer(handoff_output, new_agent) await gather_with_cancel( hooks.on_handoff( @@ -714,6 +725,12 @@ def nest_history( # No filtering or nesting - session_step_items not needed. session_step_items = None + if handoff_output_committer is None and ( + handoff_output in new_step_items + or (session_step_items is not None and handoff_output in session_step_items) + ): + _register_tool_call_items(context_wrapper, [handoff_output]) + return SingleStepResult( original_input=original_input, model_response=new_response, @@ -771,6 +788,7 @@ async def execute_tools_and_side_effects( run_config: RunConfig, error_handlers: RunErrorHandlers[TContext] | None = None, server_manages_conversation: bool = False, + precomputed_skipped_raw_item_ids: set[int] | None = None, ) -> SingleStepResult: """Run one turn of the loop, coordinating tools, approvals, guardrails, and handoffs.""" public_agent = bindings.public_agent @@ -779,6 +797,25 @@ async def execute_tools_and_side_effects( execute_handoffs_call = execute_handoffs pre_step_items = list(pre_step_items) + _register_tool_call_items( + context_wrapper, + pre_step_items, + validate_invocations=False, + ) + skipped_raw_item_ids = ( + precomputed_skipped_raw_item_ids + if precomputed_skipped_raw_item_ids is not None + else _dedupe_processed_response_invocations( + processed_response, + context_wrapper=context_wrapper, + existing_items=pre_step_items, + ) + ) + _register_tool_call_items(context_wrapper, processed_response.new_items) + _validate_unresolved_function_calls( + context_wrapper, + processed_response.function_tools_not_found, + ) approval_items_by_call_id = index_approval_items_by_call_id(pre_step_items) plan = _build_plan_for_fresh_turn( @@ -787,10 +824,10 @@ async def execute_tools_and_side_effects( context_wrapper=context_wrapper, approval_items_by_call_id=approval_items_by_call_id, ) - new_step_items = _dedupe_tool_call_items( existing_items=pre_step_items, new_items=processed_response.new_items, + skipped_raw_item_ids=skipped_raw_item_ids, ) ( @@ -840,6 +877,8 @@ async def execute_tools_and_side_effects( interruptions.extend(plan.pending_interruptions) new_step_items.extend(plan.pending_interruptions) + _register_tool_call_items(context_wrapper, new_step_items) + processed_response.interruptions = interruptions if interruptions: @@ -860,6 +899,7 @@ async def execute_tools_and_side_effects( context_wrapper=context_wrapper, append_item=new_step_items.append, ) + _register_tool_call_items(context_wrapper, new_step_items) if run_handoffs := processed_response.handoffs: return await execute_handoffs_call( @@ -900,7 +940,7 @@ async def execute_tools_and_side_effects( if not processed_response.has_tools_or_approvals_to_run(): has_tool_activity_without_message = not message_items and bool( - processed_response.tools_used + processed_response.tools_used or skipped_raw_item_ids ) if not has_tool_activity_without_message: if refusal: @@ -1094,6 +1134,12 @@ async def resolve_interrupted_turn( execute_handoffs_call = execute_handoffs + _register_tool_call_items( + context_wrapper, + original_pre_step_items, + validate_invocations=False, + ) + def _pending_approvals_from_state() -> list[ToolApprovalItem]: if ( run_state is not None @@ -1120,6 +1166,7 @@ async def _record_function_rejection( rejection_message = await resolve_approval_rejection_message( context_wrapper=context_wrapper, run_config=run_config, + tool_call=tool_call, tool_type="function", tool_name=get_tool_call_trace_name(tool_call) or function_tool.name, call_id=call_id, @@ -1211,6 +1258,7 @@ async def _build_shell_rejection(run: ToolRunShellCall, call_id: str) -> RunItem rejection_message = await resolve_approval_rejection_message( context_wrapper=context_wrapper, run_config=run_config, + tool_call=run.tool_call, tool_type="shell", tool_name=run.shell_tool.name, call_id=call_id, @@ -1229,6 +1277,7 @@ async def _build_apply_patch_rejection(run: ToolRunApplyPatchCall, call_id: str) rejection_message = await resolve_approval_rejection_message( context_wrapper=context_wrapper, run_config=run_config, + tool_call=run.tool_call, tool_type="apply_patch", tool_name=run.apply_patch_tool.name, call_id=call_id, @@ -1248,6 +1297,7 @@ async def _build_custom_rejection(run: ToolRunCustom, call_id: str) -> RunItem: rejection_message = await resolve_approval_rejection_message( context_wrapper=context_wrapper, run_config=run_config, + tool_call=run.tool_call, tool_type="custom", tool_name=run.custom_tool.name, call_id=call_id, @@ -1342,6 +1392,11 @@ def _function_output_exists(run: ToolRunFunction) -> bool: if not call_id: return False + if call_id not in approval_items_by_call_id and _has_output_item( + call_id, "function_call_output" + ): + return True + pending_run_result = peek_agent_tool_run_result( run.tool_call, scope_id=tool_state_scope_id, @@ -1353,7 +1408,14 @@ def _function_output_exists(run: ToolRunFunction) -> bool: return False return True - return _has_output_item(call_id, "function_call_output") + binding_status = context_wrapper._approved_tool_invocation_status( + run.tool_call, + tool_lookup_key=get_function_tool_lookup_key_for_tool(run.function_tool), + ) + if binding_status is not None: + return binding_status[1] + + return False def _add_pending_interruption(item: ToolApprovalItem | None) -> None: if item is None: @@ -1415,6 +1477,45 @@ def _approval_persisted_lookup_key( approval.tool_namespace, ) + deferred_binding_validation_raw_item_ids = { + id(run.tool_call) + for run in processed_response.functions + if ( + ( + nested_result := peek_agent_tool_run_result( + run.tool_call, + scope_id=tool_state_scope_id, + ) + ) + is not None + and getattr(nested_result, "interruptions", None) + ) + } + for function_run in processed_response.functions: + if id(function_run.tool_call) not in deferred_binding_validation_raw_item_ids: + continue + approval_item = approval_items_by_call_id.get(function_run.tool_call.call_id) + persisted_lookup_key = ( + _approval_persisted_lookup_key(approval_item) + if approval_item is not None + else get_function_tool_lookup_key_for_tool(function_run.function_tool) + ) + current_lookup_key = get_function_tool_lookup_key_for_tool(function_run.function_tool) + if persisted_lookup_key != current_lookup_key: + # Preserve the more specific interrupted Agent.as_tool() replacement error below. + continue + context_wrapper._approved_tool_invocation_status( + function_run.tool_call, + tool_lookup_key=persisted_lookup_key, + ) + _dedupe_processed_response_invocations( + processed_response, + context_wrapper=context_wrapper, + existing_items=original_pre_step_items, + deferred_binding_validation_raw_item_ids=deferred_binding_validation_raw_item_ids, + filter_completed=False, + ) + queued_call_id_counts: dict[str, int] = {} queued_call_items = [ *(run.tool_call for run in processed_response.functions), @@ -1831,6 +1932,12 @@ def _rebind_function_run( call_id, tool_namespace=approval_record.tool_namespace, existing_pending=approval_record, + current_invocation=ToolApprovalItem( + agent=public_agent, + raw_item=call, + tool_name=call.name, + tool_namespace=get_tool_call_namespace(call), + ), ) if approval_record is not None else True @@ -1842,6 +1949,19 @@ def _rebind_function_run( continue current_handoff = current_handoffs.get(call_id) + if current_handoff is not None and approval_record is not None: + approval_status = context_wrapper.get_approval_status( + approval_record.tool_name or call.name, + call_id, + tool_namespace=approval_record.tool_namespace, + existing_pending=approval_record, + current_invocation=ToolApprovalItem( + agent=public_agent, + raw_item=current_handoff.tool_call, + tool_name=current_handoff.tool_call.name, + tool_namespace=get_tool_call_namespace(current_handoff.tool_call), + ), + ) if current_handoff is not None and approval_status is True: if stale_function is not None: _reject_nested_replacement(stale_function) @@ -1888,6 +2008,7 @@ def _rebind_function_run( rejection_message = await resolve_approval_rejection_message( context_wrapper=context_wrapper, run_config=run_config, + tool_call=rejection_call, tool_type="function", tool_name=get_tool_call_trace_name(rejection_call) or rejection_call.name, call_id=call_id, @@ -1953,6 +2074,21 @@ def _rebind_function_run( for run in reconciled_functions if run.tool_call.call_id not in missing_function_call_ids ] + + # Validate every current execution candidate before any dynamic approval callback or + # output-based replay suppression can run. This keeps a changed invocation under an approved + # call ID from triggering sibling user code or hiding behind a previously committed output. + for function_run in selectable_function_runs: + context_wrapper._approved_tool_invocation_status( + function_run.tool_call, + tool_lookup_key=get_function_tool_lookup_key_for_tool(function_run.function_tool), + ) + for handoff_run in reconciled_handoffs: + context_wrapper._approved_tool_invocation_status( + handoff_run.tool_call, + invocation_role="handoff", + ) + _validate_unresolved_function_calls(context_wrapper, missing_function_tools) function_tool_runs = await _select_function_tool_runs_for_resume( selectable_function_runs, approval_items_by_call_id=function_approval_items_by_call_id, @@ -2034,7 +2170,6 @@ def _rebind_function_run( shell_calls=approved_shell_calls, apply_patch_calls=approved_apply_patch_calls, ) - missing_output_items = await _build_tool_not_found_output_items( agent=public_agent, calls=missing_function_tools, @@ -2058,6 +2193,29 @@ def _rebind_function_run( dropped_nested_call_ids.add(id(stale_call)) _drop_stable_nested_result(stale_call) + call_positions = dict(response_call_positions) + next_call_position = len(new_response.output) + for call in calls_to_reconcile: + if call.call_id not in call_positions: + call_positions[call.call_id] = next_call_position + next_call_position += 1 + + committed_tool_outputs: list[RunItem] = [] + + def _commit_tool_output(item: RunItem) -> None: + if any(existing is item for existing in committed_tool_outputs): + return + committed_tool_outputs.append(item) + committed_tool_outputs.sort( + key=lambda output: call_positions.get( + extract_tool_call_id(getattr(output, "raw_item", None)) or "", + len(call_positions), + ) + ) + if run_state is not None: + run_state._generated_items = [*original_pre_step_items, *committed_tool_outputs] + _register_tool_call_items(context_wrapper, [item]) + ( function_results, tool_input_guardrail_results, @@ -2073,6 +2231,7 @@ def _rebind_function_run( hooks=hooks, context_wrapper=context_wrapper, run_config=run_config, + tool_output_committer=_commit_tool_output, ) for interruption in _collect_tool_interruptions( @@ -2093,12 +2252,6 @@ def _rebind_function_run( apply_patch_results=[], local_shell_results=[], ) - call_positions = dict(response_call_positions) - next_call_position = len(new_response.output) - for call in calls_to_reconcile: - if call.call_id not in call_positions: - call_positions[call.call_id] = next_call_position - next_call_position += 1 function_outcomes = [ *function_result_items, *missing_output_items, @@ -2133,7 +2286,15 @@ def _rebind_function_run( for approved_response in plan.approved_mcp_responses: append_if_new(approved_response) + def _checkpoint_new_items() -> None: + if run_state is not None: + run_state._generated_items = [*original_pre_step_items, *new_items] + _register_tool_call_items(context_wrapper, new_items) + + _checkpoint_new_items() + def _commit_missing_state(result: SingleStepResult) -> SingleStepResult: + _checkpoint_new_items() if missing_function_call_ids: processed_response.functions = [ run @@ -2167,9 +2328,26 @@ def _commit_missing_state(result: SingleStepResult) -> SingleStepResult: context_wrapper=context_wrapper, append_item=append_if_new, ) + _checkpoint_new_items() + ( + pending_hosted_mcp_approvals, + pending_hosted_mcp_approval_ids, + ) = process_hosted_mcp_approvals( + original_pre_step_items=original_pre_step_items, + mcp_approval_requests=processed_response.mcp_approval_requests, + context_wrapper=context_wrapper, + agent=public_agent, + append_item=append_if_new, + ) - pre_step_items: list[RunItem] = [ - item for item in original_pre_step_items if not isinstance(item, ToolApprovalItem) + pre_step_items = [ + item + for item in original_pre_step_items + if should_keep_hosted_mcp_item( + item, + pending_hosted_mcp_approvals=pending_hosted_mcp_approvals, + pending_hosted_mcp_approval_ids=pending_hosted_mcp_approval_ids, + ) ] if rejected_function_call_ids: @@ -2205,6 +2383,15 @@ def _commit_missing_state(result: SingleStepResult) -> SingleStepResult: ] if pending_handoffs: + + def _commit_handoff_output( + _handoff_output: HandoffOutputItem, + new_agent: Agent[Any], + ) -> None: + _checkpoint_new_items() + if run_state is not None: + run_state._current_agent = new_agent + return _commit_missing_state( await execute_handoffs_call( public_agent=public_agent, @@ -2220,6 +2407,7 @@ def _commit_missing_state(result: SingleStepResult) -> SingleStepResult: nest_handoff_history_fn=nest_handoff_history_fn, tool_input_guardrail_results=tool_input_guardrail_results, tool_output_guardrail_results=tool_output_guardrail_results, + handoff_output_committer=_commit_handoff_output, ) ) @@ -2448,7 +2636,6 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: "created_by": get_mapping_or_attr(output, "created_by"), } shell_call_raw.pop("created_by", None) - items.append(ToolCallItem(raw_item=cast(Any, shell_call_raw), agent=agent)) if not shell_tool: tools_used.append("shell") _error_tracing.attach_error_to_current_span( @@ -2458,6 +2645,13 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: ) ) raise ModelBehaviorError("Model produced shell call without a shell tool.") + items.append( + ToolCallItem( + raw_item=cast(Any, shell_call_raw), + agent=agent, + _resolved_tool_name=shell_tool.name, + ) + ) ensure_tool_caller_allowed( tool_call=output, allowed_callers=shell_tool.allowed_callers, @@ -2535,7 +2729,13 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: tool_name=apply_patch_tool.name, agent_name=agent.name, ) - items.append(ToolCallItem(raw_item=cast(Any, apply_patch_call_raw), agent=agent)) + items.append( + ToolCallItem( + raw_item=cast(Any, apply_patch_call_raw), + agent=agent, + _resolved_tool_name=apply_patch_tool.name, + ) + ) tools_used.append(apply_patch_tool.name) call_identifier = get_mapping_or_attr(apply_patch_call_raw, "call_id") logger.debug("Queuing apply_patch_call %s", call_identifier) @@ -2640,7 +2840,13 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: tool_name=computer_tool.name, agent_name=agent.name, ) - items.append(ToolCallItem(raw_item=output, agent=agent)) + items.append( + ToolCallItem( + raw_item=output, + agent=agent, + _resolved_tool_name=computer_tool.name, + ) + ) tools_used.append(computer_tool.name) computer_actions.append( ToolRunComputerAction(tool_call=output, computer_tool=computer_tool) @@ -2747,7 +2953,13 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: tool_name=local_shell_tool.name, agent_name=agent.name, ) - items.append(ToolCallItem(raw_item=output, agent=agent)) + items.append( + ToolCallItem( + raw_item=output, + agent=agent, + _resolved_tool_name=local_shell_tool.name, + ) + ) tools_used.append("local_shell") local_shell_calls.append( ToolRunLocalShellCall(tool_call=output, local_shell_tool=local_shell_tool) @@ -2759,7 +2971,13 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: tool_name=shell_tool.name, agent_name=agent.name, ) - items.append(ToolCallItem(raw_item=output, agent=agent)) + items.append( + ToolCallItem( + raw_item=output, + agent=agent, + _resolved_tool_name=shell_tool.name, + ) + ) tools_used.append(shell_tool.name) shell_calls.append(ToolRunShellCall(tool_call=output, shell_tool=shell_tool)) else: @@ -2786,13 +3004,8 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: tools_used.append(custom_tool.name) custom_tool_calls.append(ToolRunCustom(tool_call=output, custom_tool=custom_tool)) elif is_apply_patch_name(output.name, apply_patch_tool): - parsed_operation = parse_apply_patch_custom_input(output.input) - pseudo_call = { - "type": "apply_patch_call", - "call_id": output.call_id, - **parsed_operation, - } - ItemHelpers.copy_tool_call_caller(output, pseudo_call) + pseudo_call = normalize_apply_patch_fallback_call(output) + assert pseudo_call is not None if apply_patch_tool: ensure_tool_caller_allowed( tool_call=pseudo_call, @@ -2800,7 +3013,13 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: tool_name=apply_patch_tool.name, agent_name=agent.name, ) - items.append(ToolCallItem(raw_item=cast(Any, pseudo_call), agent=agent)) + items.append( + ToolCallItem( + raw_item=cast(Any, pseudo_call), + agent=agent, + _resolved_tool_name=apply_patch_tool.name, + ) + ) tools_used.append(apply_patch_tool.name) apply_patch_calls.append( ToolRunApplyPatchCall( @@ -2834,13 +3053,8 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: and is_apply_patch_name(output.name, apply_patch_tool) and get_function_tool_lookup_key_for_call(output) not in function_map ): - parsed_operation = parse_apply_patch_function_args(output.arguments) - pseudo_call = { - "type": "apply_patch_call", - "call_id": output.call_id, - "operation": parsed_operation, - } - ItemHelpers.copy_tool_call_caller(output, pseudo_call) + pseudo_call = normalize_apply_patch_fallback_call(output) + assert pseudo_call is not None if apply_patch_tool: ensure_tool_caller_allowed( tool_call=pseudo_call, @@ -2848,7 +3062,13 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: tool_name=apply_patch_tool.name, agent_name=agent.name, ) - items.append(ToolCallItem(raw_item=cast(Any, pseudo_call), agent=agent)) + items.append( + ToolCallItem( + raw_item=cast(Any, pseudo_call), + agent=agent, + _resolved_tool_name=apply_patch_tool.name, + ) + ) tools_used.append(apply_patch_tool.name) apply_patch_calls.append( ToolRunApplyPatchCall(tool_call=pseudo_call, apply_patch_tool=apply_patch_tool) @@ -2974,6 +3194,104 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: ) +def _preflight_response_invocations_after_processing_error( + *, + response: ModelResponse, + all_tools: Sequence[Tool], + handoffs: Sequence[Handoff], + context_wrapper: RunContextWrapper[Any], +) -> None: + """Reject call-ID reuse before reporting an unrelated response-processing error.""" + handoff_map = {handoff.tool_name: handoff for handoff in handoffs} + function_map = build_function_tool_lookup_map( + [tool for tool in all_tools if isinstance(tool, FunctionTool)] + ) + custom_tool_map = {tool.name: tool for tool in all_tools if isinstance(tool, CustomTool)} + computer_tool = next((tool for tool in all_tools if isinstance(tool, ComputerTool)), None) + local_shell_tool = next((tool for tool in all_tools if isinstance(tool, LocalShellTool)), None) + shell_tool = next((tool for tool in all_tools if isinstance(tool, ShellTool)), None) + apply_patch_tool = next((tool for tool in all_tools if isinstance(tool, ApplyPatchTool)), None) + response_identities: dict[str, tuple[str, str, str, str] | None] = {} + + for output in response.output: + raw_item: Any = output + output_type = get_mapping_or_attr(output, "type") + tool_lookup_key: FunctionToolLookupKey | None = None + tool_name: str | None = None + invocation_role: str | None = None + + if isinstance(output, ResponseFunctionToolCall): + tool_lookup_key = get_function_tool_lookup_key_for_call(output) + if is_handoff_tool_call(output, handoff_map): + invocation_role = "handoff" + elif ( + is_apply_patch_name(output.name, apply_patch_tool) + and tool_lookup_key not in function_map + ): + raw_item = normalize_apply_patch_fallback_call(output) or output + tool_name = apply_patch_tool.name if apply_patch_tool is not None else None + elif isinstance(output, ResponseCustomToolCall): + custom_tool = custom_tool_map.get(output.name) + if custom_tool is not None: + tool_name = custom_tool.name + elif is_apply_patch_name(output.name, apply_patch_tool): + raw_item = normalize_apply_patch_fallback_call(output) or output + tool_name = apply_patch_tool.name if apply_patch_tool is not None else None + elif output_type == "shell_call": + tool_name = shell_tool.name if shell_tool is not None else None + elif isinstance(output, LocalShellCall): + selected_shell_tool = local_shell_tool or shell_tool + tool_name = selected_shell_tool.name if selected_shell_tool is not None else None + elif output_type == "apply_patch_call": + tool_name = apply_patch_tool.name if apply_patch_tool is not None else None + elif isinstance(output, ResponseComputerToolCall): + tool_name = computer_tool.name if computer_tool is not None else None + + call_identity = tool_invocation_call_id(raw_item) + if call_identity is None: + continue + _, call_id = call_identity + if call_id is None: + raise ModelBehaviorError( + "Tool invocations require a non-empty string call ID before execution." + ) + identity = tool_invocation_identity_and_scope( + raw_item, + tool_lookup_key=tool_lookup_key, + tool_name=tool_name, + invocation_role=invocation_role, + ) + if call_id in response_identities: + previous_identity = response_identities[call_id] + if previous_identity is None or identity is None or previous_identity != identity: + raise ModelBehaviorError( + "Model reused a tool call ID for a different invocation in one response. " + "Use a unique call ID for each tool invocation." + ) + response_identities[call_id] = identity + + record = context_wrapper._tool_invocations.get(call_id) + if record is None: + continue + if identity is None: + if not isinstance(output, McpApprovalRequest): + raise ModelBehaviorError( + "Model reused a tool call ID for a different invocation. " + "Use a unique call ID for each tool invocation." + ) + continue + invocation_type, _, approval_scope, fingerprint = identity + if ( + record.invocation_type != invocation_type + or record.approval_scope != approval_scope + or record.fingerprint != fingerprint + ): + raise ModelBehaviorError( + "Model reused a tool call ID for a different invocation. " + "Use a unique call ID for each tool invocation." + ) + + async def get_single_step_result_from_response( *, bindings: AgentBindings[TContext], @@ -2989,38 +3307,56 @@ async def get_single_step_result_from_response( tool_use_tracker, error_handlers: RunErrorHandlers[TContext] | None = None, server_manages_conversation: bool = False, - event_queue: asyncio.Queue[StreamEvent | QueueCompleteSentinel] | None = None, + after_invocation_validation: Callable[[list[RunItem] | None], Awaitable[None]] | None = None, before_side_effects: Callable[[], Awaitable[None]] | None = None, ) -> SingleStepResult: item_agent = bindings.public_agent - processed_response = process_model_response( - agent=item_agent, - all_tools=all_tools, - response=new_response, - output_schema=output_schema, - handoffs=handoffs, + try: + processed_response = process_model_response( + agent=item_agent, + all_tools=all_tools, + response=new_response, + output_schema=output_schema, + handoffs=handoffs, + existing_items=pre_step_items, + run_config=run_config, + server_manages_conversation=server_manages_conversation, + server_managed_input_items=( + ItemHelpers.input_to_new_input_list(original_input) + if server_manages_conversation + else None + ), + ) + except ModelBehaviorError: + _preflight_response_invocations_after_processing_error( + response=new_response, + all_tools=all_tools, + handoffs=handoffs, + context_wrapper=context_wrapper, + ) + if after_invocation_validation is not None: + await after_invocation_validation(None) + raise + + _register_tool_call_items( + context_wrapper, + pre_step_items, + validate_invocations=False, + ) + skipped_raw_item_ids = _dedupe_processed_response_invocations( + processed_response, + context_wrapper=context_wrapper, existing_items=pre_step_items, - run_config=run_config, - server_manages_conversation=server_manages_conversation, - server_managed_input_items=( - ItemHelpers.input_to_new_input_list(original_input) - if server_manages_conversation - else None - ), ) + if after_invocation_validation is not None: + await after_invocation_validation(processed_response.new_items) + if before_side_effects is not None: await before_side_effects() tool_use_tracker.record_processed_response(item_agent, processed_response) - if event_queue is not None and processed_response.new_items: - handoff_items = [ - item for item in processed_response.new_items if isinstance(item, HandoffCallItem) - ] - if handoff_items: - stream_step_items_to_queue(cast(list[RunItem], handoff_items), event_queue) - return await execute_tools_and_side_effects( bindings=bindings, original_input=original_input, @@ -3033,4 +3369,5 @@ async def get_single_step_result_from_response( run_config=run_config, error_handlers=error_handlers, server_manages_conversation=server_manages_conversation, + precomputed_skipped_raw_item_ids=skipped_raw_item_ids, ) diff --git a/src/agents/run_state.py b/src/agents/run_state.py index 243a6d2c9e..36df53f097 100644 --- a/src/agents/run_state.py +++ b/src/agents/run_state.py @@ -50,8 +50,13 @@ get_function_tool_qualified_name, serialize_function_tool_lookup_key, ) +from ._tool_invocation import ( + tool_invocation_call_id, + tool_invocation_identity, + tool_output_identity, +) from .agent import Agent -from .exceptions import UserError +from .exceptions import ModelBehaviorError, UserError from .guardrail import ( GuardrailFunctionOutput, InputGuardrail, @@ -150,7 +155,7 @@ # 3. to_json() always emits CURRENT_SCHEMA_VERSION. # 4. Forward compatibility is intentionally fail-fast (older SDKs reject newer or unsupported # versions). -CURRENT_SCHEMA_VERSION = "1.14" +CURRENT_SCHEMA_VERSION = "1.15" _PROGRAMMATIC_TOOL_CALLING_MIN_SCHEMA_VERSION = "1.13" _HOSTED_MCP_APPROVALS_MIN_SCHEMA_VERSION = "1.14" # Keep this mapping in chronological order. Every schema bump must add a one-line summary here. @@ -176,6 +181,7 @@ "flows." ), "1.14": "Scopes hosted MCP approvals and restored requests by server label.", + "1.15": "Persists canonical tool invocation identity and lifecycle across resume flows.", } SUPPORTED_SCHEMA_VERSIONS = frozenset(SCHEMA_VERSION_SUMMARIES) @@ -414,8 +420,25 @@ def _serialize_approvals(self) -> dict[str, dict[str, Any]]: approvals_dict[tool_name]["sticky_rejection_message"] = ( record.sticky_rejection_message ) + if record.sticky_scope is not None: + approvals_dict[tool_name]["sticky_scope"] = record.sticky_scope return approvals_dict + def _serialize_tool_invocations(self) -> dict[str, dict[str, Any]]: + """Serialize the run-owned canonical tool invocation ledger.""" + if self._context is None: + return {} + return { + call_id: { + "type": invocation.invocation_type, + "approval_scope": invocation.approval_scope, + "fingerprint": invocation.fingerprint, + "executed": invocation.executed, + "completed": invocation.completed, + } + for call_id, invocation in self._context._tool_invocations.items() + } + def _serialize_hosted_mcp_approvals(self) -> list[dict[str, Any]]: """Serialize hosted MCP approvals with explicit typed identities.""" if self._context is None: @@ -456,6 +479,8 @@ def _serialize_hosted_mcp_approvals(self) -> list[dict[str, Any]]: decision["rejection_messages"] = dict(record.rejection_messages) if record.sticky_rejection_message is not None: decision["sticky_rejection_message"] = record.sticky_rejection_message + if record.sticky_scope is not None: + decision["sticky_scope"] = record.sticky_scope serialized.append({"identity": identity_data, "decision": decision}) return serialized @@ -802,6 +827,7 @@ def to_json( raise UserError("Cannot serialize RunState: No context") approvals_dict = self._serialize_approvals() + tool_invocations = self._serialize_tool_invocations() hosted_mcp_approvals = self._serialize_hosted_mcp_approvals() model_responses = self._serialize_model_responses() original_input_serialized = self._serialize_original_input() @@ -813,6 +839,7 @@ def to_json( context_entry: dict[str, Any] = { "usage": serialize_usage(self._context.usage), "approvals": approvals_dict, + "tool_invocations": tool_invocations, "context": context_payload, # Preserve metadata so deserialization can warn when context types were erased. "context_meta": context_meta, @@ -2837,12 +2864,20 @@ async def _build_run_state_from_json( else: raise UserError("Serialized run state context must be a mapping. Please provide one.") context.usage = usage + context._restored_unbound_approval_call_ids = set() + context._allow_legacy_approval_binding_reconstruction = (schema_major, schema_minor) < (1, 15) context._rebuild_approvals(context_data.get("approvals", {})) + if (schema_major, schema_minor) >= (1, 15): + context._rebuild_tool_invocations(context_data.get("tool_invocations", {})) + else: + context._tool_invocations = {} hosted_mcp_major, hosted_mcp_minor = ( int(part) for part in _HOSTED_MCP_APPROVALS_MIN_SCHEMA_VERSION.split(".", maxsplit=1) ) if (schema_major, schema_minor) >= (hosted_mcp_major, hosted_mcp_minor): context._rebuild_hosted_mcp_approvals(context_data.get("hosted_mcp_approvals", [])) + if (schema_major, schema_minor) >= (1, 15): + context._mark_restored_unbound_approval_call_ids() serialized_tool_input = context_data.get("tool_input") if ( context_override is None @@ -3061,6 +3096,8 @@ async def _build_run_state_from_json( state._current_step = NextStepInterruption( interruptions=[item for item in interruptions if isinstance(item, ToolApprovalItem)] ) + for approval_item in state._current_step.interruptions: + context._mark_restored_unbound_pending_approval(approval_item) state._current_turn_persisted_item_count = state_json.get( "current_turn_persisted_item_count", 0 @@ -3083,9 +3120,252 @@ async def _build_run_state_from_json( sandbox_data = state_json.get("sandbox") state._sandbox = dict(sandbox_data) if isinstance(sandbox_data, Mapping) else None + _validate_completed_tool_invocations( + state, + reconstruct_legacy=(schema_major, schema_minor) < (1, 15), + ) + return state +def _validate_completed_tool_invocations( + state: RunState[Any, Agent[Any]], + *, + reconstruct_legacy: bool = False, +) -> None: + """Reconcile invocation bindings with restored calls and outputs.""" + if state._context is None: + return + from .run_internal.tool_execution import ( + is_apply_patch_name, + normalize_apply_patch_fallback_call, + ) + + completed_records = { + call_id: record + for call_id, record in state._context._tool_invocations.items() + if record.completed + } + starting_agent = state._starting_agent + assert starting_agent is not None + apply_patch_tools = [ + tool + for agent in _iter_agent_graph(starting_agent) + for tool in agent.tools + if isinstance(tool, ApplyPatchTool) + ] + resolved_tool_names_by_call_id: dict[str, str] = {} + + def collect_resolved_tool_name(run_item: RunItem) -> None: + tool_name = getattr(run_item, "tool_name", None) + call_identity = tool_invocation_call_id(run_item.raw_item) + if isinstance(tool_name, str) and tool_name and call_identity is not None: + _, call_id = call_identity + if call_id is not None: + resolved_tool_names_by_call_id.setdefault(call_id, tool_name) + + for run_item in state._generated_items: + collect_resolved_tool_name(run_item) + for run_item in state._session_items: + collect_resolved_tool_name(run_item) + if state._last_processed_response is not None: + for run_item in state._last_processed_response.new_items: + collect_resolved_tool_name(run_item) + + restored_call_occurrences: list[ + dict[ + tuple[str, str, str], + tuple[Any, FunctionToolLookupKey | None, str | None, str | None], + ] + ] = [] + restored_outputs: dict[tuple[str, str], Any] = {} + uncanonical_call_ids: set[str] = set() + + def record_raw_item( + raw_item: Any, + *, + tool_lookup_key: FunctionToolLookupKey | None = None, + tool_name: str | None = None, + invocation_role: str | None = None, + allow_handoff_alternative: bool = False, + ) -> None: + output_identity = tool_output_identity(raw_item) + if output_identity is not None: + restored_outputs.setdefault(output_identity, raw_item) + + occurrence: dict[ + tuple[str, str, str], + tuple[Any, FunctionToolLookupKey | None, str | None, str | None], + ] = {} + + call_identity = tool_invocation_call_id(raw_item) + if tool_name is None: + if call_identity is not None and call_identity[1] is not None: + tool_name = resolved_tool_names_by_call_id.get(call_identity[1]) + + def add_identity(role: str | None) -> None: + identity = tool_invocation_identity( + raw_item, + tool_lookup_key=tool_lookup_key, + tool_name=tool_name, + invocation_role=role, + ) + if identity is not None: + occurrence.setdefault(identity, (raw_item, tool_lookup_key, tool_name, role)) + + add_identity(invocation_role) + if allow_handoff_alternative and invocation_role is None: + add_identity("handoff") + raw_name = getattr(raw_item, "name", None) + if isinstance(raw_item, Mapping): + raw_name = raw_item.get("name") + if any(is_apply_patch_name(raw_name, tool) for tool in apply_patch_tools): + try: + fallback_call = normalize_apply_patch_fallback_call(raw_item) + except ModelBehaviorError: + fallback_call = None + if fallback_call is not None: + fallback_identity = tool_invocation_identity( + fallback_call, + tool_name=tool_name, + ) + if fallback_identity is not None: + occurrence.setdefault( + fallback_identity, + (fallback_call, None, tool_name, None), + ) + if occurrence: + restored_call_occurrences.append(occurrence) + elif call_identity is not None and call_identity[1] is not None: + uncanonical_call_ids.add(call_identity[1]) + + def record_run_item(run_item: RunItem) -> None: + record_raw_item( + run_item.raw_item, + tool_lookup_key=getattr(run_item, "tool_lookup_key", None), + tool_name=getattr(run_item, "tool_name", None), + invocation_role="handoff" if isinstance(run_item, HandoffCallItem) else None, + ) + + for run_item in state._generated_items: + record_run_item(run_item) + for run_item in state._session_items: + record_run_item(run_item) + if state._last_processed_response is not None: + for run_item in state._last_processed_response.new_items: + record_run_item(run_item) + for response in state._model_responses: + for raw_item in response.output: + record_raw_item(raw_item, allow_handoff_alternative=True) + if isinstance(state._original_input, list): + for raw_item in state._original_input: + record_raw_item(raw_item, allow_handoff_alternative=True) + + occurrences_by_call_id: dict[ + str, + list[ + dict[ + tuple[str, str, str], + tuple[Any, FunctionToolLookupKey | None, str | None, str | None], + ] + ], + ] = {} + for occurrence in restored_call_occurrences: + call_ids = {call_id for _, call_id, _ in occurrence} + if len(call_ids) == 1: + occurrences_by_call_id.setdefault(next(iter(call_ids)), []).append(occurrence) + + if reconstruct_legacy: + for call_id, occurrences in occurrences_by_call_id.items(): + if call_id in state._context._tool_invocations or call_id in uncanonical_call_ids: + continue + output_types = { + invocation_type + for invocation_type, output_call_id in restored_outputs + if output_call_id == call_id + } + if not output_types: + continue + common_identities = set(occurrences[0]) + for occurrence in occurrences[1:]: + common_identities.intersection_update(occurrence) + completed_identities = [ + identity for identity in common_identities if identity[0] in output_types + ] + if completed_identities: + identity = next( + ( + candidate + for candidate in completed_identities + if occurrences[0][candidate][3] is None + ), + completed_identities[0], + ) + details = next( + occurrence[identity] for occurrence in occurrences if identity in occurrence + ) + else: + identity, details = next( + candidate for occurrence in occurrences for candidate in occurrence.items() + ) + raw_item, tool_lookup_key, tool_name, invocation_role = details + invocation_type, _, _ = identity + status = state._context._tool_invocation_status( + raw_item, + tool_lookup_key=tool_lookup_key, + tool_name=tool_name, + invocation_role=invocation_role, + ) + if status is not None: + if completed_identities: + state._context._mark_tool_call_completed( + restored_outputs[(invocation_type, call_id)] + ) + else: + state._context._mark_tool_invocation_executed( + raw_item, + tool_lookup_key=tool_lookup_key, + tool_name=tool_name, + invocation_role=invocation_role, + ) + + interruptions = getattr(state._current_step, "interruptions", ()) + for approval_item in interruptions: + if isinstance(approval_item, ToolApprovalItem): + try: + state._context._restore_pending_approval_binding(approval_item) + except ModelBehaviorError: + pending_call_id = state._context._resolve_call_id(approval_item) + if pending_call_id is not None: + state._context._restored_unbound_approval_call_ids.add(pending_call_id) + + state._context._mark_restored_unbound_approval_call_ids() + + state._context._restored_unbound_approval_call_ids.update( + { + call_id + for call_id in occurrences_by_call_id + if call_id not in state._context._tool_invocations + } + | uncanonical_call_ids + ) + + for call_id, record in completed_records.items(): + expected_call = (record.invocation_type, call_id, record.fingerprint) + expected_output = (record.invocation_type, call_id) + occurrences = occurrences_by_call_id.get(call_id, []) + if ( + not occurrences + or call_id in uncanonical_call_ids + or any(expected_call not in occurrence for occurrence in occurrences) + or expected_output not in restored_outputs + ): + raise UserError( + f"RunState completed tool invocation {call_id!r} does not match a restored " + "tool call and output." + ) + + def _iter_agent_graph(initial_agent: Agent[Any]) -> Iterator[Agent[Any]]: """Yield agents reachable from the starting agent in breadth-first order.""" queue: deque[Agent[Any]] = deque([initial_agent]) @@ -3731,6 +4011,11 @@ def _resolve_agent_info( description=description, title=title, tool_origin=tool_origin, + _resolved_tool_name=( + item_data.get("tool_name") + if isinstance(item_data.get("tool_name"), str) + else None + ), ) ) diff --git a/src/agents/tool_context.py b/src/agents/tool_context.py index 1ab2dd29f3..c8fd64f6e6 100644 --- a/src/agents/tool_context.py +++ b/src/agents/tool_context.py @@ -134,7 +134,9 @@ def from_agent_context( """ # Grab the names of the RunContextWrapper's init=True fields base_values: dict[str, Any] = { - f.name: getattr(context, f.name) for f in fields(RunContextWrapper) if f.init + f.name: getattr(context, f.name) + for f in fields(RunContextWrapper) + if f.init and f.name != "_approvals" } resolved_tool_name = ( tool_name @@ -174,5 +176,6 @@ def from_agent_context( run_config=tool_run_config, **base_values, ) + context._share_tool_state_with(tool_context) set_agent_tool_state_scope(tool_context, get_agent_tool_state_scope(context)) return tool_context diff --git a/tests/mcp/test_mcp_tracing.py b/tests/mcp/test_mcp_tracing.py index 7654ab948f..2ebcf83cb1 100644 --- a/tests/mcp/test_mcp_tracing.py +++ b/tests/mcp/test_mcp_tracing.py @@ -26,7 +26,10 @@ async def test_mcp_tracing(): model.add_multiple_turn_outputs( [ # First turn: a message and tool call - [get_text_message("a_message"), get_function_tool_call("test_tool_1", "")], + [ + get_text_message("a_message"), + get_function_tool_call("test_tool_1", "", call_id="mcp_call_1"), + ], # Second turn: text message [get_text_message("done")], ] @@ -88,8 +91,8 @@ async def test_mcp_tracing(): # First turn: a message and tool call [ get_text_message("a_message"), - get_function_tool_call("non_mcp_tool", ""), - get_function_tool_call("test_tool_2", ""), + get_function_tool_call("non_mcp_tool", "", call_id="function_call_1"), + get_function_tool_call("test_tool_2", "", call_id="mcp_call_2"), ], # Second turn: text message [get_text_message("done")], diff --git a/tests/realtime/test_session.py b/tests/realtime/test_session.py index a52b89f241..435dba4647 100644 --- a/tests/realtime/test_session.py +++ b/tests/realtime/test_session.py @@ -12,6 +12,7 @@ from pydantic import BaseModel, ConfigDict import agents._debug as _debug +from agents._tool_identity import get_function_tool_lookup_key_for_tool from agents.agent import AgentBase from agents.exceptions import ModelBehaviorError, ToolTimeoutError, UserError from agents.guardrail import GuardrailFunctionOutput, OutputGuardrail @@ -2458,11 +2459,44 @@ async def test_duplicate_function_tool_call_id_is_ignored( mock_function_tool.on_invoke_tool.assert_called_once() assert len(mock_model.sent_tool_outputs) == 1 + @pytest.mark.asyncio + async def test_approved_function_tool_failure_replay_does_not_rerun( + self, mock_model, mock_agent, mock_function_tool + ): + mock_function_tool.needs_approval = True + mock_function_tool.on_invoke_tool.side_effect = RuntimeError("failed after side effect") + mock_agent.get_all_tools.return_value = [mock_function_tool] + session = RealtimeSession( + mock_model, + mock_agent, + None, + run_config={"async_tool_calls": False}, + ) + tool_call_event = RealtimeModelToolCallEvent( + name="test_function", call_id="call_failed", arguments="{}" + ) + + await session._handle_tool_call(tool_call_event) + with pytest.raises(RuntimeError, match="failed after side effect"): + await session.approve_tool_call(tool_call_event.call_id) + + with pytest.raises(ModelBehaviorError, match="already executed"): + await session._handle_tool_call(tool_call_event) + + mock_function_tool.on_invoke_tool.assert_awaited_once() + assert len(mock_model.sent_tool_outputs) == 0 + + @pytest.mark.parametrize("always", [False, True], ids=["per-call", "sticky"]) + @pytest.mark.parametrize("changed_field", ["arguments", "tool_name"]) @pytest.mark.asyncio async def test_function_tool_send_failure_retries_cached_output_without_rerun( - self, mock_agent, mock_function_tool + self, + mock_agent, + mock_function_tool, + always: bool, + changed_field: str, ): - """A post-execution send failure should retry output without rerunning the tool.""" + """An approved call should retry cached output only for the same invocation.""" class FailingToolOutputModel(MockRealtimeModel): def __init__(self): @@ -2475,29 +2509,92 @@ async def send_event(self, event): raise RuntimeError("send failed") await super().send_event(event) + mock_function_tool.needs_approval = True mock_agent.get_all_tools.return_value = [mock_function_tool] mock_model = FailingToolOutputModel() - session = RealtimeSession(mock_model, mock_agent, None) + session = RealtimeSession( + mock_model, + mock_agent, + None, + run_config={"async_tool_calls": False}, + ) tool_call_event = RealtimeModelToolCallEvent( name="test_function", call_id="call_retry_output", arguments="{}" ) + await session._handle_tool_call(tool_call_event) with pytest.raises(RuntimeError, match="send failed"): - await session._handle_tool_call(tool_call_event) + await session.approve_tool_call(tool_call_event.call_id, always=always) mock_function_tool.on_invoke_tool.assert_called_once() assert len(mock_model.sent_tool_outputs) == 0 + changed_event = RealtimeModelToolCallEvent( + name="other_function" if changed_field == "tool_name" else tool_call_event.name, + call_id=tool_call_event.call_id, + arguments=( + tool_call_event.arguments if changed_field == "tool_name" else '{"changed":true}' + ), + ) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await session._handle_tool_call(changed_event) await session._handle_tool_call(tool_call_event) mock_function_tool.on_invoke_tool.assert_called_once() assert len(mock_model.sent_tool_outputs) == 1 + @pytest.mark.asyncio + async def test_tool_end_cancellation_after_output_send_does_not_resend( + self, mock_model, mock_agent, mock_function_tool + ) -> None: + """Provider delivery commits the output before local end-event publication.""" + mock_agent.get_all_tools.return_value = [mock_function_tool] + session = RealtimeSession( + mock_model, + mock_agent, + None, + run_config={"async_tool_calls": False}, + ) + tool_call_event = RealtimeModelToolCallEvent( + name="test_function", + call_id="call_tool_end_cancelled", + arguments="{}", + ) + original_put_event_nowait = session._put_event_nowait + + def cancel_tool_end(event: Any) -> bool: + if isinstance(event, RealtimeToolEnd): + raise asyncio.CancelledError + return original_put_event_nowait(event) + + session._put_event_nowait = cancel_tool_end # type: ignore[method-assign] + with pytest.raises(asyncio.CancelledError): + await session._handle_tool_call(tool_call_event) + + invocation = session._context_wrapper._tool_invocations[tool_call_event.call_id] + assert invocation.executed is True + assert invocation.completed is True + assert tool_call_event.call_id not in session._pending_tool_outputs + mock_function_tool.on_invoke_tool.assert_called_once() + assert len(mock_model.sent_tool_outputs) == 1 + + session._put_event_nowait = original_put_event_nowait # type: ignore[method-assign] + await session._handle_tool_call(tool_call_event) + + mock_function_tool.on_invoke_tool.assert_called_once() + assert len(mock_model.sent_tool_outputs) == 1 + + @pytest.mark.parametrize("always", [False, True], ids=["per-call", "sticky"]) + @pytest.mark.parametrize("changed_field", ["arguments", "tool_name"]) @pytest.mark.asyncio async def test_async_function_tool_send_failure_retries_cached_output_without_rerun( - self, mock_agent, mock_function_tool + self, + mock_agent, + mock_function_tool, + always: bool, + changed_field: str, ): - """The async task path should keep cached outputs retryable after send failure.""" + """The async approval path should bind retries to the original invocation.""" class FailingToolOutputModel(MockRealtimeModel): def __init__(self): @@ -2510,6 +2607,7 @@ async def send_event(self, event): raise RuntimeError("send failed") await super().send_event(event) + mock_function_tool.needs_approval = True mock_agent.get_all_tools.return_value = [mock_function_tool] mock_model = FailingToolOutputModel() session = RealtimeSession(mock_model, mock_agent, None) @@ -2517,7 +2615,8 @@ async def send_event(self, event): name="test_function", call_id="call_async_retry_output", arguments="{}" ) - await session.on_event(tool_call_event) + await session._handle_tool_call(tool_call_event) + await session.approve_tool_call(tool_call_event.call_id, always=always) tool_call_tasks = list(session._tool_call_tasks) assert len(tool_call_tasks) == 1 task_results = await asyncio.gather(*tool_call_tasks, return_exceptions=True) @@ -2530,6 +2629,15 @@ async def send_event(self, event): mock_function_tool.on_invoke_tool.assert_called_once() assert len(mock_model.sent_tool_outputs) == 0 + changed_event = RealtimeModelToolCallEvent( + name="other_function" if changed_field == "tool_name" else tool_call_event.name, + call_id=tool_call_event.call_id, + arguments=( + tool_call_event.arguments if changed_field == "tool_name" else '{"changed":true}' + ), + ) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await session._handle_tool_call(changed_event) await session.on_event(tool_call_event) tool_call_tasks = list(session._tool_call_tasks) assert len(tool_call_tasks) == 1 @@ -2806,8 +2914,8 @@ async def test_handoff_validation_failure_keeps_current_agent(self, mock_model): assert session._current_agent is first_agent assert mock_model.sent_events == [] assert mock_model.sent_tool_outputs == [] - assert "call_invalid" not in session._active_tool_call_ids - assert "call_invalid" not in session._completed_tool_call_ids + assert "call_invalid" not in session._active_tool_invocations + assert not session._context_wrapper._tool_invocations["call_invalid"].completed @pytest.mark.asyncio async def test_handoff_session_update_preserves_custom_voice(self, mock_model): @@ -3024,6 +3132,43 @@ async def invoke_tool(_ctx: ToolContext[Any], _arguments: str) -> str: assert sent_output == "blocked before execution" assert start_response is True + @pytest.mark.asyncio + async def test_realtime_tool_contexts_share_session_tool_state(self, mock_model): + """Realtime guardrails and callbacks receive the session-owned tool state.""" + observed_contexts: list[ToolContext[Any]] = [] + + @tool_input_guardrail + def capture_guardrail(data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput: + observed_contexts.append(data.context) + return ToolGuardrailFunctionOutput.allow() + + async def invoke_tool(context: ToolContext[Any], _arguments: str) -> str: + observed_contexts.append(context) + return "ok" + + guarded_tool = FunctionTool( + name="test_function", + description="guarded", + params_json_schema={"type": "object", "properties": {}}, + on_invoke_tool=invoke_tool, + tool_input_guardrails=[capture_guardrail], + ) + agent = RealtimeAgent(name="agent", tools=[guarded_tool]) + session = RealtimeSession(mock_model, agent, None, run_config={"async_tool_calls": False}) + + await session._handle_tool_call( + RealtimeModelToolCallEvent( + name="test_function", + call_id="call_shared_context", + arguments="{}", + ) + ) + + assert len(observed_contexts) == 2 + for context in observed_contexts: + assert context._approvals is session._context_wrapper._approvals + assert context._tool_invocations is session._context_wrapper._tool_invocations + @pytest.mark.asyncio async def test_realtime_pending_approval_skips_tool_input_guardrails_by_default( self, mock_model @@ -3177,6 +3322,14 @@ async def test_duplicate_pending_approval_call_id_is_ignored_and_approval_runs_o await session._handle_tool_call(tool_call_event) await session._handle_tool_call(tool_call_event) + changed_event = RealtimeModelToolCallEvent( + name="test_function", + call_id=tool_call_event.call_id, + arguments='{"changed":true}', + ) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await session._handle_tool_call(changed_event) + assert list(session._pending_tool_calls) == [tool_call_event.call_id] approval_events = [] while not session._event_queue.empty(): @@ -3187,10 +3340,185 @@ async def test_duplicate_pending_approval_call_id_is_ignored_and_approval_runs_o await session.approve_tool_call(tool_call_event.call_id) await session._handle_tool_call(tool_call_event) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await session._handle_tool_call(changed_event) mock_function_tool.on_invoke_tool.assert_called_once() assert len(mock_model.sent_tool_outputs) == 1 + @pytest.mark.asyncio + async def test_changed_realtime_call_id_fails_while_dispatch_resolution_is_pending( + self, mock_model + ) -> None: + """Concurrent Realtime events compare identities before dispatch resolution awaits.""" + dispatch_started = asyncio.Event() + release_dispatch = asyncio.Event() + executed: list[str] = [] + + async def invoke_tool(_ctx: ToolContext[Any], arguments: str) -> str: + executed.append(arguments) + return "ok" + + tool = FunctionTool( + name="test_function", + description="test", + params_json_schema={"type": "object", "properties": {}}, + on_invoke_tool=invoke_tool, + ) + agent = RealtimeAgent(name="agent", tools=[tool]) + session = RealtimeSession(mock_model, agent, None, run_config={"async_tool_calls": False}) + original_resolver = session._resolve_dispatch_snapshot + + async def delayed_resolver( + resolver_agent: RealtimeAgent[Any], + dispatch_snapshot: Any, + ) -> Any: + dispatch_started.set() + await release_dispatch.wait() + return await original_resolver(resolver_agent, dispatch_snapshot) + + session._resolve_dispatch_snapshot = delayed_resolver # type: ignore[assignment] + first_event = RealtimeModelToolCallEvent( + name="test_function", + call_id="call_dispatch_pending", + arguments='{"value":"safe"}', + ) + first_task = asyncio.create_task(session._handle_tool_call(first_event)) + await dispatch_started.wait() + + changed_event = RealtimeModelToolCallEvent( + name="test_function", + call_id=first_event.call_id, + arguments='{"value":"changed"}', + ) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await session._handle_tool_call(changed_event) + + release_dispatch.set() + await first_task + + assert executed == ['{"value":"safe"}'] + + @pytest.mark.parametrize( + ("failure_stage", "error_type"), + [ + ("dispatch", RuntimeError), + ("dispatch", asyncio.CancelledError), + ("enablement", RuntimeError), + ("enablement", asyncio.CancelledError), + ], + ) + @pytest.mark.asyncio + async def test_changed_realtime_call_id_fails_after_dispatch_await_failure( + self, + mock_model: Any, + failure_stage: str, + error_type: type[BaseException], + ) -> None: + """A failed dispatch await retains the provisional invocation identity.""" + invoke_tool = AsyncMock(return_value="ok") + tool = FunctionTool( + name="test_function", + description="test", + params_json_schema={"type": "object", "properties": {}}, + on_invoke_tool=invoke_tool, + ) + agent = RealtimeAgent(name="agent", tools=[tool]) + session = RealtimeSession(mock_model, agent, None, run_config={"async_tool_calls": False}) + stage_calls = 0 + + async def fail_stage(*_args: Any, **_kwargs: Any) -> Any: + nonlocal stage_calls + stage_calls += 1 + raise error_type() + + if failure_stage == "dispatch": + session._resolve_dispatch_snapshot = fail_stage # type: ignore[method-assign] + else: + session._filter_enabled_dispatch_snapshot = fail_stage # type: ignore[method-assign] + + first_event = RealtimeModelToolCallEvent( + name="test_function", + call_id="call_failed_dispatch", + arguments='{"value":"safe"}', + ) + with pytest.raises(error_type): + await session._handle_tool_call(first_event) + + changed_event = RealtimeModelToolCallEvent( + name=first_event.name, + call_id=first_event.call_id, + arguments='{"value":"changed"}', + ) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await session._handle_tool_call(changed_event) + + assert stage_calls == 1 + invoke_tool.assert_not_called() + + @pytest.mark.asyncio + async def test_namespaced_realtime_call_rebinds_active_identity_after_resolution( + self, mock_model + ) -> None: + """Resolved routing replaces the provisional identity before later awaits.""" + approval_started = asyncio.Event() + release_approval = asyncio.Event() + executed: list[str] = [] + + async def needs_approval(_ctx: Any, _params: dict[str, Any], _call_id: str) -> bool: + approval_started.set() + await release_approval.wait() + return False + + async def invoke_tool(_ctx: ToolContext[Any], arguments: str) -> str: + executed.append(arguments) + return "ok" + + namespaced_tool = tool_namespace( + name="crm", + description="CRM tools", + tools=[ + FunctionTool( + name="lookup_account", + description="Look up an account.", + params_json_schema={"type": "object", "properties": {}}, + on_invoke_tool=invoke_tool, + needs_approval=needs_approval, + ) + ], + )[0] + agent = RealtimeAgent(name="agent", tools=[namespaced_tool]) + session = RealtimeSession(mock_model, agent, None, run_config={"async_tool_calls": False}) + event = RealtimeModelToolCallEvent( + name="lookup_account", + call_id="call_namespaced_active", + arguments="{}", + ) + first_task = asyncio.create_task(session._handle_tool_call(event)) + approval_wait = asyncio.create_task(approval_started.wait()) + done, _ = await asyncio.wait( + {first_task, approval_wait}, + return_when=asyncio.FIRST_COMPLETED, + ) + if first_task in done: + approval_wait.cancel() + await first_task + + await session._handle_tool_call(event) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await session._handle_tool_call( + RealtimeModelToolCallEvent( + name=event.name, + call_id=event.call_id, + arguments='{"changed":true}', + ) + ) + + release_approval.set() + await first_task + + assert executed == ["{}"] + @pytest.mark.asyncio async def test_approve_pending_tool_call_runs_tool( self, mock_model, mock_agent, mock_function_tool @@ -3264,7 +3592,7 @@ async def invoke_duplicate_tool(_ctx: ToolContext[Any], _arguments: str) -> str: await session._handle_tool_call(tool_call_event) await session.approve_tool_call(tool_call_event.call_id) - assert tool_call_event.call_id in session._active_tool_call_ids + assert tool_call_event.call_id in session._active_tool_invocations await session._handle_tool_call(tool_call_event, agent_snapshot=duplicate_agent) tool_call_tasks = list(session._tool_call_tasks) @@ -3477,6 +3805,60 @@ def fail_formatter(_args): assert message mock_logger.error.assert_called_once_with("%s", "Tool error formatter failed", stacklevel=3) + @pytest.mark.asyncio + async def test_cancelled_rejection_formatter_leaves_invocation_executed( + self, mock_model, mock_agent + ): + formatter_entered = asyncio.Event() + + @function_tool + def approval_tool() -> str: + return "done" + + async def blocking_formatter(_args): + formatter_entered.set() + await asyncio.Event().wait() + return "rejected" + + session = RealtimeSession( + mock_model, + mock_agent, + None, + run_config={"tool_error_formatter": blocking_formatter}, + ) + tool_call = RealtimeModelToolCallEvent( + name=approval_tool.name, + call_id="call_rejected_cancelled", + arguments="{}", + ) + canonical_call = session._build_tool_approval_item( # noqa: SLF001 + approval_tool, + tool_call, + mock_agent, + ).raw_item + lookup_key = get_function_tool_lookup_key_for_tool(approval_tool) + assert session._context_wrapper._tool_invocation_status( # noqa: SLF001 + canonical_call, + tool_lookup_key=lookup_key, + ) == (("function_call", "call_rejected_cancelled"), False, False) + + task = asyncio.create_task( + session._resolve_approval_rejection_message( # noqa: SLF001 + tool=approval_tool, + call_id=tool_call.call_id, + tool_call=canonical_call, + ) + ) + await formatter_entered.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert session._context_wrapper._tool_invocation_status( # noqa: SLF001 + canonical_call, + tool_lookup_key=lookup_key, + ) == (("function_call", "call_rejected_cancelled"), False, True) + @pytest.mark.asyncio async def test_reject_pending_tool_call_prefers_explicit_message( self, mock_model, mock_agent, mock_function_tool @@ -3568,6 +3950,270 @@ async def invoke_tool(_ctx: ToolContext[Any], _arguments: str) -> str: ] assert tool_calls == [] + @pytest.mark.asyncio + async def test_sticky_rejection_does_not_bind_duplicate_call_id_payload( + self, mock_model, mock_agent, mock_function_tool + ): + mock_function_tool.needs_approval = True + mock_agent.get_all_tools.return_value = [mock_function_tool] + session = RealtimeSession(mock_model, mock_agent, None) + first_call = RealtimeModelToolCallEvent( + name="test_function", call_id="call-sticky-reject", arguments="{}" + ) + changed_call = RealtimeModelToolCallEvent( + name="test_function", + call_id=first_call.call_id, + arguments='{"changed":true}', + ) + + await session._handle_tool_call(first_call) + await session.reject_tool_call(first_call.call_id, always=True) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await session._handle_tool_call(changed_call) + + mock_function_tool.on_invoke_tool.assert_not_called() + assert len(mock_model.sent_tool_outputs) == 1 + + @pytest.mark.asyncio + async def test_changed_completed_non_approval_call_id_fails_before_execution( + self, mock_model, mock_agent, mock_function_tool + ): + mock_agent.get_all_tools.return_value = [mock_function_tool] + session = RealtimeSession( + mock_model, + mock_agent, + None, + run_config={"async_tool_calls": False}, + ) + first_call = RealtimeModelToolCallEvent( + name="test_function", call_id="call-reused", arguments='{"value":"safe"}' + ) + changed_call = RealtimeModelToolCallEvent( + name="test_function", call_id="call-reused", arguments='{"value":"changed"}' + ) + + await session._handle_tool_call(first_call) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await session._handle_tool_call(changed_call) + + mock_function_tool.on_invoke_tool.assert_called_once() + assert len(mock_model.sent_tool_outputs) == 1 + + @pytest.mark.asyncio + async def test_changed_completed_function_call_id_fails_for_handoff_role(self, mock_model): + function_calls: list[str] = [] + + async def invoke_function(_ctx: ToolContext[Any], _arguments: str) -> str: + function_calls.append("function") + return "function result" + + function_tool = FunctionTool( + name="route", + description="Run a function.", + params_json_schema={"type": "object", "properties": {}}, + on_invoke_tool=invoke_function, + ) + function_agent = RealtimeAgent(name="function", tools=[function_tool]) + target = RealtimeAgent(name="target") + route_name = Handoff.default_tool_name(target) + function_tool.name = route_name + handoff_agent = RealtimeAgent(name="handoff", handoffs=[target]) + session = RealtimeSession( + mock_model, + function_agent, + None, + run_config={"async_tool_calls": False}, + ) + event = RealtimeModelToolCallEvent(name=route_name, call_id="shared", arguments="{}") + + await session._handle_tool_call(event) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await session._handle_tool_call(event, agent_snapshot=handoff_agent) + + assert function_calls == ["function"] + + @pytest.mark.asyncio + async def test_async_changed_completed_function_call_id_fails_for_handoff_role( + self, mock_model + ): + function_calls: list[str] = [] + + async def invoke_function(_ctx: ToolContext[Any], _arguments: str) -> str: + function_calls.append("function") + return "function result" + + function_tool = FunctionTool( + name="route", + description="Run a function.", + params_json_schema={"type": "object", "properties": {}}, + on_invoke_tool=invoke_function, + ) + function_agent = RealtimeAgent(name="function", tools=[function_tool]) + target = RealtimeAgent(name="target") + route_name = Handoff.default_tool_name(target) + function_tool.name = route_name + handoff_agent = RealtimeAgent(name="handoff", handoffs=[target]) + session = RealtimeSession(mock_model, function_agent, None) + event = RealtimeModelToolCallEvent(name=route_name, call_id="shared", arguments="{}") + + await session.on_event(event) + await asyncio.gather(*list(session._tool_call_tasks)) + session._current_agent = handoff_agent + session._current_dispatch_snapshot = None + await session.on_event(event) + results = await asyncio.gather( + *list(session._tool_call_tasks), + return_exceptions=True, + ) + + assert any(isinstance(result, ModelBehaviorError) for result in results) + assert function_calls == ["function"] + + @pytest.mark.asyncio + async def test_pending_function_output_rejects_handoff_role_reuse(self): + class FailingToolOutputModel(MockRealtimeModel): + async def send_event(self, event): + if isinstance(event, RealtimeModelSendToolOutput): + raise RuntimeError("send failed") + await super().send_event(event) + + function_callback = AsyncMock(return_value="function result") + function_tool = FunctionTool( + name="route", + description="Run a function.", + params_json_schema={"type": "object", "properties": {}}, + on_invoke_tool=function_callback, + ) + function_agent = RealtimeAgent(name="function", tools=[function_tool]) + target = RealtimeAgent(name="target") + route_name = Handoff.default_tool_name(target) + function_tool.name = route_name + handoff_agent = RealtimeAgent(name="handoff", handoffs=[target]) + session = RealtimeSession( + FailingToolOutputModel(), + function_agent, + None, + run_config={"async_tool_calls": False}, + ) + event = RealtimeModelToolCallEvent(name=route_name, call_id="shared", arguments="{}") + + with pytest.raises(RuntimeError, match="send failed"): + await session._handle_tool_call(event) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await session._handle_tool_call(event, agent_snapshot=handoff_agent) + + function_callback.assert_awaited_once() + + @pytest.mark.parametrize( + "failure", + [RuntimeError("settings failed"), asyncio.CancelledError()], + ids=["failure", "cancellation"], + ) + @pytest.mark.asyncio + async def test_exact_handoff_retry_after_settings_failure_does_not_repeat_callback( + self, + mock_model, + failure: BaseException, + ): + target = RealtimeAgent(name="target") + callback = AsyncMock(return_value=target) + route = Handoff( + tool_name="route", + tool_description="Route to target.", + input_json_schema={}, + on_invoke_handoff=callback, + input_filter=None, + agent_name=target.name, + is_enabled=True, + ) + agent = RealtimeAgent(name="source", handoffs=[route]) + session = RealtimeSession( + mock_model, + agent, + None, + run_config={"async_tool_calls": False}, + ) + event = RealtimeModelToolCallEvent(name="route", call_id="shared", arguments="{}") + + with patch.object( + session, + "_get_updated_model_settings_from_agent", + AsyncMock(side_effect=failure), + ): + with pytest.raises(type(failure)): + await session._handle_tool_call(event) + + with pytest.raises(ModelBehaviorError, match="already executed"): + await session._handle_tool_call(event) + + callback.assert_awaited_once() + + @pytest.mark.asyncio + async def test_async_exact_function_retry_after_serialization_failure_does_not_repeat_callback( + self, + mock_model, + ): + callback = AsyncMock(return_value={"result": "ok"}) + tool = FunctionTool( + name="run_function", + description="Run a function.", + params_json_schema={"type": "object", "properties": {}}, + on_invoke_tool=callback, + ) + agent = RealtimeAgent(name="agent", tools=[tool]) + session = RealtimeSession(mock_model, agent, None) + event = RealtimeModelToolCallEvent( + name=tool.name, + call_id="shared", + arguments="{}", + ) + + with patch( + "agents.realtime.session._serialize_tool_output", + side_effect=RuntimeError("serialization failed"), + ): + await session.on_event(event) + first_results = await asyncio.gather( + *list(session._tool_call_tasks), + return_exceptions=True, + ) + + await session.on_event(event) + retry_results = await asyncio.gather( + *list(session._tool_call_tasks), + return_exceptions=True, + ) + + assert any( + isinstance(result, RuntimeError) and str(result) == "serialization failed" + for result in first_results + ) + assert any(isinstance(result, ModelBehaviorError) for result in retry_results) + callback.assert_awaited_once() + + @pytest.mark.asyncio + async def test_empty_handoff_call_id_fails_before_callback(self, mock_model): + target = RealtimeAgent(name="target") + callback = AsyncMock(return_value=target) + route = Handoff( + tool_name="route", + tool_description="Route to target.", + input_json_schema={}, + on_invoke_handoff=callback, + input_filter=None, + agent_name=target.name, + is_enabled=True, + ) + agent = RealtimeAgent(name="source", handoffs=[route]) + session = RealtimeSession(mock_model, agent, None) + + with pytest.raises(ModelBehaviorError, match="non-empty string call ID"): + await session._handle_tool_call( + RealtimeModelToolCallEvent(name=route.tool_name, call_id="", arguments="{}") + ) + + callback.assert_not_awaited() + @pytest.mark.asyncio async def test_sticky_rejection_skips_dynamic_approval_checker(self, mock_model): checker_calls: list[str] = [] diff --git a/tests/sandbox/capabilities/test_apply_patch_tool.py b/tests/sandbox/capabilities/test_apply_patch_tool.py index a69d34ddf3..0d4b847c00 100644 --- a/tests/sandbox/capabilities/test_apply_patch_tool.py +++ b/tests/sandbox/capabilities/test_apply_patch_tool.py @@ -303,6 +303,7 @@ async def test_custom_tool_input_create_update_move_delete(self) -> None: await _execute_custom_tool_call( tool, context_wrapper=context_wrapper, + call_id="call_create", raw_input=("*** Begin Patch\n*** Add File: notes.txt\n+hello\n+world\n*** End Patch\n"), ) assert session.files[Path("/workspace/notes.txt")] == b"hello\nworld" @@ -310,6 +311,7 @@ async def test_custom_tool_input_create_update_move_delete(self) -> None: result = await _execute_custom_tool_call( tool, context_wrapper=context_wrapper, + call_id="call_update", raw_input=( "*** Begin Patch\n" "*** Update File: notes.txt\n" @@ -329,6 +331,7 @@ async def test_custom_tool_input_create_update_move_delete(self) -> None: await _execute_custom_tool_call( tool, context_wrapper=context_wrapper, + call_id="call_delete", raw_input="*** Begin Patch\n*** Delete File: moved.txt\n*** End Patch\n", ) assert Path("/workspace/moved.txt") not in session.files @@ -339,6 +342,7 @@ async def _execute_custom_tool_call( *, context_wrapper: RunContextWrapper[Any], raw_input: str, + call_id: str = "call_apply", ) -> Any: result = await CustomToolAction.execute( agent=Agent(name="patcher", tools=[tool]), @@ -347,7 +351,7 @@ async def _execute_custom_tool_call( tool_call={ "type": "custom_tool_call", "name": "apply_patch", - "call_id": "call_apply", + "call_id": call_id, "input": raw_input, }, ), diff --git a/tests/sandbox/test_runtime.py b/tests/sandbox/test_runtime.py index 59a64ff643..55810a5b52 100644 --- a/tests/sandbox/test_runtime.py +++ b/tests/sandbox/test_runtime.py @@ -2811,7 +2811,7 @@ def approval_tool() -> str: call_id="call_write", ) ], - [get_handoff_tool_call(second)], + [get_handoff_tool_call(second, call_id="handoff_to_second")], [ get_function_tool_call( "read_file", @@ -2825,7 +2825,7 @@ def approval_tool() -> str: second_model.add_multiple_turn_outputs( [ [get_function_tool_call("approval_tool", json.dumps({}), call_id="call_approval")], - [get_handoff_tool_call(first)], + [get_handoff_tool_call(first, call_id="handoff_to_first")], ] ) @@ -4027,7 +4027,7 @@ def approval_tool() -> str: call_id="call_write", ) ], - [get_handoff_tool_call(second)], + [get_handoff_tool_call(second, call_id="handoff_to_second")], ] ) second_model.add_multiple_turn_outputs( @@ -4059,7 +4059,9 @@ def approval_tool() -> str: ) resumed_first.handoffs = [resumed_second] resumed_second.handoffs = [resumed_first] - resumed_second_model.add_multiple_turn_outputs([[get_handoff_tool_call(resumed_first)]]) + resumed_second_model.add_multiple_turn_outputs( + [[get_handoff_tool_call(resumed_first, call_id="handoff_to_first")]] + ) resumed_first_model.add_multiple_turn_outputs( [ [ diff --git a/tests/test_agent_as_tool.py b/tests/test_agent_as_tool.py index 96012ac71a..5d2bd5254d 100644 --- a/tests/test_agent_as_tool.py +++ b/tests/test_agent_as_tool.py @@ -1564,6 +1564,8 @@ async def test_agent_as_tool_wrapped_hosted_mcp_exact_decision_resumes_run( "type": "mcp_approval_request", "id": "inner-1", "name": "lookup_account", + "server_label": "accounts", + "arguments": "{}", }, }, tool_name="lookup_account", @@ -1919,6 +1921,7 @@ def __init__(self) -> None: approved=True, rejected=[], ) + tool_context._allow_legacy_approval_binding_reconstruction = True resume_state = DummyState(nested_context) pending_result = DummyPendingResult() record_agent_tool_run_result(tool_call, cast(Any, pending_result)) diff --git a/tests/test_agent_hooks.py b/tests/test_agent_hooks.py index 750566009b..a580fb2633 100644 --- a/tests/test_agent_hooks.py +++ b/tests/test_agent_hooks.py @@ -103,7 +103,7 @@ async def test_non_streamed_agent_hooks(): model.add_multiple_turn_outputs( [ - [get_function_tool_call("some_function", json.dumps({"a": "b"}))], + [get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")], [get_text_message("done")], ] ) @@ -136,11 +136,11 @@ async def test_non_streamed_agent_hooks(): model.add_multiple_turn_outputs( [ # First turn: a tool call - [get_function_tool_call("some_function", json.dumps({"a": "b"}))], + [get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")], # Second turn: a message, another tool call, and a handoff [ get_text_message("a_message"), - get_function_tool_call("some_function", json.dumps({"a": "b"})), + get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_2"), get_handoff_tool_call(agent_1), ], # Third turn: a message and a handoff back to the orig agent @@ -210,11 +210,11 @@ async def test_streamed_agent_hooks(): model.add_multiple_turn_outputs( [ # First turn: a tool call - [get_function_tool_call("some_function", json.dumps({"a": "b"}))], + [get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")], # Second turn: a message, another tool call, and a handoff [ get_text_message("a_message"), - get_function_tool_call("some_function", json.dumps({"a": "b"})), + get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_2"), get_handoff_tool_call(agent_1), ], # Third turn: a message and a handoff back to the orig agent @@ -287,11 +287,11 @@ async def test_structured_output_non_streamed_agent_hooks(): model.add_multiple_turn_outputs( [ # First turn: a tool call - [get_function_tool_call("some_function", json.dumps({"a": "b"}))], + [get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")], # Second turn: a message, another tool call, and a handoff [ get_text_message("a_message"), - get_function_tool_call("some_function", json.dumps({"a": "b"})), + get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_2"), get_handoff_tool_call(agent_1), ], # Third turn: a message and a handoff back to the orig agent @@ -359,11 +359,11 @@ async def test_structured_output_streamed_agent_hooks(): model.add_multiple_turn_outputs( [ # First turn: a tool call - [get_function_tool_call("some_function", json.dumps({"a": "b"}))], + [get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")], # Second turn: a message, another tool call, and a handoff [ get_text_message("a_message"), - get_function_tool_call("some_function", json.dumps({"a": "b"})), + get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_2"), get_handoff_tool_call(agent_1), ], # Third turn: a message and a handoff back to the orig agent diff --git a/tests/test_agent_runner.py b/tests/test_agent_runner.py index d37e8cf608..79d53db07f 100644 --- a/tests/test_agent_runner.py +++ b/tests/test_agent_runner.py @@ -1742,13 +1742,23 @@ async def test_structured_output(): model.add_multiple_turn_outputs( [ # First turn: a tool call - [get_function_tool_call("foo", json.dumps({"bar": "baz"}))], + [ + get_function_tool_call( + "foo", + json.dumps({"bar": "baz"}), + call_id="call_foo", + ) + ], # Second turn: a message and a handoff [get_text_message("a_message"), get_handoff_tool_call(agent_1)], # Third turn: tool call with preamble message [ get_text_message(json.dumps(Foo(bar="preamble"))), - get_function_tool_call("bar", json.dumps({"bar": "baz"})), + get_function_tool_call( + "bar", + json.dumps({"bar": "baz"}), + call_id="call_bar", + ), ], # Fourth turn: structured output [get_final_output_message(json.dumps(Foo(bar="baz")))], @@ -4357,8 +4367,8 @@ async def test_tool_use_behavior_first_output(): # First turn: a message and tool call [ get_text_message("a_message"), - get_function_tool_call("test_tool_one", None), - get_function_tool_call("test_tool_two", None), + get_function_tool_call("test_tool_one", None, call_id="tool-one"), + get_function_tool_call("test_tool_two", None, call_id="tool-two"), ], ] ) @@ -4394,13 +4404,13 @@ async def test_tool_use_behavior_custom_function(): # First turn: a message and tool call [ get_text_message("a_message"), - get_function_tool_call("test_tool_two", None), + get_function_tool_call("test_tool_two", None, call_id="call-tool-two-first"), ], # Second turn: a message and tool call [ get_text_message("a_message"), - get_function_tool_call("test_tool_one", None), - get_function_tool_call("test_tool_two", None), + get_function_tool_call("test_tool_one", None, call_id="call-tool-one"), + get_function_tool_call("test_tool_two", None, call_id="call-tool-two-second"), ], ] ) @@ -4607,9 +4617,19 @@ async def test_conversation_id_only_sends_new_items_multi_turn(): model.add_multiple_turn_outputs( [ # First turn: a message and tool call - [get_text_message("a_message"), get_function_tool_call("test_func", '{"arg": "foo"}')], + [ + get_text_message("a_message"), + get_function_tool_call( + "test_func", '{"arg": "foo"}', call_id="call-test-func-first" + ), + ], # Second turn: another message and tool call - [get_text_message("b_message"), get_function_tool_call("test_func", '{"arg": "bar"}')], + [ + get_text_message("b_message"), + get_function_tool_call( + "test_func", '{"arg": "bar"}', call_id="call-test-func-second" + ), + ], # Third turn: final text message [get_text_message("done")], ] @@ -4657,9 +4677,19 @@ async def test_conversation_id_only_sends_new_items_multi_turn_streamed(): model.add_multiple_turn_outputs( [ # First turn: a message and tool call - [get_text_message("a_message"), get_function_tool_call("test_func", '{"arg": "foo"}')], + [ + get_text_message("a_message"), + get_function_tool_call( + "test_func", '{"arg": "foo"}', call_id="call-test-func-first" + ), + ], # Second turn: another message and tool call - [get_text_message("b_message"), get_function_tool_call("test_func", '{"arg": "bar"}')], + [ + get_text_message("b_message"), + get_function_tool_call( + "test_func", '{"arg": "bar"}', call_id="call-test-func-second" + ), + ], # Third turn: final text message [get_text_message("done")], ] @@ -5402,8 +5432,8 @@ async def add_tool() -> str: model.add_multiple_turn_outputs( [ - [get_function_tool_call("add_tool", json.dumps({}))], - [get_function_tool_call("tool2", json.dumps({}))], + [get_function_tool_call("add_tool", json.dumps({}), call_id="call-add-tool")], + [get_function_tool_call("tool2", json.dumps({}), call_id="call-tool-two")], [get_text_message("done")], ] ) @@ -5730,6 +5760,44 @@ async def test_tool() -> str: assert not tool_called # Tool should not have been executed +@pytest.mark.asyncio +async def test_execute_approved_tools_rejects_changed_pending_invocation() -> None: + """A decision for one payload must not authorize a changed interruption.""" + tool_called = False + + async def test_tool(value: str) -> str: + nonlocal tool_called + tool_called = True + return value + + tool = function_tool(test_tool, name_override="test_tool") + _, agent = make_model_and_agent(tools=[tool]) + approved_call = get_function_tool_call( + "test_tool", + '{"value":"safe"}', + call_id="call-shared", + ) + changed_call = get_function_tool_call( + "test_tool", + '{"value":"changed"}', + call_id="call-shared", + ) + assert isinstance(approved_call, ResponseFunctionToolCall) + assert isinstance(changed_call, ResponseFunctionToolCall) + approved_item = ToolApprovalItem(agent=agent, raw_item=approved_call) + changed_item = ToolApprovalItem(agent=agent, raw_item=changed_call) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await run_execute_approved_tools( + agent=agent, + approval_item=changed_item, + approve=None, + mutate_state=lambda state, _item: state.approve(approved_item), + ) + + assert tool_called is False + + @pytest.mark.asyncio async def test_execute_approved_tools_with_rejected_tool_uses_run_level_formatter(): """Rejected tools should prefer RunConfig tool error formatter output.""" @@ -6145,21 +6213,18 @@ async def second_lookup() -> str: @pytest.mark.asyncio -async def test_execute_approved_tools_with_missing_call_id(): - """Test _execute_approved_tools handles tool approvals without call IDs.""" +async def test_execute_approved_tools_rejects_missing_call_id(): + """Test _execute_approved_tools rejects tool approvals without call IDs.""" _, agent = make_model_and_agent() tool_call = {"type": "function_call", "name": "test_tool"} approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call) - generated_items = await run_execute_approved_tools( - agent=agent, - approval_item=approval_item, - approve=True, - ) - - assert len(generated_items) == 1 - assert isinstance(generated_items[0], ToolCallOutputItem) - assert "missing call id" in generated_items[0].output.lower() + with pytest.raises(ModelBehaviorError, match="non-empty call ID"): + await run_execute_approved_tools( + agent=agent, + approval_item=approval_item, + approve=True, + ) @pytest.mark.asyncio @@ -6171,7 +6236,12 @@ async def test_tool() -> str: tool = function_tool(test_tool, name_override="test_tool") _, agent = make_model_and_agent(tools=[tool]) - tool_call = {"type": "function_call", "name": "test_tool", "call_id": "call-1"} + tool_call = { + "type": "function_call", + "name": "test_tool", + "call_id": "call-1", + "arguments": "{}", + } approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call) generated_items = await run_execute_approved_tools( diff --git a/tests/test_agent_runner_streamed.py b/tests/test_agent_runner_streamed.py index 2a3c605817..a9491fe5c8 100644 --- a/tests/test_agent_runner_streamed.py +++ b/tests/test_agent_runner_streamed.py @@ -932,13 +932,23 @@ async def test_structured_output(): model.add_multiple_turn_outputs( [ # First turn: a tool call - [get_function_tool_call("foo", json.dumps({"bar": "baz"}))], + [ + get_function_tool_call( + "foo", + json.dumps({"bar": "baz"}), + call_id="call_foo", + ) + ], # Second turn: a message and a handoff [get_text_message("a_message"), get_handoff_tool_call(agent_1)], # Third turn: tool call with preamble message [ get_text_message(json.dumps(Foo(bar="preamble"))), - get_function_tool_call("bar", json.dumps({"bar": "baz"})), + get_function_tool_call( + "bar", + json.dumps({"bar": "baz"}), + call_id="call_bar", + ), ], # Fourth turn: structured output [get_final_output_message(json.dumps(Foo(bar="baz")))], @@ -1778,11 +1788,23 @@ async def test_streaming_events(): model.add_multiple_turn_outputs( [ # First turn: a tool call - [get_function_tool_call("foo", json.dumps({"bar": "baz"}))], + [ + get_function_tool_call( + "foo", + json.dumps({"bar": "baz"}), + call_id="call_foo", + ) + ], # Second turn: a message and a handoff [get_text_message("a_message"), get_handoff_tool_call(agent_1)], # Third turn: tool call - [get_function_tool_call("bar", json.dumps({"bar": "baz"}))], + [ + get_function_tool_call( + "bar", + json.dumps({"bar": "baz"}), + call_id="call_bar", + ) + ], # Fourth turn: structured output [get_final_output_message(json.dumps(Foo(bar="baz")))], ] @@ -1874,8 +1896,8 @@ async def add_tool() -> str: model.add_multiple_turn_outputs( [ - [get_function_tool_call("add_tool", json.dumps({}))], - [get_function_tool_call("tool2", json.dumps({}))], + [get_function_tool_call("add_tool", json.dumps({}), call_id="call-add-tool")], + [get_function_tool_call("tool2", json.dumps({}), call_id="call-tool-two")], [get_text_message("done")], ] ) diff --git a/tests/test_apply_patch_tool.py b/tests/test_apply_patch_tool.py index 1e66312c4a..569fb09632 100644 --- a/tests/test_apply_patch_tool.py +++ b/tests/test_apply_patch_tool.py @@ -65,6 +65,13 @@ class DummyApplyPatchCall: call_id: str operation: dict[str, Any] + def model_dump(self, **_kwargs: Any) -> dict[str, Any]: + return { + "type": self.type, + "call_id": self.call_id, + "operation": self.operation, + } + class RecordingEditor: def __init__(self) -> None: diff --git a/tests/test_example_workflows.py b/tests/test_example_workflows.py index bab6f8eb74..757478b416 100644 --- a/tests/test_example_workflows.py +++ b/tests/test_example_workflows.py @@ -1075,7 +1075,11 @@ def european_enabled(ctx: RunContextWrapper[AppContext], _agent: AgentBase) -> b orchestrator_model = FakeModel() # Build tool calls only for expected tools to avoid missing-tool errors. tool_calls = [ - get_function_tool_call(tool_name, json.dumps({"input": "Hi"})) + get_function_tool_call( + tool_name, + json.dumps({"input": "Hi"}), + call_id=f"call_{tool_name}", + ) for tool_name in sorted(expected_tools) ] orchestrator_model.add_multiple_turn_outputs([tool_calls, [get_text_message("Done")]]) @@ -1139,8 +1143,20 @@ async def test_agents_as_tools_orchestrator_runs_multiple_translations() -> None orchestrator_model = FakeModel() orchestrator_model.add_multiple_turn_outputs( [ - [get_function_tool_call("translate_to_spanish", json.dumps({"input": "Hi"}))], - [get_function_tool_call("translate_to_french", json.dumps({"input": "Hi"}))], + [ + get_function_tool_call( + "translate_to_spanish", + json.dumps({"input": "Hi"}), + call_id="translate_spanish", + ) + ], + [ + get_function_tool_call( + "translate_to_french", + json.dumps({"input": "Hi"}), + call_id="translate_french", + ) + ], [get_text_message("Summary complete")], ] ) diff --git a/tests/test_global_hooks.py b/tests/test_global_hooks.py index f4ec6dfe1f..0b4bada0c0 100644 --- a/tests/test_global_hooks.py +++ b/tests/test_global_hooks.py @@ -130,11 +130,11 @@ async def test_non_streamed_agent_hooks(): model.add_multiple_turn_outputs( [ # First turn: a tool call - [get_function_tool_call("some_function", json.dumps({"a": "b"}))], + [get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")], # Second turn: a message, another tool call, and a handoff [ get_text_message("a_message"), - get_function_tool_call("some_function", json.dumps({"a": "b"})), + get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_2"), get_handoff_tool_call(agent_1), ], # Third turn: a message and a handoff back to the orig agent @@ -207,11 +207,11 @@ async def test_streamed_agent_hooks(): model.add_multiple_turn_outputs( [ # First turn: a tool call - [get_function_tool_call("some_function", json.dumps({"a": "b"}))], + [get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")], # Second turn: a message, another tool call, and a handoff [ get_text_message("a_message"), - get_function_tool_call("some_function", json.dumps({"a": "b"})), + get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_2"), get_handoff_tool_call(agent_1), ], # Third turn: a message and a handoff back to the orig agent @@ -287,11 +287,11 @@ async def test_structured_output_non_streamed_agent_hooks(): model.add_multiple_turn_outputs( [ # First turn: a tool call - [get_function_tool_call("some_function", json.dumps({"a": "b"}))], + [get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")], # Second turn: a message, another tool call, and a handoff [ get_text_message("a_message"), - get_function_tool_call("some_function", json.dumps({"a": "b"})), + get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_2"), get_handoff_tool_call(agent_1), ], # Third turn: a message and a handoff back to the orig agent @@ -363,11 +363,11 @@ async def test_structured_output_streamed_agent_hooks(): model.add_multiple_turn_outputs( [ # First turn: a tool call - [get_function_tool_call("some_function", json.dumps({"a": "b"}))], + [get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")], # Second turn: a message, another tool call, and a handoff [ get_text_message("a_message"), - get_function_tool_call("some_function", json.dumps({"a": "b"})), + get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_2"), get_handoff_tool_call(agent_1), ], # Third turn: a message and a handoff back to the orig agent diff --git a/tests/test_hitl_error_scenarios.py b/tests/test_hitl_error_scenarios.py index 7f936d4ca0..483346c8b9 100644 --- a/tests/test_hitl_error_scenarios.py +++ b/tests/test_hitl_error_scenarios.py @@ -21,6 +21,7 @@ from agents import ( Agent, + AgentBase, ApplyPatchTool, ComputerTool, CustomTool, @@ -32,6 +33,7 @@ ToolApprovalItem, ToolExecutionConfig, function_tool, + handoff, tool_namespace, ) from agents._public_agent import set_public_agent @@ -47,6 +49,7 @@ ) from agents.lifecycle import RunHooks from agents.run import RunConfig +from agents.run_context import RunContextWrapper from agents.run_internal import run_loop from agents.run_internal.agent_bindings import bind_execution_agent, bind_public_agent from agents.run_internal.run_loop import ( @@ -56,6 +59,7 @@ ToolRunApplyPatchCall, ToolRunComputerAction, ToolRunFunction, + ToolRunHandoff, ToolRunMCPApprovalRequest, ToolRunShellCall, extract_tool_call_id, @@ -66,13 +70,16 @@ from agents.run_internal.tool_planning import ( _collect_runs_by_approval, _select_function_tool_runs_for_resume, + execute_mcp_approval_requests, ) from agents.run_state import RunState as RunStateClass from agents.tool import FunctionTool, HostedMCPTool from agents.tool_guardrails import ( ToolGuardrailFunctionOutput, ToolInputGuardrailData, + ToolOutputGuardrailData, tool_input_guardrail, + tool_output_guardrail, ) from agents.usage import Usage @@ -369,8 +376,63 @@ async def inner_hitl_tool() -> str: @pytest.mark.asyncio -async def test_nested_agent_tool_interruptions_dont_collide_on_duplicate_call_ids() -> None: - """Nested agent tool interruptions should survive duplicate outer call IDs.""" +async def test_changed_nested_parent_fails_before_tool_inventory_callbacks() -> None: + enabled_calls: list[str] = [] + + async def enabled(_context: RunContextWrapper[Any], agent: AgentBase[Any]) -> bool: + enabled_calls.append(agent.name) + return True + + @function_tool(needs_approval=True) + async def inner_hitl_tool() -> str: + return "ok" + + @function_tool(is_enabled=enabled) + async def observer() -> str: + return "unused" + + inner_model = FakeModel() + inner_model.add_multiple_turn_outputs( + [[make_function_tool_call(inner_hitl_tool.name, call_id="inner-1")]] + ) + inner_agent = Agent(name="Inner", model=inner_model, tools=[inner_hitl_tool]) + agent_tool = inner_agent.as_tool( + tool_name="inner_agent_tool", + tool_description="Inner agent tool with HITL", + needs_approval=True, + ) + outer_model = FakeModel( + initial_output=[ + make_function_tool_call( + agent_tool.name, + call_id="outer-1", + arguments='{"input":"safe"}', + ) + ] + ) + outer_agent = Agent(name="Outer", model=outer_model, tools=[agent_tool, observer]) + + first = await Runner.run(outer_agent, "start") + first_state = first.to_state() + first_state.approve(first.interruptions[0]) + second = await Runner.run(outer_agent, first_state) + assert second.interruptions[0].tool_name == inner_hitl_tool.name + + resume_state = second.to_state() + assert resume_state._last_processed_response is not None + enabled_calls.clear() + resume_state._last_processed_response.functions[0].tool_call.arguments = '{"input":"evil"}' + resume_state.approve(second.interruptions[0]) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await Runner.run(outer_agent, resume_state) + + assert enabled_calls == [] + + +@pytest.mark.asyncio +async def test_nested_agent_tool_interruptions_remain_distinct_across_outer_calls() -> None: + """Nested agent tool interruptions should survive multiple outer calls.""" @function_tool(needs_approval=True) async def inner_hitl_tool() -> str: @@ -397,10 +459,10 @@ async def inner_hitl_tool() -> str: [ [ make_function_tool_call( - agent_tool.name, call_id="outer-dup", arguments='{"input":"a"}' + agent_tool.name, call_id="outer-a", arguments='{"input":"a"}' ), make_function_tool_call( - agent_tool.name, call_id="outer-dup", arguments='{"input":"b"}' + agent_tool.name, call_id="outer-b", arguments='{"input":"b"}' ), ] ] @@ -698,8 +760,8 @@ async def test_deserialize_interruptions_preserve_mcp_tools( @pytest.mark.asyncio -async def test_hosted_mcp_approval_matches_unknown_tool_key() -> None: - """Approved hosted MCP interruptions should resume even when the tool name is missing.""" +async def test_hosted_mcp_approval_with_unknown_legacy_identity_requires_reapproval() -> None: + """Incomplete legacy MCP approvals cannot receive an authorization decision.""" agent = make_agent() context_wrapper = make_context_wrapper() @@ -711,51 +773,8 @@ async def test_hosted_mcp_approval_matches_unknown_tool_key() -> None: include_name=False, use_call_id=False, ) - context_wrapper.approve_tool(approval_item) - - class DummyMcpTool: - on_approval_request: Any = None - - processed_response = ProcessedResponse( - new_items=[], - handoffs=[], - functions=[], - computer_actions=[], - local_shell_calls=[], - shell_calls=[], - apply_patch_calls=[], - tools_used=[], - mcp_approval_requests=[ - ToolRunMCPApprovalRequest( - request_item=McpApprovalRequest( - id="mcp-123", - type="mcp_approval_request", - server_label="test_server", - arguments="{}", - name="hosted_mcp", - ), - mcp_tool=cast(Any, DummyMcpTool()), - ) - ], - interruptions=[], - ) - - result = await _resolve_interrupted_turn( - agent=agent, - original_input="test", - original_pre_step_items=[approval_item], - new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"), - processed_response=processed_response, - hooks=RunHooks(), - context_wrapper=context_wrapper, - run_config=RunConfig(), - run_state=None, - ) - - assert any( - isinstance(item, MCPApprovalResponseItem) and item.raw_item.get("approve") is True - for item in result.new_step_items - ), "Approved hosted MCP call should emit an approval response" + with pytest.raises(ModelBehaviorError, match="canonical invocation identity"): + context_wrapper.approve_tool(approval_item) @pytest.mark.asyncio @@ -1092,6 +1111,137 @@ async def get_current_timestamp() -> str: assert tool_calls == ["called"] +@pytest.mark.asyncio +@pytest.mark.parametrize("streamed", [False, True]) +async def test_nested_agent_tool_continuation_runs_outer_callbacks_once(streamed: bool) -> None: + """Nested resume re-enters only the saved agent run, not the outer callback pipeline.""" + nested_model, nested_agent = make_model_and_agent(name="nested_agent") + inner_calls: list[str] = [] + counts = { + "input_guardrail": 0, + "start": 0, + "output_guardrail": 0, + "custom_output": 0, + "extractor": 0, + "end": 0, + } + + @function_tool(needs_approval=True) + async def inner_tool() -> str: + inner_calls.append("called") + return "inner output" + + nested_agent.tools = [inner_tool] + nested_model.add_multiple_turn_outputs( + [ + [ + make_function_tool_call( + "inner_tool", + call_id=f"inner-call-{streamed}", + ) + ], + [get_text_message("nested done")], + ] + ) + + @tool_input_guardrail + def track_input(_data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput: + counts["input_guardrail"] += 1 + return ToolGuardrailFunctionOutput.allow() + + @tool_output_guardrail + def track_output(_data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput: + counts["output_guardrail"] += 1 + return ToolGuardrailFunctionOutput.allow() + + def extract_custom_data(_context: Any) -> dict[str, Any]: + counts["extractor"] += 1 + return {"nested": True} + + async def extract_custom_output(result: Any) -> str: + counts["custom_output"] += 1 + return cast(str, result.final_output) + + outer_tool = nested_agent.as_tool( + tool_name="delegate", + tool_description="Delegate to the nested agent", + custom_output_extractor=extract_custom_output, + ) + outer_tool.tool_input_guardrails = [track_input] + outer_tool.tool_output_guardrails = [track_output] + outer_tool.custom_data_extractor = extract_custom_data + + outer_model = FakeModel() + outer_model.add_multiple_turn_outputs( + [ + [ + make_function_tool_call( + "delegate", + call_id=f"outer-call-{streamed}", + arguments='{"input":"hello"}', + ) + ], + [get_text_message("outer done")], + ] + ) + outer_agent = Agent(name="outer_agent", model=outer_model, tools=[outer_tool]) + + class CountingHooks(RunHooks[Any]): + async def on_tool_start( + self, + _context: Any, + _agent: Agent[Any], + tool: Any, + ) -> None: + if tool.name == "delegate": + counts["start"] += 1 + + async def on_tool_end( + self, + _context: Any, + _agent: Agent[Any], + tool: Any, + _result: object, + ) -> None: + if tool.name == "delegate": + counts["end"] += 1 + + hooks = CountingHooks() + + async def run(input_value: Any) -> Any: + if not streamed: + return await Runner.run(outer_agent, input_value, hooks=hooks) + result = Runner.run_streamed(outer_agent, input_value, hooks=hooks) + async for _event in result.stream_events(): + pass + return result + + interrupted = await run("start") + assert interrupted.interruptions + assert counts["input_guardrail"] <= 1 + assert counts["start"] <= 1 + assert counts["output_guardrail"] == 0 + assert counts["custom_output"] == 0 + assert counts["extractor"] == 0 + assert counts["end"] == 0 + + state = interrupted.to_state() + state.approve(state.get_interruptions()[0]) + restored = await RunState.from_json(outer_agent, state.to_json()) + final = await run(restored) + + assert final.final_output == "outer done" + assert inner_calls == ["called"] + assert counts == { + "input_guardrail": 1, + "start": 1, + "output_guardrail": 1, + "custom_output": 1, + "extractor": 1, + "end": 1, + } + + @pytest.mark.asyncio async def test_resume_rebuilds_function_runs_from_pending_approvals() -> None: """Resuming with only pending approvals should reconstruct and run function calls.""" @@ -1464,6 +1614,56 @@ async def _record_rejection( assert rejections == [tool_call.call_id] +@pytest.mark.asyncio +async def test_resume_rejects_changed_handoff_under_approved_function_call_id() -> None: + """A resumed handoff must match the invocation that received approval.""" + target = Agent(name="target") + route_handoff = handoff(target, tool_name_override="route") + agent = Agent(name="agent", handoffs=[route_handoff]) + approved_call = make_function_tool_call( + "route", + call_id="call-shared", + arguments='{"destination":"safe"}', + ) + changed_call = make_function_tool_call( + "route", + call_id="call-shared", + arguments='{"destination":"safe"}', + ) + approval_item = ToolApprovalItem(agent=agent, raw_item=approved_call, tool_name="route") + context_wrapper = make_context_wrapper() + context_wrapper.approve_tool(approval_item, always_approve=True) + processed_response = ProcessedResponse( + new_items=[], + handoffs=[ToolRunHandoff(handoff=route_handoff, tool_call=changed_call)], + functions=[], + computer_actions=[], + local_shell_calls=[], + shell_calls=[], + apply_patch_calls=[], + tools_used=[], + mcp_approval_requests=[], + interruptions=[], + ) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await _resolve_interrupted_turn( + agent=agent, + original_input="resume handoff", + original_pre_step_items=[approval_item], + new_response=ModelResponse( + output=[changed_call], + usage=Usage(), + response_id="resp", + ), + processed_response=processed_response, + hooks=RunHooks(), + context_wrapper=context_wrapper, + run_config=RunConfig(), + run_state=make_state_with_interruptions(agent, [approval_item]), + ) + + @pytest.mark.parametrize("approved", [True, False], ids=["approved", "rejected"]) @pytest.mark.asyncio async def test_execute_path_prefers_decision_resolved_during_rejecting_guardrail( @@ -1568,6 +1768,53 @@ async def sensitive(value: str) -> str: ) +@pytest.mark.asyncio +async def test_resume_checkpoints_tool_output_before_tool_use_behavior_failure() -> None: + """A failed post-tool callback must leave the exact output replayable without reexecution.""" + executions: list[str] = [] + + @function_tool(needs_approval=True) + async def sensitive(value: str) -> str: + executions.append(value) + return f"ran:{value}" + + def failing_behavior(_ctx: Any, _results: Any) -> Any: + raise RuntimeError("tool use behavior failed") + + model = FakeModel() + agent = Agent( + name="agent", + model=model, + tools=[sensitive], + tool_use_behavior=failing_behavior, + ) + model.add_multiple_turn_outputs( + [ + [make_function_tool_call(sensitive.name, call_id="call-1", arguments='{"value":"x"}')], + [get_text_message("done")], + ] + ) + + first = await Runner.run(agent, "hello") + state = first.to_state() + state.approve(first.interruptions[0]) + + with pytest.raises(RuntimeError, match="tool use behavior failed"): + await Runner.run(agent, state) + + assert executions == ["x"] + assert any( + isinstance(item, ToolCallOutputItem) and item.output == "ran:x" + for item in state._generated_items + ) + + agent.tool_use_behavior = "run_llm_again" + resumed = await Runner.run(agent, state) + + assert resumed.final_output == "done" + assert executions == ["x"] + + @pytest.mark.parametrize("tool_kind", ["function", "shell", "custom", "apply_patch"]) @pytest.mark.asyncio async def test_execute_path_honors_sticky_rejection_before_checker(tool_kind: str) -> None: @@ -1711,6 +1958,120 @@ async def invoke_custom(_ctx: Any, _raw: str) -> str: assert executed == [] +@pytest.mark.parametrize("tool_kind", ["shell", "custom", "apply_patch"]) +@pytest.mark.asyncio +async def test_tool_execution_rejects_changed_approval_recorded_while_policy_waits( + tool_kind: str, +) -> None: + """A concurrent decision for changed content must not authorize the waiting call.""" + checker_started = asyncio.Event() + release_checker = asyncio.Event() + executed: list[str] = [] + context_wrapper = make_context_wrapper() + tool: Any + current_raw: Any + changed_raw: Any + execution_task: asyncio.Task[RunItem] + + async def needs_approval(_ctx: Any, _payload: Any, _call_id: str) -> bool: + checker_started.set() + await release_checker.wait() + return True + + if tool_kind == "shell": + + def execute_shell(_request: Any) -> str: + executed.append("shell") + return "should-not-run" + + tool = ShellTool(executor=execute_shell, needs_approval=needs_approval) + agent = Agent(name="agent", tools=[tool]) + current_raw = cast(dict[str, Any], make_shell_call("call-shared", commands=["safe"])) + changed_raw = cast(dict[str, Any], make_shell_call("call-shared", commands=["changed"])) + execution_task = asyncio.create_task( + ShellAction.execute( + agent=agent, + call=ToolRunShellCall(tool_call=current_raw, shell_tool=tool), + hooks=RunHooks(), + context_wrapper=context_wrapper, + config=RunConfig(), + ) + ) + elif tool_kind == "custom": + + async def invoke_custom(_ctx: Any, _raw: str) -> str: + executed.append("custom") + return "should-not-run" + + tool = CustomTool( + name="raw_editor", + description="Edit raw text.", + on_invoke_tool=invoke_custom, + format={"type": "text"}, + needs_approval=needs_approval, + ) + agent = Agent(name="agent", tools=[tool]) + current_raw = ResponseCustomToolCall( + type="custom_tool_call", + name=tool.name, + call_id="call-shared", + input="safe", + ) + changed_raw = ResponseCustomToolCall( + type="custom_tool_call", + name=tool.name, + call_id="call-shared", + input="changed", + ) + execution_task = asyncio.create_task( + CustomToolAction.execute( + agent=agent, + call=ToolRunCustom(tool_call=current_raw, custom_tool=tool), + hooks=RunHooks(), + context_wrapper=context_wrapper, + config=RunConfig(), + ) + ) + else: + editor = RecordingEditor() + tool = ApplyPatchTool(editor=editor, needs_approval=needs_approval) + agent = Agent(name="agent", tools=[tool]) + current_raw = { + "type": "apply_patch_call", + "call_id": "call-shared", + "operation": {"type": "delete_file", "path": "safe.txt"}, + } + changed_raw = { + "type": "apply_patch_call", + "call_id": "call-shared", + "operation": {"type": "delete_file", "path": "changed.txt"}, + } + execution_task = asyncio.create_task( + ApplyPatchAction.execute( + agent=agent, + call=ToolRunApplyPatchCall(tool_call=current_raw, apply_patch_tool=tool), + hooks=RunHooks(), + context_wrapper=context_wrapper, + config=RunConfig(), + ) + ) + + try: + await asyncio.wait_for(checker_started.wait(), timeout=1) + context_wrapper.approve_tool( + ToolApprovalItem(agent=agent, raw_item=changed_raw, tool_name=tool.name) + ) + release_checker.set() + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await execution_task + finally: + release_checker.set() + + assert executed == [] + if tool_kind == "apply_patch": + assert editor.operations == [] + + @pytest.mark.asyncio async def test_collect_runs_by_approval_skips_checker_when_status_resolved() -> None: """Approved/rejected shell calls must not invoke needs_approval_checker. @@ -2507,6 +2868,100 @@ async def test_resume_skips_shell_calls_with_existing_output() -> None: assert not result.new_step_items, "Shell call should not run when output already exists" +@pytest.mark.asyncio +async def test_resume_validates_changed_shell_before_sibling_approval_callback() -> None: + """Changed completed calls must fail before sibling approval callbacks run.""" + checker_calls: list[str] = [] + + async def needs_approval(_ctx: Any, _args: dict[str, Any], call_id: str) -> bool: + checker_calls.append(call_id) + return False + + @function_tool(needs_approval=needs_approval) + async def sibling_tool() -> str: + return "should-not-run" + + shell_tool = ShellTool(executor=lambda _request: "should-not-run", needs_approval=True) + enabled_calls: list[str] = [] + target = Agent(name="target") + + def handoff_is_enabled(_ctx: Any, _agent: Agent[Any]) -> bool: + enabled_calls.append("handoff") + return True + + agent = Agent( + name="agent", + tools=[sibling_tool, shell_tool], + handoffs=[handoff(target, is_enabled=handoff_is_enabled)], + ) + context_wrapper = make_context_wrapper() + approved_shell_call = cast( + dict[str, Any], + make_shell_call("call-reused", commands=["echo safe"], status="completed"), + ) + changed_shell_call = cast( + dict[str, Any], + make_shell_call("call-reused", commands=["echo changed"], status="completed"), + ) + context_wrapper.approve_tool( + ToolApprovalItem( + agent=agent, + raw_item=approved_shell_call, + tool_name=shell_tool.name, + ) + ) + processed_response = ProcessedResponse( + new_items=[], + handoffs=[], + functions=[ + ToolRunFunction( + tool_call=make_function_tool_call(sibling_tool.name, call_id="call-sibling"), + function_tool=sibling_tool, + ) + ], + computer_actions=[], + local_shell_calls=[], + shell_calls=[ + ToolRunShellCall(tool_call=changed_shell_call, shell_tool=shell_tool), + ], + apply_patch_calls=[], + tools_used=[], + mcp_approval_requests=[], + interruptions=[], + ) + original_pre_step_items = [ + ToolCallOutputItem( + agent=agent, + raw_item=cast( + dict[str, Any], + { + "type": "shell_call_output", + "call_id": "call-reused", + "status": "completed", + "output": "prior run", + }, + ), + output="prior run", + ) + ] + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await _resolve_interrupted_turn( + agent=agent, + original_input="resume run", + original_pre_step_items=cast(list[RunItem], original_pre_step_items), + new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"), + processed_response=processed_response, + hooks=RunHooks(), + context_wrapper=context_wrapper, + run_config=RunConfig(), + run_state=None, + ) + + assert checker_calls == [] + assert enabled_calls == [] + + @pytest.mark.asyncio async def test_resume_keeps_approved_shell_outputs_with_pending_interruptions() -> None: """Approved shell outputs should be emitted even when other approvals are still pending.""" @@ -2631,6 +3086,84 @@ async def test_resume_executes_pending_computer_actions() -> None: assert isinstance(result.next_step, NextStepRunAgain) +@pytest.mark.asyncio +async def test_resume_checkpoints_computer_output_before_custom_data_failure() -> None: + """A failed extractor must not make a completed computer side effect retryable.""" + + computer = TrackingComputer() + + def fail_custom_data(_context: Any) -> dict[str, Any]: + raise RuntimeError("custom data failed") + + computer_tool = ComputerTool( + computer=computer, + custom_data_extractor=fail_custom_data, + ) + _model, agent = make_model_and_agent(tools=[computer_tool]) + computer_call = ResponseComputerToolCall( + type="computer_call", + id="comp_checkpoint", + call_id="comp_checkpoint", + status="in_progress", + action=ActionScreenshot(type="screenshot"), + pending_safety_checks=[], + ) + processed_response = ProcessedResponse( + new_items=[], + handoffs=[], + functions=[], + computer_actions=[ + ToolRunComputerAction(tool_call=computer_call, computer_tool=computer_tool) + ], + local_shell_calls=[], + shell_calls=[], + apply_patch_calls=[], + tools_used=[computer_tool.name], + mcp_approval_requests=[], + interruptions=[], + ) + context_wrapper = make_context_wrapper() + run_state = make_state_with_interruptions(agent, []) + run_state._context = context_wrapper + + with pytest.raises(RuntimeError, match="custom data failed"): + await _resolve_interrupted_turn( + agent=agent, + original_input="resume computer", + original_pre_step_items=[], + new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"), + processed_response=processed_response, + hooks=RunHooks(), + context_wrapper=context_wrapper, + run_config=RunConfig(), + run_state=run_state, + ) + + checkpointed_items = list(run_state._generated_items) + assert len(checkpointed_items) == 1 + assert isinstance(checkpointed_items[0], ToolCallOutputItem) + assert checkpointed_items[0].call_id == "comp_checkpoint" + assert checkpointed_items[0].custom_data is None + + computer_tool.custom_data_extractor = None + resumed = await _resolve_interrupted_turn( + agent=agent, + original_input="resume computer", + original_pre_step_items=checkpointed_items, + new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"), + processed_response=processed_response, + hooks=RunHooks(), + context_wrapper=context_wrapper, + run_config=RunConfig(), + run_state=run_state, + ) + + assert computer.calls == ["screenshot"] + assert resumed.new_step_items == [] + assert run_state._generated_items == checkpointed_items + assert isinstance(resumed.next_step, NextStepRunAgain) + + @pytest.mark.asyncio async def test_resume_skips_computer_actions_with_existing_output() -> None: """Computer actions with persisted output should not execute again when resuming.""" @@ -3016,3 +3549,157 @@ def __init__(self) -> None: for item in result.new_step_items ), "MCP callback approvals should emit approval responses" assert isinstance(result.next_step, NextStepRunAgain) + + +@pytest.mark.asyncio +async def test_mcp_callback_exact_retry_reuses_stored_decision() -> None: + """An exact uncommitted MCP retry must not invoke the approval callback twice.""" + callback_calls: list[str] = [] + agent = make_agent() + context_wrapper = make_context_wrapper() + + class DummyMcpTool: + def on_approval_request(self, request: Any) -> dict[str, Any]: + callback_calls.append(request.data.id) + return {"approve": True, "reason": "ok"} + + approval_request = ToolRunMCPApprovalRequest( + request_item=McpApprovalRequest( + id="mcp-callback-retry", + type="mcp_approval_request", + server_label="server", + arguments="{}", + name="hosted_mcp", + ), + mcp_tool=cast(HostedMCPTool, DummyMcpTool()), + ) + + first = await execute_mcp_approval_requests( + agent=agent, + approval_requests=[approval_request], + context_wrapper=context_wrapper, + ) + second = await execute_mcp_approval_requests( + agent=agent, + approval_requests=[approval_request], + context_wrapper=context_wrapper, + ) + + assert callback_calls == ["mcp-callback-retry"] + responses = [item for item in [*first, *second] if isinstance(item, MCPApprovalResponseItem)] + assert [item.raw_item["approve"] for item in responses] == [True, True] + + +@pytest.mark.asyncio +async def test_mcp_callback_failure_is_not_retried_for_same_request() -> None: + """A callback that started without a committed response fails closed on retry.""" + callback_calls: list[str] = [] + agent = make_agent() + context_wrapper = make_context_wrapper() + + class DummyMcpTool: + def on_approval_request(self, request: Any) -> dict[str, Any]: + callback_calls.append(request.data.id) + raise RuntimeError("callback failed") + + approval_request = ToolRunMCPApprovalRequest( + request_item=McpApprovalRequest( + id="mcp-callback-failure", + type="mcp_approval_request", + server_label="server", + arguments="{}", + name="hosted_mcp", + ), + mcp_tool=cast(HostedMCPTool, DummyMcpTool()), + ) + + with pytest.raises(RuntimeError, match="callback failed"): + await execute_mcp_approval_requests( + agent=agent, + approval_requests=[approval_request], + context_wrapper=context_wrapper, + ) + with pytest.raises(ModelBehaviorError, match="already ran"): + await execute_mcp_approval_requests( + agent=agent, + approval_requests=[approval_request], + context_wrapper=context_wrapper, + ) + + assert callback_calls == ["mcp-callback-failure"] + + +@pytest.mark.asyncio +async def test_mcp_callback_exact_siblings_invoke_callback_once() -> None: + """Exact same-ID MCP siblings must share one approval callback result.""" + callback_calls: list[str] = [] + agent = make_agent() + context_wrapper = make_context_wrapper() + + class DummyMcpTool: + async def on_approval_request(self, request: Any) -> dict[str, Any]: + callback_calls.append(request.data.id) + await asyncio.sleep(0) + return {"approve": True, "reason": "ok"} + + mcp_tool = cast(HostedMCPTool, DummyMcpTool()) + approval_requests = [ + ToolRunMCPApprovalRequest( + request_item=McpApprovalRequest( + id="mcp-callback-sibling", + type="mcp_approval_request", + server_label="server", + arguments="{}", + name="hosted_mcp", + ), + mcp_tool=mcp_tool, + ) + for _ in range(2) + ] + + responses = await execute_mcp_approval_requests( + agent=agent, + approval_requests=approval_requests, + context_wrapper=context_wrapper, + ) + + assert callback_calls == ["mcp-callback-sibling"] + assert len(responses) == 1 + + +@pytest.mark.asyncio +async def test_mcp_callback_changed_same_id_siblings_fail_before_callbacks() -> None: + """Changed same-ID MCP siblings must fail before invoking approval callbacks.""" + callback_calls: list[str] = [] + agent = make_agent() + context_wrapper = make_context_wrapper() + + class DummyMcpTool: + async def on_approval_request(self, request: Any) -> dict[str, Any]: + callback_calls.append(request.data.arguments) + await asyncio.sleep(0) + return {"approve": True, "reason": "ok"} + + mcp_tool = cast(HostedMCPTool, DummyMcpTool()) + approval_requests = [ + ToolRunMCPApprovalRequest( + request_item=McpApprovalRequest( + id="mcp-callback-sibling", + type="mcp_approval_request", + server_label="server", + arguments=arguments, + name="hosted_mcp", + ), + mcp_tool=mcp_tool, + ) + for arguments in ('{"q":1}', '{"q":2}') + ] + + with pytest.raises(ModelBehaviorError, match="reused an approval-gated tool call ID"): + await execute_mcp_approval_requests( + agent=agent, + approval_requests=approval_requests, + context_wrapper=context_wrapper, + ) + + assert callback_calls == [] diff --git a/tests/test_max_turns.py b/tests/test_max_turns.py index 7e6de97001..e192b14e83 100644 --- a/tests/test_max_turns.py +++ b/tests/test_max_turns.py @@ -41,11 +41,11 @@ async def test_non_streamed_max_turns(): model.add_multiple_turn_outputs( [ - [get_text_message("1"), get_function_tool_call("some_function", func_output)], - [get_text_message("2"), get_function_tool_call("some_function", func_output)], - [get_text_message("3"), get_function_tool_call("some_function", func_output)], - [get_text_message("4"), get_function_tool_call("some_function", func_output)], - [get_text_message("5"), get_function_tool_call("some_function", func_output)], + [get_text_message("1"), get_function_tool_call("some_function", func_output, "1")], + [get_text_message("2"), get_function_tool_call("some_function", func_output, "2")], + [get_text_message("3"), get_function_tool_call("some_function", func_output, "3")], + [get_text_message("4"), get_function_tool_call("some_function", func_output, "4")], + [get_text_message("5"), get_function_tool_call("some_function", func_output, "5")], ] ) with pytest.raises(MaxTurnsExceeded): @@ -65,10 +65,10 @@ async def test_non_streamed_max_turns_none_disables_limit(): model.add_multiple_turn_outputs( [ - [get_text_message("1"), get_function_tool_call("some_function", func_output)], - [get_text_message("2"), get_function_tool_call("some_function", func_output)], - [get_text_message("3"), get_function_tool_call("some_function", func_output)], - [get_text_message("4"), get_function_tool_call("some_function", func_output)], + [get_text_message("1"), get_function_tool_call("some_function", func_output, "1")], + [get_text_message("2"), get_function_tool_call("some_function", func_output, "2")], + [get_text_message("3"), get_function_tool_call("some_function", func_output, "3")], + [get_text_message("4"), get_function_tool_call("some_function", func_output, "4")], [get_text_message("done")], ] ) @@ -93,23 +93,23 @@ async def test_streamed_max_turns(): [ [ get_text_message("1"), - get_function_tool_call("some_function", func_output), + get_function_tool_call("some_function", func_output, "1"), ], [ get_text_message("2"), - get_function_tool_call("some_function", func_output), + get_function_tool_call("some_function", func_output, "2"), ], [ get_text_message("3"), - get_function_tool_call("some_function", func_output), + get_function_tool_call("some_function", func_output, "3"), ], [ get_text_message("4"), - get_function_tool_call("some_function", func_output), + get_function_tool_call("some_function", func_output, "4"), ], [ get_text_message("5"), - get_function_tool_call("some_function", func_output), + get_function_tool_call("some_function", func_output, "5"), ], ] ) @@ -131,10 +131,10 @@ async def test_streamed_max_turns_none_disables_limit(): model.add_multiple_turn_outputs( [ - [get_text_message("1"), get_function_tool_call("some_function", func_output)], - [get_text_message("2"), get_function_tool_call("some_function", func_output)], - [get_text_message("3"), get_function_tool_call("some_function", func_output)], - [get_text_message("4"), get_function_tool_call("some_function", func_output)], + [get_text_message("1"), get_function_tool_call("some_function", func_output, "1")], + [get_text_message("2"), get_function_tool_call("some_function", func_output, "2")], + [get_text_message("3"), get_function_tool_call("some_function", func_output, "3")], + [get_text_message("4"), get_function_tool_call("some_function", func_output, "4")], [get_text_message("done")], ] ) diff --git a/tests/test_responses.py b/tests/test_responses.py index 944fba596f..fbb38e4072 100644 --- a/tests/test_responses.py +++ b/tests/test_responses.py @@ -80,10 +80,14 @@ def get_function_tool_call( def get_handoff_tool_call( - to_agent: Agent[Any], override_name: str | None = None, args: str | None = None + to_agent: Agent[Any], + override_name: str | None = None, + args: str | None = None, + *, + call_id: str | None = None, ) -> ResponseOutputItem: name = override_name or Handoff.default_tool_name(to_agent) - return get_function_tool_call(name, args) + return get_function_tool_call(name, args, call_id=call_id or f"handoff_{to_agent.name}") def get_final_output_message(args: str) -> ResponseOutputItem: diff --git a/tests/test_run_context_approvals.py b/tests/test_run_context_approvals.py index 2b9df0a6ac..675852d6a2 100644 --- a/tests/test_run_context_approvals.py +++ b/tests/test_run_context_approvals.py @@ -3,7 +3,7 @@ import pytest from openai.types.responses.response_output_item import McpApprovalRequest -from agents import Agent, RunContextWrapper, ToolApprovalItem, UserError +from agents import Agent, ModelBehaviorError, RunContextWrapper, ToolApprovalItem, UserError from .utils.factories import make_tool_approval_item @@ -184,7 +184,7 @@ def test_hosted_mcp_exact_query_does_not_inherit_function_rejection_reason( context_wrapper = RunContextWrapper(context=None) function_item = make_tool_approval_item( agent, - call_id="shared-call", + call_id="function-call", name="lookup_account", ) hosted_item = _make_hosted_mcp_approval_item( @@ -321,6 +321,7 @@ def test_hosted_mcp_legacy_exact_call_decisions_remain_usable() -> None: } } ) + context_wrapper._allow_legacy_approval_binding_reconstruction = True # noqa: SLF001 assert ( context_wrapper.get_approval_status( @@ -352,7 +353,7 @@ def test_hosted_mcp_legacy_exact_call_decisions_remain_usable() -> None: "request-rejected", existing_pending=rejected_without_raw_name, ) - is False + is None ) @@ -417,35 +418,10 @@ def test_incomplete_hosted_mcp_uses_only_exact_call_decisions() -> None: is None ) - context_wrapper.approve_tool(malformed) - assert context_wrapper.is_tool_approved("lookup_account", "request-a-1") is True - assert ( - context_wrapper.get_approval_status( - "lookup_account", - "request-a-1", - existing_pending=malformed, - ) - is True - ) - - context_wrapper.reject_tool(malformed, rejection_message="exact denial") - assert context_wrapper.is_tool_approved("lookup_account", "request-a-1") is False - assert ( - context_wrapper.get_approval_status( - "lookup_account", - "request-a-1", - existing_pending=malformed, - ) - is False - ) - assert ( - context_wrapper.get_rejection_message( - "lookup_account", - "request-a-1", - existing_pending=malformed, - ) - == "exact denial" - ) + with pytest.raises(ModelBehaviorError, match="canonical invocation identity"): + context_wrapper.approve_tool(malformed) + with pytest.raises(ModelBehaviorError, match="canonical invocation identity"): + context_wrapper.reject_tool(malformed, rejection_message="exact denial") def test_hosted_mcp_decision_requires_request_id() -> None: @@ -681,6 +657,7 @@ def test_deferred_top_level_legacy_permanent_approval_key_still_restores() -> No context_wrapper._rebuild_approvals( # noqa: SLF001 {"get_weather.get_weather": {"approved": True, "rejected": []}} ) + context_wrapper._allow_legacy_approval_binding_reconstruction = True # noqa: SLF001 assert ( context_wrapper.get_approval_status( diff --git a/tests/test_run_context_wrapper.py b/tests/test_run_context_wrapper.py index 159027d1e0..6623675a19 100644 --- a/tests/test_run_context_wrapper.py +++ b/tests/test_run_context_wrapper.py @@ -28,7 +28,15 @@ def test_run_context_resolve_tool_name_and_call_id_fallbacks() -> None: def test_run_context_scopes_approvals_to_call_ids() -> None: wrapper: RunContextWrapper[dict[str, object]] = RunContextWrapper(context={}) agent = make_agent() - approval = ToolApprovalItem(agent=agent, raw_item={"type": "tool_call", "call_id": "call-1"}) + approval = ToolApprovalItem( + agent=agent, + raw_item={ + "type": "function_call", + "name": "tool_call", + "call_id": "call-1", + "arguments": "{}", + }, + ) wrapper.approve_tool(approval) assert wrapper.is_tool_approved("tool_call", "call-1") is True @@ -40,7 +48,15 @@ def test_run_context_scopes_approvals_to_call_ids() -> None: def test_run_context_scopes_rejections_to_call_ids() -> None: wrapper: RunContextWrapper[dict[str, object]] = RunContextWrapper(context={}) agent = make_agent() - approval = ToolApprovalItem(agent=agent, raw_item={"type": "tool_call", "call_id": "call-1"}) + approval = ToolApprovalItem( + agent=agent, + raw_item={ + "type": "function_call", + "name": "tool_call", + "call_id": "call-1", + "arguments": "{}", + }, + ) wrapper.reject_tool(approval) assert wrapper.is_tool_approved("tool_call", "call-1") is False @@ -52,7 +68,15 @@ def test_run_context_scopes_rejections_to_call_ids() -> None: def test_run_context_honors_global_approval_and_rejection() -> None: wrapper: RunContextWrapper[dict[str, object]] = RunContextWrapper(context={}) agent = make_agent() - approval = ToolApprovalItem(agent=agent, raw_item={"type": "tool_call", "call_id": "call-1"}) + approval = ToolApprovalItem( + agent=agent, + raw_item={ + "type": "function_call", + "name": "tool_call", + "call_id": "call-1", + "arguments": "{}", + }, + ) wrapper.approve_tool(approval, always_approve=True) assert wrapper.is_tool_approved("tool_call", "call-2") is True @@ -64,7 +88,15 @@ def test_run_context_honors_global_approval_and_rejection() -> None: def test_run_context_stores_per_call_rejection_messages() -> None: wrapper: RunContextWrapper[dict[str, object]] = RunContextWrapper(context={}) agent = make_agent() - approval = ToolApprovalItem(agent=agent, raw_item={"type": "tool_call", "call_id": "call-1"}) + approval = ToolApprovalItem( + agent=agent, + raw_item={ + "type": "function_call", + "name": "tool_call", + "call_id": "call-1", + "arguments": "{}", + }, + ) wrapper.reject_tool(approval, rejection_message="Denied by policy") @@ -75,7 +107,15 @@ def test_run_context_stores_per_call_rejection_messages() -> None: def test_run_context_stores_sticky_rejection_messages_for_always_reject() -> None: wrapper: RunContextWrapper[dict[str, object]] = RunContextWrapper(context={}) agent = make_agent() - approval = ToolApprovalItem(agent=agent, raw_item={"type": "tool_call", "call_id": "call-1"}) + approval = ToolApprovalItem( + agent=agent, + raw_item={ + "type": "function_call", + "name": "tool_call", + "call_id": "call-1", + "arguments": "{}", + }, + ) wrapper.reject_tool(approval, always_reject=True, rejection_message="") @@ -86,7 +126,15 @@ def test_run_context_stores_sticky_rejection_messages_for_always_reject() -> Non def test_run_context_clears_rejection_message_after_approval() -> None: wrapper: RunContextWrapper[dict[str, object]] = RunContextWrapper(context={}) agent = make_agent() - approval = ToolApprovalItem(agent=agent, raw_item={"type": "tool_call", "call_id": "call-1"}) + approval = ToolApprovalItem( + agent=agent, + raw_item={ + "type": "function_call", + "name": "tool_call", + "call_id": "call-1", + "arguments": "{}", + }, + ) wrapper.reject_tool(approval, rejection_message="Denied by policy") wrapper.approve_tool(approval) diff --git a/tests/test_run_hooks.py b/tests/test_run_hooks.py index c37ca2b5d0..e580651b8c 100644 --- a/tests/test_run_hooks.py +++ b/tests/test_run_hooks.py @@ -375,10 +375,13 @@ async def test_streamed_run_hooks_count_tool_and_handoff_invocations(): model.add_multiple_turn_outputs( [ [ - get_function_tool_call("some_function", json.dumps({"a": "b"})), - get_function_tool_call("some_function", json.dumps({"a": "b"})), + get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="call_1"), + get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="call_2"), + ], + [ + get_text_message("a_message"), + get_handoff_tool_call(agent_1, call_id="handoff_1"), ], - [get_text_message("a_message"), get_handoff_tool_call(agent_1)], [get_text_message("done")], ] ) diff --git a/tests/test_run_state.py b/tests/test_run_state.py index 83bbdb7c1f..8b16348d38 100644 --- a/tests/test_run_state.py +++ b/tests/test_run_state.py @@ -38,7 +38,7 @@ from openai.types.responses.tool_param import Mcp from pydantic import BaseModel -from agents import Agent, Model, ModelSettings, RunConfig, Runner, handoff, trace +from agents import Agent, Model, ModelSettings, RunConfig, RunHooks, Runner, handoff, trace from agents.computer import Computer from agents.exceptions import ModelBehaviorError, UserError from agents.guardrail import ( @@ -1618,6 +1618,906 @@ async def test_serializes_and_restores_approvals(self): assert new_state._context.is_tool_approved(tool_name="tool2", call_id="cid2") is False assert new_state._context.get_rejection_message("tool2", "cid2") is None + async def test_schema_1_13_restores_pending_approval_binding_from_interruption(self): + """A 1.13 snapshot may resume only the exact invocation that was approved.""" + agent = Agent(name="ApprovalLegacyAgent") + approved_call = make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"safe"}', + ) + approval_item = ToolApprovalItem(agent=agent, raw_item=approved_call) + state = make_state_with_interruptions(agent, [approval_item]) + state.approve(approval_item) + json_data = state.to_json() + json_data["$schemaVersion"] = "1.13" + json_data["context"].pop("tool_invocations", None) + + restored = await RunState.from_json(agent, json_data) + + assert restored._context is not None + restored_item = restored.get_interruptions()[0] + assert ( + restored._context.get_approval_status( + "tool1", + "cid1", + existing_pending=restored_item, + ) + is True + ) + changed_item = ToolApprovalItem( + agent=agent, + raw_item=make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"changed"}', + ), + ) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + restored._context.get_approval_status( + "tool1", + "cid1", + existing_pending=restored_item, + current_invocation=changed_item, + ) + + @pytest.mark.parametrize("schema_version", ["1.13", "1.14"]) + async def test_legacy_schema_sticky_approval_binds_pending_function_invocation( + self, + schema_version: str, + ): + """A legacy sticky decision cannot authorize changed resumed arguments.""" + agent = Agent(name="ApprovalLegacyAgent") + approved_call = make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"safe"}', + ) + approval_item = ToolApprovalItem(agent=agent, raw_item=approved_call) + state = make_state_with_interruptions(agent, [approval_item]) + state.approve(approval_item, always_approve=True) + json_data = state.to_json() + json_data["$schemaVersion"] = schema_version + json_data["context"].pop("tool_invocations", None) + + restored = await RunState.from_json(agent, json_data) + + assert restored._context is not None + restored_item = restored.get_interruptions()[0] + changed_item = ToolApprovalItem( + agent=agent, + raw_item=make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"changed"}', + ), + ) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + restored._context.get_approval_status( + "tool1", + "cid1", + existing_pending=restored_item, + current_invocation=changed_item, + ) + + async def test_schema_1_14_sticky_approval_binds_pending_hosted_mcp_invocation(self): + """A restored hosted MCP sticky decision binds the pending request payload.""" + agent = Agent(name="ApprovalLegacyAgent") + approval_item = ToolApprovalItem( + agent=agent, + raw_item=McpApprovalRequest( + id="request-a", + type="mcp_approval_request", + arguments='{"value":"safe"}', + name="lookup_account", + server_label="server-a", + ), + ) + state = make_state_with_interruptions(agent, [approval_item]) + state.approve(approval_item, always_approve=True) + json_data = state.to_json() + json_data["$schemaVersion"] = "1.14" + json_data["context"].pop("tool_invocations", None) + + restored = await RunState.from_json(agent, json_data) + + assert restored._context is not None + restored_item = restored.get_interruptions()[0] + changed_item = ToolApprovalItem( + agent=agent, + raw_item=McpApprovalRequest( + id="request-a", + type="mcp_approval_request", + arguments='{"value":"changed"}', + name="lookup_account", + server_label="server-a", + ), + ) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + restored._context.get_approval_status( + "lookup_account", + "request-a", + existing_pending=restored_item, + current_invocation=changed_item, + ) + + async def test_current_schema_does_not_reconstruct_missing_approval_binding(self): + """A malformed current snapshot must require a new approval decision.""" + agent = Agent(name="ApprovalCurrentAgent") + approved_call = make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"safe"}', + ) + approval_item = ToolApprovalItem(agent=agent, raw_item=approved_call) + state = make_state_with_interruptions(agent, [approval_item]) + state.approve(approval_item) + json_data = state.to_json() + json_data["context"].pop("tool_invocations", None) + + restored = await RunState.from_json(agent, json_data) + + assert restored._context is not None + restored_item = restored.get_interruptions()[0] + assert ( + restored._context.get_approval_status( + "tool1", + "cid1", + existing_pending=restored_item, + ) + is None + ) + + async def test_current_schema_sticky_approval_requires_restored_pending_binding(self): + """A malformed sticky snapshot cannot treat a resumed call ID as fresh.""" + agent = Agent(name="ApprovalCurrentAgent") + approved_call = make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"safe"}', + ) + approval_item = ToolApprovalItem(agent=agent, raw_item=approved_call) + state = make_state_with_interruptions(agent, [approval_item]) + state.approve(approval_item, always_approve=True) + json_data = state.to_json() + json_data["context"].pop("tool_invocations", None) + + restored = await RunState.from_json(agent, json_data) + + assert restored._context is not None + restored_item = restored.get_interruptions()[0] + assert ( + restored._context.get_approval_status( + "tool1", + "cid1", + existing_pending=restored_item, + ) + is None + ) + changed_item = ToolApprovalItem( + agent=agent, + raw_item=make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"changed"}', + ), + ) + assert ( + restored._context.get_approval_status( + "tool1", + "cid1", + existing_pending=restored_item, + current_invocation=changed_item, + ) + is None + ) + fresh_item = ToolApprovalItem( + agent=agent, + raw_item=make_function_tool_call( + "tool1", + call_id="cid-fresh", + arguments='{"value":"fresh"}', + ), + ) + assert ( + restored._context.get_approval_status( + "tool1", + "cid-fresh", + current_invocation=fresh_item, + ) + is True + ) + + tool_context = ToolContext.from_agent_context( + restored._context, + tool_call_id="cid1", + tool_call=approved_call, + ) + assert ( + tool_context.get_approval_status( + "tool1", + "cid1", + existing_pending=restored_item, + ) + is None + ) + + hook_statuses: list[bool | None] = [] + + class ApprovalProbeHooks(RunHooks[Any]): + async def on_agent_start(self, context: Any, _agent: Agent[Any]) -> None: + hook_statuses.append( + context.get_approval_status( + "tool1", + "cid1", + existing_pending=restored_item, + ) + ) + + probe_agent = Agent( + name="ApprovalProbeAgent", + model=FakeModel(initial_output=[get_text_message("done")]), + ) + await Runner.run( + probe_agent, + "probe approval state", + context=restored._context, + hooks=ApprovalProbeHooks(), + ) + + assert hook_statuses == [None] + assert "cid1" not in restored._context._tool_invocations + + @pytest.mark.parametrize( + ("field", "value"), + [ + ("type", "unknown_tool_call"), + ("approval_scope", "not-a-digest"), + ("fingerprint", 123), + ("fingerprint", "A" * 64), + ], + ) + async def test_current_schema_rejects_malformed_tool_invocation_ledger( + self, + field: str, + value: Any, + ): + """Current snapshots fail closed when canonical invocation data is malformed.""" + agent = Agent(name="ApprovalCurrentAgent") + approved_call = make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"safe"}', + ) + approval_item = ToolApprovalItem(agent=agent, raw_item=approved_call) + state = make_state_with_interruptions(agent, [approval_item]) + state.approve(approval_item) + json_data = state.to_json() + json_data["context"]["tool_invocations"]["cid1"][field] = value + + with pytest.raises(UserError, match="invalid lifecycle data"): + await RunState.from_json(agent, json_data) + + @pytest.mark.parametrize("missing_field", ["executed", "completed"]) + async def test_current_schema_requires_tool_invocation_lifecycle_fields( + self, + missing_field: str, + ): + """Current snapshots must preserve explicit monotonic lifecycle evidence.""" + agent = Agent(name="ApprovalCurrentAgent") + approved_call = make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"safe"}', + ) + approval_item = ToolApprovalItem(agent=agent, raw_item=approved_call) + state = make_state_with_interruptions(agent, [approval_item]) + state.approve(approval_item) + json_data = state.to_json() + invocation = json_data["context"]["tool_invocations"]["cid1"] + invocation["executed"] = True + invocation["completed"] = False + del invocation[missing_field] + + with pytest.raises(UserError, match="invalid lifecycle data"): + await RunState.from_json(agent, json_data) + + async def test_current_schema_rejects_null_tool_invocation_ledger(self): + """A present current-schema ledger must be a mapping.""" + agent = Agent(name="ApprovalCurrentAgent") + state = make_state(agent, context=RunContextWrapper(context=None)) + json_data = state.to_json() + json_data["context"]["tool_invocations"] = None + + with pytest.raises(UserError, match="tool_invocations must be a mapping"): + await RunState.from_json(agent, json_data) + + async def test_output_item_id_does_not_complete_unrelated_invocation(self): + """Only an output call_id can commit a tool invocation.""" + context: RunContextWrapper[Any] = RunContextWrapper(context=None) + approved_call = make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"safe"}', + ) + context._tool_invocation_status(approved_call) + + context._mark_tool_call_completed( + { + "type": "function_call_output", + "call_id": "", + "id": "cid1", + "output": "forged", + } + ) + + assert context._tool_invocation_status(approved_call) == ( + ("function_call", "cid1"), + False, + False, + ) + + async def test_current_schema_rejects_completed_invocation_with_only_output_item_id(self): + """An output item ID cannot satisfy completed-call reconciliation.""" + agent = Agent(name="ApprovalCurrentAgent") + approved_call = make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"safe"}', + ) + approval_item = ToolApprovalItem(agent=agent, raw_item=approved_call) + state = make_state_with_interruptions(agent, [approval_item]) + state.approve(approval_item) + json_data = state.to_json() + invocation = json_data["context"]["tool_invocations"]["cid1"] + invocation["executed"] = True + invocation["completed"] = True + json_data["original_input"] = [ + { + "type": "function_call_output", + "call_id": "", + "id": "cid1", + "output": "forged", + } + ] + + with pytest.raises(UserError, match="does not match a restored tool call and output"): + await RunState.from_json(agent, json_data) + + async def test_current_schema_rejects_completed_invocation_without_committed_output(self): + """A completed ledger entry must have a matching restored call and output.""" + agent = Agent(name="ApprovalCurrentAgent") + approved_call = make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"safe"}', + ) + approval_item = ToolApprovalItem(agent=agent, raw_item=approved_call) + state = make_state_with_interruptions(agent, [approval_item]) + state.approve(approval_item) + json_data = state.to_json() + invocation = json_data["context"]["tool_invocations"]["cid1"] + invocation["executed"] = True + invocation["completed"] = True + + with pytest.raises(UserError, match="does not match a restored tool call and output"): + await RunState.from_json(agent, json_data) + + async def test_current_schema_rejects_completed_cross_paired_same_id_invocations(self): + """A historical output cannot complete changed arguments under the same call ID.""" + agent = Agent(name="ApprovalCurrentAgent") + changed_call = make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"changed"}', + ) + approval_item = ToolApprovalItem(agent=agent, raw_item=changed_call) + state = make_state_with_interruptions(agent, [approval_item]) + state.approve(approval_item) + json_data = state.to_json() + invocation = json_data["context"]["tool_invocations"]["cid1"] + invocation["executed"] = True + invocation["completed"] = True + historical_call = make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"safe"}', + ) + json_data["original_input"] = [ + historical_call.model_dump(exclude_none=True), + { + "type": "function_call_output", + "call_id": "cid1", + "output": "safe", + }, + ] + + with pytest.raises(UserError, match="does not match a restored tool call and output"): + await RunState.from_json(agent, json_data) + + async def test_current_schema_rejects_completed_id_with_malformed_call_occurrence(self): + """A malformed same-ID occurrence invalidates completed-ledger authority.""" + agent = Agent(name="ApprovalCurrentAgent") + approved_call = make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"safe"}', + ) + approval_item = ToolApprovalItem(agent=agent, raw_item=approved_call) + state = make_state_with_interruptions(agent, [approval_item]) + state.approve(approval_item) + json_data = state.to_json() + invocation = json_data["context"]["tool_invocations"]["cid1"] + invocation["executed"] = True + invocation["completed"] = True + json_data["original_input"] = [ + approved_call.model_dump(exclude_none=True), + { + "type": "function_call", + "name": "missing", + "call_id": "cid1", + }, + { + "type": "function_call_output", + "call_id": "cid1", + "output": "safe", + }, + ] + + with pytest.raises(UserError, match="does not match a restored tool call and output"): + await RunState.from_json(agent, json_data) + + async def test_current_schema_missing_call_id_cannot_create_sticky_approval(self): + """Approving a malformed current interruption must not authorize later calls.""" + agent = Agent(name="ApprovalCurrentAgent") + approval_item = ToolApprovalItem( + agent=agent, + raw_item={ + "type": "function_call", + "name": "tool1", + "arguments": '{"value":"safe"}', + }, + ) + state = make_state_with_interruptions(agent, [approval_item]) + restored = await RunState.from_json(agent, state.to_json()) + + assert restored._context is not None + with pytest.raises(ModelBehaviorError, match="non-empty call ID"): + restored.approve(restored.get_interruptions()[0]) + + assert restored._context._approvals == {} + fresh_item = ToolApprovalItem( + agent=agent, + raw_item=make_function_tool_call( + "tool1", + call_id="cid-fresh", + arguments='{"value":"safe"}', + ), + ) + assert ( + restored._context.get_approval_status( + "tool1", + "cid-fresh", + current_invocation=fresh_item, + ) + is None + ) + + @pytest.mark.parametrize( + "raw_item", + [ + { + "type": "function_call", + "name": "tool1", + "call_id": "cid1", + }, + { + "type": "mcp_approval_request", + "name": "lookup_account", + "server_label": "server-a", + "id": "request-a", + }, + { + "type": "unknown_tool_call", + "name": "tool1", + "call_id": "cid1", + }, + { + "type": "unknown_tool_call", + "name": "tool1", + "id": "provider-id", + }, + { + "type": "mcp_approval_request", + "name": "", + "server_label": "server-a", + "arguments": "{}", + "id": "request-empty-name", + }, + { + "type": "mcp_approval_request", + "name": "lookup_account", + "server_label": None, + "arguments": "{}", + "id": "request-null-server", + }, + { + "type": "hosted_tool_call", + "call_id": "request-wrapped-empty-name", + "provider_data": { + "type": "mcp_approval_request", + "name": "", + "server_label": "server-a", + "arguments": "{}", + }, + }, + ], + ) + async def test_approval_decision_requires_canonical_invocation(self, raw_item: dict[str, Any]): + """An unbindable recognized item cannot create approval authority.""" + agent = Agent(name="ApprovalCurrentAgent") + approval_item = ToolApprovalItem(agent=agent, raw_item=raw_item) + state = make_state_with_interruptions(agent, [approval_item]) + + with pytest.raises(ModelBehaviorError, match="canonical invocation identity"): + state.approve(approval_item) + + assert state._context is not None + assert state._context._approvals == {} + + async def test_current_schema_orphaned_per_call_approval_requires_reapproval(self): + """A restored per-call decision without a ledger entry cannot bind a new payload.""" + agent = Agent(name="ApprovalCurrentAgent") + approved_call = make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"safe"}', + ) + state: RunState[Any, Agent[Any]] = make_state(agent, context=RunContextWrapper(context={})) + state.approve(ToolApprovalItem(agent=agent, raw_item=approved_call)) + serialized = state.to_json() + serialized["context"]["tool_invocations"] = {} + + restored = await RunState.from_json(agent, serialized) + + assert restored._context is not None + changed_item = ToolApprovalItem( + agent=agent, + raw_item=make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"changed"}', + ), + ) + assert ( + restored._context.get_approval_status( + "tool1", + "cid1", + current_invocation=changed_item, + ) + is None + ) + assert "cid1" not in restored._context._tool_invocations + + @pytest.mark.parametrize("schema_version", ["1.13", "1.14"]) + @pytest.mark.parametrize("arguments", ['{"value":"safe"}', '{"value":"changed"}']) + async def test_legacy_schema_orphaned_per_call_approval_requires_reapproval( + self, + schema_version: str, + arguments: str, + ): + """A legacy per-call decision without a reconstructable call is not authority.""" + agent = Agent(name="ApprovalLegacyAgent") + approved_call = make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"safe"}', + ) + state: RunState[Any, Agent[Any]] = make_state(agent, context=RunContextWrapper(context={})) + state.approve(ToolApprovalItem(agent=agent, raw_item=approved_call)) + serialized = state.to_json() + serialized["$schemaVersion"] = schema_version + serialized["context"].pop("tool_invocations", None) + + restored = await RunState.from_json(agent, serialized) + + assert restored._context is not None + current_item = ToolApprovalItem( + agent=agent, + raw_item=make_function_tool_call( + "tool1", + call_id="cid1", + arguments=arguments, + ), + ) + assert ( + restored._context.get_approval_status( + "tool1", + "cid1", + current_invocation=current_item, + ) + is None + ) + assert "cid1" not in restored._context._tool_invocations + + restored.approve(current_item) + + assert ( + restored._context.get_approval_status( + "tool1", + "cid1", + current_invocation=current_item, + ) + is True + ) + + async def test_current_schema_missing_ledger_marks_historical_sticky_call_unbound(self): + """A historical ID cannot borrow sticky authority when its ledger entry is missing.""" + agent = Agent(name="ApprovalCurrentAgent") + approved_call = make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"safe"}', + ) + state: RunState[Any, Agent[Any]] = make_state( + agent, + context=RunContextWrapper(context={}), + original_input=[approved_call.model_dump(exclude_none=True)], + ) + state.approve( + ToolApprovalItem(agent=agent, raw_item=approved_call), + always_approve=True, + ) + serialized = state.to_json() + serialized["context"].pop("tool_invocations") + + restored = await RunState.from_json(agent, serialized) + + assert restored._context is not None + changed_item = ToolApprovalItem( + agent=agent, + raw_item=make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"changed"}', + ), + ) + assert ( + restored._context.get_approval_status( + "tool1", + "cid1", + current_invocation=changed_item, + ) + is None + ) + fresh_item = ToolApprovalItem( + agent=agent, + raw_item=make_function_tool_call( + "tool1", + call_id="cid-fresh", + arguments='{"value":"fresh"}', + ), + ) + assert ( + restored._context.get_approval_status( + "tool1", + "cid-fresh", + current_invocation=fresh_item, + ) + is True + ) + + @pytest.mark.parametrize("missing_field", ["arguments", "server_label"]) + async def test_current_schema_unbindable_pending_approval_cannot_bind_replacement( + self, + missing_field: str, + ): + """A malformed current pending item cannot lend authority to a replacement payload.""" + agent = Agent(name="ApprovalCurrentAgent") + approval_item = ToolApprovalItem( + agent=agent, + raw_item=McpApprovalRequest( + id="request-a", + type="mcp_approval_request", + arguments='{"value":"safe"}', + name="lookup_account", + server_label="server-a", + ), + ) + state = make_state_with_interruptions(agent, [approval_item]) + assert state._context is not None + state._context._rebuild_approvals( # noqa: SLF001 + { + "lookup_account": { + "approved": ["request-a"], + "rejected": [], + } + } + ) + serialized = state.to_json() + serialized["context"].pop("tool_invocations", None) + serialized["current_step"]["data"]["interruptions"][0]["raw_item"].pop(missing_field) + + restored = await RunState.from_json(agent, serialized) + + assert restored._context is not None + restored_item = restored.get_interruptions()[0] + current_item = ToolApprovalItem( + agent=agent, + raw_item=McpApprovalRequest( + id="request-a", + type="mcp_approval_request", + arguments='{"value":"changed"}', + name="lookup_account", + server_label="server-a", + ), + ) + assert ( + restored._context.get_approval_status( + "lookup_account", + "request-a", + existing_pending=restored_item, + current_invocation=current_item, + ) + is None + ) + + async def test_current_schema_unbindable_pending_with_ledger_requires_reapproval(self): + """An unbindable pending item overrides even a matching serialized ledger entry.""" + agent = Agent(name="ApprovalCurrentAgent") + approved_item = ToolApprovalItem( + agent=agent, + raw_item=McpApprovalRequest( + id="request-a", + type="mcp_approval_request", + arguments='{"value":"safe"}', + name="lookup_account", + server_label="server-a", + ), + ) + state = make_state_with_interruptions(agent, [approved_item]) + state.approve(approved_item) + serialized = state.to_json() + serialized["current_step"]["data"]["interruptions"][0]["raw_item"].pop("arguments") + + restored = await RunState.from_json(agent, serialized) + + assert restored._context is not None + restored_pending = restored.get_interruptions()[0] + safe_item = ToolApprovalItem( + agent=agent, + raw_item=McpApprovalRequest( + id="request-a", + type="mcp_approval_request", + arguments='{"value":"safe"}', + name="lookup_account", + server_label="server-a", + ), + ) + assert ( + restored._context.get_approval_status( + "lookup_account", + "request-a", + existing_pending=restored_pending, + current_invocation=safe_item, + ) + is None + ) + + changed_item = ToolApprovalItem( + agent=agent, + raw_item=McpApprovalRequest( + id="request-a", + type="mcp_approval_request", + arguments='{"value":"changed"}', + name="lookup_account", + server_label="server-a", + ), + ) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + restored._context.approve_tool(changed_item) + + assert ( + restored._context.get_approval_status( + "lookup_account", + "request-a", + existing_pending=restored_pending, + current_invocation=safe_item, + ) + is None + ) + + restored._context.approve_tool(safe_item) + + assert ( + restored._context.get_approval_status( + "lookup_account", + "request-a", + existing_pending=restored_pending, + current_invocation=safe_item, + ) + is True + ) + + async def test_current_schema_missing_ledger_rejects_malformed_current_authority(self): + """A malformed current call cannot consume a decision whose binding is missing.""" + agent = Agent(name="ApprovalCurrentAgent") + approved_call = make_function_tool_call( + "tool1", + call_id="cid1", + arguments='{"value":"safe"}', + ) + approval_item = ToolApprovalItem(agent=agent, raw_item=approved_call) + state = make_state_with_interruptions(agent, [approval_item]) + state.approve(approval_item) + serialized = state.to_json() + serialized["context"]["tool_invocations"] = {} + + restored = await RunState.from_json(agent, serialized) + + assert restored._context is not None + restored_pending = restored.get_interruptions()[0] + malformed_current = ToolApprovalItem( + agent=agent, + raw_item=ResponseFunctionToolCall.model_construct( + type="function_call", + name="tool1", + call_id="cid1", + ), + ) + assert ( + restored._context.get_approval_status( + "tool1", + "cid1", + existing_pending=restored_pending, + current_invocation=malformed_current, + ) + is None + ) + assert restored._context._tool_invocations == {} + + @pytest.mark.parametrize("always_approve", [False, True]) + async def test_serialized_apply_patch_approval_binds_plural_operations( + self, + always_approve: bool, + ): + """Changed plural apply-patch operations cannot reuse a restored decision.""" + agent = Agent(name="ApprovalCurrentAgent") + approval_item = ToolApprovalItem( + agent=agent, + raw_item={ + "type": "apply_patch_call", + "name": "apply_patch", + "call_id": "patch-call", + "operations": [{"type": "delete_file", "path": "safe.txt"}], + }, + tool_name="apply_patch", + ) + state = make_state_with_interruptions(agent, [approval_item]) + state.approve(approval_item, always_approve=always_approve) + + restored = await RunState.from_json(agent, state.to_json()) + + assert restored._context is not None + restored_item = restored.get_interruptions()[0] + changed_item = ToolApprovalItem( + agent=agent, + raw_item={ + "type": "apply_patch_call", + "name": "apply_patch", + "call_id": "patch-call", + "operations": [{"type": "delete_file", "path": "important.txt"}], + }, + tool_name="apply_patch", + ) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + restored._context.get_approval_status( + "apply_patch", + "patch-call", + existing_pending=restored_item, + current_invocation=changed_item, + ) + async def test_serializes_and_restores_rejection_messages(self): """Test that rejection messages are preserved through serialization.""" context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={}) @@ -1705,6 +2605,40 @@ async def test_from_json_with_context_override_uses_serialized_rejection_message assert restored._context.get_rejection_message("tool2", "cid2") == "Denied by reviewer" assert restored._context.get_rejection_message("tool2", "cid3") == "Denied by reviewer" + async def test_context_override_discards_unbound_ids_from_previous_restore(self): + """Each restore rebuilds derived approval state on a reused context wrapper.""" + agent = Agent(name="ApprovalOverrideAgent") + approval_item = ToolApprovalItem( + agent=agent, + raw_item=make_function_tool_call( + "tool1", + call_id="shared", + arguments='{"value":"safe"}', + ), + ) + state = make_state_with_interruptions(agent, [approval_item]) + state.approve(approval_item) + malformed = state.to_json() + malformed["context"]["tool_invocations"] = {} + valid = state.to_json() + override_context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={}) + + await RunState.from_json(agent, malformed, context_override=override_context) + assert override_context._restored_unbound_approval_call_ids == {"shared"} + + restored = await RunState.from_json(agent, valid, context_override=override_context) + + assert restored._context is override_context + assert override_context._restored_unbound_approval_call_ids == set() + assert ( + override_context.get_approval_status( + "tool1", + "shared", + current_invocation=approval_item, + ) + is True + ) + class TestBuildAgentMap: """Test agent map building for handoff resolution.""" @@ -5587,6 +6521,7 @@ def test_supported_schema_versions_match_released_boundary(self): "1.11", "1.12", "1.13", + "1.14", CURRENT_SCHEMA_VERSION, } ) @@ -6446,8 +7381,8 @@ def test_approve_tool_with_explicit_tool_name(self): assert context.is_tool_approved(tool_name="explicit_name", call_id="call123") is True - def test_approve_tool_extracts_call_id_from_dict(self): - """Test that approve_tool extracts call_id from dict raw_item.""" + def test_approve_tool_rejects_uncanonical_hosted_call_dict(self): + """A generic hosted call cannot create approval authority from its item ID.""" context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={}) agent = Agent(name="TestAgent") # Dict with hosted tool identifiers (id instead of call_id) @@ -6458,9 +7393,10 @@ def test_approve_tool_extracts_call_id_from_dict(self): } approval_item = ToolApprovalItem(agent=agent, raw_item=raw_item) - context.approve_tool(approval_item) + with pytest.raises(ModelBehaviorError, match="canonical invocation identity"): + context.approve_tool(approval_item) - assert context.is_tool_approved(tool_name="hosted_tool", call_id="hosted_call_123") is True + assert context.is_tool_approved(tool_name="hosted_tool", call_id="hosted_call_123") is None def test_reject_tool_with_explicit_tool_name(self): """Test that reject_tool works with explicit tool_name.""" @@ -7604,24 +8540,33 @@ async def test_hosted_mcp_approval_round_trip_uses_typed_identity_records() -> N serialized = state.to_json() assert serialized["context"]["approvals"] == {} - assert serialized["context"]["hosted_mcp_approvals"] == [ + hosted_approvals = serialized["context"]["hosted_mcp_approvals"] + assert [entry["identity"] for entry in hosted_approvals] == [ { - "identity": { - "type": "server_tool", - "server_label": "server-a", - "tool_name": "lookup_account", - }, - "decision": {"approved": True, "rejected": []}, + "type": "server_tool", + "server_label": "server-a", + "tool_name": "lookup_account", }, { - "identity": { - "type": "query", - "tool_name": "lookup_account", - "request_id": "request-a", - }, - "decision": {"approved": ["request-a"], "rejected": []}, + "type": "query", + "tool_name": "lookup_account", + "request_id": "request-a", }, ] + server_decision = hosted_approvals[0]["decision"] + assert server_decision["approved"] is True + assert server_decision["rejected"] == [] + assert isinstance(server_decision["sticky_scope"], str) + server_binding = serialized["context"]["tool_invocations"]["request-a"] + assert server_binding["type"] == "mcp_approval_request" + assert server_binding["approval_scope"] == server_decision["sticky_scope"] + assert isinstance(server_binding["fingerprint"], str) + assert server_binding["executed"] is False + assert server_binding["completed"] is False + query_decision = hosted_approvals[1]["decision"] + assert query_decision["approved"] == ["request-a"] + assert query_decision["rejected"] == [] + assert "invocations" not in query_decision restored = await RunState.from_json(agent, serialized) assert restored._context is not None @@ -7647,7 +8592,7 @@ async def test_hosted_mcp_approval_round_trip_uses_typed_identity_records() -> N @pytest.mark.asyncio -async def test_incomplete_hosted_mcp_query_round_trip_preserves_exact_decision() -> None: +async def test_incomplete_hosted_mcp_query_cannot_create_approval_authority() -> None: agent = Agent(name="test") context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={}) state = make_state(agent, context=context) @@ -7662,41 +8607,11 @@ async def test_incomplete_hosted_mcp_query_round_trip_preserves_exact_decision() }, tool_name="lookup_account", ) - state.reject(approval, rejection_message="exact denial") + with pytest.raises(ModelBehaviorError, match="canonical invocation identity"): + state.reject(approval, rejection_message="exact denial") - serialized = state.to_json() - - assert serialized["context"]["hosted_mcp_approvals"] == [ - { - "identity": { - "type": "request", - "request_id": "request-a", - }, - "decision": { - "approved": [], - "rejected": ["request-a"], - "rejection_messages": {"request-a": "exact denial"}, - }, - }, - { - "identity": { - "type": "query", - "tool_name": "lookup_account", - "request_id": "request-a", - }, - "decision": { - "approved": [], - "rejected": ["request-a"], - "rejection_messages": {"request-a": "exact denial"}, - }, - }, - ] - restored = await RunState.from_json(agent, serialized) - - assert restored._context is not None - assert restored._context.is_tool_approved("lookup_account", "request-a") is False - assert restored._context.get_rejection_message("lookup_account", "request-a") == "exact denial" - assert restored._context.is_tool_approved("lookup_account", "request-next") is None + assert context._approvals == {} + assert state._serialize_hosted_mcp_approvals() == [] @pytest.mark.asyncio @@ -7794,7 +8709,7 @@ async def test_schema_1_13_ignores_typed_hosted_mcp_approval_records() -> None: @pytest.mark.asyncio -async def test_schema_1_13_hosted_mcp_exact_call_decisions_remain_usable() -> None: +async def test_schema_1_13_hosted_mcp_orphaned_call_decisions_require_reapproval() -> None: agent = Agent(name="test") context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={}) context._rebuild_approvals( # noqa: SLF001 @@ -7834,6 +8749,15 @@ async def test_schema_1_13_hosted_mcp_exact_call_decisions_remain_usable() -> No }, tool_name="lookup_account", ) + assert ( + restored._context.get_approval_status( + "lookup_account", + "request-approved", + existing_pending=approved, + ) + is None + ) + restored._context.approve_tool(approved) assert ( restored._context.get_approval_status( "lookup_account", @@ -7848,7 +8772,7 @@ async def test_schema_1_13_hosted_mcp_exact_call_decisions_remain_usable() -> No "request-rejected", existing_pending=rejected, ) - is False + is None ) assert ( restored._context.get_rejection_message( diff --git a/tests/test_run_step_execution.py b/tests/test_run_step_execution.py index 16f70c074f..329917e509 100644 --- a/tests/test_run_step_execution.py +++ b/tests/test_run_step_execution.py @@ -31,6 +31,8 @@ HostedMCPTool, MCPApprovalRequestItem, MCPApprovalResponseItem, + MCPToolApprovalFunctionResult, + MCPToolApprovalRequest, MessageOutputItem, ModelBehaviorError, ModelRefusalError, @@ -562,8 +564,8 @@ async def test_multiple_tool_calls(): response = ModelResponse( output=[ get_text_message("Hello, world!"), - get_function_tool_call("test_1"), - get_function_tool_call("test_2"), + get_function_tool_call("test_1", call_id="test-1"), + get_function_tool_call("test_2", call_id="test-2"), ], usage=Usage(), response_id=None, @@ -3187,6 +3189,51 @@ def _apply_patch_tool_approval_run() -> ToolApprovalRun: ) +@pytest.mark.parametrize("tool_kind", ["shell", "apply_patch"]) +@pytest.mark.asyncio +async def test_empty_action_call_id_fails_before_approval_callback(tool_kind: str) -> None: + approval_calls: list[str] = [] + + async def approve(_context: RunContextWrapper[Any], _item: ToolApprovalItem) -> Any: + approval_calls.append(tool_kind) + return {"approve": True} + + if tool_kind == "shell": + shell_tool = ShellTool( + executor=lambda _request: "output", + needs_approval=True, + on_approval=approve, + ) + agent = make_agent(tools=[shell_tool]) + tool_call = cast(dict[str, Any], make_shell_call("")) + tool_call["id"] = "item-shell" + processed_response = make_processed_response( + shell_calls=[ToolRunShellCall(tool_call=tool_call, shell_tool=shell_tool)] + ) + else: + apply_patch_tool = ApplyPatchTool( + editor=RecordingEditor(), + needs_approval=True, + on_approval=approve, + ) + agent = make_agent(tools=[apply_patch_tool]) + tool_call = cast(dict[str, Any], make_apply_patch_dict("")) + tool_call["id"] = "item-apply" + processed_response = make_processed_response( + apply_patch_calls=[ + ToolRunApplyPatchCall( + tool_call=tool_call, + apply_patch_tool=apply_patch_tool, + ) + ] + ) + + with pytest.raises(ModelBehaviorError, match="non-empty string call ID"): + await run_execute_with_processed_response(agent, processed_response) + + assert approval_calls == [] + + @pytest.mark.parametrize( "setup_fn", [ @@ -3314,6 +3361,80 @@ async def test_execute_tools_runs_hosted_mcp_callback_when_present(): assert not result.processed_response or not result.processed_response.interruptions +@pytest.mark.parametrize("with_callback", [False, True], ids=["manual", "callback"]) +@pytest.mark.asyncio +async def test_execute_tools_omits_completed_mcp_approval_request_replay( + with_callback: bool, +) -> None: + """A committed MCP approval replay must not emit a request or invoke its callback.""" + callback_calls = 0 + + def approve_request(_request: MCPToolApprovalRequest) -> MCPToolApprovalFunctionResult: + nonlocal callback_calls + callback_calls += 1 + return {"approve": True} + + mcp_tool = HostedMCPTool( + tool_config={ + "type": "mcp", + "server_label": "test_mcp_server", + "server_url": "https://example.com", + "require_approval": "always", + }, + on_approval_request=approve_request if with_callback else None, + ) + agent = make_agent(tools=[mcp_tool]) + request_item = McpApprovalRequest( + id="mcp-approval-replay", + type="mcp_approval_request", + server_label="test_mcp_server", + arguments='{"path":"src"}', + name="list_files", + ) + context_wrapper = make_context_wrapper() + context_wrapper.approve_tool( + ToolApprovalItem( + agent=agent, + raw_item=request_item, + tool_name="list_files", + ) + ) + context_wrapper._mark_tool_call_completed( + { + "type": "mcp_approval_response", + "approval_request_id": request_item.id, + "approve": True, + } + ) + processed_response = make_processed_response( + new_items=[MCPApprovalRequestItem(raw_item=request_item, agent=agent)], + mcp_approval_requests=[ + ToolRunMCPApprovalRequest( + request_item=request_item, + mcp_tool=mcp_tool, + ) + ], + ) + + result = await run_loop.execute_tools_and_side_effects( + bindings=_bind_agent(agent), + original_input="test", + pre_step_items=[], + new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"), + processed_response=processed_response, + output_schema=None, + hooks=RunHooks(), + context_wrapper=context_wrapper, + run_config=RunConfig(), + ) + + assert callback_calls == 0 + assert not any( + isinstance(item, MCPApprovalRequestItem | MCPApprovalResponseItem) + for item in result.new_step_items + ) + + @pytest.mark.asyncio async def test_execute_tools_uses_public_agent_for_hosted_mcp_callback_results(): """Hosted MCP callback responses should expose the public agent when execution uses a clone.""" @@ -3440,17 +3561,15 @@ def test_manual_hosted_mcp_approval_does_not_reuse_stale_pending_identity(): context_wrapper._rebuild_approvals( # noqa: SLF001 {"lookup_account": {"approved": ["shared-request"], "rejected": []}} ) + context_wrapper._allow_legacy_approval_binding_reconstruction = True # noqa: SLF001 - approved, pending = tool_execution.collect_manual_mcp_approvals( - agent=agent, - requests=[request_run], - context_wrapper=context_wrapper, - existing_pending_by_call_id={"shared-request": pending_a}, - ) - - assert approved == [] - assert len(pending) == 1 - assert pending[0].raw_item is current_b + with pytest.raises(ModelBehaviorError, match="unique call ID"): + tool_execution.collect_manual_mcp_approvals( + agent=agent, + requests=[request_run], + context_wrapper=context_wrapper, + existing_pending_by_call_id={"shared-request": pending_a}, + ) def test_hosted_mcp_approval_does_not_reuse_legacy_name_for_a_different_current_tool(): @@ -3657,7 +3776,7 @@ async def test_resolve_interrupted_turn_keeps_callback_owned_hosted_mcp_request_ assert not any(isinstance(item, ToolApprovalItem) for item in result.new_step_items) -def test_manual_hosted_mcp_approval_keeps_incomplete_exact_call_decision(): +def test_manual_hosted_mcp_approval_rejects_incomplete_exact_call_decision(): server_a = HostedMCPTool( tool_config={ "type": "mcp", @@ -3677,30 +3796,13 @@ def test_manual_hosted_mcp_approval_keeps_incomplete_exact_call_decision(): }, }, ) - current = McpApprovalRequest( - id="shared-request", - type="mcp_approval_request", - server_label="server-a", - arguments="{}", - name="lookup_account", - ) - request_run = ToolRunMCPApprovalRequest(request_item=current, mcp_tool=server_a) context_wrapper = make_context_wrapper() - context_wrapper.approve_tool(pending_unknown) - approved, pending = tool_execution.collect_manual_mcp_approvals( - agent=agent, - requests=[request_run], - context_wrapper=context_wrapper, - existing_pending_by_call_id={"shared-request": pending_unknown}, - ) - - assert pending == [] - assert len(approved) == 1 - assert approved[0].raw_item["approve"] is True + with pytest.raises(ModelBehaviorError, match="canonical invocation identity"): + context_wrapper.approve_tool(pending_unknown) -def test_manual_hosted_mcp_approval_prefers_complete_current_scoped_identity(): +def test_manual_hosted_mcp_approval_reprompts_for_partial_pending_identity(): server_a = HostedMCPTool( tool_config={ "type": "mcp", @@ -3741,6 +3843,7 @@ def test_manual_hosted_mcp_approval_prefers_complete_current_scoped_identity(): } ] ) + context_wrapper._allow_legacy_approval_binding_reconstruction = True # noqa: SLF001 approved, pending = tool_execution.collect_manual_mcp_approvals( agent=agent, @@ -3749,9 +3852,9 @@ def test_manual_hosted_mcp_approval_prefers_complete_current_scoped_identity(): existing_pending_by_call_id={"shared-request": pending_partial}, ) - assert pending == [] - assert len(approved) == 1 - assert approved[0].raw_item["approve"] is True + assert approved == [] + assert len(pending) == 1 + assert pending[0].raw_item is current def test_manual_hosted_mcp_approval_does_not_apply_legacy_exact_without_pending(): @@ -3786,8 +3889,7 @@ def test_manual_hosted_mcp_approval_does_not_apply_legacy_exact_without_pending( assert pending[0].raw_item is current -@pytest.mark.asyncio -async def test_resolve_interrupted_turn_prefers_wrapped_pending_exact_over_legacy(): +def test_resolve_interrupted_turn_rejects_incomplete_pending_decision(): server_a = HostedMCPTool( tool_config={ "type": "mcp", @@ -3808,46 +3910,12 @@ async def test_resolve_interrupted_turn_prefers_wrapped_pending_exact_over_legac }, tool_name="lookup_account", ) - current = McpApprovalRequest( - id="shared-request", - type="mcp_approval_request", - server_label="server-a", - arguments="{}", - name="lookup_account", - ) context_wrapper = make_context_wrapper() context_wrapper._rebuild_approvals( # noqa: SLF001 {"lookup_account": {"approved": ["shared-request"], "rejected": []}} ) - context_wrapper.reject_tool(pending_partial, rejection_message="new exact denial") - processed_response = make_processed_response( - new_items=[MCPApprovalRequestItem(raw_item=current, agent=agent)], - mcp_approval_requests=[ToolRunMCPApprovalRequest(request_item=current, mcp_tool=server_a)], - ) - - result = await turn_resolution.resolve_interrupted_turn( - bindings=_bind_agent(agent), - original_input="test", - original_pre_step_items=[pending_partial], - new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"), - processed_response=processed_response, - hooks=RunHooks(), - context_wrapper=context_wrapper, - run_config=RunConfig(), - ) - - assert not isinstance(result.next_step, NextStepInterruption) - responses = [ - item - for item in result.new_step_items - if isinstance(item, MCPApprovalResponseItem) - and item.raw_item.get("approval_request_id") == "shared-request" - ] - assert len(responses) == 1 - assert responses[0].raw_item["approve"] is False - assert responses[0].raw_item["reason"] == "new exact denial" - assert not any(isinstance(item, ToolApprovalItem) for item in result.pre_step_items) - assert not any(isinstance(item, ToolApprovalItem) for item in result.new_step_items) + with pytest.raises(ModelBehaviorError, match="canonical invocation identity"): + context_wrapper.reject_tool(pending_partial, rejection_message="new exact denial") def test_incomplete_current_hosted_mcp_request_does_not_reuse_scoped_pending_identity(): @@ -4022,8 +4090,12 @@ async def test_execute_handoffs_uses_public_agent_for_ignored_extra_handoffs(): public_agent = Agent(name="triage", handoffs=[first_target, second_target]) execution_agent = public_agent.clone() set_public_agent(execution_agent, public_agent) + first_call = cast(ResponseFunctionToolCall, get_handoff_tool_call(first_target)) + first_call.call_id = "handoff-alpha" + second_call = cast(ResponseFunctionToolCall, get_handoff_tool_call(second_target)) + second_call.call_id = "handoff-beta" response = ModelResponse( - output=[get_handoff_tool_call(first_target), get_handoff_tool_call(second_target)], + output=[first_call, second_call], usage=Usage(), response_id="resp", ) diff --git a/tests/test_soft_cancel.py b/tests/test_soft_cancel.py index 1ece9e3e2e..3941c85523 100644 --- a/tests/test_soft_cancel.py +++ b/tests/test_soft_cancel.py @@ -453,8 +453,8 @@ async def test_soft_cancel_with_multiple_tool_calls(): model.add_multiple_turn_outputs( [ [ - get_function_tool_call("tool1", "{}"), - get_function_tool_call("tool2", "{}"), + get_function_tool_call("tool1", "{}", call_id="tool_1"), + get_function_tool_call("tool2", "{}", call_id="tool_2"), ], [get_text_message("Both tools executed")], ] @@ -679,8 +679,8 @@ async def test_soft_cancel_with_session_and_multiple_turns(): # Setup 3 turns model.add_multiple_turn_outputs( [ - [get_function_tool_call("tool1", "{}")], - [get_function_tool_call("tool1", "{}")], + [get_function_tool_call("tool1", "{}", call_id="tool_1")], + [get_function_tool_call("tool1", "{}", call_id="tool_2")], [get_text_message("Final")], ] ) diff --git a/tests/test_stream_events.py b/tests/test_stream_events.py index 5cdc026f66..27e482d55a 100644 --- a/tests/test_stream_events.py +++ b/tests/test_stream_events.py @@ -1,5 +1,6 @@ import asyncio import time +from copy import deepcopy from typing import Any, cast import pytest @@ -32,10 +33,11 @@ from openai.types.responses.response_reasoning_item import ResponseReasoningItem, Summary from agents import Agent, HandoffCallItem, Runner, function_tool -from agents.extensions.handoff_filters import remove_all_tools -from agents.handoffs import handoff +from agents.extensions.handoff_filters import nest_handoff_history, remove_all_tools +from agents.handoffs import HandoffInputData, handoff from agents.items import ( CompactionItem, + ItemHelpers, MCPApprovalRequestItem, MCPApprovalResponseItem, MCPListToolsItem, @@ -400,35 +402,35 @@ async def test_complete_streaming_events(): assert events[8].type == "raw_response_event" assert isinstance(events[8].data, ResponseOutputItemDoneEvent) - # Event 9: ReasoningItem run_item_stream_event - assert events[9].type == "run_item_stream_event" - assert events[9].name == "reasoning_item_created" - assert isinstance(events[9].item, ReasoningItem) + # Event 9: ResponseOutputItemAddedEvent (function call) + assert events[9].type == "raw_response_event" + assert isinstance(events[9].data, ResponseOutputItemAddedEvent) - # Event 10: ResponseOutputItemAddedEvent (function call) + # Event 10: ResponseFunctionCallArgumentsDeltaEvent assert events[10].type == "raw_response_event" - assert isinstance(events[10].data, ResponseOutputItemAddedEvent) + assert isinstance(events[10].data, ResponseFunctionCallArgumentsDeltaEvent) - # Event 11: ResponseFunctionCallArgumentsDeltaEvent + # Event 11: ResponseFunctionCallArgumentsDoneEvent assert events[11].type == "raw_response_event" - assert isinstance(events[11].data, ResponseFunctionCallArgumentsDeltaEvent) + assert isinstance(events[11].data, ResponseFunctionCallArgumentsDoneEvent) - # Event 12: ResponseFunctionCallArgumentsDoneEvent + # Event 12: ResponseOutputItemDoneEvent (function call) assert events[12].type == "raw_response_event" - assert isinstance(events[12].data, ResponseFunctionCallArgumentsDoneEvent) + assert isinstance(events[12].data, ResponseOutputItemDoneEvent) - # Event 13: ResponseOutputItemDoneEvent (function call) + # Event 13: ResponseCompletedEvent (first turn ended) assert events[13].type == "raw_response_event" - assert isinstance(events[13].data, ResponseOutputItemDoneEvent) + assert isinstance(events[13].data, ResponseCompletedEvent) - # Event 14: ToolCallItem run_item_stream_event + # Event 14: ReasoningItem after the complete response passes canonical validation assert events[14].type == "run_item_stream_event" - assert events[14].name == "tool_called" - assert isinstance(events[14].item, ToolCallItem) + assert events[14].name == "reasoning_item_created" + assert isinstance(events[14].item, ReasoningItem) - # Event 15: ResponseCompletedEvent (first turn ended) - assert events[15].type == "raw_response_event" - assert isinstance(events[15].data, ResponseCompletedEvent) + # Event 15: ToolCallItem after the complete response passes canonical validation + assert events[15].type == "run_item_stream_event" + assert events[15].name == "tool_called" + assert isinstance(events[15].item, ToolCallItem) # Event 16: ToolCallOutputItem run_item_stream_event assert events[16].type == "run_item_stream_event" @@ -477,6 +479,97 @@ async def test_complete_streaming_events(): assert isinstance(events[26].item, MessageOutputItem) +@pytest.mark.asyncio +async def test_tool_call_event_preserves_order_before_later_reasoning_item() -> None: + model = FakeModel() + model.add_multiple_turn_outputs( + [ + [ + get_function_tool_call("foo", '{"arg": "value"}'), + get_reasoning_item(), + ], + [get_text_message("Final response")], + ] + ) + agent = Agent(name="TestAgent", model=model, tools=[foo]) + + result = Runner.run_streamed(agent, input="Hello") + semantic_event_names = [ + event.name + async for event in result.stream_events() + if event.type == "run_item_stream_event" + ] + + assert semantic_event_names[:3] == [ + "tool_called", + "reasoning_item_created", + "tool_output", + ] + + +@pytest.mark.asyncio +async def test_handoff_event_preserves_order_before_later_reasoning_item() -> None: + english_agent = Agent( + name="EnglishAgent", + model=FakeModel(initial_output=[get_text_message("Done")]), + ) + model = FakeModel( + initial_output=[ + get_handoff_tool_call(english_agent), + get_reasoning_item(), + ] + ) + triage_agent = Agent(name="TriageAgent", model=model, handoffs=[english_agent]) + + result = Runner.run_streamed(triage_agent, input="Start") + semantic_event_names = [ + event.name + async for event in result.stream_events() + if event.type == "run_item_stream_event" + ] + + assert semantic_event_names[:2] == [ + "handoff_requested", + "reasoning_item_created", + ] + + +@pytest.mark.asyncio +async def test_handoff_filter_copy_does_not_duplicate_streamed_model_items() -> None: + def copied_filter(data: HandoffInputData) -> HandoffInputData: + nested = nest_handoff_history(data) + return nested.clone(new_items=deepcopy(nested.new_items)) + + english_agent = Agent( + name="EnglishAgent", + model=FakeModel(initial_output=[get_text_message("Done")]), + ) + model = FakeModel( + initial_output=[ + get_text_message("Transferring"), + get_handoff_tool_call(english_agent), + ] + ) + triage_agent = Agent( + name="TriageAgent", + model=model, + handoffs=[handoff(english_agent, input_filter=copied_filter)], + ) + + result = Runner.run_streamed(triage_agent, input="Start") + item_events = [ + event async for event in result.stream_events() if event.type == "run_item_stream_event" + ] + + message_texts = [ + ItemHelpers.text_message_output(event.item) + for event in item_events + if isinstance(event.item, MessageOutputItem) + ] + assert message_texts == ["Transferring", "Done"] + assert sum(event.name == "handoff_requested" for event in item_events) == 1 + + @pytest.mark.asyncio async def test_stream_events_emit_tool_search_items() -> None: model = FakeModel() diff --git a/tests/test_tool_approval_call_id_reuse.py b/tests/test_tool_approval_call_id_reuse.py new file mode 100644 index 0000000000..a54c51ce8c --- /dev/null +++ b/tests/test_tool_approval_call_id_reuse.py @@ -0,0 +1,2683 @@ +from __future__ import annotations + +import asyncio +import json +from types import SimpleNamespace +from typing import Any, Literal, cast + +import pytest +from openai.types.responses import ResponseCustomToolCall, ResponseFunctionToolCall +from openai.types.responses.response_computer_tool_call import ( + ActionScreenshot, + PendingSafetyCheck, + ResponseComputerToolCall, +) +from openai.types.responses.response_output_item import McpApprovalRequest +from openai.types.responses.response_reasoning_item import ResponseReasoningItem + +from agents import ( + Agent, + ApplyPatchTool, + ComputerTool, + CustomTool, + HostedMCPTool, + MCPToolApprovalFunctionResult, + MCPToolApprovalRequest, + RunConfig, + Runner, + ShellTool, + ToolGuardrailFunctionOutput, + ToolOutputGuardrailData, + ToolOutputGuardrailTripwireTriggered, + function_tool, + handoff, + tool_output_guardrail, +) +from agents._tool_invocation import tool_invocation_identity +from agents.editor import ApplyPatchOperation, ApplyPatchResult +from agents.exceptions import ModelBehaviorError, UserError +from agents.items import ModelResponse, ToolApprovalItem +from agents.lifecycle import RunHooks +from agents.models.interface import Model, ModelProvider +from agents.run_context import RunContextWrapper +from agents.run_internal.run_loop import ToolRunFunction +from agents.run_internal.tool_execution import ( + collect_manual_mcp_approvals, + process_hosted_mcp_approvals, + resolve_approval_rejection_message, +) +from agents.run_internal.tool_planning import _collect_runs_by_approval +from agents.run_state import RunState +from agents.stream_events import RunItemStreamEvent +from agents.tool import Tool +from agents.tool_context import ToolContext +from tests.fake_model import FakeModel +from tests.test_computer_tool_lifecycle import FakeComputer +from tests.test_responses import get_function_tool_call, get_handoff_tool_call, get_text_message +from tests.utils.hitl import make_apply_patch_dict, make_shell_call, make_state_with_interruptions + + +class _ScriptedProvider(ModelProvider): + def __init__(self, model: Model) -> None: + self.model = model + + def get_model(self, model_name: str | None) -> Model: + assert model_name == "scripted-provider-model" + return self.model + + +def test_canonical_shell_identity_ignores_stripped_provider_metadata() -> None: + provider_call = { + "type": "shell_call", + "call_id": "shell_0", + "action": {"commands": ["echo safe"]}, + "created_by": "server", + } + persisted_call = dict(provider_call) + persisted_call.pop("created_by") + + assert tool_invocation_identity(provider_call) == tool_invocation_identity(persisted_call) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fallback_type", ["custom", "function"]) +async def test_completed_apply_patch_fallback_run_state_round_trip(fallback_type: str) -> None: + class RecordingEditor: + def __init__(self) -> None: + self.paths: list[str] = [] + + def update_file(self, operation: ApplyPatchOperation) -> ApplyPatchResult: + self.paths.append(operation.path) + return ApplyPatchResult(status="completed", output="updated") + + editor = RecordingEditor() + tool = ApplyPatchTool(editor=cast(Any, editor)) + operation = {"type": "update_file", "path": "safe.txt", "diff": "-old\n+new\n"} + if fallback_type == "custom": + call: Any = ResponseCustomToolCall( + type="custom_tool_call", + name="apply_patch", + call_id="patch_0", + input=json.dumps(operation), + ) + else: + call = ResponseFunctionToolCall( + type="function_call", + name="apply_patch", + call_id="patch_0", + arguments=json.dumps(operation), + ) + model = FakeModel(initial_output=[call]) + model.set_next_output([get_text_message("done")]) + agent = Agent(name="agent", model=model, tools=[tool]) + + result = await Runner.run(agent, "update the file") + restored = await RunState.from_json(agent, result.to_state().to_json()) + + assert result.final_output == "done" + assert editor.paths == ["safe.txt"] + assert restored._context is not None + assert restored._context._tool_invocations["patch_0"].completed is True + + +@pytest.mark.asyncio +async def test_streamed_function_apply_patch_replay_emits_one_tool_called_event() -> None: + class RecordingEditor: + def __init__(self) -> None: + self.paths: list[str] = [] + + def update_file(self, operation: ApplyPatchOperation) -> ApplyPatchResult: + self.paths.append(operation.path) + return ApplyPatchResult(status="completed", output="updated") + + editor = RecordingEditor() + operation = {"type": "update_file", "path": "safe.txt", "diff": "-old\n+new\n"} + call = ResponseFunctionToolCall( + type="function_call", + name="apply_patch", + call_id="patch_0", + arguments=json.dumps(operation), + ) + model = FakeModel() + model.add_multiple_turn_outputs( + [[call], [call.model_copy(deep=True)], [get_text_message("done")]] + ) + agent = Agent( + name="agent", + model=model, + tools=[ApplyPatchTool(editor=cast(Any, editor))], + ) + + streamed = Runner.run_streamed(agent, "update the file") + events = [event async for event in streamed.stream_events()] + + tool_called_events = [ + event + for event in events + if isinstance(event, RunItemStreamEvent) and event.name == "tool_called" + ] + assert streamed.final_output == "done" + assert editor.paths == ["safe.txt"] + assert len(tool_called_events) == 1 + assert tool_called_events[0].item.call_id == "patch_0" + + +@pytest.mark.parametrize( + ("raw_item", "first_tool_name", "replacement_tool_name"), + [ + (make_shell_call("call_0", commands=["echo safe"]), "safe_shell", "other_shell"), + (make_apply_patch_dict("call_0"), "safe_patch", "other_patch"), + ], + ids=["shell", "apply_patch"], +) +def test_native_tool_approval_scope_rejects_resolved_tool_replacement( + raw_item: Any, + first_tool_name: str, + replacement_tool_name: str, +) -> None: + context: RunContextWrapper[Any] = RunContextWrapper(context=None) + approval_item = ToolApprovalItem( + agent=Agent(name="agent"), + raw_item=cast(Any, raw_item), + tool_name=first_tool_name, + ) + context.approve_tool(approval_item) + + assert ( + context._approved_tool_invocation_status( # noqa: SLF001 + raw_item, + tool_name=first_tool_name, + ) + is not None + ) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + context._approved_tool_invocation_status( # noqa: SLF001 + raw_item, + tool_name=replacement_tool_name, + ) + + +@pytest.mark.asyncio +async def test_custom_named_shell_sticky_approval_applies_to_fresh_call_ids() -> None: + executed: list[str] = [] + + def execute(request: Any) -> str: + executed.extend(request.data.action.commands) + return "ok" + + first_call = make_shell_call("call_0", commands=["echo first"]) + second_call = make_shell_call("call_1", commands=["echo second"]) + model = FakeModel() + model.add_multiple_turn_outputs([[first_call], [second_call], [get_text_message("done")]]) + tool = ShellTool( + executor=execute, + name="safe_shell", + needs_approval=True, + ) + agent = Agent(name="agent", model=model, tools=[tool]) + + first = await Runner.run(agent, "run commands") + state = first.to_state() + state.approve(first.interruptions[0], always_approve=True) + resumed = await Runner.run(agent, state) + + assert resumed.final_output == "done" + assert executed == ["echo first", "echo second"] + + +@pytest.mark.asyncio +async def test_custom_named_shell_replacement_rejects_completed_call_id_replay() -> None: + first_executions: list[str] = [] + replacement_executions: list[str] = [] + + def run_first(_request: Any) -> str: + first_executions.append("ran") + return "ok" + + def run_replacement(_request: Any) -> str: + replacement_executions.append("ran") + return "ok" + + call = make_shell_call("call_0", commands=["echo safe"]) + model = FakeModel(initial_output=[call]) + original_tool = ShellTool( + executor=run_first, + name="safe_shell", + needs_approval=True, + ) + agent = Agent(name="agent", model=model, tools=[original_tool]) + + first = await Runner.run(agent, "run command") + state = first.to_state() + state.approve(first.interruptions[0]) + model.set_next_output([get_text_message("done")]) + completed = await Runner.run(agent, state) + + agent.tools = [ + ShellTool( + executor=run_replacement, + name="other_shell", + needs_approval=False, + ) + ] + model.set_next_output([cast(Any, dict(cast(dict[str, Any], call)))]) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await Runner.run(agent, completed.to_state()) + + assert first_executions == ["ran"] + assert replacement_executions == [] + + +@pytest.mark.parametrize( + "raw_item", + [ + {"type": "custom_tool_call", "name": "raw_editor", "call_id": "call_0"}, + {"type": "computer_call", "call_id": "call_0"}, + {"type": "local_shell_call", "call_id": "call_0"}, + {"type": "shell_call", "call_id": "call_0"}, + {"type": "apply_patch_call", "call_id": "call_0"}, + ], +) +def test_canonical_identity_requires_each_tool_payload(raw_item: dict[str, Any]) -> None: + assert tool_invocation_identity(raw_item) is None + + +@pytest.mark.asyncio +async def test_cancelled_rejection_formatter_leaves_invocation_executed() -> None: + tool_call = { + "type": "function_call", + "name": "approval_tool", + "call_id": "call_rejected", + "arguments": "{}", + } + context: RunContextWrapper[Any] = RunContextWrapper(context=None) + assert context._tool_invocation_status(tool_call) == ( # noqa: SLF001 + ("function_call", "call_rejected"), + False, + False, + ) + formatter_entered = asyncio.Event() + + async def blocking_formatter(_args: Any) -> str: + formatter_entered.set() + await asyncio.Event().wait() + return "rejected" + + task = asyncio.create_task( + resolve_approval_rejection_message( + context_wrapper=context, + run_config=RunConfig(tool_error_formatter=blocking_formatter), + tool_call=tool_call, + tool_type="function", + tool_name="approval_tool", + call_id="call_rejected", + ) + ) + await formatter_entered.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert context._tool_invocation_status(tool_call) == ( # noqa: SLF001 + ("function_call", "call_rejected"), + False, + True, + ) + + +async def _run( + agent: Agent[Any], + input_value: Any, + *, + run_config: RunConfig, + mode: Literal["non_streamed", "streamed"], + hooks: RunHooks[Any] | None = None, + events: list[Any] | None = None, +) -> Any: + if mode == "non_streamed": + return await Runner.run(agent, input_value, run_config=run_config, hooks=hooks) + result = Runner.run_streamed(agent, input_value, run_config=run_config, hooks=hooks) + async for event in result.stream_events(): + if events is not None: + events.append(event) + return result + + +def _build_scenario( + second_call_id: str, + second_value: str, +) -> tuple[Agent[Any], RunConfig, list[str]]: + executed: list[str] = [] + + @function_tool(name_override="record_value", needs_approval=True) + def record_value(value: str) -> str: + executed.append(value) + return value + + model = FakeModel() + model.add_multiple_turn_outputs( + [ + [ + get_function_tool_call( + "record_value", + json.dumps({"value": "safe"}), + call_id="call_0", + ) + ], + [ + get_function_tool_call( + "record_value", + json.dumps({"value": second_value}), + call_id=second_call_id, + ) + ], + [get_text_message("done")], + ] + ) + provider = _ScriptedProvider(model) + agent = Agent( + name="custom-provider-agent", + model="scripted-provider-model", + tools=[record_value], + ) + return agent, RunConfig(model_provider=provider), executed + + +@pytest.mark.asyncio +async def test_empty_custom_tool_call_id_fails_before_approval_or_execution() -> None: + callbacks: list[str] = [] + executed: list[str] = [] + + async def approve(_context: RunContextWrapper[Any], _item: ToolApprovalItem) -> Any: + callbacks.append("approval") + return {"approve": True} + + async def invoke(_context: Any, raw_input: str) -> str: + executed.append(raw_input) + return raw_input + + tool = CustomTool( + name="raw_editor", + description="Edit raw text.", + on_invoke_tool=invoke, + format={"type": "text"}, + needs_approval=True, + on_approval=approve, + ) + model = FakeModel( + initial_output=[ + ResponseCustomToolCall( + type="custom_tool_call", + name=tool.name, + call_id="", + input="changed", + ) + ] + ) + agent = Agent(name="agent", model=model, tools=[tool]) + + with pytest.raises(ModelBehaviorError, match="non-empty string call ID"): + await Runner.run(agent, "run it") + + assert callbacks == [] + assert executed == [] + + +@pytest.mark.asyncio +async def test_empty_unresolved_function_call_id_fails_before_error_formatter() -> None: + formatter_calls: list[str] = [] + + def format_tool_error(args: Any) -> str: + formatter_calls.append(args.tool_name) + return "error" + + model = FakeModel( + initial_output=[ + ResponseFunctionToolCall( + id="item_0", + type="function_call", + name="missing", + arguments="{}", + call_id="", + ) + ] + ) + agent = Agent(name="agent", model=model) + + with pytest.raises(ModelBehaviorError, match="non-empty string call ID"): + await Runner.run( + agent, + "run it", + run_config=RunConfig( + tool_error_formatter=format_tool_error, + tool_not_found_behavior="return_error_to_model", + ), + ) + + assert formatter_calls == [] + + +@pytest.mark.asyncio +async def test_bound_call_id_with_missing_arguments_fails_before_error_formatter() -> None: + executed: list[str] = [] + formatter_calls: list[str] = [] + + @function_tool + def record_value(context: ToolContext[Any], value: str) -> str: + executed.append(value) + context._restored_unbound_approval_call_ids.add("shared") + return value + + def format_tool_error(args: Any) -> str: + formatter_calls.append(args.tool_name) + return "error" + + valid_call = get_function_tool_call( + "record_value", + json.dumps({"value": "safe"}), + call_id="shared", + ) + malformed_replacement = ResponseFunctionToolCall.model_construct( + type="function_call", + name="missing", + call_id="shared", + ) + model = FakeModel() + model.add_multiple_turn_outputs([[valid_call], [malformed_replacement]]) + agent = Agent(name="agent", model=model, tools=[record_value]) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await Runner.run( + agent, + "run it", + run_config=RunConfig( + tool_error_formatter=format_tool_error, + tool_not_found_behavior="return_error_to_model", + ), + ) + + assert executed == ["safe"] + assert formatter_calls == [] + + +@pytest.mark.asyncio +async def test_empty_handoff_call_id_fails_before_handoff_callback() -> None: + handoff_calls: list[str] = [] + target = Agent(name="target") + route = handoff( + target, + tool_name_override="route", + on_handoff=lambda _context: handoff_calls.append("route"), + ) + model = FakeModel( + initial_output=[ + ResponseFunctionToolCall( + id="item_0", + type="function_call", + name="route", + arguments="{}", + call_id="", + ) + ] + ) + agent = Agent(name="agent", model=model, handoffs=[route]) + + with pytest.raises(ModelBehaviorError, match="non-empty string call ID"): + await Runner.run(agent, "route it") + + assert handoff_calls == [] + + +@pytest.mark.asyncio +async def test_failed_handoff_hook_does_not_commit_output_or_repeat_callback() -> None: + hook_calls: list[str] = [] + target = Agent(name="target") + call = get_handoff_tool_call(target, call_id="handoff_0") + + class FailingHooks(RunHooks[Any]): + async def on_handoff( + self, + context: RunContextWrapper[Any], + from_agent: Agent[Any], + to_agent: Agent[Any], + ) -> None: + hook_calls.append(to_agent.name) + raise RuntimeError("handoff hook failed") + + model = FakeModel(initial_output=[call]) + model.set_next_output([call.model_copy(deep=True)]) + agent = Agent(name="source", model=model, handoffs=[target]) + context = RunContextWrapper(context=None) + hooks = FailingHooks() + + with pytest.raises(RuntimeError, match="handoff hook failed"): + await Runner.run(agent, "handoff", context=context, hooks=hooks) + + assert context._tool_invocation_status(call, invocation_role="handoff") == ( # noqa: SLF001 + ("function_call", "handoff_0"), + False, + True, + ) + with pytest.raises(ModelBehaviorError, match="already executed"): + await Runner.run(agent, "handoff", context=context, hooks=hooks) + + assert hook_calls == ["target"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["non_streamed", "streamed"]) +async def test_changed_arguments_under_reused_call_id_fail_before_second_side_effect( + mode: Literal["non_streamed", "streamed"], +) -> None: + agent, run_config, executed = _build_scenario("call_0", "changed") + + first = await _run(agent, "record a value", run_config=run_config, mode=mode) + assert len(first.interruptions) == 1 + state = first.to_state() + state.approve(first.interruptions[0]) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await _run(agent, state, run_config=run_config, mode=mode) + + assert executed == ["safe"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["non_streamed", "streamed"]) +async def test_exact_replay_reuses_committed_output_without_executing_again( + mode: Literal["non_streamed", "streamed"], +) -> None: + agent, run_config, executed = _build_scenario("call_0", "safe") + + first = await _run(agent, "record a value", run_config=run_config, mode=mode) + state = first.to_state() + state.approve(first.interruptions[0]) + + resumed = await _run(agent, state, run_config=run_config, mode=mode) + + assert resumed.final_output == "done" + assert resumed.interruptions == [] + assert executed == ["safe"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["non_streamed", "streamed"]) +async def test_non_approval_identical_siblings_execute_once( + mode: Literal["non_streamed", "streamed"], +) -> None: + executed: list[str] = [] + events: list[Any] = [] + + @function_tool + async def record_value(value: str) -> str: + executed.append(value) + return value + + duplicate = get_function_tool_call( + "record_value", + '{"value":"safe"}', + call_id="call_0", + ) + model = FakeModel(initial_output=[duplicate, duplicate.model_copy(deep=True)]) + model.set_next_output([get_text_message("done")]) + agent = Agent(name="agent", model=model, tools=[record_value]) + + result = await _run( + agent, + "record a value", + run_config=RunConfig(), + mode=mode, + events=events, + ) + + assert result.final_output == "done" + assert executed == ["safe"] + if mode == "streamed": + assert [ + event.name for event in events if getattr(event, "name", None) == "tool_called" + ] == ["tool_called"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["non_streamed", "streamed"]) +async def test_non_approval_completed_replay_does_not_execute_again( + mode: Literal["non_streamed", "streamed"], +) -> None: + executed: list[str] = [] + + @function_tool + async def record_value(value: str) -> str: + executed.append(value) + return value + + model = FakeModel() + model.add_multiple_turn_outputs( + [ + [get_function_tool_call("record_value", '{"value":"safe"}', call_id="call_0")], + [get_function_tool_call("record_value", '{ "value" : "safe" }', call_id="call_0")], + [get_text_message("done")], + ] + ) + agent = Agent(name="agent", model=model, tools=[record_value]) + + result = await _run(agent, "record a value", run_config=RunConfig(), mode=mode) + + assert result.final_output == "done" + assert executed == ["safe"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["non_streamed", "streamed"]) +async def test_non_approval_changed_completed_call_id_fails_before_execution( + mode: Literal["non_streamed", "streamed"], +) -> None: + executed: list[str] = [] + events: list[Any] = [] + + class RecordingHooks(RunHooks[Any]): + def __init__(self) -> None: + self.llm_end_calls = 0 + + async def on_llm_end( + self, + context: RunContextWrapper[Any], + agent: Agent[Any], + response: ModelResponse, + ) -> None: + self.llm_end_calls += 1 + + hooks = RecordingHooks() + + @function_tool + async def record_value(value: str) -> str: + executed.append(value) + return value + + model = FakeModel() + model.add_multiple_turn_outputs( + [ + [get_function_tool_call("record_value", '{"value":"safe"}', call_id="call_0")], + [get_function_tool_call("record_value", '{"value":"changed"}', call_id="call_0")], + ] + ) + agent = Agent(name="agent", model=model, tools=[record_value]) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await _run( + agent, + "record a value", + run_config=RunConfig(), + mode=mode, + hooks=hooks, + events=events, + ) + + assert executed == ["safe"] + assert hooks.llm_end_calls == 1 + if mode == "streamed": + assert [ + event.name for event in events if getattr(event, "name", None) == "tool_called" + ] == ["tool_called"] + + +@pytest.mark.asyncio +async def test_non_approval_failed_tool_body_does_not_reexecute() -> None: + attempts: list[str] = [] + + @function_tool(failure_error_function=None) + async def perform_side_effect() -> str: + attempts.append("ran") + raise RuntimeError("failed after side effect") + + call = get_function_tool_call("perform_side_effect", "{}", call_id="call_0") + model = FakeModel() + model.add_multiple_turn_outputs([[call], [call.model_copy(deep=True)]]) + agent = Agent(name="agent", model=model, tools=[perform_side_effect]) + context = RunContextWrapper(context=None) + + with pytest.raises(UserError, match="failed after side effect"): + await Runner.run(agent, "run it", context=context) + + with pytest.raises(ModelBehaviorError, match="already executed"): + await Runner.run(agent, "run it", context=context) + + assert attempts == ["ran"] + + +@pytest.mark.asyncio +async def test_non_approval_custom_tool_identical_siblings_execute_once() -> None: + executed: list[str] = [] + + async def invoke(_context: Any, value: str) -> str: + executed.append(value) + return value + + tool = CustomTool( + name="raw_editor", + description="Edit raw text.", + on_invoke_tool=invoke, + format={"type": "text"}, + ) + duplicate = ResponseCustomToolCall( + type="custom_tool_call", + name=tool.name, + call_id="call_0", + input="safe", + ) + model = FakeModel(initial_output=[duplicate, duplicate.model_copy(deep=True)]) + model.set_next_output([get_text_message("done")]) + agent = Agent(name="agent", model=model, tools=[tool]) + + result = await Runner.run(agent, "edit text") + + assert result.final_output == "done" + assert executed == ["safe"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["non_streamed", "streamed"]) +async def test_changed_same_id_siblings_fail_before_approval_callback( + mode: Literal["non_streamed", "streamed"], +) -> None: + approval_calls: list[str] = [] + + async def needs_approval(_context: Any, arguments: dict[str, Any], _call_id: str) -> bool: + approval_calls.append(arguments["value"]) + return True + + @function_tool(needs_approval=needs_approval) + async def record_value(value: str) -> str: + return value + + model = FakeModel( + initial_output=[ + get_function_tool_call("record_value", '{"value":"one"}', call_id="call_0"), + get_function_tool_call("record_value", '{"value":"two"}', call_id="call_0"), + ] + ) + agent = Agent(name="agent", model=model, tools=[record_value]) + + with pytest.raises(ModelBehaviorError, match="one response"): + await _run(agent, "record values", run_config=RunConfig(), mode=mode) + + assert approval_calls == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["non_streamed", "streamed"]) +async def test_response_processing_error_still_invokes_llm_end( + mode: Literal["non_streamed", "streamed"], +) -> None: + class CountingHooks(RunHooks[Any]): + def __init__(self) -> None: + self.llm_end_calls = 0 + + async def on_llm_end( + self, + context: RunContextWrapper[Any], + agent: Agent[Any], + response: ModelResponse, + ) -> None: + _ = (context, agent, response) + self.llm_end_calls += 1 + + hooks = CountingHooks() + model = FakeModel(initial_output=[make_shell_call("call_0", commands=["echo safe"])]) + agent = Agent(name="agent", model=model) + + with pytest.raises(ModelBehaviorError, match="without a shell tool"): + await _run(agent, "run command", run_config=RunConfig(), mode=mode, hooks=hooks) + + assert hooks.llm_end_calls == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["non_streamed", "streamed"]) +async def test_processing_error_with_changed_call_id_still_suppresses_llm_end( + mode: Literal["non_streamed", "streamed"], +) -> None: + class CountingHooks(RunHooks[Any]): + def __init__(self) -> None: + self.llm_end_calls = 0 + + async def on_llm_end( + self, + context: RunContextWrapper[Any], + agent: Agent[Any], + response: ModelResponse, + ) -> None: + _ = (context, agent, response) + self.llm_end_calls += 1 + + @function_tool + async def record_value(value: str) -> str: + return value + + model = FakeModel() + model.add_multiple_turn_outputs( + [ + [get_function_tool_call("record_value", '{"value":"safe"}', call_id="shared")], + [ + get_function_tool_call( + "record_value", + '{"value":"changed"}', + call_id="shared", + ), + make_shell_call("shell_0", commands=["echo safe"]), + ], + ] + ) + hooks = CountingHooks() + agent = Agent(name="agent", model=model, tools=[record_value]) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await _run(agent, "record value", run_config=RunConfig(), mode=mode, hooks=hooks) + + assert hooks.llm_end_calls == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["non_streamed", "streamed"]) +async def test_processing_error_with_repeated_uncanonical_id_suppresses_llm_end( + mode: Literal["non_streamed", "streamed"], +) -> None: + class CountingHooks(RunHooks[Any]): + def __init__(self) -> None: + self.llm_end_calls = 0 + + async def on_llm_end( + self, + context: RunContextWrapper[Any], + agent: Agent[Any], + response: ModelResponse, + ) -> None: + _ = (context, agent, response) + self.llm_end_calls += 1 + + first_call = ResponseCustomToolCall.model_construct( + type="custom_tool_call", + name="first_missing_tool", + call_id="shared", + ) + second_call = ResponseCustomToolCall.model_construct( + type="custom_tool_call", + name="second_missing_tool", + call_id="shared", + ) + hooks = CountingHooks() + model = FakeModel(initial_output=[first_call, second_call]) + agent = Agent(name="agent", model=model) + + with pytest.raises(ModelBehaviorError, match="one response"): + await _run(agent, "run tool", run_config=RunConfig(), mode=mode, hooks=hooks) + + assert hooks.llm_end_calls == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["non_streamed", "streamed"]) +async def test_changed_same_id_siblings_fail_before_non_approval_execution( + mode: Literal["non_streamed", "streamed"], +) -> None: + executed: list[str] = [] + + @function_tool + async def record_value(value: str) -> str: + executed.append(value) + return value + + model = FakeModel( + initial_output=[ + get_function_tool_call("record_value", '{"value":"one"}', call_id="call_0"), + get_function_tool_call("record_value", '{"value":"two"}', call_id="call_0"), + ] + ) + agent = Agent(name="agent", model=model, tools=[record_value]) + + with pytest.raises(ModelBehaviorError, match="one response"): + await _run(agent, "record values", run_config=RunConfig(), mode=mode) + + assert executed == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["non_streamed", "streamed"]) +async def test_changed_computer_safety_checks_fail_before_same_response_effects( + mode: Literal["non_streamed", "streamed"], +) -> None: + safety_checks: list[str] = [] + screenshots: list[str] = [] + + class RecordingComputer(FakeComputer): + def screenshot(self) -> str: + screenshots.append("screenshot") + return "img" + + def acknowledge_safety_check(data: Any) -> bool: + safety_checks.append(data.safety_check.id) + return True + + tool = ComputerTool( + computer=RecordingComputer(), + on_safety_check=acknowledge_safety_check, + ) + first_call = ResponseComputerToolCall( + id="computer-item-1", + type="computer_call", + action=ActionScreenshot(type="screenshot"), + call_id="computer-call", + pending_safety_checks=[PendingSafetyCheck(id="safety-1", code="code-1", message="first")], + status="completed", + ) + changed_call = first_call.model_copy( + update={ + "id": "computer-item-2", + "pending_safety_checks": [ + PendingSafetyCheck(id="safety-2", code="code-2", message="changed") + ], + } + ) + agent = Agent( + name="computer-agent", + model=FakeModel(initial_output=[first_call, changed_call]), + tools=[tool], + ) + + with pytest.raises(ModelBehaviorError, match="one response"): + await _run(agent, "use computer", run_config=RunConfig(), mode=mode) + + assert safety_checks == [] + assert screenshots == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["non_streamed", "streamed"]) +async def test_changed_computer_safety_checks_fail_before_completed_replay_effects( + mode: Literal["non_streamed", "streamed"], +) -> None: + safety_checks: list[str] = [] + screenshots: list[str] = [] + + class RecordingComputer(FakeComputer): + def screenshot(self) -> str: + screenshots.append("screenshot") + return "img" + + def acknowledge_safety_check(data: Any) -> bool: + safety_checks.append(data.safety_check.id) + return True + + tool = ComputerTool( + computer=RecordingComputer(), + on_safety_check=acknowledge_safety_check, + ) + first_call = ResponseComputerToolCall( + id="computer-item-1", + type="computer_call", + action=ActionScreenshot(type="screenshot"), + call_id="computer-call", + pending_safety_checks=[PendingSafetyCheck(id="safety-1", code="code-1", message="first")], + status="completed", + ) + changed_call = first_call.model_copy( + update={ + "id": "computer-item-2", + "pending_safety_checks": [ + PendingSafetyCheck(id="safety-2", code="code-2", message="changed") + ], + } + ) + model = FakeModel() + model.add_multiple_turn_outputs([[first_call], [changed_call]]) + agent = Agent(name="computer-agent", model=model, tools=[tool]) + + with pytest.raises(ModelBehaviorError, match="completed tool call ID"): + await _run(agent, "use computer", run_config=RunConfig(), mode=mode) + + assert safety_checks == ["safety-1"] + assert screenshots == ["screenshot"] + + +@pytest.mark.asyncio +async def test_computer_hook_failure_does_not_repeat_side_effect() -> None: + screenshots: list[str] = [] + + class RecordingComputer(FakeComputer): + def screenshot(self) -> str: + screenshots.append("screenshot") + return "img" + + class FailOnceHooks(RunHooks[Any]): + def __init__(self) -> None: + self.failed = False + + async def on_tool_end( + self, + context: RunContextWrapper[Any], + agent: Agent[Any], + tool: Tool, + result: object, + ) -> None: + if not self.failed: + self.failed = True + raise RuntimeError("end hook failed") + + tool = ComputerTool(computer=RecordingComputer()) + call = ResponseComputerToolCall( + id="computer-item", + type="computer_call", + action=ActionScreenshot(type="screenshot"), + call_id="computer-call", + pending_safety_checks=[], + status="completed", + ) + model = FakeModel() + model.add_multiple_turn_outputs([[call], [call.model_copy(deep=True)]]) + agent = Agent(name="computer-agent", model=model, tools=[tool]) + context = RunContextWrapper(context=None) + hooks = FailOnceHooks() + + with pytest.raises(RuntimeError, match="end hook failed"): + await Runner.run(agent, "use computer", context=context, hooks=hooks) + + with pytest.raises(ModelBehaviorError, match="already executed"): + await Runner.run(agent, "use computer", context=context, hooks=hooks) + + assert screenshots == ["screenshot"] + + +@pytest.mark.asyncio +async def test_exact_replay_drops_tied_reasoning_item() -> None: + executed: list[str] = [] + + @function_tool(needs_approval=True) + async def record_value(value: str) -> str: + executed.append(value) + return value + + model = FakeModel() + model.add_multiple_turn_outputs( + [ + [get_function_tool_call("record_value", '{"value":"safe"}', call_id="call_0")], + [ + ResponseReasoningItem(id="rs_replay", summary=[], type="reasoning"), + get_function_tool_call( + "record_value", + '{ "value" : "safe" }', + call_id="call_0", + ), + ], + [get_text_message("done")], + ] + ) + agent = Agent(name="agent", model=model, tools=[record_value]) + first = await Runner.run(agent, "record a value") + state = first.to_state() + state.approve(first.interruptions[0]) + + resumed = await Runner.run(agent, state) + + assert resumed.final_output == "done" + assert executed == ["safe"] + model_input = model.last_turn_args["input"] + assert isinstance(model_input, list) + assert not any(item.get("id") == "rs_replay" for item in model_input) + assert ( + sum( + item.get("type") == "function_call" and item.get("call_id") == "call_0" + for item in model_input + ) + == 1 + ) + + +@pytest.mark.asyncio +async def test_streamed_exact_replay_does_not_emit_tied_reasoning_item() -> None: + executed: list[str] = [] + + @function_tool(needs_approval=True) + async def record_value(value: str) -> str: + executed.append(value) + return value + + model = FakeModel() + model.add_multiple_turn_outputs( + [ + [get_function_tool_call("record_value", '{"value":"safe"}', call_id="call_0")], + [ + ResponseReasoningItem(id="rs_replay", summary=[], type="reasoning"), + get_function_tool_call( + "record_value", + '{ "value" : "safe" }', + call_id="call_0", + ), + ], + [get_text_message("done")], + ] + ) + agent = Agent(name="agent", model=model, tools=[record_value]) + first = await Runner.run(agent, "record a value") + state = first.to_state() + state.approve(first.interruptions[0]) + + resumed = Runner.run_streamed(agent, state) + item_events = [ + event async for event in resumed.stream_events() if isinstance(event, RunItemStreamEvent) + ] + + assert resumed.final_output == "done" + assert executed == ["safe"] + assert not any( + event.name == "reasoning_item_created" + and getattr(event.item.raw_item, "id", None) == "rs_replay" + for event in item_events + ) + assert not any(event.name == "tool_called" for event in item_events) + + +@pytest.mark.asyncio +async def test_failed_tool_end_hook_does_not_reexecute_approved_call() -> None: + agent, run_config, executed = _build_scenario("call_0", "safe") + + class FailOnceHooks(RunHooks[Any]): + def __init__(self) -> None: + self.failed = False + + async def on_tool_end( + self, + context: RunContextWrapper[Any], + agent: Agent[Any], + tool: Tool, + result: object, + ) -> None: + if not self.failed: + self.failed = True + raise RuntimeError("end hook failed") + + hooks = FailOnceHooks() + first = await Runner.run(agent, "record a value", run_config=run_config) + state = first.to_state() + state.approve(first.interruptions[0]) + + with pytest.raises(UserError, match="end hook failed"): + await Runner.run(agent, state, run_config=run_config, hooks=hooks) + + resumed = await Runner.run(agent, state, run_config=run_config, hooks=hooks) + + assert resumed.final_output == "done" + assert executed == ["safe"] + + +@pytest.mark.asyncio +async def test_failed_output_guardrail_does_not_reexecute_approved_call() -> None: + executed: list[str] = [] + + @tool_output_guardrail + async def reject_output(_data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput: + return ToolGuardrailFunctionOutput.raise_exception(output_info="blocked") + + @function_tool(needs_approval=True, tool_output_guardrails=[reject_output]) + async def record_value() -> str: + executed.append("ran") + return "sensitive" + + model = FakeModel( + initial_output=[get_function_tool_call("record_value", "{}", call_id="call_0")] + ) + agent = Agent(name="agent", model=model, tools=[record_value]) + first = await Runner.run(agent, "record a value") + state = first.to_state() + state.approve(first.interruptions[0]) + + with pytest.raises(ToolOutputGuardrailTripwireTriggered): + await Runner.run(agent, state) + + restored = await RunState.from_json(agent, state.to_json()) + with pytest.raises(ModelBehaviorError, match="already executed"): + await Runner.run(agent, restored) + + assert executed == ["ran"] + + +@pytest.mark.asyncio +async def test_failed_approved_tool_body_does_not_reexecute() -> None: + attempts: list[str] = [] + + @function_tool(needs_approval=True, failure_error_function=None) + async def perform_side_effect() -> str: + attempts.append("ran") + raise RuntimeError("failed after side effect") + + model = FakeModel( + initial_output=[get_function_tool_call("perform_side_effect", "{}", call_id="call_0")] + ) + agent = Agent(name="agent", model=model, tools=[perform_side_effect]) + first = await Runner.run(agent, "run it") + state = first.to_state() + state.approve(first.interruptions[0]) + + with pytest.raises(UserError, match="failed after side effect"): + await Runner.run(agent, state) + + with pytest.raises(ModelBehaviorError, match="already executed"): + await Runner.run(agent, state) + + assert attempts == ["ran"] + + +@pytest.mark.asyncio +async def test_cancelled_approved_tool_body_does_not_reexecute() -> None: + attempts: list[str] = [] + started = asyncio.Event() + keep_running = asyncio.Event() + + @function_tool(needs_approval=True, failure_error_function=None) + async def perform_side_effect() -> str: + attempts.append("ran") + started.set() + await keep_running.wait() + return "done" + + model = FakeModel( + initial_output=[get_function_tool_call("perform_side_effect", "{}", call_id="call_0")] + ) + agent = Agent(name="agent", model=model, tools=[perform_side_effect]) + first = await Runner.run(agent, "run it") + state = first.to_state() + state.approve(first.interruptions[0]) + resume_task = asyncio.create_task(Runner.run(agent, state)) + await started.wait() + resume_task.cancel() + + with pytest.raises(asyncio.CancelledError): + await resume_task + + with pytest.raises(ModelBehaviorError, match="already executed"): + await Runner.run(agent, state) + + assert attempts == ["ran"] + + +@pytest.mark.asyncio +async def test_failed_approved_agent_tool_start_does_not_reexecute() -> None: + hook_calls: list[str] = [] + inner_agent = Agent( + name="inner", + model=FakeModel(initial_output=[get_text_message("inner done")]), + ) + agent_tool = inner_agent.as_tool( + tool_name="delegate", + tool_description="Delegate work.", + needs_approval=True, + ) + + class FailingHooks(RunHooks[Any]): + async def on_tool_start( + self, + context: RunContextWrapper[Any], + agent: Agent[Any], + tool: Tool, + ) -> None: + if tool is agent_tool: + hook_calls.append("ran") + raise RuntimeError("failed after side effect") + + outer_model = FakeModel( + initial_output=[get_function_tool_call("delegate", '{"input":"hi"}', call_id="call_0")] + ) + outer_agent = Agent(name="outer", model=outer_model, tools=[agent_tool]) + first = await Runner.run(outer_agent, "delegate") + state = first.to_state() + state.approve(first.interruptions[0]) + + with pytest.raises(UserError, match="failed after side effect"): + await Runner.run(outer_agent, state, hooks=FailingHooks()) + + with pytest.raises(ModelBehaviorError, match="already executed"): + await Runner.run(outer_agent, state, hooks=FailingHooks()) + + assert hook_calls == ["ran"] + + +@pytest.mark.asyncio +async def test_failed_parallel_tool_end_hook_checkpoints_outputs_in_model_order() -> None: + executed: list[str] = [] + first_finished = asyncio.Event() + + @function_tool(needs_approval=True) + async def first_tool() -> str: + await asyncio.sleep(0.01) + executed.append("first") + first_finished.set() + return "first" + + @function_tool(needs_approval=True) + async def second_tool() -> str: + executed.append("second") + return "second" + + class FailSecondHookOnce(RunHooks[Any]): + def __init__(self) -> None: + self.failed = False + + async def on_tool_end( + self, + context: RunContextWrapper[Any], + agent: Agent[Any], + tool: Tool, + result: object, + ) -> None: + if tool.name == "second_tool" and not self.failed: + await first_finished.wait() + self.failed = True + raise RuntimeError("second end hook failed") + + model = FakeModel( + initial_output=[ + get_function_tool_call("first_tool", "{}", call_id="call_first"), + get_function_tool_call("second_tool", "{}", call_id="call_second"), + ] + ) + model.set_next_output([get_text_message("done")]) + agent = Agent(name="agent", model=model, tools=[first_tool, second_tool]) + hooks = FailSecondHookOnce() + + first = await Runner.run(agent, "run tools") + state = first.to_state() + for interruption in first.interruptions: + state.approve(interruption) + + with pytest.raises(UserError, match="second end hook failed"): + await Runner.run(agent, state, hooks=hooks) + + resumed = await Runner.run(agent, state, hooks=hooks) + + assert resumed.final_output == "done" + assert sorted(executed) == ["first", "second"] + model_input = model.last_turn_args["input"] + assert isinstance(model_input, list) + output_call_ids = [ + item["call_id"] + for item in model_input + if isinstance(item, dict) and item.get("type") == "function_call_output" + ] + assert output_call_ids == ["call_first", "call_second"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["non_streamed", "streamed"]) +async def test_identical_approval_bound_siblings_execute_once( + mode: Literal["non_streamed", "streamed"], +) -> None: + executed: list[str] = [] + + @function_tool(needs_approval=True) + def record_value(value: str) -> str: + executed.append(value) + return value + + duplicated_call = get_function_tool_call( + "record_value", + json.dumps({"value": "safe"}), + call_id="call-duplicate", + ) + model = FakeModel() + model.add_multiple_turn_outputs( + [[duplicated_call, duplicated_call.model_copy(deep=True)], [get_text_message("done")]] + ) + run_config = RunConfig(model_provider=_ScriptedProvider(model)) + agent = Agent( + name="custom-provider-agent", + model="scripted-provider-model", + tools=[record_value], + ) + + first = await _run(agent, "record a value", run_config=run_config, mode=mode) + assert len(first.interruptions) == 1 + state = first.to_state() + state.approve(first.interruptions[0]) + + resumed = await _run(agent, state, run_config=run_config, mode=mode) + + assert resumed.final_output == "done" + assert executed == ["safe"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["non_streamed", "streamed"]) +async def test_new_call_id_requires_a_new_per_call_approval( + mode: Literal["non_streamed", "streamed"], +) -> None: + agent, run_config, executed = _build_scenario("call_1", "changed") + + first = await _run(agent, "record a value", run_config=run_config, mode=mode) + state = first.to_state() + state.approve(first.interruptions[0]) + + resumed = await _run(agent, state, run_config=run_config, mode=mode) + + assert len(resumed.interruptions) == 1 + assert resumed.interruptions[0].arguments == json.dumps({"value": "changed"}) + assert executed == ["safe"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("approve", [True, False], ids=["always-approve", "always-reject"]) +async def test_serialized_sticky_decision_rejects_changed_reused_call_id( + approve: bool, +) -> None: + agent, run_config, executed = _build_scenario("call_0", "changed") + + first = await Runner.run(agent, "record a value", run_config=run_config) + state = first.to_state() + if approve: + state.approve(first.interruptions[0], always_approve=True) + else: + state.reject(first.interruptions[0], always_reject=True) + restored_state = await RunState.from_json(agent, state.to_json()) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await Runner.run(agent, restored_state, run_config=run_config) + + assert executed == (["safe"] if approve else []) + + +@pytest.mark.asyncio +async def test_sticky_upgrade_rejects_changed_reuse_of_prior_call_id() -> None: + agent, run_config, executed = _build_scenario("call_0", "changed") + + first = await Runner.run(agent, "record a value", run_config=run_config) + state = first.to_state() + state.approve(first.interruptions[0]) + state.approve(first.interruptions[0], always_approve=True) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await Runner.run(agent, state, run_config=run_config) + + assert executed == ["safe"] + + +@pytest.mark.asyncio +async def test_serialized_sticky_identical_siblings_execute_once() -> None: + executed: list[str] = [] + + @function_tool(needs_approval=True) + async def record_value(value: int) -> str: + executed.append(str(value)) + return str(value) + + duplicate = get_function_tool_call( + "record_value", + '{"value":1}', + call_id="call_0", + ) + model = FakeModel( + initial_output=[ + duplicate, + duplicate.model_copy(deep=True), + ] + ) + model.set_next_output([get_text_message("done")]) + agent = Agent(name="agent", model=model, tools=[record_value]) + + first = await Runner.run(agent, "record a value") + state = first.to_state() + state.approve(first.interruptions[0], always_approve=True) + restored = await RunState.from_json(agent, state.to_json()) + + resumed = await Runner.run(agent, restored) + + assert resumed.final_output == "done" + assert executed == ["1"] + + +@pytest.mark.parametrize("approve", [True, False], ids=["always-approve", "always-reject"]) +def test_sticky_decision_preserves_prior_per_call_binding(approve: bool) -> None: + agent = Agent(name="agent") + context = RunContextWrapper(context=None) + first = ToolApprovalItem( + agent=agent, + raw_item=cast( + ResponseFunctionToolCall, + get_function_tool_call("tool_a", '{"value":1}', call_id="call_0"), + ), + tool_name="tool_a", + ) + sticky = ToolApprovalItem( + agent=agent, + raw_item=cast( + ResponseFunctionToolCall, + get_function_tool_call("tool_a", '{"value":2}', call_id="call_1"), + ), + tool_name="tool_a", + ) + changed = ToolApprovalItem( + agent=agent, + raw_item=cast( + ResponseFunctionToolCall, + get_function_tool_call("tool_a", '{"value":3}', call_id="call_0"), + ), + tool_name="tool_a", + ) + context.approve_tool(first) + if approve: + context.approve_tool(sticky, always_approve=True) + else: + context.reject_tool(sticky, always_reject=True) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + context.get_approval_status( + "tool_a", + "call_0", + current_invocation=changed, + ) + + +@pytest.mark.parametrize("approve", [True, False], ids=["always-approve", "always-reject"]) +def test_sticky_decision_does_not_disable_other_tool_binding(approve: bool) -> None: + agent = Agent(name="agent") + context = RunContextWrapper(context=None) + other = ToolApprovalItem( + agent=agent, + raw_item=cast( + ResponseFunctionToolCall, + get_function_tool_call("tool_b", '{"value":1}', call_id="call_0"), + ), + tool_name="tool_b", + ) + sticky = ToolApprovalItem( + agent=agent, + raw_item=cast( + ResponseFunctionToolCall, + get_function_tool_call("tool_a", '{"value":2}', call_id="call_1"), + ), + tool_name="tool_a", + ) + changed_other = ToolApprovalItem( + agent=agent, + raw_item=cast( + ResponseFunctionToolCall, + get_function_tool_call("tool_b", '{"value":3}', call_id="call_0"), + ), + tool_name="tool_b", + ) + context.approve_tool(other) + if approve: + context.approve_tool(sticky, always_approve=True) + else: + context.reject_tool(sticky, always_reject=True) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + context.get_approval_status( + "tool_b", + "call_0", + current_invocation=changed_other, + ) + + +@pytest.mark.parametrize("approve", [True, False], ids=["always-approve", "always-reject"]) +def test_matching_sticky_decision_does_not_mask_other_tool_binding(approve: bool) -> None: + agent = Agent(name="agent") + context = RunContextWrapper(context=None) + other = ToolApprovalItem( + agent=agent, + raw_item=cast( + ResponseFunctionToolCall, + get_function_tool_call("tool_b", '{"value":1}', call_id="call_0"), + ), + tool_name="tool_b", + ) + sticky = ToolApprovalItem( + agent=agent, + raw_item=cast( + ResponseFunctionToolCall, + get_function_tool_call("tool_a", '{"value":2}', call_id="call_1"), + ), + tool_name="tool_a", + ) + current = ToolApprovalItem( + agent=agent, + raw_item=cast( + ResponseFunctionToolCall, + get_function_tool_call("tool_a", '{"value":3}', call_id="call_0"), + ), + tool_name="tool_a", + ) + context.approve_tool(other) + if approve: + context.approve_tool(sticky, always_approve=True) + else: + context.reject_tool(sticky, always_reject=True) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + context.get_approval_status( + "tool_a", + "call_0", + current_invocation=current, + ) + + +@pytest.mark.parametrize("approve", [True, False], ids=["always-approve", "always-reject"]) +def test_deferred_sticky_decision_preserves_mirrored_per_call_binding(approve: bool) -> None: + agent = Agent(name="agent") + context = RunContextWrapper(context=None) + + def approval_item(call_id: str, value: int) -> ToolApprovalItem: + raw_item = cast( + ResponseFunctionToolCall, + get_function_tool_call( + "lookup", + json.dumps({"value": value}), + call_id=call_id, + ), + ) + return ToolApprovalItem( + agent=agent, + raw_item=raw_item, + tool_name="lookup", + tool_namespace="lookup", + tool_lookup_key=("deferred_top_level", "lookup"), + _allow_bare_name_alias=True, + ) + + first = approval_item("call_0", 1) + sticky = approval_item("call_1", 2) + changed = approval_item("call_0", 3) + context.approve_tool(first) + if approve: + context.approve_tool(sticky, always_approve=True) + else: + context.reject_tool(sticky, always_reject=True) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + context.get_approval_status( + "lookup", + "call_0", + tool_namespace="lookup", + tool_lookup_key=("deferred_top_level", "lookup"), + current_invocation=changed, + ) + + +def test_sticky_function_approval_rejects_same_id_shell_call() -> None: + agent = Agent(name="agent") + context = RunContextWrapper(context=None) + approval_item = ToolApprovalItem( + agent=agent, + raw_item=cast( + ResponseFunctionToolCall, + get_function_tool_call("shell", "{}", call_id="call_0"), + ), + tool_name="shell", + ) + context.approve_tool(approval_item, always_approve=True) + shell_item = ToolApprovalItem( + agent=agent, + raw_item={ + "type": "shell_call", + "call_id": "call_0", + "action": {"commands": ["echo safe"]}, + }, + tool_name="shell", + ) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + context.get_approval_status( + "shell", + "call_0", + current_invocation=shell_item, + ) + + +@pytest.mark.asyncio +async def test_changed_tool_under_approved_call_id_fails_before_second_tool_starts() -> None: + executed: list[str] = [] + + @function_tool(needs_approval=True) + def first_tool(value: str) -> str: + executed.append(f"first:{value}") + return value + + @function_tool(needs_approval=True) + def second_tool(value: str) -> str: + executed.append(f"second:{value}") + return value + + model = FakeModel() + model.add_multiple_turn_outputs( + [ + [ + get_function_tool_call( + "first_tool", + json.dumps({"value": "same"}), + call_id="call_0", + ) + ], + [ + get_function_tool_call( + "second_tool", + json.dumps({"value": "same"}), + call_id="call_0", + ) + ], + ] + ) + run_config = RunConfig(model_provider=_ScriptedProvider(model)) + agent = Agent( + name="custom-provider-agent", + model="scripted-provider-model", + tools=[first_tool, second_tool], + ) + + first = await Runner.run(agent, "run tools", run_config=run_config) + state = first.to_state() + state.approve(first.interruptions[0]) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await Runner.run(agent, state, run_config=run_config) + + assert executed == ["first:same"] + + +@pytest.mark.asyncio +async def test_approved_call_id_reused_for_another_invocation_type_fails_before_execution() -> None: + executed: list[str] = [] + + @function_tool(needs_approval=True) + def approved_tool(value: str) -> str: + executed.append(f"function:{value}") + return value + + async def invoke_custom(_ctx: Any, raw_input: str) -> str: + executed.append(f"custom:{raw_input}") + return raw_input + + custom_tool = CustomTool( + name="raw_editor", + description="Edit raw text.", + on_invoke_tool=invoke_custom, + format={"type": "text"}, + ) + model = FakeModel() + model.add_multiple_turn_outputs( + [ + [ + get_function_tool_call( + "approved_tool", + json.dumps({"value": "safe"}), + call_id="call_0", + ) + ], + [ + ResponseCustomToolCall( + type="custom_tool_call", + name="raw_editor", + call_id="call_0", + input="changed-kind", + ) + ], + ] + ) + run_config = RunConfig(model_provider=_ScriptedProvider(model)) + agent = Agent( + name="custom-provider-agent", + model="scripted-provider-model", + tools=[approved_tool, custom_tool], + ) + + first = await Runner.run(agent, "run tools", run_config=run_config) + state = first.to_state() + state.approve(first.interruptions[0]) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await Runner.run(agent, state, run_config=run_config) + + assert executed == ["function:safe"] + + +@pytest.mark.asyncio +async def test_changed_missing_tool_under_approved_call_id_fails_before_sibling_tool() -> None: + executed: list[str] = [] + approval_checks: list[str] = [] + + @function_tool(needs_approval=True) + def approved_tool(value: str) -> str: + executed.append(f"approved:{value}") + return value + + async def sibling_needs_approval(_ctx: Any, _args: dict[str, Any], call_id: str) -> bool: + approval_checks.append(call_id) + return False + + @function_tool(needs_approval=sibling_needs_approval) + def sibling_tool() -> str: + executed.append("sibling") + return "sibling" + + model = FakeModel() + model.add_multiple_turn_outputs( + [ + [ + get_function_tool_call( + "approved_tool", + json.dumps({"value": "safe"}), + call_id="call_0", + ) + ], + [ + get_function_tool_call("sibling_tool", "{}", call_id="call_1"), + get_function_tool_call("missing_tool", "{}", call_id="call_0"), + ], + ] + ) + run_config = RunConfig( + model_provider=_ScriptedProvider(model), + tool_not_found_behavior="return_error_to_model", + ) + agent = Agent( + name="custom-provider-agent", + model="scripted-provider-model", + tools=[approved_tool, sibling_tool], + ) + + first = await Runner.run(agent, "run tools", run_config=run_config) + state = first.to_state() + state.approve(first.interruptions[0]) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await Runner.run(agent, state, run_config=run_config) + + assert executed == ["approved:safe"] + assert approval_checks == [] + + +def _build_serialized_replay_scenario( + replay_value: str, +) -> tuple[Agent[Any], RunConfig, list[str]]: + executed: list[str] = [] + + @function_tool(needs_approval=True) + def record_value(value: str) -> str: + executed.append(f"record:{value}") + return value + + @function_tool(needs_approval=True) + def approval_gate(value: str) -> str: + executed.append(f"gate:{value}") + return value + + model = FakeModel() + model.add_multiple_turn_outputs( + [ + [ + get_function_tool_call( + "record_value", + json.dumps({"value": "safe"}), + call_id="call_0", + ) + ], + [ + get_function_tool_call( + "approval_gate", + json.dumps({"value": "pause"}), + call_id="call_1", + ) + ], + [ + get_function_tool_call( + "record_value", + json.dumps({"value": replay_value}, separators=(",", ":")), + call_id="call_0", + ) + ], + [get_text_message("done")], + ] + ) + run_config = RunConfig(model_provider=_ScriptedProvider(model)) + agent = Agent( + name="custom-provider-agent", + model="scripted-provider-model", + tools=[record_value, approval_gate], + ) + return agent, run_config, executed + + +@pytest.mark.asyncio +async def test_serialized_pending_approval_keeps_its_invocation_binding() -> None: + agent, run_config, executed = _build_scenario("call_0", "changed") + + first = await Runner.run(agent, "record a value", run_config=run_config) + state = first.to_state() + state.approve(first.interruptions[0]) + restored = await RunState.from_string(agent, state.to_string()) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await Runner.run(agent, restored, run_config=run_config) + + assert executed == ["safe"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["non_streamed", "streamed"]) +async def test_serialized_completed_approval_rejects_changed_replay( + mode: Literal["non_streamed", "streamed"], +) -> None: + agent, run_config, executed = _build_serialized_replay_scenario("changed") + + first = await _run(agent, "record a value", run_config=run_config, mode=mode) + state = first.to_state() + state.approve(first.interruptions[0]) + second = await _run(agent, state, run_config=run_config, mode=mode) + restored = await RunState.from_string(agent, second.to_state().to_string()) + restored.approve(restored.get_interruptions()[0]) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await _run(agent, restored, run_config=run_config, mode=mode) + + assert executed == ["record:safe", "gate:pause"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["non_streamed", "streamed"]) +async def test_serialized_completed_approval_skips_exact_replay( + mode: Literal["non_streamed", "streamed"], +) -> None: + agent, run_config, executed = _build_serialized_replay_scenario("safe") + + first = await _run(agent, "record a value", run_config=run_config, mode=mode) + state = first.to_state() + state.approve(first.interruptions[0]) + second = await _run(agent, state, run_config=run_config, mode=mode) + restored = await RunState.from_string(agent, second.to_state().to_string()) + restored.approve(restored.get_interruptions()[0]) + + result = await _run(agent, restored, run_config=run_config, mode=mode) + + assert result.final_output == "done" + assert executed == ["record:safe", "gate:pause"] + provider = run_config.model_provider + assert isinstance(provider, _ScriptedProvider) + model = provider.model + assert isinstance(model, FakeModel) + model_input = model.last_turn_args["input"] + assert isinstance(model_input, list) + for call_id in ("call_0", "call_1"): + calls = [ + item + for item in model_input + if item.get("type") == "function_call" and item.get("call_id") == call_id + ] + outputs = [ + item + for item in model_input + if item.get("type") == "function_call_output" and item.get("call_id") == call_id + ] + assert len(calls) == 1 + assert len(outputs) == 1 + + +async def _restore_as_schema_1_13( + agent: Agent[Any], + state: RunState[Any, Agent[Any]], +) -> RunState[Any, Agent[Any]]: + json_data = state.to_json() + json_data["$schemaVersion"] = "1.13" + json_data["context"].pop("tool_invocations", None) + return await RunState.from_json(agent, json_data) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("schema_version", ["1.13", "1.14"]) +@pytest.mark.parametrize("mode", ["non_streamed", "streamed"]) +@pytest.mark.parametrize("replay_value", ["safe", "changed"]) +async def test_legacy_schema_historical_sticky_call_id_is_not_reexecuted( + schema_version: str, + mode: Literal["non_streamed", "streamed"], + replay_value: str, +) -> None: + executed: list[str] = [] + + @function_tool(needs_approval=True) + async def record_value(value: str) -> str: + executed.append(value) + return value + + historical_call = cast( + ResponseFunctionToolCall, + get_function_tool_call( + "record_value", + '{"value":"safe"}', + call_id="call_0", + ), + ) + model = FakeModel( + initial_output=[ + get_function_tool_call( + "record_value", + json.dumps({"value": replay_value}), + call_id="call_0", + ) + ] + ) + model.set_next_output([get_text_message("done")]) + agent = Agent(name="agent", model=model, tools=[record_value]) + context: RunContextWrapper[Any] = RunContextWrapper(context=None) + context.approve_tool( + ToolApprovalItem(agent=agent, raw_item=historical_call), + always_approve=True, + ) + state = RunState( + context=context, + original_input=[ + historical_call.model_dump(exclude_none=True), + { + "type": "function_call_output", + "call_id": "call_0", + "output": "safe", + }, + ], + starting_agent=agent, + ) + serialized = state.to_json() + serialized["$schemaVersion"] = schema_version + serialized["context"].pop("tool_invocations", None) + restored = await RunState.from_json(agent, serialized) + + if replay_value == "changed": + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await _run(agent, restored, run_config=RunConfig(), mode=mode) + else: + result = await _run(agent, restored, run_config=RunConfig(), mode=mode) + assert result.final_output == "done" + + assert executed == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("schema_version", ["1.13", "1.14"]) +async def test_legacy_changed_pending_call_fails_before_approval_callback( + schema_version: str, +) -> None: + approval_calls: list[str] = [] + + async def needs_approval(_context: Any, arguments: dict[str, Any], _call_id: str) -> bool: + approval_calls.append(arguments["value"]) + return True + + @function_tool(needs_approval=needs_approval) + async def record_value(value: str) -> str: + return value + + historical_call = cast( + ResponseFunctionToolCall, + get_function_tool_call( + "record_value", + '{"value":"safe"}', + call_id="call_0", + ), + ) + changed_call = cast( + ResponseFunctionToolCall, + get_function_tool_call( + "record_value", + '{"value":"changed"}', + call_id="call_0", + ), + ) + agent = Agent(name="agent", tools=[record_value]) + context: RunContextWrapper[Any] = RunContextWrapper(context=None) + context.approve_tool(ToolApprovalItem(agent=agent, raw_item=historical_call)) + pending_item = ToolApprovalItem(agent=agent, raw_item=changed_call) + state = make_state_with_interruptions( + agent, + [pending_item], + original_input=cast( + Any, + [ + historical_call.model_dump(exclude_none=True), + { + "type": "function_call_output", + "call_id": "call_0", + "output": "safe", + }, + ], + ), + ) + state._context = context + serialized = state.to_json() + serialized["$schemaVersion"] = schema_version + serialized["context"].pop("tool_invocations", None) + restored = await RunState.from_json(agent, serialized) + restored_pending = restored.get_interruptions()[0] + run = ToolRunFunction(tool_call=changed_call, function_tool=record_value) + + async def build_rejection(_run: ToolRunFunction, _call_id: str) -> ToolApprovalItem: + return restored_pending + + async def check_approval(_run: ToolRunFunction) -> bool: + return await needs_approval(None, {"value": "changed"}, "call_0") + + assert restored._context is not None + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await _collect_runs_by_approval( + [run], + call_id_extractor=lambda item: item.tool_call.call_id, + tool_name_resolver=lambda item: item.function_tool.name, + rejection_builder=build_rejection, + context_wrapper=restored._context, + approval_items_by_call_id={"call_0": restored_pending}, + agent=agent, + pending_interruption_adder=lambda _item: None, + needs_approval_checker=check_approval, + ) + + assert approval_calls == [] + + +@pytest.mark.asyncio +async def test_schema_1_13_completed_approval_rejects_changed_replay() -> None: + agent, run_config, executed = _build_serialized_replay_scenario("changed") + + first = await Runner.run(agent, "record a value", run_config=run_config) + state = first.to_state() + state.approve(first.interruptions[0]) + second = await Runner.run(agent, state, run_config=run_config) + restored = await _restore_as_schema_1_13(agent, second.to_state()) + restored.approve(restored.get_interruptions()[0]) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await Runner.run(agent, restored, run_config=run_config) + + assert executed == ["record:safe", "gate:pause"] + + +@pytest.mark.asyncio +async def test_schema_1_13_completed_approval_skips_exact_replay() -> None: + agent, run_config, executed = _build_serialized_replay_scenario("safe") + + first = await Runner.run(agent, "record a value", run_config=run_config) + state = first.to_state() + state.approve(first.interruptions[0]) + second = await Runner.run(agent, state, run_config=run_config) + restored = await _restore_as_schema_1_13(agent, second.to_state()) + restored.approve(restored.get_interruptions()[0]) + + result = await Runner.run(agent, restored, run_config=run_config) + + assert result.final_output == "done" + assert executed == ["record:safe", "gate:pause"] + + +def _build_mcp_approval_request( + agent: Agent[Any], + *, + name: str = "lookup", + server_label: str = "test_server", + call_id: str = "mcp_call_0", + arguments: str | None = None, +) -> tuple[Any, ToolApprovalItem]: + request_item = McpApprovalRequest( + id=call_id, + type="mcp_approval_request", + name=name, + server_label=server_label, + arguments=arguments or json.dumps({"query": "safe"}), + ) + request = SimpleNamespace( + request_item=request_item, + mcp_tool=SimpleNamespace(name=name, on_approval_request=None), + ) + approval_item = ToolApprovalItem( + agent=agent, + raw_item=request_item, + tool_name=name, + ) + return request, approval_item + + +def test_wrapped_mcp_approval_uses_the_same_canonical_request_id() -> None: + agent = Agent(name="mcp-approval-agent") + approval_item = ToolApprovalItem( + agent=agent, + raw_item={ + "type": "hosted_tool_call", + "id": "outer-item-id", + "call_id": "shared-request", + "name": "lookup", + "provider_data": { + "type": "mcp_approval_request", + "name": "lookup", + "server_label": "test_server", + "arguments": '{"query":"safe"}', + }, + }, + tool_name="lookup", + ) + changed_request, _ = _build_mcp_approval_request( + agent, + call_id="shared-request", + arguments='{"query":"changed"}', + ) + context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={}) + context.approve_tool(approval_item) + + assert set(context._tool_invocations) == {"shared-request"} + with pytest.raises(ModelBehaviorError, match="unique call ID"): + collect_manual_mcp_approvals( + agent=agent, + requests=[changed_request], + context_wrapper=context, + existing_pending_by_call_id={"shared-request": approval_item}, + ) + + +def test_sticky_mcp_approval_rejects_same_id_on_another_server() -> None: + agent = Agent(name="mcp-approval-agent") + _, approval_item = _build_mcp_approval_request(agent, server_label="server_a") + changed_request, _ = _build_mcp_approval_request(agent, server_label="server_b") + context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={}) + context.approve_tool(approval_item, always_approve=True) + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + collect_manual_mcp_approvals( + agent=agent, + requests=[changed_request], + context_wrapper=context, + existing_pending_by_call_id={"mcp_call_0": approval_item}, + ) + + +def test_sticky_mcp_approval_reprompts_new_id_on_another_server() -> None: + agent = Agent(name="mcp-approval-agent") + _, approval_item = _build_mcp_approval_request(agent, server_label="server_a") + changed_request, _ = _build_mcp_approval_request( + agent, + server_label="server_b", + call_id="mcp_call_1", + ) + context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={}) + context.approve_tool(approval_item, always_approve=True) + + responses, pending = collect_manual_mcp_approvals( + agent=agent, + requests=[changed_request], + context_wrapper=context, + existing_pending_by_call_id={}, + ) + + assert responses == [] + assert len(pending) == 1 + assert pending[0].raw_item is changed_request.request_item + + +@pytest.mark.asyncio +async def test_serialized_sticky_mcp_scope_reprompts_another_server() -> None: + agent = Agent(name="mcp-approval-agent") + _, approval_item = _build_mcp_approval_request(agent, server_label="server_a") + context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={}) + state = RunState(context=context, original_input="", starting_agent=agent) + state.approve(approval_item, always_approve=True) + restored = await RunState.from_json(agent, state.to_json()) + assert restored._context is not None + changed_request, _ = _build_mcp_approval_request( + agent, + server_label="server_b", + call_id="mcp_call_1", + ) + + responses, pending = collect_manual_mcp_approvals( + agent=agent, + requests=[changed_request], + context_wrapper=restored._context, + existing_pending_by_call_id={}, + ) + + assert responses == [] + assert len(pending) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("schema_version", ["1.13", "1.14"]) +@pytest.mark.parametrize("missing_field", ["arguments", "server_label"]) +async def test_unbindable_legacy_mcp_approval_requires_current_reapproval( + schema_version: str, + missing_field: str, +) -> None: + agent = Agent(name="mcp-approval-agent") + current_request, _ = _build_mcp_approval_request(agent) + approval_item = ToolApprovalItem( + agent=agent, + raw_item=current_request.request_item, + tool_name="lookup", + ) + state = make_state_with_interruptions(agent, [approval_item]) + assert state._context is not None + state._context._rebuild_approvals( # noqa: SLF001 + { + "lookup": { + "approved": ["mcp_call_0"], + "rejected": [], + } + } + ) + serialized = state.to_json() + serialized["$schemaVersion"] = schema_version + serialized["context"].pop("tool_invocations", None) + serialized["current_step"]["data"]["interruptions"][0]["raw_item"].pop(missing_field) + + restored = await RunState.from_json(agent, serialized) + assert restored._context is not None + restored_item = restored.get_interruptions()[0] + + responses, pending = collect_manual_mcp_approvals( + agent=agent, + requests=[current_request], + context_wrapper=restored._context, + existing_pending_by_call_id={"mcp_call_0": restored_item}, + ) + + assert responses == [] + assert len(pending) == 1 + assert pending[0].raw_item is current_request.request_item + + +def _build_unbindable_current_mcp_request() -> Any: + request_item = McpApprovalRequest.model_construct( + id="mcp_call_0", + type="mcp_approval_request", + name="lookup", + server_label="test_server", + ) + return SimpleNamespace( + request_item=request_item, + mcp_tool=SimpleNamespace(name="lookup", on_approval_request=None), + ) + + +@pytest.mark.asyncio +async def test_unbindable_mcp_callback_request_requires_manual_reapproval() -> None: + callback_calls = 0 + + def approve_request(_request: MCPToolApprovalRequest) -> MCPToolApprovalFunctionResult: + nonlocal callback_calls + callback_calls += 1 + return {"approve": True} + + mcp_tool = HostedMCPTool( + tool_config={ + "type": "mcp", + "server_label": "test_server", + "server_url": "https://example.com", + "require_approval": "always", + }, + on_approval_request=approve_request, + ) + request = McpApprovalRequest.model_construct( + id="mcp_call_0", + type="mcp_approval_request", + name="lookup", + server_label="test_server", + ) + agent = Agent( + name="mcp-approval-agent", + model=FakeModel(initial_output=[request]), + tools=[mcp_tool], + ) + + result = await Runner.run(agent, "lookup") + + assert callback_calls == 0 + assert len(result.interruptions) == 1 + assert result.interruptions[0].raw_item is request + + +@pytest.mark.parametrize("always_approve", [False, True], ids=["per-call", "sticky"]) +def test_unbindable_current_manual_mcp_request_requires_reapproval( + always_approve: bool, +) -> None: + agent = Agent(name="mcp-approval-agent") + _, approval_item = _build_mcp_approval_request(agent) + current_request = _build_unbindable_current_mcp_request() + context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={}) + context.approve_tool(approval_item, always_approve=always_approve) + + responses, pending = collect_manual_mcp_approvals( + agent=agent, + requests=[current_request], + context_wrapper=context, + existing_pending_by_call_id={"mcp_call_0": approval_item}, + ) + + assert responses == [] + assert len(pending) == 1 + assert pending[0].raw_item is current_request.request_item + + +def test_unbindable_current_hosted_mcp_request_requires_reapproval() -> None: + agent = Agent(name="mcp-approval-agent") + _, approval_item = _build_mcp_approval_request(agent) + current_request = _build_unbindable_current_mcp_request() + context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={}) + context.approve_tool(approval_item) + appended: list[Any] = [] + + pending, pending_ids = process_hosted_mcp_approvals( + original_pre_step_items=[approval_item], + mcp_approval_requests=[current_request], + context_wrapper=context, + agent=agent, + append_item=appended.append, + ) + + assert len(pending) == 1 + assert pending[0].raw_item is current_request.request_item + assert pending_ids == {"mcp_call_0"} + assert appended == pending + + +@pytest.mark.parametrize("with_callback", [False, True], ids=["manual", "callback"]) +@pytest.mark.asyncio +async def test_runner_omits_completed_mcp_approval_request_replay( + with_callback: bool, +) -> None: + callback_calls = 0 + + def approve_request(_request: MCPToolApprovalRequest) -> MCPToolApprovalFunctionResult: + nonlocal callback_calls + callback_calls += 1 + return {"approve": True} + + mcp_tool = HostedMCPTool( + tool_config={ + "type": "mcp", + "server_label": "test_server", + "server_url": "https://example.com", + "require_approval": "always", + }, + on_approval_request=approve_request if with_callback else None, + ) + first_request = McpApprovalRequest( + id="mcp_call_0", + type="mcp_approval_request", + name="lookup", + server_label="test_server", + arguments='{"query": "safe", "limit": 1}', + ) + replayed_request = McpApprovalRequest( + id="mcp_call_0", + type="mcp_approval_request", + name="lookup", + server_label="test_server", + arguments='{"limit":1,"query":"safe"}', + ) + model = FakeModel() + model.add_multiple_turn_outputs( + [[first_request], [replayed_request], [get_text_message("done")]] + ) + agent = Agent(name="mcp-approval-agent", model=model, tools=[mcp_tool]) + + first = await Runner.run(agent, "lookup") + if with_callback: + result = first + else: + assert len(first.interruptions) == 1 + state = first.to_state() + state.approve(first.interruptions[0]) + result = await Runner.run(agent, state) + + assert result.final_output == "done" + assert callback_calls == int(with_callback) + model_input = model.last_turn_args["input"] + assert isinstance(model_input, list) + replay_items = [ + item + for item in model_input + if item.get("type") == "mcp_approval_request" and item.get("id") == "mcp_call_0" + ] + approval_responses = [ + item + for item in model_input + if item.get("type") == "mcp_approval_response" + and item.get("approval_request_id") == "mcp_call_0" + ] + assert len(replay_items) == 1 + assert len(approval_responses) == 1 + + +@pytest.mark.asyncio +async def test_serialized_completed_manual_mcp_approval_skips_exact_replay() -> None: + agent = Agent(name="mcp-approval-agent") + context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={}) + request, approval_item = _build_mcp_approval_request(agent) + state = RunState(context=context, original_input="", starting_agent=agent) + state.approve(approval_item) + + first_responses, first_pending = collect_manual_mcp_approvals( + agent=agent, + requests=[request], + context_wrapper=context, + existing_pending_by_call_id={"mcp_call_0": approval_item}, + ) + + assert len(first_responses) == 1 + assert first_pending == [] + context._mark_tool_call_completed(first_responses[0].raw_item) + state._generated_items = [approval_item, first_responses[0]] + + restored = await RunState.from_string(agent, state.to_string()) + assert restored._context is not None + replayed_responses, replayed_pending = collect_manual_mcp_approvals( + agent=agent, + requests=[request], + context_wrapper=restored._context, + existing_pending_by_call_id={"mcp_call_0": approval_item}, + ) + + assert replayed_responses == [] + assert replayed_pending == [] + + +def test_completed_hosted_mcp_approval_reconciliation_skips_exact_replay() -> None: + agent = Agent(name="mcp-approval-agent") + context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={}) + request, approval_item = _build_mcp_approval_request(agent) + context.approve_tool(approval_item) + appended: list[Any] = [] + + process_hosted_mcp_approvals( + original_pre_step_items=[approval_item], + mcp_approval_requests=[request], + context_wrapper=context, + agent=agent, + append_item=appended.append, + ) + + assert len(appended) == 1 + context._mark_tool_call_completed(appended[0].raw_item) + appended.clear() + + pending, pending_ids = process_hosted_mcp_approvals( + original_pre_step_items=[approval_item], + mcp_approval_requests=[request], + context_wrapper=context, + agent=agent, + append_item=appended.append, + ) + + assert appended == [] + assert pending == [] + assert pending_ids == set() + + +def test_sticky_manual_mcp_approval_rejects_same_id_for_changed_tool_name() -> None: + agent = Agent(name="mcp-approval-agent") + context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={}) + _, approval_item = _build_mcp_approval_request(agent) + context.approve_tool(approval_item, always_approve=True) + + same_tool, _ = _build_mcp_approval_request( + agent, + arguments=json.dumps({"query": "changed"}), + ) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + collect_manual_mcp_approvals( + agent=agent, + requests=[same_tool], + context_wrapper=context, + existing_pending_by_call_id={"mcp_call_0": approval_item}, + ) + changed_context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={}) + changed_context.approve_tool(approval_item, always_approve=True) + changed_tool, _ = _build_mcp_approval_request( + agent, + name="delete_all", + arguments=json.dumps({"confirm": True}), + ) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + collect_manual_mcp_approvals( + agent=agent, + requests=[changed_tool], + context_wrapper=changed_context, + existing_pending_by_call_id={"mcp_call_0": approval_item}, + ) + + +def test_sticky_hosted_mcp_approval_rejects_same_id_for_changed_tool_name() -> None: + agent = Agent(name="mcp-approval-agent") + context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={}) + _, approval_item = _build_mcp_approval_request(agent) + context.approve_tool(approval_item, always_approve=True) + + same_tool, _ = _build_mcp_approval_request( + agent, + arguments=json.dumps({"query": "changed"}), + ) + same_appended: list[Any] = [] + with pytest.raises(ModelBehaviorError, match="unique call ID"): + process_hosted_mcp_approvals( + original_pre_step_items=[approval_item], + mcp_approval_requests=[same_tool], + context_wrapper=context, + agent=agent, + append_item=same_appended.append, + ) + changed_context: RunContextWrapper[dict[str, str]] = RunContextWrapper(context={}) + changed_context.approve_tool(approval_item, always_approve=True) + changed_tool, _ = _build_mcp_approval_request( + agent, + name="delete_all", + arguments=json.dumps({"confirm": True}), + ) + changed_appended: list[Any] = [] + with pytest.raises(ModelBehaviorError, match="unique call ID"): + process_hosted_mcp_approvals( + original_pre_step_items=[approval_item], + mcp_approval_requests=[changed_tool], + context_wrapper=changed_context, + agent=agent, + append_item=changed_appended.append, + ) + + assert same_appended == [] + assert changed_appended == [] diff --git a/tests/test_tool_guardrails.py b/tests/test_tool_guardrails.py index 9402edf247..6819ba885d 100644 --- a/tests/test_tool_guardrails.py +++ b/tests/test_tool_guardrails.py @@ -539,8 +539,12 @@ def guarded(query: str) -> str: guarded.tool_output_guardrails = output_guardrails or [] model = FakeModel() - tool_call = [get_function_tool_call("guarded", '{"query": "secret"}')] - model.add_multiple_turn_outputs([tool_call, tool_call]) + model.add_multiple_turn_outputs( + [ + [get_function_tool_call("guarded", '{"query": "secret"}', call_id="guarded_1")], + [get_function_tool_call("guarded", '{"query": "secret"}', call_id="guarded_2")], + ] + ) return Agent(name="guarded_tool_agent", model=model, tools=[guarded]) diff --git a/tests/test_tool_name_collision_policy.py b/tests/test_tool_name_collision_policy.py index 9fa342890b..3eae9ba5f8 100644 --- a/tests/test_tool_name_collision_policy.py +++ b/tests/test_tool_name_collision_policy.py @@ -24,6 +24,7 @@ tool_namespace, ) from agents.items import ToolCallOutputItem +from agents.lifecycle import RunHooks from agents.tool import Tool, function_tool from .fake_model import FakeModel @@ -98,7 +99,7 @@ async def test_resume_error_mode_rejects_current_collision_before_side_effects() @pytest.mark.parametrize("deserialize", [False, True]) @pytest.mark.asyncio -async def test_resume_reclassifies_function_call_to_current_handoff( +async def test_resume_rejects_function_approval_reclassified_as_handoff( deserialize: bool, ) -> None: calls: list[str] = [] @@ -144,15 +145,61 @@ def route_function() -> str: state._model_responses[-1] = replace(state._model_responses[-1], output=[]) state.approve(state.get_interruptions()[0]) - resumed_result = await Runner.run(agent, state) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await Runner.run(agent, state) + + assert calls == [] + assert filter_calls == [] + + +@pytest.mark.asyncio +async def test_reclassified_handoff_is_rejected_before_run_hook() -> None: + calls: list[str] = [] + + route_tool = function_tool( + lambda: "function", + name_override="route", + needs_approval=True, + ) + target = Agent( + name="target", + model=FakeModel(initial_output=[get_text_message("target done")]), + ) + route_handoff = handoff( + target, + tool_name_override="route", + on_handoff=lambda _: calls.append("handoff"), + ) + model = FakeModel(initial_output=[get_function_tool_call("route", "{}", call_id="route")]) + agent = Agent(name="agent", model=model, tools=[route_tool]) + + first = await Runner.run(agent, "Route this request") + state = first.to_state() + state.approve(state.get_interruptions()[0]) + state._model_responses[-1] = replace(state._model_responses[-1], output=[]) + agent.tools = [] + agent.handoffs = [route_handoff] - assert resumed_result.final_output == "target done" - assert calls == ["handoff"] - assert filter_calls == ["filter"] + hook_calls: list[str] = [] + + class RecordingHandoffHooks(RunHooks[Any]): + async def on_handoff( + self, + context: RunContextWrapper[Any], + from_agent: Agent[Any], + to_agent: Agent[Any], + ) -> None: + hook_calls.append("handoff") + + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await Runner.run(agent, state, hooks=RecordingHandoffHooks()) + + assert calls == [] + assert hook_calls == [] @pytest.mark.asyncio -async def test_resume_reclassifies_queued_handoff_to_current_function() -> None: +async def test_resume_rejects_queued_handoff_reclassified_as_function() -> None: calls: list[str] = [] def approved_function() -> str: @@ -192,10 +239,10 @@ def route_function() -> str: agent.handoffs = [] state._model_responses[-1] = replace(state._model_responses[-1], output=[]) - resumed_result = await Runner.run(agent, state) + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await Runner.run(agent, state) - assert resumed_result.final_output == "done" - assert calls == ["approved", "route"] + assert calls == [] @pytest.mark.asyncio @@ -441,7 +488,9 @@ async def test_replacing_interrupted_agent_tool_fails_before_side_effects() -> N ) inner_agent = Agent( name="inner", - model=FakeModel(initial_output=[get_function_tool_call("sensitive", "{}")]), + model=FakeModel( + initial_output=[get_function_tool_call("sensitive", "{}", call_id="call_sensitive")] + ), tools=[sensitive_tool], ) nested_tool = inner_agent.as_tool( @@ -450,7 +499,15 @@ async def test_replacing_interrupted_agent_tool_fails_before_side_effects() -> N ) outer_agent = Agent( name="outer", - model=FakeModel(initial_output=[get_function_tool_call("lookup", '{"input":"hi"}')]), + model=FakeModel( + initial_output=[ + get_function_tool_call( + "lookup", + '{"input":"hi"}', + call_id="call_lookup", + ) + ] + ), tools=[nested_tool], ) @@ -533,7 +590,7 @@ async def test_resume_preserves_model_order_for_function_outcomes() -> None: @pytest.mark.asyncio -async def test_resume_preserves_duplicate_agent_tool_calls() -> None: +async def test_resume_preserves_multiple_agent_tool_calls() -> None: inner_calls: list[str] = [] @function_tool(needs_approval=True) @@ -561,12 +618,12 @@ async def inner_hitl_tool() -> str: get_function_tool_call( agent_tool.name, '{"input":"a"}', - call_id="outer-dup", + call_id="outer-a", ), get_function_tool_call( agent_tool.name, '{"input":"b"}', - call_id="outer-dup", + call_id="outer-b", ), ] ) @@ -587,7 +644,7 @@ async def inner_hitl_tool() -> str: if isinstance(item, ToolCallOutputItem) and isinstance(item.raw_item, dict) and item.raw_item.get("type") == "function_call_output" - and item.raw_item.get("call_id") == "outer-dup" + and item.raw_item.get("call_id") in {"outer-a", "outer-b"} ] assert len(outer_outputs) == 2 @@ -1201,7 +1258,7 @@ async def test_nested_rebind_is_not_committed_before_later_strict_missing_error( @pytest.mark.asyncio -async def test_resume_preserves_cross_kind_duplicate_call_id_baseline() -> None: +async def test_cross_kind_duplicate_call_id_fails_before_execution() -> None: calls: list[str] = [] missing_tool = function_tool( lambda: _record(calls, "missing"), @@ -1213,10 +1270,6 @@ async def test_resume_preserves_cross_kind_duplicate_call_id_baseline() -> None: name_override="lookup", needs_approval=True, ) - replacement_tool = function_tool( - lambda: _record(calls, "replacement"), - name_override="lookup", - ) shell_tool = ShellTool( executor=lambda _request: _record(calls, "shell"), ) @@ -1241,36 +1294,16 @@ async def test_resume_preserves_cross_kind_duplicate_call_id_baseline() -> None: shell_call, ] ) - model.set_next_output([get_text_message("done")]) agent = Agent( name="agent", model=model, tools=[missing_tool, original_tool, shell_tool], ) - initial_result = await Runner.run(agent, "Look this up") - assert calls == ["shell"] - state = initial_result.to_state() - for interruption in state.get_interruptions(): - state.approve(interruption) - agent.tools = [replacement_tool, shell_tool] + with pytest.raises(ModelBehaviorError, match="unique call ID"): + await Runner.run(agent, "Look this up") - resumed_result = await Runner.run( - agent, - state, - run_config=RunConfig(tool_not_found_behavior="return_error_to_model"), - ) - - assert resumed_result.final_output == "done" - assert calls == ["shell", "original"] - output_ids = [ - cast(dict[str, Any], item.raw_item)["call_id"] - for item in resumed_result.new_items - if isinstance(item, ToolCallOutputItem) - and isinstance(item.raw_item, dict) - and item.raw_item.get("type") == "function_call_output" - ] - assert output_ids == ["missing_call", "shared_call"] + assert calls == [] @pytest.mark.parametrize( @@ -1363,7 +1396,10 @@ async def test_resume_rejects_cross_kind_approval_identity_before_sibling_effect @pytest.mark.asyncio -async def test_missing_formatter_cancellation_precedes_sibling_side_effects() -> None: +@pytest.mark.parametrize("streamed", [False, True], ids=["non-streamed", "streamed"]) +async def test_missing_formatter_cancellation_precedes_sibling_side_effects( + streamed: bool, +) -> None: calls: list[str] = [] formatter_started = asyncio.Event() keep_formatter_waiting = asyncio.Event() @@ -1397,14 +1433,20 @@ async def blocking_formatter(_args: Any) -> str: await keep_formatter_waiting.wait() return "missing" + async def resume(run_config: RunConfig) -> Any: + if not streamed: + return await Runner.run(agent, state, run_config=run_config) + result = Runner.run_streamed(agent, state, run_config=run_config) + async for _event in result.stream_events(): + pass + return result + resume_task = asyncio.create_task( - Runner.run( - agent, - state, - run_config=RunConfig( + resume( + RunConfig( tool_not_found_behavior="return_error_to_model", tool_error_formatter=blocking_formatter, - ), + ) ) ) await formatter_started.wait() @@ -1414,11 +1456,7 @@ async def blocking_formatter(_args: Any) -> str: assert calls == [] - resumed_result = await Runner.run( - agent, - state, - run_config=RunConfig(tool_not_found_behavior="return_error_to_model"), - ) + with pytest.raises(ModelBehaviorError, match="already executed"): + await resume(RunConfig(tool_not_found_behavior="return_error_to_model")) - assert resumed_result.final_output == "done" - assert calls == ["available"] + assert calls == [] diff --git a/tests/test_tracing_errors.py b/tests/test_tracing_errors.py index e256f90cc8..b37622ef90 100644 --- a/tests/test_tracing_errors.py +++ b/tests/test_tracing_errors.py @@ -236,8 +236,8 @@ async def test_multiple_handoff_doesnt_error(): # Second turn: a message and 2 handoff [ get_text_message("a_message"), - get_handoff_tool_call(agent_1), - get_handoff_tool_call(agent_2), + get_handoff_tool_call(agent_1, call_id="handoff_1"), + get_handoff_tool_call(agent_2, call_id="handoff_2"), ], # Third turn: text message [get_text_message("done")], @@ -363,7 +363,7 @@ async def test_handoffs_lead_to_correct_agent_spans(): model.add_multiple_turn_outputs( [ # First turn: a tool call - [get_function_tool_call("some_function", json.dumps({"a": "b"}))], + [get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")], # Second turn: a message and 2 handoff [ get_text_message("a_message"), @@ -371,7 +371,7 @@ async def test_handoffs_lead_to_correct_agent_spans(): get_handoff_tool_call(agent_2), ], # Third turn: tool call - [get_function_tool_call("some_function", json.dumps({"a": "b"}))], + [get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_2")], # Fourth turn: handoff [get_handoff_tool_call(agent_3)], # Fifth turn: text message @@ -477,11 +477,11 @@ async def test_max_turns_exceeded(): model.add_multiple_turn_outputs( [ - [get_function_tool_call("foo")], - [get_function_tool_call("foo")], - [get_function_tool_call("foo")], - [get_function_tool_call("foo")], - [get_function_tool_call("foo")], + [get_function_tool_call("foo", call_id="tool_1")], + [get_function_tool_call("foo", call_id="tool_2")], + [get_function_tool_call("foo", call_id="tool_3")], + [get_function_tool_call("foo", call_id="tool_4")], + [get_function_tool_call("foo", call_id="tool_5")], ] ) diff --git a/tests/test_tracing_errors_streamed.py b/tests/test_tracing_errors_streamed.py index 69e65fdadb..52b6b50a58 100644 --- a/tests/test_tracing_errors_streamed.py +++ b/tests/test_tracing_errors_streamed.py @@ -292,8 +292,8 @@ async def test_multiple_handoff_doesnt_error(): # Second turn: a message and 2 handoff [ get_text_message("a_message"), - get_handoff_tool_call(agent_1), - get_handoff_tool_call(agent_2), + get_handoff_tool_call(agent_1, call_id="handoff_1"), + get_handoff_tool_call(agent_2, call_id="handoff_2"), ], # Third turn: text message [get_text_message("done")], @@ -421,7 +421,7 @@ async def test_handoffs_lead_to_correct_agent_spans(): model.add_multiple_turn_outputs( [ # First turn: a tool call - [get_function_tool_call("some_function", json.dumps({"a": "b"}))], + [get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_1")], # Second turn: a message and 2 handoff [ get_text_message("a_message"), @@ -429,7 +429,7 @@ async def test_handoffs_lead_to_correct_agent_spans(): get_handoff_tool_call(agent_2), ], # Third turn: tool call - [get_function_tool_call("some_function", json.dumps({"a": "b"}))], + [get_function_tool_call("some_function", json.dumps({"a": "b"}), call_id="tool_2")], # Fourth turn: handoff [get_handoff_tool_call(agent_3)], # Fifth turn: text message @@ -532,11 +532,11 @@ async def test_max_turns_exceeded(): model.add_multiple_turn_outputs( [ - [get_function_tool_call("foo")], - [get_function_tool_call("foo")], - [get_function_tool_call("foo")], - [get_function_tool_call("foo")], - [get_function_tool_call("foo")], + [get_function_tool_call("foo", call_id="tool_1")], + [get_function_tool_call("foo", call_id="tool_2")], + [get_function_tool_call("foo", call_id="tool_3")], + [get_function_tool_call("foo", call_id="tool_4")], + [get_function_tool_call("foo", call_id="tool_5")], ] )