diff --git a/.changeset/agent-update-instructions.md b/.changeset/agent-update-instructions.md new file mode 100644 index 000000000..69ecaebb4 --- /dev/null +++ b/.changeset/agent-update-instructions.md @@ -0,0 +1,6 @@ +--- +'@livekit/agents': patch +'@livekit/agents-plugin-openai': patch +--- + +Add `Agent.updateInstructions()` to update an agent's instructions mid-session (parity with Python). The change propagates the new instructions through the active `AgentActivity`, records an `AgentConfigUpdate` in the chat and session history, and syncs the realtime/stateless contexts. For OpenAI realtime, per-response instructions now preserve the session-level instructions instead of replacing them. diff --git a/.changeset/bargein-default-threshold-drop-http.md b/.changeset/bargein-default-threshold-drop-http.md new file mode 100644 index 000000000..2d937eb5c --- /dev/null +++ b/.changeset/bargein-default-threshold-drop-http.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Adaptive interruption detection now omits the threshold from `session.create` unless the user explicitly overrides it, letting the gateway apply its fetched default (surfaced via `default_threshold` on `session.created`). The HTTP transport has been dropped — detection always connects over WebSocket and always requires LiveKit credentials, and its base URL now defaults from `LIVEKIT_INFERENCE_URL` instead of `LIVEKIT_REMOTE_EOT_URL`. Inference requests also send an `X-LiveKit-Worker-Token` header when `LIVEKIT_WORKER_TOKEN` is set (hosted agents); a token supplied via the `--worker-token` CLI flag is now re-exported into the environment so forked job subprocesses inherit it and include the header. The `X-LiveKit-Agent-Id` header is now only attached once the room is connected to avoid leaking an unset local-participant SID. The interruption WebSocket is now closed deterministically on stream teardown (including error and cancel paths) instead of only on graceful completion — previously an orphaned socket leaked per session/activity and accumulated for the worker's lifetime. Mid-session threshold/duration changes via `updateOptions` now reconnect the WebSocket in place rather than closing it and letting the next send error the stream — so option changes no longer consume a failover retry (previously enough updates in a session could exhaust the retry budget and stop interruption detection). diff --git a/.changeset/cold-avocados-behave.md b/.changeset/cold-avocados-behave.md new file mode 100644 index 000000000..a2eef8a7b --- /dev/null +++ b/.changeset/cold-avocados-behave.md @@ -0,0 +1,5 @@ +--- +"@livekit/agents": patch +--- + +Add Agent.create method diff --git a/.changeset/gemini-provider-tools.md b/.changeset/gemini-provider-tools.md new file mode 100644 index 000000000..3b1093432 --- /dev/null +++ b/.changeset/gemini-provider-tools.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-google': minor +--- + +Add Gemini provider tools for Google Search, Google Maps, URL context, File Search, code execution, and Vertex RAG retrieval, and serialize them from `ToolContext` for Google LLM and realtime sessions. diff --git a/.changeset/honest-swans-drum.md b/.changeset/honest-swans-drum.md new file mode 100644 index 000000000..e4782071e --- /dev/null +++ b/.changeset/honest-swans-drum.md @@ -0,0 +1,5 @@ +--- +"@livekit/agents": patch +--- + +Don't retain recorded events when recording is disabled diff --git a/.changeset/inworld-delivery-mode.md b/.changeset/inworld-delivery-mode.md new file mode 100644 index 000000000..c02a8c94e --- /dev/null +++ b/.changeset/inworld-delivery-mode.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Add Inworld `delivery_mode` to inference TTS model options. diff --git a/.changeset/list-syntax-toolcontext.md b/.changeset/list-syntax-toolcontext.md new file mode 100644 index 000000000..5e854a813 --- /dev/null +++ b/.changeset/list-syntax-toolcontext.md @@ -0,0 +1,16 @@ +--- +'@livekit/agents': minor +--- + +**BREAKING**: `Agent({ tools })` and `agent.updateTools()` now accept a flat list `(FunctionTool | ProviderTool | Toolset)[]` instead of a `Record` map, and `llm.tool({ ... })` requires a `name` field. `ToolContext` is now a Python-parity class with `functionTools` / `providerTools` / `toolsets` accessors, plus `flatten()`, `hasTool(id)`, `getFunctionTool(id)`, `updateTools()`, `copy()`, and `equals()`. To match the Python reference, registering two **different** function-tool instances under the same `name` now throws `duplicate function name: ` instead of silently overriding the earlier entry; passing the **same instance** twice is a no-op. `agent.toolCtx` returns a defensive copy so callers can no longer mutate the agent's internal state. `LLM.chat({ toolCtx })` accepts either a `ToolContext` instance or a raw `(FunctionTool | ProviderTool | Toolset)[]` array (`ToolCtxInput`) and normalizes it internally, so callers don't have to construct a `ToolContext` themselves. + +Tools also expose an `id: string` field on the base `Tool` interface (parity with Python's `Tool.id` property): for `FunctionTool` it mirrors `name`, for `ProviderTool` it is the provider tool id. `ToolContext` keys and equality now use `tool.id` consistently. + +**BREAKING**: Provider tools are now modeled to match Python's `ProviderTool`: + +- `ProviderDefinedTool` is renamed to `ProviderTool`, and `isProviderDefinedTool` is renamed to `isProviderTool`. +- `ProviderTool` is now an **abstract class** (Python parity). Plugins must subclass it (`class WebSearch extends ProviderTool { ... }`) to attach provider-specific fields and serializers; bare `new ProviderTool(...)` is rejected at compile time. +- The `tool({ id })` factory overload is removed; `tool({ ... })` only creates function tools now. Construct provider tools by instantiating a `ProviderTool` subclass. +- The `ToolType` literal for provider tools is renamed from `'provider-defined'` to `'provider'`. + +`Toolset` now carries a `TOOLSET_SYMBOL` marker and is detected via a new `isToolset()` guard (consistent with `isFunctionTool` / `isProviderTool`). Existing `instanceof Toolset` checks still work, but symbol-based detection is preferred for cross-realm safety. diff --git a/.changeset/odd-languages-speak.md b/.changeset/odd-languages-speak.md new file mode 100644 index 000000000..b70a32109 --- /dev/null +++ b/.changeset/odd-languages-speak.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-elevenlabs': patch +--- + +Avoid persisting per-call STT language overrides on ElevenLabs STT instances. diff --git a/.changeset/openai-provider-tools.md b/.changeset/openai-provider-tools.md new file mode 100644 index 000000000..8e793a935 --- /dev/null +++ b/.changeset/openai-provider-tools.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-openai': minor +--- + +Add OpenAI Responses provider tools for web search, file search, and code interpreter. diff --git a/.changeset/polite-avatars-swap.md b/.changeset/polite-avatars-swap.md new file mode 100644 index 000000000..4ed64af3f --- /dev/null +++ b/.changeset/polite-avatars-swap.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +fix(avatar): preserve audio wrappers across avatar hot-swaps diff --git a/.changeset/port-end-call-tool.md b/.changeset/port-end-call-tool.md new file mode 100644 index 000000000..f1646f943 --- /dev/null +++ b/.changeset/port-end-call-tool.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': minor +--- + +Add beta EndCallTool for ending calls from agent tools diff --git a/.changeset/quick-meals-breathe.md b/.changeset/quick-meals-breathe.md new file mode 100644 index 000000000..d32233b93 --- /dev/null +++ b/.changeset/quick-meals-breathe.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Adds base `Toolset` support: a stateful container for a group of tools with `setup()` / `aclose()` lifecycle hooks. Toolsets can be passed directly into `Agent({ tools: [...] })` alongside individual function tools; their tools are flattened into the agent's `ToolContext` and the runtime drives `setup()` on activity start, `aclose()` on close, and a setup/close diff when `agent.updateTools()` adds or removes Toolsets mid-session. Per-toolset `setup()` errors are logged but do not abort the activity. The `IGNORE_ON_ENTER` flag is also respected for function tools nested inside a Toolset. Every LLM and realtime plugin tool builder iterates `ToolContext.flatten()` so toolset-contributed tools are correctly advertised. Also exports `ToolCalledEvent` / `ToolCompletedEvent` payload types. diff --git a/.changeset/scope-forward-audio-playback-started.md b/.changeset/scope-forward-audio-playback-started.md new file mode 100644 index 000000000..5c5cd277e --- /dev/null +++ b/.changeset/scope-forward-audio-playback-started.md @@ -0,0 +1,15 @@ +--- +'@livekit/agents': patch +--- + +fix(voice): scope forwardAudio's playback-started listener to its own segment + +When a speech is interrupted, the scheduling loop immediately authorizes the next +speech, so the new segment's `forwardAudio` registers its `playback_started` +listener on the shared audio output while the interrupted segment is still +emitting events during teardown. The stray event resolved the new segment's +`firstFrameFut` before its first frame was captured, which skipped resampler +creation and pushed an unresampled frame straight to the `AudioSource` +(`RtcError: sample_rate and num_channels don't match`) and corrupted playback +bookkeeping. The listener now only resolves `firstFrameFut` after the segment has +captured its own first frame. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a527b4fe4..5681c1fca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,5 +48,13 @@ jobs: cache: pnpm - name: Install dependencies run: pnpm install --frozen-lockfile --ignore-scripts + - name: Cache turbo + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: .turbo + key: turbo-${{ runner.os }}-node20-${{ hashFiles('pnpm-lock.yaml') }}-${{ github.run_id }} + restore-keys: | + turbo-${{ runner.os }}-node20-${{ hashFiles('pnpm-lock.yaml') }}- + turbo-${{ runner.os }}-node20- - name: Build run: pnpm build diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index de5e3535e..d205e932d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,6 +29,14 @@ jobs: cache: pnpm - name: Install dependencies run: pnpm install --frozen-lockfile --ignore-scripts + - name: Cache turbo + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: .turbo + key: turbo-${{ runner.os }}-node20-${{ hashFiles('pnpm-lock.yaml') }}-${{ github.run_id }} + restore-keys: | + turbo-${{ runner.os }}-node20-${{ hashFiles('pnpm-lock.yaml') }}- + turbo-${{ runner.os }}-node20- - name: Build run: pnpm build - name: Check which tests to run diff --git a/agents/src/beta/index.ts b/agents/src/beta/index.ts index 98ac382e9..d5141be55 100644 --- a/agents/src/beta/index.ts +++ b/agents/src/beta/index.ts @@ -12,3 +12,10 @@ export { type WarmTransferTaskOptions, } from './workflows/index.js'; export { Instructions } from '../llm/index.js'; +export { + END_CALL_DESCRIPTION, + createEndCallTool, + type EndCallToolCalledEvent, + type EndCallToolCompletedEvent, + type EndCallToolOptions, +} from './tools/index.js'; diff --git a/agents/src/beta/tools/end_call.ts b/agents/src/beta/tools/end_call.ts new file mode 100644 index 000000000..962deda36 --- /dev/null +++ b/agents/src/beta/tools/end_call.ts @@ -0,0 +1,181 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { type EventEmitter, once } from 'node:events'; +import { setTimeout as waitFor } from 'node:timers/promises'; +import { getJobContext } from '../../job.js'; +import { + RealtimeModel, + type ToolCalledEvent, + type ToolCompletedEvent, + Toolset, + tool, +} from '../../llm/index.js'; +import { log } from '../../log.js'; +import type { AgentSession, AgentSessionCallbacks } from '../../voice/agent_session.js'; +import { AgentSessionEventTypes } from '../../voice/events.js'; +import type { UnknownUserData } from '../../voice/run_context.js'; + +/** How long to wait for the agent's goodbye reply to play out before forcing shutdown. */ +const END_CALL_REPLY_TIMEOUT = 5000; + +/** Typed wrapper around `events.once`; abort resolves to `undefined`, other errors propagate. */ +function onceEvent( + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- callbacks don't depend on UserData + session: AgentSession, + event: E, + options?: { signal?: AbortSignal }, +): Promise[0] | undefined> { + return ( + once(session as unknown as EventEmitter, event, options) as Promise< + Parameters + > + ).then( + ([payload]) => payload, + (err) => { + if (options?.signal?.aborted) return undefined; + throw err; + }, + ); +} + +export const END_CALL_DESCRIPTION = ` +Ends the current call and disconnects immediately. + +Call when: +- The user clearly indicates they are done (e.g., "that's all, bye"). + +Do not call when: +- The user asks to pause, hold, or transfer. +- Intent is unclear. + +This is the final action the agent can take. +Once called, no further interaction is possible with the user. +Don't generate any other text or response when the tool is called. +`; + +export type EndCallToolCalledEvent = ToolCalledEvent; + +export type EndCallToolCompletedEvent = ToolCompletedEvent; + +export type EndCallToolOptions = { + /** Additional description to add to the end call tool. */ + extraDescription?: string; + /** + * Whether to delete the room when the user ends the call. + * Deleting the room disconnects all remote users, including SIP callers. + */ + deleteRoom?: boolean; + /** Tool output to the LLM for generating the tool response. */ + endInstructions?: string | null; + /** Callback to call when the tool is called. */ + onToolCalled?: (event: EndCallToolCalledEvent) => Promise | void; + /** Callback to call when the tool is completed. */ + onToolCompleted?: (event: EndCallToolCompletedEvent) => Promise | void; +}; + +/** + * Allows the agent to end the call and disconnect from the room. + */ +export function createEndCallTool({ + extraDescription = '', + deleteRoom = true, + endInstructions = 'say goodbye to the user', + onToolCalled, + onToolCompleted, +}: EndCallToolOptions = {}): Toolset { + // For a realtime LLM that generates the goodbye reply itself, wait for that reply to play out + // (bounded by END_CALL_REPLY_TIMEOUT) before shutting down. `signal` is aborted when the call + // ends or the toolset is torn down, which cancels whichever of the two races is still pending. + const delayedSessionShutdown = async ( + session: AgentSession, + signal: AbortSignal, + ): Promise => { + const speech = onceEvent(session, AgentSessionEventTypes.SpeechCreated, { signal }).then( + (event) => event?.speechHandle, + ); + const timeout = waitFor(END_CALL_REPLY_TIMEOUT, 'timeout' as const, { signal }).catch( + () => undefined, + ); + + const winner = await Promise.race([speech, timeout]); + if (signal.aborted) return; // session already closed or toolset torn down + + if (winner === 'timeout') { + log().warn('tool reply timed out, shutting down session'); + session.shutdown(); + } else if (winner) { + await winner.waitForPlayout(); + session.shutdown(); + } + }; + + return Toolset.create({ + id: 'end_call', + tools: [ + tool({ + name: 'end_call', + description: `${END_CALL_DESCRIPTION}\n${extraDescription}`, + execute: async (_args, { ctx, abortSignal }) => { + log().debug('end_call tool called'); + const session = ctx.session; + const llm = session.currentAgent.getActivityOrThrow().llm; + + // Lifetime of this invocation: aborts when the session closes, and also when the tool + // call itself is aborted. All listeners/timers below are scoped to it. + const controller = new AbortController(); + const signal = abortSignal + ? AbortSignal.any([abortSignal, controller.signal]) + : controller.signal; + + void onceEvent(session, AgentSessionEventTypes.Close, { signal }) + .then((event) => { + if (!event) return; // signal aborted before close fired + controller.abort(); // stop the delayed-shutdown race + + const jobCtx = getJobContext(false); + if (!jobCtx) return; + + if (deleteRoom) { + jobCtx.addShutdownCallback(async () => { + log().info('deleting the room because the user ended the call'); + await jobCtx.deleteRoom(); + }); + } + + jobCtx.shutdown(String(event.reason)); + }) + .catch((error) => log().error({ error }, 'error during end call shutdown')); + + ctx.speechHandle.addDoneCallback(() => { + if (!(llm instanceof RealtimeModel) || !llm.capabilities.autoToolReplyGeneration) { + session.shutdown(); + return; + } + + void delayedSessionShutdown(session, signal).catch((error) => + log().error({ error }, 'error during delayed session shutdown'), + ); + }); + + if (onToolCalled) { + await onToolCalled({ ctx, arguments: {} }); + } + + const completedEvent = { + ctx, + output: + endInstructions === null + ? undefined + : ({ type: 'output', value: endInstructions } as const), + }; + if (onToolCompleted) { + await onToolCompleted(completedEvent); + } + + return endInstructions ?? undefined; + }, + }), + ], + }); +} diff --git a/agents/src/beta/tools/index.ts b/agents/src/beta/tools/index.ts new file mode 100644 index 000000000..8ef18b993 --- /dev/null +++ b/agents/src/beta/tools/index.ts @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +export { + END_CALL_DESCRIPTION, + createEndCallTool, + type EndCallToolCalledEvent, + type EndCallToolCompletedEvent, + type EndCallToolOptions, +} from './end_call.js'; diff --git a/agents/src/beta/workflows/task_group.ts b/agents/src/beta/workflows/task_group.ts index 8c96790dd..add6655fb 100644 --- a/agents/src/beta/workflows/task_group.ts +++ b/agents/src/beta/workflows/task_group.ts @@ -84,10 +84,7 @@ export class TaskGroup extends AgentTask { const outOfScopeTool = this.buildOutOfScopeTool(taskId); if (outOfScopeTool) { - await this._currentTask.updateTools({ - ...this._currentTask.toolCtx, - out_of_scope: outOfScopeTool, - }); + await this._currentTask.updateTools([...this._currentTask.toolCtx.tools, outOfScopeTool]); } try { @@ -190,6 +187,7 @@ export class TaskGroup extends AgentTask { const visitedTasks = this._visitedTasks; return tool({ + name: 'out_of_scope', description, flags: ToolFlag.IGNORE_ON_ENTER, parameters: z.object({ diff --git a/agents/src/beta/workflows/warm_transfer.ts b/agents/src/beta/workflows/warm_transfer.ts index b3d99dda0..3cf0016d3 100644 --- a/agents/src/beta/workflows/warm_transfer.ts +++ b/agents/src/beta/workflows/warm_transfer.ts @@ -12,9 +12,9 @@ import type { Instructions, LLM, RealtimeModel, - ToolContext, + ToolContextEntry, } from '../../llm/index.js'; -import { ToolError, ToolFlag, tool } from '../../llm/index.js'; +import { ToolContext, ToolError, ToolFlag, tool } from '../../llm/index.js'; import { log } from '../../log.js'; import type { STT } from '../../stt/index.js'; import type { TTS } from '../../tts/index.js'; @@ -72,7 +72,7 @@ export interface WarmTransferTaskOptions { instructions?: InstructionParts | string; chatCtx?: ChatContext; turnDetection?: TurnDetectionMode | null; - tools?: ToolContext; + tools?: readonly ToolContextEntry[]; stt?: STT | STTModelString | null; vad?: VAD | null; llm?: LLM | RealtimeModel | LLMModels | null; @@ -171,13 +171,13 @@ export class WarmTransferTask extends AgentTask { this._resolveHumanAgentFailed = resolve; }); - this._tools = { - ...this._tools, - connect_to_caller: this.buildConnectToCallerTool(), - decline_transfer: this.buildDeclineTransferTool(), - voicemail_detected: this.buildVoicemailDetectedTool(), - }; - this._chatCtx = this._chatCtx.copy({ toolCtx: this._tools }); + this._toolCtx = new ToolContext([ + ...this._toolCtx.tools, + this.buildConnectToCallerTool(), + this.buildDeclineTransferTool(), + this.buildVoicemailDetectedTool(), + ]); + this._chatCtx = this._chatCtx.copy({ toolCtx: this._toolCtx }); this._taskTurnDetection = turnDetection ?? undefined; this._allowInterruptions = allowInterruptions; @@ -268,6 +268,7 @@ export class WarmTransferTask extends AgentTask { private buildConnectToCallerTool() { return tool({ + name: 'connect_to_caller', description: 'Called when the human agent wants to connect to the caller.', flags: ToolFlag.IGNORE_ON_ENTER, execute: async () => { @@ -288,6 +289,7 @@ export class WarmTransferTask extends AgentTask { private buildDeclineTransferTool() { return tool({ + name: 'decline_transfer', description: 'Handles the case when the human agent explicitly declines to connect to the caller.', parameters: z.object({ @@ -304,6 +306,7 @@ export class WarmTransferTask extends AgentTask { private buildVoicemailDetectedTool() { return tool({ + name: 'voicemail_detected', description: 'Called when the call reaches voicemail. Use this tool AFTER you hear the voicemail greeting', flags: ToolFlag.IGNORE_ON_ENTER, @@ -418,7 +421,7 @@ export class WarmTransferTask extends AgentTask { vad: this.vad, llm: this.llm, tts: this.tts, - tools: this.toolCtx, + tools: this.toolCtx.tools, chatCtx: this._chatCtx.copy(), turnDetection: this._taskTurnDetection, allowInterruptions: this._allowInterruptions, diff --git a/agents/src/generator.test.ts b/agents/src/generator.test.ts new file mode 100644 index 000000000..a92ff9e58 --- /dev/null +++ b/agents/src/generator.test.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { describe, expect, it } from 'vitest'; +import { defineAgent, isAgent } from './generator.js'; + +describe('generator', () => { + it('marks definitions created with defineAgent as agents', () => { + const agent = defineAgent({ + entry: async () => {}, + }); + + expect(isAgent(agent)).toBe(true); + }); + + it('does not treat unmarked structural objects as agents', () => { + expect(isAgent({ entry: async () => {} })).toBe(false); + }); +}); diff --git a/agents/src/generator.ts b/agents/src/generator.ts index b7e7ad93f..e9bb09075 100644 --- a/agents/src/generator.ts +++ b/agents/src/generator.ts @@ -3,24 +3,22 @@ // SPDX-License-Identifier: Apache-2.0 import type { JobContext, JobProcess } from './job.js'; +export const AGENT_DEFINITION_SYMBOL = Symbol.for('livekit.agents.AgentDefinition'); + /** @see {@link defineAgent} */ -export interface Agent> { +export interface AgentDefinition> { entry: (ctx: JobContext) => Promise; prewarm?: (proc: JobProcess) => unknown; } +export type Agent> = AgentDefinition; + /** Helper to check if an object is an agent before running it. * * @internal */ -export function isAgent(obj: unknown): obj is Agent { - return ( - typeof obj === 'object' && - obj !== null && - 'entry' in obj && - typeof (obj as Agent).entry === 'function' && - (('prewarm' in obj && typeof (obj as Agent).prewarm === 'function') || !('prewarm' in obj)) - ); +export function isAgent(obj: unknown): obj is AgentDefinition { + return typeof obj === 'object' && obj !== null && AGENT_DEFINITION_SYMBOL in obj; } /** @@ -34,7 +32,10 @@ export function isAgent(obj: unknown): obj is Agent { * ``` */ export function defineAgent>( - agent: Agent, -): Agent { + agent: AgentDefinition, +): AgentDefinition { + Object.defineProperty(agent, AGENT_DEFINITION_SYMBOL, { + value: true, + }); return agent; } diff --git a/agents/src/index.test.ts b/agents/src/index.test.ts new file mode 100644 index 000000000..40b7ade44 --- /dev/null +++ b/agents/src/index.test.ts @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { describe, expect, it } from 'vitest'; +import { + Agent, + AgentSession, + ChatContext, + ModelUsageCollector, + logMetrics, + tool, +} from './index.js'; + +describe('index exports', () => { + it('exports voice, llm, and metrics APIs directly from the package root', () => { + expect(Agent).toBeDefined(); + expect(AgentSession).toBeDefined(); + expect(ChatContext).toBeDefined(); + expect(tool).toBeDefined(); + expect(ModelUsageCollector).toBeDefined(); + expect(logMetrics).toBeDefined(); + }); +}); diff --git a/agents/src/index.ts b/agents/src/index.ts index 07e4b45da..9f1519bf3 100644 --- a/agents/src/index.ts +++ b/agents/src/index.ts @@ -14,14 +14,16 @@ export * from './audio.js'; export * as beta from './beta/index.js'; export * as cli from './cli.js'; export * from './connection_pool.js'; -export * from './generator.js'; +export { defineAgent, isAgent, type AgentDefinition } from './generator.js'; export * as inference from './inference/index.js'; export * from './inference_runner.js'; export * as ipc from './ipc/index.js'; export * from './job.js'; export * from './language.js'; +export * from './llm/index.js'; export * as llm from './llm/index.js'; export * from './log.js'; +export * from './metrics/index.js'; export * as metrics from './metrics/index.js'; export * from './plugin.js'; export * as stream from './stream/index.js'; @@ -34,6 +36,6 @@ export * from './types.js'; export * from './utils.js'; export * from './vad.js'; export * from './version.js'; +export * from './voice/index.js'; export * as voice from './voice/index.js'; -export { createTimedString, isTimedString, type TimedString } from './voice/io.js'; export * from './worker.js'; diff --git a/agents/src/inference/llm.ts b/agents/src/inference/llm.ts index 7196ba998..ea2213ddc 100644 --- a/agents/src/inference/llm.ts +++ b/agents/src/inference/llm.ts @@ -277,7 +277,7 @@ export class LLM extends llm.LLM { chat({ chatCtx, - toolCtx, + toolCtx: toolCtxInput, connOptions = DEFAULT_API_CONNECT_OPTIONS, parallelToolCalls, toolChoice, @@ -285,13 +285,14 @@ export class LLM extends llm.LLM { extraKwargs, }: { chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: llm.ToolChoice; inferenceClass?: InferenceClass; extraKwargs?: Record; }): LLMStream { + const toolCtx = llm.toToolContext(toolCtxInput); let modelOptions: Record = { ...(extraKwargs || {}) }; parallelToolCalls = @@ -299,7 +300,11 @@ export class LLM extends llm.LLM { ? parallelToolCalls : this.opts.modelOptions.parallel_tool_calls; - if (toolCtx && Object.keys(toolCtx).length > 0 && parallelToolCalls !== undefined) { + if ( + toolCtx && + Object.keys(toolCtx.functionTools).length > 0 && + parallelToolCalls !== undefined + ) { modelOptions.parallel_tool_calls = parallelToolCalls; } @@ -402,6 +407,8 @@ export class LLMStream extends llm.LLMStream { this.providerFmt, )) as OpenAI.ChatCompletionMessageParam[]; + // Provider-defined tools are not supported by the inference adapter; `sortedToolEntries` + // yields only function tools (sorted by name), so they are skipped here. See AJS-112. const tools = this.toolCtx ? llm.sortedToolEntries(this.toolCtx).map(([name, func]) => { const oaiParams = { diff --git a/agents/src/inference/tts.ts b/agents/src/inference/tts.ts index 47a93bb04..78c0d2a26 100644 --- a/agents/src/inference/tts.ts +++ b/agents/src/inference/tts.ts @@ -127,6 +127,8 @@ export interface InworldOptions { speaking_rate?: number; /** Range 0-2. */ temperature?: number; + /** Controls output variation on inworld-tts-2 only. */ + delivery_mode?: 'DELIVERY_MODE_UNSPECIFIED' | 'STABLE' | 'BALANCED' | 'CREATIVE'; timestamp_type?: 'TIMESTAMP_TYPE_UNSPECIFIED' | 'WORD' | 'CHARACTER'; apply_text_normalization?: 'APPLY_TEXT_NORMALIZATION_UNSPECIFIED' | 'ON' | 'OFF'; /** @deprecated Backward-compatible alias. Use `apply_text_normalization`. */ diff --git a/agents/src/llm/chat_context.test.ts b/agents/src/llm/chat_context.test.ts index 849469e55..101a9fe46 100644 --- a/agents/src/llm/chat_context.test.ts +++ b/agents/src/llm/chat_context.test.ts @@ -19,6 +19,7 @@ import { isInstructions, renderInstructions, } from './chat_context.js'; +import { ProviderTool, ToolContext, tool } from './tool_context.js'; initializeLogger({ pretty: false, level: 'error' }); @@ -1479,3 +1480,32 @@ extra`; expect((baseCtx.items[0]! as ChatMessage).content[0]).toBe(instr); }); }); + +describe('ChatContext.copy with toolCtx filter', () => { + it('drops function calls / outputs whose tool is not in the supplied ToolContext', () => { + const known = tool({ name: 'known', description: 'k', execute: async () => 'ok' }); + const ctx = new ChatContext([ + ChatMessage.create({ role: 'user', content: ['hello'] }), + FunctionCall.create({ callId: 'c1', name: 'known', args: '{}' }), + FunctionCallOutput.create({ callId: 'c1', name: 'known', output: 'done', isError: false }), + FunctionCall.create({ callId: 'c2', name: 'removed', args: '{}' }), + FunctionCallOutput.create({ callId: 'c2', name: 'removed', output: 'x', isError: false }), + ]); + + const filtered = ctx.copy({ toolCtx: new ToolContext([known]) }); + const types = filtered.items.map((i) => `${i.type}:${'name' in i ? i.name : ''}`); + expect(types).toEqual(['message:', 'function_call:known', 'function_call_output:known']); + }); + + it('keeps provider-tool calls when the ToolContext holds a matching provider tool id', () => { + class CodeRunner extends ProviderTool {} + const provider = new CodeRunner({ id: 'code_runner' }); + const ctx = new ChatContext([ + FunctionCall.create({ callId: 'p1', name: 'code_runner', args: '{}' }), + FunctionCall.create({ callId: 'p2', name: 'other', args: '{}' }), + ]); + + const filtered = ctx.copy({ toolCtx: new ToolContext([provider]) }); + expect(filtered.items.map((i) => ('name' in i ? i.name : ''))).toEqual(['code_runner']); + }); +}); diff --git a/agents/src/llm/chat_context.ts b/agents/src/llm/chat_context.ts index 743e7efb8..34cf39200 100644 --- a/agents/src/llm/chat_context.ts +++ b/agents/src/llm/chat_context.ts @@ -835,7 +835,7 @@ export class ChatContext { continue; } - if (toolCtx !== undefined && isToolCallOrOutput(item) && toolCtx[item.name] === undefined) { + if (toolCtx !== undefined && isToolCallOrOutput(item) && !toolCtx.hasTool(item.name)) { continue; } diff --git a/agents/src/llm/fallback_adapter.test.ts b/agents/src/llm/fallback_adapter.test.ts index a9747c885..d466c9306 100644 --- a/agents/src/llm/fallback_adapter.test.ts +++ b/agents/src/llm/fallback_adapter.test.ts @@ -9,7 +9,7 @@ import { delay } from '../utils.js'; import type { ChatContext } from './chat_context.js'; import { FallbackAdapter } from './fallback_adapter.js'; import { type ChatChunk, LLM, LLMStream } from './llm.js'; -import type { ToolChoice, ToolContext } from './tool_context.js'; +import type { ToolChoice, ToolCtxInput } from './tool_context.js'; class MockLLMStream extends LLMStream { public myLLM: LLM; @@ -18,7 +18,7 @@ class MockLLMStream extends LLMStream { llm: LLM, opts: { chatCtx: ChatContext; - toolCtx?: ToolContext; + toolCtx?: ToolCtxInput; connOptions: APIConnectOptions; }, private shouldFail: boolean = false, @@ -64,7 +64,7 @@ class MockLLM extends LLM { chat(opts: { chatCtx: ChatContext; - toolCtx?: ToolContext; + toolCtx?: ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: ToolChoice; diff --git a/agents/src/llm/fallback_adapter.ts b/agents/src/llm/fallback_adapter.ts index 128c2392c..27d87d2a0 100644 --- a/agents/src/llm/fallback_adapter.ts +++ b/agents/src/llm/fallback_adapter.ts @@ -8,7 +8,7 @@ import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js import type { ChatContext } from './chat_context.js'; import type { ChatChunk } from './llm.js'; import { LLM, LLMStream } from './llm.js'; -import type { ToolChoice, ToolContext } from './tool_context.js'; +import type { ToolChoice, ToolCtxInput } from './tool_context.js'; /** * Default connection options for FallbackAdapter. @@ -113,7 +113,7 @@ export class FallbackAdapter extends LLM { chat(opts: { chatCtx: ChatContext; - toolCtx?: ToolContext; + toolCtx?: ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: ToolChoice; @@ -159,7 +159,7 @@ class FallbackLLMStream extends LLMStream { adapter: FallbackAdapter, opts: { chatCtx: ChatContext; - toolCtx?: ToolContext; + toolCtx?: ToolCtxInput; connOptions: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: ToolChoice; diff --git a/agents/src/llm/index.ts b/agents/src/llm/index.ts index 8fab0e4a7..400849be4 100644 --- a/agents/src/llm/index.ts +++ b/agents/src/llm/index.ts @@ -4,18 +4,29 @@ export { handoff, isFunctionTool, - tool, + isProviderTool, + isTool, + isToolset, + ProviderTool, sortedToolEntries, sortedToolNames, + tool, + ToolContext, ToolError, ToolFlag, + Toolset, + toToolContext, type AgentHandoff, type FunctionTool, - type ProviderDefinedTool, type Tool, + type ToolCalledEvent, type ToolChoice, - type ToolContext, + type ToolCompletedEvent, + type ToolContextEntry, + type ToolCtxInput, type ToolOptions, + type ToolsetContext, + type ToolsetCreateOptions, type ToolType, } from './tool_context.js'; diff --git a/agents/src/llm/llm.ts b/agents/src/llm/llm.ts index 553541939..828e134e9 100644 --- a/agents/src/llm/llm.ts +++ b/agents/src/llm/llm.ts @@ -11,7 +11,12 @@ import { recordException, traceTypes, tracer } from '../telemetry/index.js'; import { type APIConnectOptions, intervalForRetry } from '../types.js'; import { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js'; import { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js'; -import type { ToolChoice, ToolContext } from './tool_context.js'; +import { + type ToolChoice, + type ToolContext, + type ToolCtxInput, + toToolContext, +} from './tool_context.js'; export interface ChoiceDelta { role: ChatRole; @@ -91,7 +96,12 @@ export abstract class LLM extends (EventEmitter as new () => TypedEmitter { connOptions, }: { chatCtx: ChatContext; - toolCtx?: ToolContext; + toolCtx?: ToolCtxInput; connOptions: APIConnectOptions; }, ) { this.#llm = llm; this.#chatCtx = chatCtx; - this.#toolCtx = toolCtx; + this.#toolCtx = toToolContext(toolCtx); this._connOptions = connOptions; this.monitorMetrics(); this.abortController.signal.addEventListener('abort', () => { diff --git a/agents/src/llm/tool_context.test.ts b/agents/src/llm/tool_context.test.ts index 183c7ff2a..5f2f80cca 100644 --- a/agents/src/llm/tool_context.test.ts +++ b/agents/src/llm/tool_context.test.ts @@ -5,7 +5,15 @@ import { describe, expect, it } from 'vitest'; import { z } from 'zod'; import * as z3 from 'zod/v3'; import * as z4 from 'zod/v4'; -import { type ToolOptions, tool } from './tool_context.js'; +import { + ProviderTool, + type Tool, + ToolContext, + type ToolOptions, + Toolset, + type ToolsetContext, + tool, +} from './tool_context.js'; import { createToolOptions, oaiParams } from './utils.js'; describe('Tool Context', () => { @@ -77,6 +85,7 @@ describe('Tool Context', () => { describe('tool', () => { it('should create and execute a basic core tool', async () => { const getWeather = tool({ + name: 'getWeather', description: 'Get the weather for a given location', parameters: z.object({ location: z.string(), @@ -95,6 +104,7 @@ describe('Tool Context', () => { it('should properly type a callable function', async () => { const testFunction = tool({ + name: 'testFunction', description: 'Test function', parameters: z.object({ name: z.string().describe('The user name'), @@ -114,6 +124,7 @@ describe('Tool Context', () => { it('should handle async execution', async () => { const testFunction = tool({ + name: 'asyncTestFunction', description: 'Async test function', parameters: z.object({ delay: z.number().describe('Delay in milliseconds'), @@ -157,6 +168,7 @@ describe('Tool Context', () => { describe('optional parameters', () => { it('should create a tool without parameters', async () => { const simpleAction = tool({ + name: 'simpleAction', description: 'Perform a simple action', execute: async () => { return 'Action performed'; @@ -175,6 +187,7 @@ describe('Tool Context', () => { it('should support .optional() fields in tool parameters', async () => { const weatherTool = tool({ + name: 'weatherTool', description: 'Get weather information', parameters: z.object({ location: z.string().describe('The city or location').optional(), @@ -205,6 +218,7 @@ describe('Tool Context', () => { it('should handle tools with context but no parameters', async () => { const greetUser = tool({ + name: 'greetUser', description: 'Greet the current user', execute: async (_, { ctx }: ToolOptions<{ username: string }>) => { return `Hello, ${ctx.userData.username}!`; @@ -217,6 +231,7 @@ describe('Tool Context', () => { it('should create a tool that accesses tool call id without parameters', async () => { const getCallId = tool({ + name: 'getCallId', description: 'Get the current tool call ID', execute: async (_, { toolCallId }) => { return `Tool call ID: ${toolCallId}`; @@ -231,6 +246,7 @@ describe('Tool Context', () => { describe('Zod v3 and v4 compatibility', () => { it('should work with Zod v3 schemas', async () => { const v3Tool = tool({ + name: 'v3Tool', description: 'A tool using Zod v3 schema', parameters: z3.object({ name: z3.string(), @@ -250,6 +266,7 @@ describe('Tool Context', () => { it('should work with Zod v4 schemas', async () => { const v4Tool = tool({ + name: 'v4Tool', description: 'A tool using Zod v4 schema', parameters: z4.object({ name: z4.string(), @@ -269,6 +286,7 @@ describe('Tool Context', () => { it('should handle v4 schemas with optional fields', async () => { const v4Tool = tool({ + name: 'v4OptionalTool', description: 'Tool with optional field using v4', parameters: z4.object({ required: z4.string(), @@ -291,6 +309,7 @@ describe('Tool Context', () => { it('should handle v4 enum schemas', async () => { const v4Tool = tool({ + name: 'v4EnumTool', description: 'Tool with enum using v4', parameters: z4.object({ color: z4.enum(['red', 'blue', 'green']), @@ -306,6 +325,7 @@ describe('Tool Context', () => { it('should handle v4 array schemas', async () => { const v4Tool = tool({ + name: 'v4ArrayTool', description: 'Tool with array using v4', parameters: z4.object({ tags: z4.array(z4.string()), @@ -324,6 +344,7 @@ describe('Tool Context', () => { it('should handle v4 nested object schemas', async () => { const v4Tool = tool({ + name: 'v4NestedTool', description: 'Tool with nested object using v4', parameters: z4.object({ user: z4.object({ @@ -405,3 +426,303 @@ describe('Tool Context', () => { }); }); }); + +describe('tool() name requirement', () => { + it('throws when name is missing', () => { + expect(() => + // @ts-expect-error - name is required + tool({ + description: 'no name', + execute: async () => 'x', + }), + ).toThrow('requires a non-empty name'); + }); + + it('throws when name is empty', () => { + expect(() => + tool({ + name: '', + description: 'empty name', + execute: async () => 'x', + }), + ).toThrow('requires a non-empty name'); + }); + + it('stores the name on the returned function tool', () => { + const t = tool({ + name: 'doStuff', + description: 'd', + execute: async () => 'x', + }); + expect(t.name).toBe('doStuff'); + }); + + it('exposes id mirroring the function tool name', () => { + const t = tool({ + name: 'doStuff', + description: 'd', + execute: async () => 'x', + }); + expect(t.id).toBe('doStuff'); + expect(t.id).toBe(t.name); + }); +}); + +class TestProviderTool extends ProviderTool {} + +describe('ToolContext', () => { + const makeFn = (name: string) => + tool({ + name, + description: `${name} tool`, + execute: async () => name, + }); + + it('empty() returns an empty context', () => { + const ctx = ToolContext.empty(); + expect(ctx.functionTools).toEqual({}); + expect(ctx.providerTools).toEqual([]); + expect(ctx.toolsets).toEqual([]); + expect(ctx.flatten()).toEqual([]); + }); + + it('indexes function tools by name and supports lookup', () => { + const a = makeFn('a'); + const b = makeFn('b'); + const ctx = new ToolContext([a, b]); + + expect(ctx.functionTools).toEqual({ a, b }); + expect(ctx.getFunctionTool('a')).toBe(a); + expect(ctx.getFunctionTool('b')).toBe(b); + expect(ctx.getFunctionTool('missing')).toBeUndefined(); + }); + + it('throws on duplicate function names with different instances', () => { + // Matches Python's `if existing is not tool: raise ValueError(...)` — silently overriding + // a registered tool would mask a real bug at the caller (two distinct functions colliding + // on a single advertised name). + const a1 = makeFn('a'); + const a2 = makeFn('a'); + expect(() => new ToolContext([a1, a2])).toThrow('duplicate function name: a'); + }); + + it('silently skips the same function tool instance listed multiple times', () => { + // Matches Python's `return # same instance, skip` branch. Useful when a tool gets + // included both directly and via a future Toolset that re-exports it. + const a = makeFn('a'); + const ctx = new ToolContext([a, a]); + expect(ctx.getFunctionTool('a')).toBe(a); + expect(Object.keys(ctx.functionTools)).toEqual(['a']); + }); + + it('separates provider tools from function tools', () => { + const fnA = makeFn('a'); + const provider = new TestProviderTool({ id: 'code' }); + const ctx = new ToolContext([fnA, provider]); + + expect(ctx.functionTools).toEqual({ a: fnA }); + expect(ctx.providerTools).toEqual([provider]); + expect(ctx.flatten()).toEqual([fnA, provider]); + }); + + it('updateTools replaces the entire context', () => { + const a = makeFn('a'); + const b = makeFn('b'); + const ctx = new ToolContext([a]); + ctx.updateTools([b]); + expect(ctx.getFunctionTool('a')).toBeUndefined(); + expect(ctx.getFunctionTool('b')).toBe(b); + }); + + it('copy() yields an independent context with the same tools', () => { + const a = makeFn('a'); + const ctx = new ToolContext([a]); + const dup = ctx.copy(); + + expect(dup.getFunctionTool('a')).toBe(a); + dup.updateTools([]); + expect(ctx.getFunctionTool('a')).toBe(a); + expect(dup.getFunctionTool('a')).toBeUndefined(); + }); + + it('equals() compares function tool maps and provider lists by identity', () => { + const a = makeFn('a'); + const b = makeFn('b'); + const c = makeFn('c'); + + expect(new ToolContext([a, b]).equals(new ToolContext([a, b]))).toBe(true); + expect(new ToolContext([a, b]).equals(new ToolContext([a]))).toBe(false); + expect(new ToolContext([a, b]).equals(new ToolContext([a, c]))).toBe(false); + }); + + it('equals() is reflexive', () => { + const a = makeFn('a'); + const provider = new TestProviderTool({ id: 'code' }); + const ctx = new ToolContext([a, provider]); + expect(ctx.equals(ctx)).toBe(true); + }); + + it('equals() treats provider tool order as insignificant', () => { + // Matches Python's `set(id(t) for t in self._provider_tools)` comparison: two contexts + // that hold the same provider-tool identities in different order are still equal so + // realtime-session / preemptive-generation reuse fast paths are not invalidated. + const a = makeFn('a'); + const p1 = new TestProviderTool({ id: 'code' }); + const p2 = new TestProviderTool({ id: 'browser' }); + expect(new ToolContext([a, p1, p2]).equals(new ToolContext([a, p2, p1]))).toBe(true); + }); + + it('equals() supports contexts with only provider tools', () => { + const p1 = new TestProviderTool({ id: 'code' }); + const p2 = new TestProviderTool({ id: 'browser' }); + expect(new ToolContext([p1, p2]).equals(new ToolContext([p1, p2]))).toBe(true); + const p3 = new TestProviderTool({ id: 'code' }); // distinct identity, same id + expect(new ToolContext([p1]).equals(new ToolContext([p3]))).toBe(false); + }); + + it('hasTool() matches function tools by name and provider tools by id', () => { + const a = makeFn('a'); + const provider = new TestProviderTool({ id: 'code_runner' }); + const ctx = new ToolContext([a, provider]); + + expect(ctx.hasTool('a')).toBe(true); + expect(ctx.hasTool('code_runner')).toBe(true); + expect(ctx.hasTool('missing')).toBe(false); + }); + + it('flatten() returns function tools in insertion order followed by provider tools', () => { + // Matches Python's `flatten()`: list(self._fnc_tools_map.values()) + self._provider_tools. + const a = makeFn('a'); + const b = makeFn('b'); + const provider = new TestProviderTool({ id: 'code' }); + const ctx = new ToolContext([b, provider, a]); + + expect(ctx.flatten()).toEqual([b, a, provider]); + }); +}); + +describe('Toolset', () => { + const makeFn = (name: string) => + tool({ + name, + description: `${name} tool`, + execute: async () => name, + }); + + it('exposes its id and the tools it was constructed with', () => { + const a = makeFn('a'); + const b = makeFn('b'); + const ts = new Toolset({ id: 'set1', tools: [a, b] }); + + expect(ts.id).toBe('set1'); + expect(ts.tools).toEqual([a, b]); + }); + + const fakeToolsetContext = ( + updateTools: (tools: readonly Tool[]) => void = () => {}, + ): ToolsetContext => ({ updateTools }); + + it('default setup and aclose are no-ops', async () => { + const ts = new Toolset({ id: 'noop', tools: [] }); + await expect(ts.setup(fakeToolsetContext())).resolves.toBeUndefined(); + await expect(ts.aclose()).resolves.toBeUndefined(); + }); + + it('lets subclasses override lifecycle hooks', async () => { + const events: string[] = []; + class Recording extends Toolset { + override async setup(_ctx: ToolsetContext): Promise { + events.push(`setup:${this.id}`); + } + override async aclose(): Promise { + events.push(`close:${this.id}`); + } + } + + const ts = new Recording({ id: 'rec', tools: [] }); + await ts.setup(fakeToolsetContext()); + await ts.aclose(); + expect(events).toEqual(['setup:rec', 'close:rec']); + }); + + it('Toolset.create() resolves a static tools list eagerly and composes aclose', async () => { + const a = makeFn('a'); + const events: string[] = []; + const ts = Toolset.create({ + id: 'composed', + tools: [a], + aclose: async () => { + events.push('close'); + }, + }); + + expect(ts).toBeInstanceOf(Toolset); + expect(ts.id).toBe('composed'); + expect(ts.tools).toEqual([a]); // static tools available before activation + + await ts.setup(fakeToolsetContext()); + await ts.aclose(); + expect(events).toEqual(['close']); + }); + + it('Toolset.create() defaults aclose to a no-op when omitted', async () => { + const ts = Toolset.create({ id: 'bare', tools: [] }); + await expect(ts.setup(fakeToolsetContext())).resolves.toBeUndefined(); + await expect(ts.aclose()).resolves.toBeUndefined(); + }); + + it('lets setup push tools after activation via ctx.updateTools', async () => { + const a = makeFn('a'); + const b = makeFn('b'); + let push!: (tools: readonly Tool[]) => void; + const ts = Toolset.create({ + id: 'mcp', + setup: async ({ updateTools }) => { + push = updateTools; + }, + tools: [], + }); + + // Mimic the runtime: ctx.updateTools writes the toolset's current tools. + await ts.setup(fakeToolsetContext((tools) => ts._setTools(tools))); + expect(ts.tools).toEqual([]); + + // A dynamic source (e.g. an MCP server) pushes its tools after connecting. + push([a, b]); + expect(ts.tools).toEqual([a, b]); + }); + + it('is flattened into a ToolContext: function tools merged, toolset tracked', () => { + const a = makeFn('a'); + const b = makeFn('b'); + const ts = new Toolset({ id: 'set', tools: [a, b] }); + const direct = makeFn('direct'); + + const ctx = new ToolContext([direct, ts]); + + expect(Object.keys(ctx.functionTools).sort()).toEqual(['a', 'b', 'direct']); + expect(ctx.toolsets).toEqual([ts]); + }); + + it('throws when a Toolset contributes a duplicate function name', () => { + // Mirrors Python's `add_tool`: a name collision between top-level and toolset-contributed + // tools is an error, not silent overwrite. + const a1 = makeFn('a'); + const a2 = makeFn('a'); + const ts = new Toolset({ id: 'collides', tools: [a2] }); + + expect(() => new ToolContext([a1, ts])).toThrow(/duplicate function name: a/); + }); + + it('equals() compares toolsets as identity sets, not by order', () => { + // Matches Python's `{id(ts) for ts in self._tool_sets}` semantics. + const ts1 = new Toolset({ id: 'one', tools: [] }); + const ts2 = new Toolset({ id: 'two', tools: [] }); + + expect(new ToolContext([ts1, ts2]).equals(new ToolContext([ts2, ts1]))).toBe(true); + + const ts3 = new Toolset({ id: 'three', tools: [] }); + expect(new ToolContext([ts1, ts2]).equals(new ToolContext([ts1, ts3]))).toBe(false); + expect(new ToolContext([ts1]).equals(new ToolContext([ts1, ts2]))).toBe(false); + }); +}); diff --git a/agents/src/llm/tool_context.ts b/agents/src/llm/tool_context.ts index 0e66ebc0d..50c4220f2 100644 --- a/agents/src/llm/tool_context.ts +++ b/agents/src/llm/tool_context.ts @@ -12,7 +12,8 @@ import { isZodObjectSchema, isZodSchema } from './zod-utils.js'; const TOOL_SYMBOL = Symbol('tool'); const FUNCTION_TOOL_SYMBOL = Symbol('function_tool'); -const PROVIDER_DEFINED_TOOL_SYMBOL = Symbol('provider_defined_tool'); +const PROVIDER_TOOL_SYMBOL = Symbol('provider_tool'); +const TOOLSET_SYMBOL = Symbol('toolset'); const TOOL_ERROR_SYMBOL = Symbol('tool_error'); const HANDOFF_SYMBOL = Symbol('handoff'); @@ -57,7 +58,7 @@ export type InferToolInput = T extends { _output: infer O } ? O : any; // eslint-disable-line @typescript-eslint/no-explicit-any -- Fallback type for JSON Schema objects without type inference -export type ToolType = 'function' | 'provider-defined'; +export type ToolType = 'function' | 'provider'; export type ToolChoice = | 'auto' @@ -136,28 +137,32 @@ export type ToolExecuteFunction< export interface Tool { /** * The type of the tool. - * @internal Either user-defined core tool or provider-defined tool. + * @internal Either user-defined function tool or provider-side tool. */ type: ToolType; + /** + * Stable identifier used to key the tool inside a `ToolContext`. For function tools this + * mirrors `name`; for provider tools this is the provider tool id. + */ + id: string; + [TOOL_SYMBOL]: true; } -// TODO(AJS-112): support provider-defined tools -export interface ProviderDefinedTool extends Tool { - type: 'provider-defined'; +// TODO(AJS-112): support provider tools +export abstract class ProviderTool implements Tool { + readonly type = 'provider' as const; - /** - * The ID of the tool. - */ - id: string; + readonly id: string; - /** - * The configuration of the tool. - */ - config: Record; + readonly [TOOL_SYMBOL] = true as const; + + readonly [PROVIDER_TOOL_SYMBOL] = true as const; - [PROVIDER_DEFINED_TOOL_SYMBOL]: true; + constructor({ id }: { id: string }) { + this.id = id; + } } export interface FunctionTool< @@ -167,6 +172,12 @@ export interface FunctionTool< > extends Tool { type: 'function'; + /** + * The name of the tool. Used to identify it inside a `ToolContext` and exposed to the LLM + * as the function name to call. Also surfaced as the inherited `Tool.id`. + */ + name: string; + /** * The description of the tool. Will be used by the language model to decide whether to use the tool. */ @@ -190,51 +201,324 @@ export interface FunctionTool< [FUNCTION_TOOL_SYMBOL]: true; } -// TODO(AJS-112): support provider-defined tools in the future) -export type ToolContext = { - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Generic tool registry needs to accept any parameter/result types - [name: string]: FunctionTool; -}; +export interface ToolCalledEvent { + ctx: RunContext; + arguments: Record; +} + +export interface ToolCompletedEvent { + ctx: RunContext; + output?: { type: 'output'; value: unknown } | { type: 'error'; value: Error }; +} -/** @internal */ +/** Context passed to a {@link Toolset}'s `setup` hook when it activates. */ +export interface ToolsetContext { + /** + * Replace the toolset's tools. Useful for dynamic sources + * (e.g. an MCP server) whose tools are discovered after `setup` or change at runtime. + */ + updateTools(tools: readonly Tool[]): void; +} + +/** + * Function tools of a `ToolContext`, sorted by name for deterministic provider payloads. + * Provider tools are intentionally excluded — callers that need them iterate `flatten()`. + * @internal + */ export function sortedToolEntries( toolCtx: ToolContext, -): Array<[string, ToolContext[string]]> { - return Object.entries(toolCtx).sort(([nameA], [nameB]) => nameA.localeCompare(nameB)); + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- entries are generic function tools +): Array<[string, FunctionTool]> { + return Object.entries(toolCtx.functionTools).sort(([nameA], [nameB]) => + nameA.localeCompare(nameB), + ); } -/** @internal */ +/** Function tool names of a `ToolContext`, sorted for deterministic output. @internal */ export function sortedToolNames(toolCtx: ToolContext | undefined): string[] { if (!toolCtx) return []; - return Object.keys(toolCtx).sort((nameA, nameB) => nameA.localeCompare(nameB)); + return Object.keys(toolCtx.functionTools).sort((nameA, nameB) => nameA.localeCompare(nameB)); } -export function isSameToolContext(ctx1: ToolContext, ctx2: ToolContext): boolean { - const toolNames = new Set(Object.keys(ctx1)); - const toolNames2 = new Set(Object.keys(ctx2)); +/** + * A stateful collection of tools sharing a lifecycle. Tools registered through a `Toolset` are + * flattened into the surrounding `ToolContext`, while the `Toolset` itself is tracked so its + * `setup()` / `aclose()` hooks can be invoked by the agent runtime. + */ +export class Toolset { + readonly #id: string; - if (toolNames.size !== toolNames2.size) { - return false; + #tools: readonly Tool[]; + + readonly [TOOLSET_SYMBOL] = true as const; + + constructor({ id, tools }: { id: string; tools: readonly Tool[] }) { + this.#id = id; + this.#tools = [...tools]; + } + + /** + * For when your tools share something that needs setup or cleanup, like a DB pool, an open MCP + * client, or listeners on a shared bus. `setup` runs once at activation, `aclose` once at + * teardown. If the tool list itself is dynamic (e.g. an MCP server), push it from `setup` via + * {@link ToolsetContext.updateTools}. + * + * @example Static tool list with a shared backing resource + * ```ts + * function createPostgresToolset(connectionUrl: string): Toolset { + * const pool = new pg.Pool({ connectionString: connectionUrl }); + * return Toolset.create({ + * id: 'postgres', + * tools: [queryOrders, queryCustomers], + * aclose: () => pool.end(), + * }); + * } + * ``` + * + * @example Dynamic tool list bound to an external source + * ```ts + * function createMcpToolset(url: string): Toolset { + * const client = new MCPClient({ url }); + * return Toolset.create({ + * id: 'mcp_remote', + * // setup connects and wires listeners that push the server's tools whenever they change; + * // the runtime re-advertises without re-running anything. + * setup: async ({ updateTools }) => { + * const sync = async () => updateTools(await client.listTools()); + * client.on('connect', sync); + * client.on('tool_list_changed', sync); + * await client.connect(); + * }, + * tools: [], + * aclose: () => client.disconnect(), + * }); + * } + * ``` + */ + static create(options: ToolsetCreateOptions): Toolset { + return new ToolsetFactory(options); + } + + get id(): string { + return this.#id; } - for (const name of toolNames) { - if (!toolNames2.has(name)) { + get tools(): readonly Tool[] { + return this.#tools; + } + + /** + * Replace the toolset's current tools. Backs {@link ToolsetContext.updateTools}; the runtime + * re-flattens and re-advertises after calling it. + * + * @internal + */ + _setTools(tools: readonly Tool[]): void { + this.#tools = [...tools]; + } + + async setup(_ctx: ToolsetContext): Promise {} + + async aclose(): Promise {} +} + +/** Options accepted by `Toolset.create()` — id + tools plus optional setup/teardown hooks. */ +export interface ToolsetCreateOptions { + id: string; + /** + * One-time async initialization run when the toolset activates — e.g. connecting to a server + * and wiring listeners. Push a changed tool list via {@link ToolsetContext.updateTools}. + */ + setup?: (ctx: ToolsetContext) => Promise; + /** The toolset's initial tools. */ + tools: readonly Tool[]; + /** Invoked when the toolset is being torn down. Release awaitable resources here. */ + aclose?: () => Promise; +} + +/** Backing implementation of `Toolset.create()`. Kept private so callers go through the factory. */ +class ToolsetFactory extends Toolset { + readonly #setupFn?: (ctx: ToolsetContext) => Promise; + + readonly #acloseFn?: () => Promise; + + constructor({ id, setup, tools, aclose }: ToolsetCreateOptions) { + super({ id, tools }); + this.#setupFn = setup; + this.#acloseFn = aclose; + } + + override async setup(ctx: ToolsetContext): Promise { + if (this.#setupFn) await this.#setupFn(ctx); + } + + override async aclose(): Promise { + if (this.#acloseFn) await this.#acloseFn(); + } +} + +/** + * Convenience input shape accepted by APIs that want to take a list of tools directly without + * forcing callers to wrap them in `new ToolContext(...)`. + */ +export type ToolCtxInput = + | ToolContext + | readonly ToolContextEntry[]; + +export function toToolContext( + input: ToolCtxInput, +): ToolContext; +export function toToolContext( + input: ToolCtxInput | undefined, +): ToolContext | undefined; +export function toToolContext( + input: ToolCtxInput | undefined, +): ToolContext | undefined { + if (input === undefined) return undefined; + return input instanceof ToolContext ? input : new ToolContext(input); +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any -- ToolContext entries accept any function-tool parameter/result types +export type ToolContextEntry = + // eslint-disable-next-line @typescript-eslint/no-explicit-any + FunctionTool | ProviderTool | Toolset; + +export class ToolContext { + private _tools: ToolContextEntry[] = []; + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- ToolContext stores generic function tools + private _functionToolsMap: Map> = new Map(); + private _providerTools: ProviderTool[] = []; + private _toolsets: Toolset[] = []; + + constructor(tools: readonly ToolContextEntry[] = []) { + this.updateTools(tools); + } + + static empty(): ToolContext { + return new ToolContext([]); + } + + /** A copy of all function tools in the tool context, including those in tool sets. */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + get functionTools(): Record> { + return Object.fromEntries(this._functionToolsMap); + } + + /** A copy of all provider tools in the tool context, including those in tool sets. */ + get providerTools(): ProviderTool[] { + return [...this._providerTools]; + } + + /** A copy of all toolsets registered in the context. */ + get toolsets(): readonly Toolset[] { + return [...this._toolsets]; + } + + /** + * A copy of the raw tool list this context was constructed with. + */ + get tools(): readonly ToolContextEntry[] { + return [...this._tools]; + } + + /** Flatten the tool context to a list of tools. */ + flatten(): Tool[] { + return [...this._functionToolsMap.values(), ...this._providerTools]; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Generic registry over any parameter/result types + getFunctionTool(id: string): FunctionTool | undefined { + return this._functionToolsMap.get(id); + } + + hasTool(id: string): boolean { + if (this._functionToolsMap.has(id)) { + return true; + } + return this._providerTools.some((tool) => tool.id === id); + } + + updateTools(tools: readonly ToolContextEntry[]): void { + this._tools = [...tools]; + this._functionToolsMap = new Map(); + this._providerTools = []; + this._toolsets = []; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts any tool shape + const addTool = (tool: any): void => { + if (isToolset(tool)) { + for (const inner of tool.tools) { + addTool(inner); + } + this._toolsets.push(tool); + return; + } + + if (isProviderTool(tool)) { + this._providerTools.push(tool); + return; + } + + if (isFunctionTool(tool)) { + const existing = this._functionToolsMap.get(tool.id); + if (existing !== undefined) { + if (existing !== tool) { + throw new Error(`duplicate function name: ${tool.id}`); + } + return; // same instance, skip + } + this._functionToolsMap.set(tool.id, tool); + return; + } + + throw new Error(`unknown tool type: ${typeof tool}`); + }; + + for (const tool of tools) { + addTool(tool); + } + } + + copy(): ToolContext { + return new ToolContext([...this._tools]); + } + + equals(other: ToolContext): boolean { + if (this._functionToolsMap.size !== other._functionToolsMap.size) { return false; } - const tool1 = ctx1[name]; - const tool2 = ctx2[name]; + for (const [id, tool] of this._functionToolsMap) { + if (other._functionToolsMap.get(id) !== tool) { + return false; + } + } - if (!tool1 || !tool2) { + if (this._providerTools.length !== other._providerTools.length) { return false; } - if (tool1.description !== tool2.description) { + // Provider tools compare as identity sets to match Python's `set(id(t) for t in ...)` + // semantics — order is not significant. + const otherProviderIds = new Set(other._providerTools); + for (const tool of this._providerTools) { + if (!otherProviderIds.has(tool)) { + return false; + } + } + + if (this._toolsets.length !== other._toolsets.length) { return false; } - } - return true; + const otherToolsets = new Set(other._toolsets); + for (const ts of this._toolsets) { + if (!otherToolsets.has(ts)) { + return false; + } + } + return true; + } } export function isSameToolChoice(choice1: ToolChoice | null, choice2: ToolChoice | null): boolean { @@ -261,11 +545,13 @@ export function tool< UserData = UnknownUserData, Result = unknown, >({ + name, description, parameters, execute, flags, }: { + name: string; description: string; parameters: Schema; execute: ToolExecuteFunction, UserData, Result>; @@ -276,95 +562,76 @@ export function tool< * Create a function tool without parameters. */ export function tool({ + name, description, execute, flags, }: { + name: string; description: string; parameters?: never; execute: ToolExecuteFunction, UserData, Result>; flags?: number; }): FunctionTool, UserData, Result>; -/** - * Create a provider-defined tool. - * - * @param id - The ID of the tool. - * @param config - The configuration of the tool. - */ -export function tool({ - id, - config, -}: { - id: string; - config: Record; -}): ProviderDefinedTool; - // eslint-disable-next-line @typescript-eslint/no-explicit-any export function tool(tool: any): any { - if (tool.execute !== undefined) { - // Default parameters to z.object({}) if not provided - const parameters = tool.parameters ?? z.object({}); - - // if parameters is a Zod schema, ensure it's an object schema - if (isZodSchema(parameters) && !isZodObjectSchema(parameters)) { - throw new Error('Tool parameters must be a Zod object schema (z.object(...))'); - } + if (typeof tool.name !== 'string' || tool.name.length === 0) { + throw new Error('tool({ name, ... }) requires a non-empty name'); + } - // Ensure parameters is either a Zod schema or a plain object (JSON schema) - if (!isZodSchema(parameters) && !(typeof parameters === 'object')) { - throw new Error('Tool parameters must be a Zod object schema or a raw JSON schema'); - } + // Default parameters to z.object({}) if not provided + const parameters = tool.parameters ?? z.object({}); - return { - type: 'function', - description: tool.description, - parameters, - execute: tool.execute, - flags: tool.flags ?? ToolFlag.NONE, - [TOOL_SYMBOL]: true, - [FUNCTION_TOOL_SYMBOL]: true, - }; + // if parameters is a Zod schema, ensure it's an object schema + if (isZodSchema(parameters) && !isZodObjectSchema(parameters)) { + throw new Error('Tool parameters must be a Zod object schema (z.object(...))'); } - if (tool.config !== undefined && tool.id !== undefined) { - return { - type: 'provider-defined', - id: tool.id, - config: tool.config, - [TOOL_SYMBOL]: true, - [PROVIDER_DEFINED_TOOL_SYMBOL]: true, - }; + // Ensure parameters is either a Zod schema or a plain object (JSON schema) + if (!isZodSchema(parameters) && !(typeof parameters === 'object')) { + throw new Error('Tool parameters must be a Zod object schema or a raw JSON schema'); } - throw new Error('Invalid tool'); + return { + type: 'function', + id: tool.name, + name: tool.name, + description: tool.description, + parameters, + execute: tool.execute, + flags: tool.flags ?? ToolFlag.NONE, + [TOOL_SYMBOL]: true, + [FUNCTION_TOOL_SYMBOL]: true, + }; } // eslint-disable-next-line @typescript-eslint/no-explicit-any export function isTool(tool: any): tool is Tool { - return tool && tool[TOOL_SYMBOL] === true; + return !!tool && tool[TOOL_SYMBOL] === true; } // eslint-disable-next-line @typescript-eslint/no-explicit-any export function isFunctionTool(tool: any): tool is FunctionTool { - const isTool = tool && tool[TOOL_SYMBOL] === true; - const isFunctionTool = tool[FUNCTION_TOOL_SYMBOL] === true; - return isTool && isFunctionTool; + return isTool(tool) && (tool as FunctionTool)[FUNCTION_TOOL_SYMBOL] === true; +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function isProviderTool(tool: any): tool is ProviderTool { + return isTool(tool) && (tool as ProviderTool)[PROVIDER_TOOL_SYMBOL] === true; } // eslint-disable-next-line @typescript-eslint/no-explicit-any -export function isProviderDefinedTool(tool: any): tool is ProviderDefinedTool { - const isTool = tool && tool[TOOL_SYMBOL] === true; - const isProviderDefinedTool = tool[PROVIDER_DEFINED_TOOL_SYMBOL] === true; - return isTool && isProviderDefinedTool; +export function isToolset(value: any): value is Toolset { + return !!value && value[TOOLSET_SYMBOL] === true; } // eslint-disable-next-line @typescript-eslint/no-explicit-any export function isToolError(error: any): error is ToolError { - return error && error[TOOL_ERROR_SYMBOL] === true; + return !!error && error[TOOL_ERROR_SYMBOL] === true; } // eslint-disable-next-line @typescript-eslint/no-explicit-any export function isAgentHandoff(handoff: any): handoff is AgentHandoff { - return handoff && handoff[HANDOFF_SYMBOL] === true; + return !!handoff && handoff[HANDOFF_SYMBOL] === true; } diff --git a/agents/src/llm/tool_context.type.test.ts b/agents/src/llm/tool_context.type.test.ts index 27cf9fe55..5d33124ad 100644 --- a/agents/src/llm/tool_context.type.test.ts +++ b/agents/src/llm/tool_context.type.test.ts @@ -3,11 +3,12 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, expectTypeOf, it } from 'vitest'; import { z } from 'zod'; -import { type FunctionTool, type ProviderDefinedTool, type ToolOptions, tool } from './index.js'; +import { type FunctionTool, ProviderTool, type ToolOptions, tool } from './index.js'; describe('tool type inference', () => { it('should infer argument type from zod schema', () => { const toolType = tool({ + name: 'test', description: 'test', parameters: z.object({ number: z.number() }), execute: async () => 'test' as const, @@ -16,19 +17,20 @@ describe('tool type inference', () => { expectTypeOf(toolType).toEqualTypeOf>(); }); - it('should infer provider defined tool type', () => { - const toolType = tool({ - id: 'code-interpreter', - config: { - language: 'python', - }, - }); + it('rejects direct instantiation of the abstract ProviderTool base', () => { + // @ts-expect-error - ProviderTool is abstract; plugins must subclass it. + new ProviderTool({ id: 'code-interpreter' }); - expectTypeOf(toolType).toEqualTypeOf(); + class CodeInterpreter extends ProviderTool {} + const providerTool = new CodeInterpreter({ id: 'code-interpreter' }); + expectTypeOf(providerTool).toMatchTypeOf(); + expect(providerTool.id).toBe('code-interpreter'); + expect(providerTool.type).toBe('provider'); }); it('should infer run context type', () => { const toolType = tool({ + name: 'test', description: 'test', parameters: z.object({ number: z.number() }), execute: async ({ number }, { ctx }: ToolOptions<{ name: string }>) => { @@ -43,7 +45,6 @@ describe('tool type inference', () => { it('should not accept primitive zod schemas', () => { expect(() => { - // @ts-expect-error - Testing that non-object schemas are rejected tool({ name: 'test', description: 'test', @@ -55,7 +56,6 @@ describe('tool type inference', () => { it('should not accept array schemas', () => { expect(() => { - // @ts-expect-error - Testing that array schemas are rejected tool({ name: 'test', description: 'test', @@ -67,7 +67,6 @@ describe('tool type inference', () => { it('should not accept union schemas', () => { expect(() => { - // @ts-expect-error - Testing that union schemas are rejected tool({ name: 'test', description: 'test', @@ -91,6 +90,7 @@ describe('tool type inference', () => { it('should infer empty object type when parameters are omitted', () => { const toolType = tool({ + name: 'simpleAction', description: 'Simple action without parameters', execute: async () => 'done' as const, }); @@ -100,6 +100,7 @@ describe('tool type inference', () => { it('should infer correct types with context but no parameters', () => { const toolType = tool({ + name: 'actionWithCtx', description: 'Action with context', execute: async (args, { ctx }: ToolOptions<{ userId: number }>) => { expectTypeOf(args).toEqualTypeOf>(); diff --git a/agents/src/llm/utils.test.ts b/agents/src/llm/utils.test.ts index 154c76348..b6b41b266 100644 --- a/agents/src/llm/utils.test.ts +++ b/agents/src/llm/utils.test.ts @@ -13,7 +13,7 @@ import { FunctionCallOutput, type ImageContent, } from './chat_context.js'; -import { tool } from './tool_context.js'; +import { ToolContext, tool } from './tool_context.js'; import { computeChatCtxDiff, executeToolCall, @@ -182,6 +182,7 @@ describe('parseFunctionArguments', () => { describe('executeToolCall', () => { it('should canonicalize repaired arguments before returning', async () => { const removeOrderItem = tool({ + name: 'removeOrderItem', description: 'remove order item', parameters: z.object({ orderId: z.array(z.string()) }), execute: async ({ orderId }) => orderId.join(','), @@ -194,7 +195,7 @@ describe('executeToolCall', () => { args: rawArgs, }); - const result = await executeToolCall(toolCall, { removeOrderItem }); + const result = await executeToolCall(toolCall, new ToolContext([removeOrderItem])); expect(result.isError).toBe(false); expect(JSON.parse(result.output)).toBe('O_WAAB70'); @@ -204,6 +205,7 @@ describe('executeToolCall', () => { it('should preserve valid argument structure during execution', async () => { const echo = tool({ + name: 'echo', description: 'echo', parameters: z.object({ arg1: z.string(), optArg2: z.string().optional() }), execute: async ({ arg1, optArg2 }) => ({ arg1, optArg2 }), @@ -216,7 +218,7 @@ describe('executeToolCall', () => { args: originalArgs, }); - const result = await executeToolCall(toolCall, { echo }); + const result = await executeToolCall(toolCall, new ToolContext([echo])); expect(result.isError).toBe(false); expect(JSON.parse(result.output)).toEqual({ arg1: 'hello', optArg2: '<|safe|>' }); diff --git a/agents/src/llm/utils.ts b/agents/src/llm/utils.ts index c9fd7682f..53619e6fc 100644 --- a/agents/src/llm/utils.ts +++ b/agents/src/llm/utils.ts @@ -176,7 +176,7 @@ export const oaiBuildFunctionInfo = ( toolName: string, rawArgs: string, ): FunctionCall => { - const tool = toolCtx[toolName]; + const tool = toolCtx.getFunctionTool(toolName); if (!tool) { throw new Error(`AI tool ${toolName} not found`); } @@ -263,8 +263,8 @@ export async function executeToolCall( toolCall: FunctionCall, toolCtx: ToolContext, ): Promise { - const tool = toolCtx[toolCall.name]!; - let args: Record | undefined; + const tool = toolCtx.getFunctionTool(toolCall.name)!; + let args: object | undefined; let params: object | undefined; // Ensure valid JSON diff --git a/agents/src/utils.test.ts b/agents/src/utils.test.ts index eff60d542..421697fb9 100644 --- a/agents/src/utils.test.ts +++ b/agents/src/utils.test.ts @@ -14,6 +14,7 @@ import { delay, isPending, resampleStream, + toStream, } from '../src/utils.js'; describe('utils', () => { @@ -36,6 +37,84 @@ describe('utils', () => { }); }); + describe('toStream', () => { + it('converts an async iterable into a ReadableStream', async () => { + async function* source() { + yield 1; + yield 2; + yield 3; + } + + const reader = toStream(source()).getReader(); + + await expect(reader.read()).resolves.toEqual({ done: false, value: 1 }); + await expect(reader.read()).resolves.toEqual({ done: false, value: 2 }); + await expect(reader.read()).resolves.toEqual({ done: false, value: 3 }); + await expect(reader.read()).resolves.toEqual({ done: true, value: undefined }); + }); + + it('propagates errors from the async iterable', async () => { + const expectedError = new Error('source failed'); + async function* source() { + yield 1; + throw expectedError; + } + + const reader = toStream(source()).getReader(); + + await expect(reader.read()).resolves.toEqual({ done: false, value: 1 }); + await expect(reader.read()).rejects.toBe(expectedError); + }); + + it('runs async iterable cleanup when the stream is canceled mid-stream', async () => { + let cleanupRan = false; + async function* source() { + try { + yield 1; + yield 2; + } finally { + cleanupRan = true; + } + } + + const reader = toStream(source()).getReader(); + + await expect(reader.read()).resolves.toEqual({ done: false, value: 1 }); + await reader.cancel('stop early'); + + expect(cleanupRan).toBe(true); + }); + + it('does not wait for a pending next value when canceled mid-stream', async () => { + let releaseNextValue: (() => void) | undefined; + let cleanupRan = false; + async function* source() { + try { + yield 1; + await new Promise((resolve) => { + releaseNextValue = resolve; + }); + yield 2; + } finally { + cleanupRan = true; + } + } + + const reader = toStream(source()).getReader(); + + await expect(reader.read()).resolves.toEqual({ done: false, value: 1 }); + const pendingRead = reader.read(); + await delay(1); + await expect( + Promise.race([reader.cancel('stop early'), delay(10).then(() => 'timeout')]), + ).resolves.not.toBe('timeout'); + releaseNextValue?.(); + await expect(pendingRead).resolves.toEqual({ done: true, value: undefined }); + + expect(cleanupRan).toBe(true); + }); + }); + describe('Task', () => { it('should execute task successfully and return result', async () => { const expectedResult = 'task completed'; @@ -160,29 +239,39 @@ describe('utils', () => { }); it('should handle task that checks abort signal manually', async () => { - const arr: number[] = []; - const task = Task.from(async (controller) => { - for (let i = 0; i < 10; i++) { - if (controller.signal.aborted) { - throw new Error('Task was aborted'); + // fake timers: with real timers the exact tick count is scheduling- + // dependent and flakes on loaded CI runners + vi.useFakeTimers(); + try { + const arr: number[] = []; + const task = Task.from(async (controller) => { + for (let i = 0; i < 10; i++) { + if (controller.signal.aborted) { + throw new Error('Task was aborted'); + } + await delay(10); + arr.push(i); } - await delay(10); - arr.push(i); - } - return 'completed'; - }); + return 'completed'; + }); - await delay(35); - task.cancel(); + await vi.advanceTimersByTimeAsync(35); + task.cancel(); - expect(arr).toEqual([0, 1, 2]); - try { - await task.result; - } catch (error: unknown) { - expect((error as Error).message).toBe('Task was aborted'); - } + expect(arr).toEqual([0, 1, 2]); + // the pending (signal-less) delay must elapse for the loop to reach + // its manual abort checkpoint + await vi.advanceTimersByTimeAsync(10); + try { + await task.result; + } catch (error: unknown) { + expect((error as Error).message).toBe('Task was aborted'); + } - expect(task.done).toBe(true); + expect(task.done).toBe(true); + } finally { + vi.useRealTimers(); + } }); it('should handle cleanup in finally block', async () => { diff --git a/agents/src/utils.ts b/agents/src/utils.ts index e537ea4e5..b242dfe3a 100644 --- a/agents/src/utils.ts +++ b/agents/src/utils.ts @@ -14,8 +14,11 @@ import { type Throws, ThrowsPromise } from '@livekit/throws-transformer/throws'; import { AsyncLocalStorage } from 'node:async_hooks'; import { randomUUID } from 'node:crypto'; import { EventEmitter, once } from 'node:events'; -import type { ReadableStream } from 'node:stream/web'; -import { TransformStream, type TransformStreamDefaultController } from 'node:stream/web'; +import { + ReadableStream, + TransformStream, + type TransformStreamDefaultController, +} from 'node:stream/web'; import { log } from './log.js'; /** @@ -1322,15 +1325,21 @@ export async function* readStream( const abortPromise = waitForAbort(signal); while (true) { const result = await ThrowsPromise.race([reader.read(), abortPromise]); - if (!result) break; + if (!result) { + break; + } const { done, value } = result; - if (done) break; + if (done) { + break; + } yield value; } } else { while (true) { const { done, value } = await reader.read(); - if (done) break; + if (done) { + break; + } yield value; } } @@ -1343,6 +1352,39 @@ export async function* readStream( } } +export function toStream(iterable: AsyncIterable): ReadableStream { + let iterator: AsyncIterator | undefined; + let cancelled = false; + + return new ReadableStream({ + async start(controller) { + iterator = iterable[Symbol.asyncIterator](); + + try { + while (true) { + const { done, value } = await iterator.next(); + if (done || cancelled) { + break; + } + controller.enqueue(value); + } + + if (!cancelled) { + controller.close(); + } + } catch (error) { + if (!cancelled) { + controller.error(error); + } + } + }, + cancel(reason) { + cancelled = true; + void iterator?.return?.(reason).catch(() => {}); + }, + }); +} + export async function waitForAbort(signal: AbortSignal) { if (signal.aborted) { return; diff --git a/agents/src/voice/agent.test.ts b/agents/src/voice/agent.test.ts index 8dd83fee3..e644ca043 100644 --- a/agents/src/voice/agent.test.ts +++ b/agents/src/voice/agent.test.ts @@ -1,9 +1,11 @@ // SPDX-FileCopyrightText: 2025 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 +import type { AudioFrame } from '@livekit/rtc-node'; +import { ReadableStream } from 'node:stream/web'; import { describe, expect, it, vi } from 'vitest'; import { z } from 'zod'; -import { tool } from '../llm/index.js'; +import { ChatContext, ChatMessage, tool } from '../llm/index.js'; import { initializeLogger } from '../log.js'; import { Task } from '../utils.js'; import { Agent, AgentTask, _setActivityTaskInfo } from './agent.js'; @@ -15,6 +17,14 @@ vi.mock('ofetch', () => ({ ofetch: vi.fn() })); initializeLogger({ pretty: false, level: 'error' }); +async function collectReadableStream(stream: ReadableStream): Promise { + const chunks: T[] = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + return chunks; +} + describe('Agent', () => { it('should create agent with basic instructions', () => { const instructions = 'You are a helpful assistant'; @@ -29,12 +39,14 @@ describe('Agent', () => { // Create mock tools using the tool function const mockTool1 = tool({ + name: 'getTool1', description: 'First test tool', parameters: z.object({}), execute: async () => 'tool1 result', }); const mockTool2 = tool({ + name: 'getTool2', description: 'Second test tool', parameters: z.object({ input: z.string().describe('Input parameter'), @@ -44,17 +56,14 @@ describe('Agent', () => { const agent = new Agent({ instructions, - tools: { - getTool1: mockTool1, - getTool2: mockTool2, - }, + tools: [mockTool1, mockTool2], }); expect(agent).toBeDefined(); expect(agent.instructions).toBe(instructions); // Assert tools are set correctly - const agentTools = agent.toolCtx; + const agentTools = agent.toolCtx.functionTools; expect(Object.keys(agentTools)).toHaveLength(2); expect(agentTools).toHaveProperty('getTool1'); expect(agentTools).toHaveProperty('getTool2'); @@ -64,27 +73,271 @@ describe('Agent', () => { expect(agentTools.getTool2?.description).toBe('Second test tool'); }); - it('should return a copy of tools, not the original reference', () => { + it('toolCtx returns a defensive copy that exposes the same tools', () => { const instructions = 'You are a helpful assistant'; const mockTool = tool({ + name: 'testTool', description: 'Test tool', parameters: z.object({}), execute: async () => 'result', }); - const tools = { testTool: mockTool }; - const agent = new Agent({ instructions, tools }); + const agent = new Agent({ instructions, tools: [mockTool] }); + + // Each call returns a fresh ToolContext so external mutation can't escape into the agent's + // internal state. + expect(agent.toolCtx).not.toBe(agent.toolCtx); + expect(agent.toolCtx.getFunctionTool('testTool')).toBe(mockTool); + }); + + describe('create', () => { + it('preserves constructor options and base Agent default id', () => { + const mockTool = tool({ + name: 'testTool', + description: 'Test tool', + parameters: z.object({}), + execute: async () => 'result', + }); + + const agent = Agent.create({ + instructions: 'factory instructions', + tools: [mockTool], + }); + + expect(agent).toBeInstanceOf(Agent); + expect(agent.instructions).toBe('factory instructions'); + expect(agent.id).toBe('default_agent'); + expect(agent.toolCtx.getFunctionTool('testTool')).toBe(mockTool); + }); + + it('passes AgentContext to lifecycle hooks', async () => { + const calls: string[] = []; + const chatCtx = ChatContext.empty(); + const newMessage = ChatMessage.create({ role: 'user', content: ['hello'] }); + const agent = Agent.create({ + id: 'factory_agent', + instructions: 'factory instructions', + minConsecutiveSpeechDelay: 12, + onEnter: (ctx) => { + expect(ctx.agent).toBe(agent); + expect(ctx.id).toBe(agent.id); + expect(ctx.instructions).toBe(agent.instructions); + expect(ctx.toolCtx.functionTools).toEqual(agent.toolCtx.functionTools); + expect(ctx.chatCtx.items).toEqual(agent.chatCtx.items); + expect(ctx.minConsecutiveSpeechDelay).toBe(agent.minConsecutiveSpeechDelay); + calls.push('enter'); + }, + onExit: async (ctx) => { + expect(ctx.agent).toBe(agent); + calls.push('exit'); + }, + onUserTurnCompleted: (ctx, receivedChatCtx, receivedMessage) => { + expect(ctx.agent).toBe(agent); + expect(receivedChatCtx).toBe(chatCtx); + expect(receivedMessage).toBe(newMessage); + calls.push('turn'); + }, + }); + + await agent.onEnter(); + await agent.onExit(); + await agent.onUserTurnCompleted(chatCtx, newMessage); + + expect(calls).toEqual(['enter', 'exit', 'turn']); + }); + + it('adapts stream node hooks between ReadableStream and AsyncIterable', async () => { + const audioFrame = 'audio' as unknown as AudioFrame; + const agent = Agent.create({ + instructions: 'factory instructions', + async sttNode(ctx, audio) { + async function* stream() { + expect(ctx.agent).toBe(agent); + const frames: AudioFrame[] = []; + for await (const frame of audio) { + frames.push(frame); + } + expect(frames).toEqual([audioFrame]); + yield 'transcript'; + } + + return stream(); + }, + }); + const audio = new ReadableStream({ + start(controller) { + controller.enqueue(audioFrame); + controller.close(); + }, + }); + + const result = await agent.sttNode(audio, {}); + + expect(result).not.toBeNull(); + await expect(collectReadableStream(result!)).resolves.toEqual(['transcript']); + }); + + it('supports async generator stream node hooks', async () => { + const audioFrame = 'audio' as unknown as AudioFrame; + const outputFrame = 'output-audio' as unknown as AudioFrame; + const agent = Agent.create({ + instructions: 'factory instructions', + async *sttNode(ctx, audio) { + expect(ctx.agent).toBe(agent); + const frames: AudioFrame[] = []; + for await (const frame of audio) { + frames.push(frame); + } + expect(frames).toEqual([audioFrame]); + yield 'transcript'; + }, + async *llmNode(ctx, chatCtx, toolCtx) { + expect(ctx.agent).toBe(agent); + expect(chatCtx).toBeInstanceOf(ChatContext); + expect(toolCtx.equals(agent.toolCtx)).toBe(true); + yield 'llm-output'; + }, + async *ttsNode(ctx, text) { + expect(ctx.agent).toBe(agent); + const chunks: string[] = []; + for await (const chunk of text) { + chunks.push(chunk); + } + expect(chunks).toEqual(['hello']); + yield outputFrame; + }, + async *realtimeAudioOutputNode(ctx, audio) { + expect(ctx.agent).toBe(agent); + const frames: AudioFrame[] = []; + for await (const frame of audio) { + frames.push(frame); + } + expect(frames).toEqual([audioFrame]); + yield outputFrame; + }, + }); + const audio = new ReadableStream({ + start(controller) { + controller.enqueue(audioFrame); + controller.close(); + }, + }); + const text = new ReadableStream({ + start(controller) { + controller.enqueue('hello'); + controller.close(); + }, + }); + + const sttResult = await agent.sttNode(audio, {}); + const llmResult = await agent.llmNode(ChatContext.empty(), agent.toolCtx, {}); + const ttsResult = await agent.ttsNode(text, {}); + const realtimeResult = await agent.realtimeAudioOutputNode( + new ReadableStream({ + start(controller) { + controller.enqueue(audioFrame); + controller.close(); + }, + }), + {}, + ); + + expect(sttResult).not.toBeNull(); + expect(llmResult).not.toBeNull(); + expect(ttsResult).not.toBeNull(); + expect(realtimeResult).not.toBeNull(); + await expect(collectReadableStream(sttResult!)).resolves.toEqual(['transcript']); + await expect(collectReadableStream(llmResult!)).resolves.toEqual(['llm-output']); + await expect(collectReadableStream(ttsResult!)).resolves.toEqual([outputFrame]); + await expect(collectReadableStream(realtimeResult!)).resolves.toEqual([outputFrame]); + }); + + it('supports stream node hooks that return async iterables', async () => { + function asyncIterableOf(...items: T[]): AsyncIterable { + return { + async *[Symbol.asyncIterator]() { + for (const item of items) { + yield item; + } + }, + }; + } + + const audioFrame = 'audio' as unknown as AudioFrame; + const outputFrame = 'output-audio' as unknown as AudioFrame; + const agent = Agent.create({ + instructions: 'factory instructions', + sttNode(ctx) { + expect(ctx.agent).toBe(agent); + return asyncIterableOf('transcript'); + }, + llmNode(ctx) { + expect(ctx.agent).toBe(agent); + return asyncIterableOf('llm-output'); + }, + ttsNode(ctx) { + expect(ctx.agent).toBe(agent); + return asyncIterableOf(outputFrame); + }, + realtimeAudioOutputNode(ctx) { + expect(ctx.agent).toBe(agent); + return asyncIterableOf(outputFrame); + }, + }); - const tools1 = agent.toolCtx; - const tools2 = agent.toolCtx; + const sttResult = await agent.sttNode( + new ReadableStream({ + start(controller) { + controller.enqueue(audioFrame); + controller.close(); + }, + }), + {}, + ); + const llmResult = await agent.llmNode(ChatContext.empty(), agent.toolCtx, {}); + const ttsResult = await agent.ttsNode( + new ReadableStream({ + start(controller) { + controller.enqueue('hello'); + controller.close(); + }, + }), + {}, + ); + const realtimeResult = await agent.realtimeAudioOutputNode( + new ReadableStream({ + start(controller) { + controller.enqueue(audioFrame); + controller.close(); + }, + }), + {}, + ); - // Should return different object references - expect(tools1).not.toBe(tools2); - expect(tools1).not.toBe(tools); + expect(sttResult).not.toBeNull(); + expect(llmResult).not.toBeNull(); + expect(ttsResult).not.toBeNull(); + expect(realtimeResult).not.toBeNull(); + await expect(collectReadableStream(sttResult!)).resolves.toEqual(['transcript']); + await expect(collectReadableStream(llmResult!)).resolves.toEqual(['llm-output']); + await expect(collectReadableStream(ttsResult!)).resolves.toEqual([outputFrame]); + await expect(collectReadableStream(realtimeResult!)).resolves.toEqual([outputFrame]); + }); + + it('falls back to existing defaults for missing hooks', async () => { + const audioFrame = 'audio' as unknown as AudioFrame; + const audio = new ReadableStream({ + start(controller) { + controller.enqueue(audioFrame); + controller.close(); + }, + }); + const agent = Agent.create({ instructions: 'factory instructions' }); + + const result = await agent.realtimeAudioOutputNode(audio, {}); - // Should contain the same set of tools - expect(tools1).toEqual(tools2); - expect(tools1).toEqual(tools); + expect(result).toBe(audio); + }); }); it('should require AgentTask to run inside task context', async () => { @@ -154,6 +407,94 @@ describe('Agent', () => { await expect(wrapper.result).resolves.toBe('ok'); }); + describe('AgentTask.create', () => { + it('exposes complete on hook context', async () => { + const task = AgentTask.create({ + instructions: 'factory task', + onEnter: (ctx) => { + expect(ctx.agent).toBe(task); + expect(ctx.id).toBe('default_agent'); + expect(ctx.instructions).toBe('factory task'); + ctx.complete('ok'); + }, + }); + const oldAgent = new Agent({ instructions: 'old agent' }); + const mockSession = { + currentAgent: oldAgent, + _globalRunState: undefined, + _updateActivity: async (agent: Agent) => { + if (agent === task) { + await agent.onEnter(); + } + }, + }; + const mockActivity = { + agent: oldAgent, + agentSession: mockSession, + _onEnterTask: undefined, + llm: undefined, + close: async () => {}, + }; + + const wrapper = Task.from(async () => { + const currentTask = Task.current(); + if (!currentTask) { + throw new Error('expected task context'); + } + _setActivityTaskInfo(currentTask, { inlineTask: true }); + return await agentActivityStorage.run(mockActivity as any, () => task.run()); + }); + + await expect(wrapper.result).resolves.toBe('ok'); + }); + + it('adapts stream node hooks between ReadableStream and AsyncIterable', async () => { + const audioFrame = 'audio' as unknown as AudioFrame; + const task = AgentTask.create({ + instructions: 'factory task', + async sttNode(ctx, audio) { + async function* stream() { + expect(ctx.agent).toBe(task); + const frames: AudioFrame[] = []; + for await (const frame of audio) { + frames.push(frame); + } + expect(frames).toEqual([audioFrame]); + yield 'transcript'; + } + + return stream(); + }, + }); + const audio = new ReadableStream({ + start(controller) { + controller.enqueue(audioFrame); + controller.close(); + }, + }); + + const result = await task.sttNode(audio, {}); + + expect(result).not.toBeNull(); + await expect(collectReadableStream(result!)).resolves.toEqual(['transcript']); + }); + + it('falls back to existing defaults for missing hooks', async () => { + const audioFrame = 'audio' as unknown as AudioFrame; + const audio = new ReadableStream({ + start(controller) { + controller.enqueue(audioFrame); + controller.close(); + }, + }); + const task = AgentTask.create({ instructions: 'factory task' }); + + const result = await task.realtimeAudioOutputNode(audio, {}); + + expect(result).toBe(audio); + }); + }); + it('should require AgentTask to run inside AgentActivity context', async () => { class TestTask extends AgentTask { constructor() { @@ -242,6 +583,7 @@ describe('Agent', () => { turnHandling: { endpointing: { minDelay: 999 }, interruption: {}, + preemptiveGeneration: {}, turnDetection: 'vad', }, allowInterruptions: false, @@ -258,6 +600,7 @@ describe('Agent', () => { turnHandling: { endpointing: { minDelay: 999, maxDelay: 4000 }, interruption: { enabled: true }, + preemptiveGeneration: {}, turnDetection: 'vad', }, allowInterruptions: false, @@ -275,6 +618,7 @@ describe('Agent', () => { turnHandling: { interruption: { mode: 'adaptive' }, endpointing: {}, + preemptiveGeneration: {}, turnDetection: undefined, }, }); @@ -287,6 +631,7 @@ describe('Agent', () => { turnHandling: { endpointing: { minDelay: 111, maxDelay: 222 }, interruption: { enabled: false }, + preemptiveGeneration: {}, turnDetection: 'manual', }, }); diff --git a/agents/src/voice/agent.ts b/agents/src/voice/agent.ts index cf5624229..7139d3588 100644 --- a/agents/src/voice/agent.ts +++ b/agents/src/voice/agent.ts @@ -20,7 +20,8 @@ import { LLM, RealtimeModel, type ToolChoice, - type ToolContext, + ToolContext, + type ToolContextEntry, } from '../llm/index.js'; import { log } from '../log.js'; import type { STT, SpeechEvent } from '../stt/index.js'; @@ -33,12 +34,27 @@ import { Future, Task } from '../utils.js'; import type { VAD } from '../vad.js'; import { type AgentActivity, agentActivityStorage } from './agent_activity.js'; import type { AgentSession, TurnDetectionMode } from './agent_session.js'; +import { + type AgentCreateOptions, + type AgentTaskCreateOptions, + createAgentTaskV2, + createAgentV2, +} from './agent_v2.js'; import type { UserTurnExceededEvent } from './events.js'; import type { TimedString } from './io.js'; import type { SpeechHandle } from './speech_handle.js'; import type { TurnHandlingOptions } from './turn_config/turn_handling.js'; import { migrateTurnHandling } from './turn_config/utils.js'; +export type { + AgentContext, + AgentCreateOptions, + AgentHookNodeResult, + AgentHooks, + AgentTaskContext, + AgentTaskCreateOptions, +} from './agent_v2.js'; + // speechHandle identifies which SpeechHandle owns the current tool call, enabling // SpeechHandle.waitForPlayout() to distinguish self-wait (deadlock) from waiting // on a different handle scheduled inside the tool. @@ -118,7 +134,7 @@ export interface AgentOptions { id?: string; instructions: string | Instructions; chatCtx?: ChatContext; - tools?: ToolContext; + tools?: readonly ToolContextEntry[]; stt?: STT | STTModelString; vad?: VAD; llm?: LLM | RealtimeModel | LLMModels; @@ -153,7 +169,11 @@ export class Agent { _instructions: string | Instructions; /** @internal */ - _tools?: ToolContext; + _toolCtx: ToolContext; + + static create(options: AgentCreateOptions): Agent { + return createAgentV2(Agent, options); + } constructor({ id, @@ -185,10 +205,10 @@ export class Agent { } this._instructions = instructions; - this._tools = { ...tools }; + this._toolCtx = new ToolContext(tools ?? []); this._chatCtx = chatCtx ? chatCtx.copy({ - toolCtx: this._tools, + toolCtx: this._toolCtx, }) : ChatContext.empty(); @@ -259,7 +279,7 @@ export class Agent { } get toolCtx(): ToolContext { - return { ...this._tools }; + return this._toolCtx.copy(); } get session(): AgentSession { @@ -346,11 +366,20 @@ export class Agent { this._agentActivity.updateChatCtx(chatCtx); } + async updateInstructions(instructions: string | Instructions): Promise { + if (!this._agentActivity) { + this._instructions = instructions; + return; + } + + await this._agentActivity.updateInstructions(instructions); + } + // TODO(parity): Add when AgentConfigUpdate is ported to ChatContext. - async updateTools(tools: ToolContext): Promise { + async updateTools(tools: readonly ToolContextEntry[]): Promise { if (!this._agentActivity) { - this._tools = { ...tools }; - this._chatCtx = this._chatCtx.copy({ toolCtx: this._tools }); + this._toolCtx = new ToolContext(tools); + this._chatCtx = this._chatCtx.copy({ toolCtx: this._toolCtx }); return; } @@ -557,6 +586,12 @@ export class AgentTask extends Agent( + options: AgentTaskCreateOptions, + ): AgentTask { + return createAgentTaskV2(AgentTask, options); + } + constructor(options: AgentTaskOptions) { const { preserveFunctionCallHistory = false, ...rest } = options; super(rest); diff --git a/agents/src/voice/agent_activity.test.ts b/agents/src/voice/agent_activity.test.ts index 5600418d3..907a83466 100644 --- a/agents/src/voice/agent_activity.test.ts +++ b/agents/src/voice/agent_activity.test.ts @@ -16,8 +16,9 @@ */ import { Heap } from 'heap-js'; import { describe, expect, it, vi } from 'vitest'; -import { ChatContext } from '../llm/chat_context.js'; +import { AgentConfigUpdate, ChatContext } from '../llm/chat_context.js'; import { LLM, type LLMStream } from '../llm/llm.js'; +import { type Tool, ToolContext, Toolset, tool } from '../llm/tool_context.js'; import { Future, Task } from '../utils.js'; import { _getActivityTaskInfo } from './agent.js'; import { AgentActivity } from './agent_activity.js'; @@ -449,15 +450,16 @@ function buildPreemptiveRunner(opts: Partial = {}) { const fakeChatCtx = new ChatContext(); + const emptyToolCtx = ToolContext.empty(); const fakeActivity = { _preemptiveGenerationCount: 0, _preemptiveGeneration: undefined, _currentSpeech: undefined as SpeechHandle | undefined, schedulingPaused: false, llm: new FakePreemptiveLLM(), - tools: {}, + tools: emptyToolCtx, toolChoice: null, - agent: { chatCtx: fakeChatCtx }, + agent: { chatCtx: fakeChatCtx, _toolCtx: emptyToolCtx }, agentSession: { sessionOptions: { turnHandling: { preemptiveGeneration: preemptiveOpts }, @@ -555,3 +557,96 @@ describe('AgentActivity - onPreemptiveGeneration guards', () => { expect(cancelPreemptiveGeneration).not.toHaveBeenCalled(); }); }); + +/** + * Regression test for the dynamic-toolset push path. + * + * When an already-activated toolset swaps its tools at runtime (e.g. an MCP server pushes a new + * tool list via `ToolsetContext.updateTools`), `setupToolsetList`'s wiring must (1) invoke + * `onToolsetToolsChanged`, which now funnels through `updateTools`, and (2) record an + * `AgentConfigUpdate` in the agent chat context + session history so a non-realtime pipeline's + * chat context reflects the new tool set on the next turn. + */ +class FakeToolsetLLM extends LLM { + label(): string { + return 'fake.toolset.LLM'; + } + chat(): LLMStream { + throw new Error('not used in these tests'); + } +} + +describe('AgentActivity - onToolsetToolsChanged (dynamic toolset push)', () => { + const makeFn = (name: string) => + tool({ name, description: `${name} tool`, execute: async () => name }); + + function buildToolsetActivity(toolset: Toolset) { + const history = new ChatContext(); + const fakeActivity = { + _toolsetsSetup: true, + realtimeSession: undefined, + llm: new FakeToolsetLLM(), + agent: { + _toolCtx: new ToolContext([toolset]), + _chatCtx: new ChatContext(), + }, + agentSession: { history }, + updateChatCtx: vi.fn(async () => {}), + logger: { info() {}, debug() {}, warn() {}, error() {} }, + }; + Object.setPrototypeOf(fakeActivity, AgentActivity.prototype); + return { fakeActivity, history }; + } + + it('fires onToolsetToolsChanged on a dynamic push and records an AgentConfigUpdate', async () => { + const toolA = makeFn('toolA'); + const toolB = makeFn('toolB'); + + // Capture the wired ctx.updateTools the framework hands the toolset during setup. + let pushTools!: (tools: readonly Tool[]) => void; + const toolset = Toolset.create({ + id: 'dynamic', + tools: [toolA], + setup: async ({ updateTools }) => { + pushTools = updateTools; + }, + }); + + const { fakeActivity, history } = buildToolsetActivity(toolset); + + const changedSpy = vi.spyOn( + AgentActivity.prototype as unknown as Record<'onToolsetToolsChanged', () => Promise>, + 'onToolsetToolsChanged', + ); + + // Activate the toolset through the real path so it captures the push channel. + const setupToolsetList = (AgentActivity.prototype as Record) + .setupToolsetList as (this: unknown, toolsets: readonly Toolset[]) => Promise; + await setupToolsetList.call(fakeActivity, [toolset]); + + expect(changedSpy).not.toHaveBeenCalled(); + + // (1) A dynamic push swaps the toolset's tools — the wiring must invoke onToolsetToolsChanged. + pushTools([toolA, toolB]); + expect(changedSpy).toHaveBeenCalledTimes(1); + await changedSpy.mock.results[0]!.value; + + // (2) An AgentConfigUpdate naming the added tool lands in the session history. + const updates = history.items.filter( + (i): i is AgentConfigUpdate => i instanceof AgentConfigUpdate, + ); + expect(updates).toHaveLength(1); + expect(updates[0]!.toolsAdded).toContain('toolB'); + expect(updates[0]!.toolsRemoved ?? []).not.toContain('toolA'); + + // The refreshed tool context advertises the new tool to the next turn, and the non-realtime + // pipeline's chat context was refreshed via updateChatCtx. + expect(Object.keys(fakeActivity.agent._toolCtx.functionTools).sort()).toEqual([ + 'toolA', + 'toolB', + ]); + expect(fakeActivity.updateChatCtx).toHaveBeenCalledTimes(1); + + changedSpy.mockRestore(); + }); +}); diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 80660536d..ee96a5454 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -23,6 +23,7 @@ import { instructionsEqual, renderInstructions, } from '../llm/chat_context.js'; +import type { Toolset } from '../llm/index.js'; import { type ChatItem, type FunctionCall, @@ -36,12 +37,16 @@ import { RealtimeModel, type RealtimeModelError, type RealtimeSession, + type Tool, type ToolChoice, - type ToolContext, + ToolContext, + type ToolContextEntry, ToolFlag, + isFunctionTool, + isToolset, } from '../llm/index.js'; import type { LLMError } from '../llm/llm.js'; -import { isSameToolChoice, isSameToolContext } from '../llm/tool_context.js'; +import { isSameToolChoice } from '../llm/tool_context.js'; import { log } from '../log.js'; import type { EOUMetrics, @@ -221,6 +226,7 @@ export class AgentActivity implements RecognitionHooks { private toolChoice: ToolChoice | null = null; private _preemptiveGeneration?: PreemptiveGeneration; private _preemptiveGenerationCount = 0; + private _toolsetsSetup = false; private interruptionDetector?: AdaptiveInterruptionDetector; private isInterruptionDetectionEnabled: boolean; private isInterruptionByAudioActivityEnabled: boolean; @@ -422,6 +428,8 @@ export class AgentActivity implements RecognitionHooks { this.agent._agentActivity = this; + await this.setupToolsets(); + if (this.llm instanceof RealtimeModel) { const rtReused = reuseResources?.rtSession !== undefined; @@ -486,7 +494,12 @@ export class AgentActivity implements RecognitionHooks { } } - const initialTools = Object.keys(this.tools); + // Surface every tool the agent advertises at start — function tools by name and provider + // tools by id. + const initialTools = [ + ...Object.keys(this.agent._toolCtx.functionTools), + ...this.agent._toolCtx.providerTools.map((t) => t.id), + ]; if (runOnEnter && (this.agent.instructions || initialTools.length > 0)) { const initialConfig = new AgentConfigUpdate({ instructions: this.agent.instructions, @@ -622,7 +635,8 @@ export class AgentActivity implements RecognitionHooks { // tools update is supported or tools are the same reusable = reusable && - (capabilities.midSessionToolsUpdate || isSameToolContext(this.tools, newActivity.tools)); + (capabilities.midSessionToolsUpdate || + this.agent._toolCtx.equals(newActivity.agent._toolCtx)); if (reusable) { // detach: remove event listeners but don't close the session @@ -777,13 +791,43 @@ export class AgentActivity implements RecognitionHooks { } } - async updateTools(tools: ToolContext): Promise { - const oldToolNames = new Set(Object.keys(this.tools)); - const newToolNames = new Set(Object.keys(tools)); + async updateInstructions(instructions: string | Instructions): Promise { + this.agent._instructions = instructions; + + const configUpdate = new AgentConfigUpdate({ instructions }); + this.agent._chatCtx.insert(configUpdate); + this.agentSession.history.insert(configUpdate); + + if (this.realtimeSession) { + await this.realtimeSession.updateInstructions(renderInstructions(instructions)); + } else { + updateInstructions({ + chatCtx: this.agent._chatCtx, + instructions, + addIfMissing: true, + }); + } + } + + async updateTools(tools: readonly ToolContextEntry[]): Promise { + const oldToolCtx = this.agent._toolCtx; + const oldToolNames = new Set(Object.keys(oldToolCtx.functionTools)); + const oldToolsets = oldToolCtx.toolsets; + const newToolCtx = new ToolContext(tools); + const newToolsets = newToolCtx.toolsets; + const addedToolsets = newToolsets.filter((ts) => !oldToolsets.includes(ts)); + const removedToolsets = oldToolsets.filter((ts) => !newToolsets.includes(ts)); + + // Resolve added factory toolsets before re-flattening, so their tools are included in the + // advertised set (newToolNames is computed below, after resolution). + await this.setupToolsetList(addedToolsets); + newToolCtx.updateTools(newToolCtx.tools); + const newToolNames = new Set(Object.keys(newToolCtx.functionTools)); const toolsAdded = [...newToolNames].filter((name) => !oldToolNames.has(name)); const toolsRemoved = [...oldToolNames].filter((name) => !newToolNames.has(name)); - this.agent._tools = { ...tools }; + this.agent._toolCtx = newToolCtx; + await this.closeToolsetList(removedToolsets); if (toolsAdded.length > 0 || toolsRemoved.length > 0) { const configUpdate = new AgentConfigUpdate({ @@ -795,12 +839,12 @@ export class AgentActivity implements RecognitionHooks { } if (this.realtimeSession) { - await this.realtimeSession.updateTools(tools); + await this.realtimeSession.updateTools(newToolCtx); } if (this.llm instanceof LLM) { // for realtime LLM, we assume the server will remove unvalid tool messages - await this.updateChatCtx(this.agent._chatCtx.copy({ toolCtx: tools })); + await this.updateChatCtx(this.agent._chatCtx.copy({ toolCtx: newToolCtx })); } } @@ -1455,7 +1499,7 @@ export class AgentActivity implements RecognitionHooks { userMessage, info, chatCtx: chatCtx.copy(), - tools: { ...this.tools }, + tools: this.agent._toolCtx.copy(), toolChoice: this.toolChoice, createdAt: Date.now(), }; @@ -1917,11 +1961,16 @@ export class AgentActivity implements RecognitionHooks { const shouldFilterTools = onEnterData?.agent === this.agent && onEnterData?.session === this.agentSession; - const tools = shouldFilterTools - ? Object.fromEntries( - Object.entries(this.agent.toolCtx).filter( - ([, fnTool]) => !(fnTool.flags & ToolFlag.IGNORE_ON_ENTER), - ), + const tools: ToolContext = shouldFilterTools + ? new ToolContext( + this.agent.toolCtx.tools.flatMap((t): ToolContextEntry[] => { + const keepFn = (fn: Tool): boolean => + !isFunctionTool(fn) || !(fn.flags & ToolFlag.IGNORE_ON_ENTER); + if (isToolset(t)) { + return t.tools.filter(keepFn) as ToolContextEntry[]; + } + return keepFn(t) ? [t] : []; + }), ) : this.agent.toolCtx; @@ -2168,7 +2217,7 @@ export class AgentActivity implements RecognitionHooks { if ( preemptive.info.newTranscript === userMessage?.textContent && preemptive.chatCtx.isEquivalent(chatCtx) && - isSameToolContext(preemptive.tools, this.tools) && + preemptive.tools.equals(this.agent._toolCtx) && isSameToolChoice(preemptive.toolChoice, this.toolChoice) ) { speechHandle = preemptive.speechHandle; @@ -4186,9 +4235,69 @@ export class AgentActivity implements RecognitionHooks { this.realtimeSpans?.clear(); await this.realtimeSession?.close(); await this.audioRecognition?.close(); + await this.closeToolsets(); this.realtimeSession = undefined; this.audioRecognition = undefined; } + + private async setupToolsets(): Promise { + // Guard against resume() re-entering _startSession on an activity whose toolsets are + // already initialized. + if (this._toolsetsSetup) return; + this._toolsetsSetup = true; + await this.setupToolsetList(this.agent.toolCtx.toolsets); + // Re-flatten now that any factory toolsets have resolved their tools, so they're advertised. + this.agent._toolCtx.updateTools(this.agent._toolCtx.tools); + } + + private async closeToolsets(): Promise { + if (!this._toolsetsSetup) return; + this._toolsetsSetup = false; + await this.closeToolsetList(this.agent.toolCtx.toolsets); + } + + /** + * Refresh the agent's tool context after a dynamic toolset pushed a new tool list (via + * `ToolsetContext.updateTools`), routing through updateTools so history, chat context, and the + * realtime session stay in sync. The next LLM turn picks up the new tools automatically. + */ + private async onToolsetToolsChanged(): Promise { + if (!this._toolsetsSetup) return; + const current = this.agent._toolCtx; + if (new ToolContext(current.tools).equals(current)) return; + // Same toolset entries, so updateTools' setup/close steps are no-ops (no re-entrancy here). + await this.updateTools(current.tools); + } + + private async setupToolsetList(toolsets: readonly Toolset[]): Promise { + const outputs = await Promise.allSettled( + toolsets.map((ts) => + ts.setup({ + // A dynamic toolset pushes a changed tool list here; re-flatten and re-advertise it. + updateTools: (tools) => { + ts._setTools(tools); + void this.onToolsetToolsChanged().catch((error) => + this.logger.error({ error }, 'error re-advertising toolset tools'), + ); + }, + }), + ), + ); + for (const output of outputs) { + if (output.status === 'rejected') { + this.logger.error({ error: output.reason }, 'error setting up toolset'); + } + } + } + + private async closeToolsetList(toolsets: readonly Toolset[]): Promise { + const outputs = await Promise.allSettled(toolsets.map((ts) => ts.aclose())); + for (const output of outputs) { + if (output.status === 'rejected') { + this.logger.error({ error: output.reason }, 'error closing toolset'); + } + } + } } function toOaiToolChoice(toolChoice: ToolChoice | null): ToolChoice | undefined { diff --git a/agents/src/voice/agent_session.ts b/agents/src/voice/agent_session.ts index 3cade1f2f..d308df854 100644 --- a/agents/src/voice/agent_session.ts +++ b/agents/src/voice/agent_session.ts @@ -480,8 +480,15 @@ export class AgentSession< event: K, ...args: Parameters ): boolean { - const eventData = args[0] as AgentEvent; - this._recordedEvents.push(eventData); + // Only retain events when recording is actually enabled. Otherwise this + // array grows unbounded for the entire (potentially hours-long) session, + // pinning every event's graph (SpeechHandle, OTel spans/contexts, streams) + // and leaking memory even though the events are never reported. The buffer + // is only consumed by makeSessionReport() when recording is enabled. + if (this._enableRecording) { + const eventData = args[0] as AgentEvent; + this._recordedEvents.push(eventData); + } return super.emit(event, ...args); } diff --git a/agents/src/voice/agent_task_handoff_eou.test.ts b/agents/src/voice/agent_task_handoff_eou.test.ts index 6006a6e97..e490852fe 100644 --- a/agents/src/voice/agent_task_handoff_eou.test.ts +++ b/agents/src/voice/agent_task_handoff_eou.test.ts @@ -213,15 +213,16 @@ describe('AgentTask handoff end-of-turn timing', () => { const makeStepTask = (step: number): AgentTask<{ step: number }> => { const task: AgentTask<{ step: number }> = new AgentTask<{ step: number }>({ instructions: `You are handling step ${step}. Wait for the caller to finish it.`, - tools: { - completeStep: tool({ + tools: [ + tool({ + name: 'completeStep', description: `Record that step ${step} is complete.`, execute: async () => { toolExecutedAt[step] = Date.now(); task.complete({ step }); }, }), - }, + ], }); return task; }; diff --git a/agents/src/voice/agent_v2.ts b/agents/src/voice/agent_v2.ts new file mode 100644 index 000000000..2f00402ce --- /dev/null +++ b/agents/src/voice/agent_v2.ts @@ -0,0 +1,472 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { AudioFrame } from '@livekit/rtc-node'; +import type { ReadableStream } from 'node:stream/web'; +import type { Instructions, ReadonlyChatContext } from '../llm/chat_context.js'; +import type { + ChatChunk, + ChatContext, + ChatMessage, + LLM, + RealtimeModel, + ToolContext, +} from '../llm/index.js'; +import type { STT, SpeechEvent } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import type { FlushSentinel } from '../types.js'; +import { readStream, toStream } from '../utils.js'; +import type { VAD } from '../vad.js'; +import type { Agent, AgentOptions, AgentTask, AgentTaskOptions, ModelSettings } from './agent.js'; +import type { AgentSession } from './agent_session.js'; +import type { TurnHandlingOptions } from './turn_config/turn_handling.js'; + +/** Context passed to hooks created with `Agent.create()`. */ +export interface AgentContext { + /** The agent instance currently running the hook. */ + agent: Agent; + /** Voice activity detector configured for the agent. */ + vad: VAD | undefined; + /** Speech-to-text model configured for the agent. */ + stt: STT | undefined; + /** LLM or realtime model configured for the agent. */ + llm: LLM | RealtimeModel | undefined; + /** Text-to-speech model configured for the agent. */ + tts: TTS | undefined; + /** Whether TTS-aligned transcripts are enabled for the agent. */ + useTtsAlignedTranscript: boolean | undefined; + /** Readonly view of the agent's current chat context. */ + chatCtx: ReadonlyChatContext; + /** Agent identifier. */ + id: string; + /** Agent instructions. */ + instructions: string | Instructions; + /** Copy of the agent tool context. */ + toolCtx: ToolContext; + /** Current session for the agent. */ + session: AgentSession; + /** Agent-level turn handling configuration. */ + turnHandling: Partial | undefined; + /** Minimum delay between consecutive speech. */ + minConsecutiveSpeechDelay: number | undefined; +} + +/** Return type for stream hooks. Returning `null` stops that pipeline node. */ +export type AgentHookNodeResult = AsyncIterable | Promise | null> | null; + +export interface AgentHooks< + UserData, + ContextT extends AgentContext = AgentContext, +> { + /** Called when the agent becomes active in a session. */ + onEnter?: (ctx: ContextT) => Promise | void; + /** Called when the agent is leaving the active session. */ + onExit?: (ctx: ContextT) => Promise | void; + /** Called after the user's turn has been committed to the chat context. */ + onUserTurnCompleted?: ( + ctx: ContextT, + chatCtx: ChatContext, + newMessage: ChatMessage, + ) => Promise | void; + /** Transforms incoming audio into speech events or transcript text for the agent. */ + sttNode?: ( + ctx: ContextT, + audio: AsyncIterable, + modelSettings: ModelSettings, + ) => AgentHookNodeResult; + /** Produces LLM chunks or text from the current chat and tool context. */ + llmNode?: ( + ctx: ContextT, + chatCtx: ChatContext, + toolCtx: ToolContext, + modelSettings: ModelSettings, + ) => AgentHookNodeResult; + /** Synthesizes agent text into audio frames for playout. */ + ttsNode?: ( + ctx: ContextT, + text: AsyncIterable, + modelSettings: ModelSettings, + ) => AgentHookNodeResult; + /** Processes realtime model audio before it is sent to the agent output. */ + realtimeAudioOutputNode?: ( + ctx: ContextT, + audio: AsyncIterable, + modelSettings: ModelSettings, + ) => AgentHookNodeResult; +} + +export interface AgentCreateOptions + extends AgentOptions, + AgentHooks {} + +/** Context passed to hooks created with `AgentTask.create()`. */ +export interface AgentTaskContext + extends AgentContext { + /** The task instance currently running the hook. */ + agent: AgentTask; + /** Complete the task with either a result or an error. */ + complete(result: ResultT | Error): void; +} + +export interface AgentTaskCreateOptions + extends AgentTaskOptions, + AgentHooks> {} + +// agent.ts passes these runtime base classes in to avoid a circular runtime import. +type AgentCtor = new (options: AgentOptions) => Agent; + +type AgentTaskCtor = new ( + options: AgentTaskOptions, +) => AgentTask; + +export function createAgentV2( + AgentBase: AgentCtor, + options: AgentCreateOptions, +): Agent { + class AgentV2 extends AgentBase { + private readonly hookAdapter: AgentHookAdapter>; + + constructor({ + onEnter, + onExit, + onUserTurnCompleted, + sttNode, + llmNode, + ttsNode, + realtimeAudioOutputNode, + ...agentOptions + }: AgentCreateOptions) { + super({ + ...agentOptions, + id: agentOptions.id ?? 'default_agent', + }); + + this.hookAdapter = new AgentHookAdapter( + { + onEnter, + onExit, + onUserTurnCompleted, + sttNode, + llmNode, + ttsNode, + realtimeAudioOutputNode, + }, + new AgentHookContext(this), + ); + } + + override async onEnter(): Promise { + return this.hookAdapter.onEnter(() => super.onEnter()); + } + + override async onExit(): Promise { + return this.hookAdapter.onExit(() => super.onExit()); + } + + override async onUserTurnCompleted( + chatCtx: ChatContext, + newMessage: ChatMessage, + ): Promise { + return this.hookAdapter.onUserTurnCompleted(chatCtx, newMessage, () => + super.onUserTurnCompleted(chatCtx, newMessage), + ); + } + + override async sttNode( + audio: ReadableStream, + modelSettings: ModelSettings, + ): Promise | null> { + return this.hookAdapter.sttNode(audio, modelSettings, () => + super.sttNode(audio, modelSettings), + ); + } + + override async llmNode( + chatCtx: ChatContext, + toolCtx: ToolContext, + modelSettings: ModelSettings, + ): Promise | null> { + return this.hookAdapter.llmNode(chatCtx, toolCtx, modelSettings, () => + super.llmNode(chatCtx, toolCtx, modelSettings), + ); + } + + override async ttsNode( + text: ReadableStream, + modelSettings: ModelSettings, + ): Promise | null> { + return this.hookAdapter.ttsNode(text, modelSettings, () => + super.ttsNode(text, modelSettings), + ); + } + + override async realtimeAudioOutputNode( + audio: ReadableStream, + modelSettings: ModelSettings, + ): Promise | null> { + return this.hookAdapter.realtimeAudioOutputNode(audio, modelSettings, () => + super.realtimeAudioOutputNode(audio, modelSettings), + ); + } + } + + return new AgentV2(options); +} + +export function createAgentTaskV2( + AgentTaskBase: AgentTaskCtor, + options: AgentTaskCreateOptions, +): AgentTask { + class AgentTaskV2 extends AgentTaskBase { + private readonly hookAdapter: AgentHookAdapter>; + + constructor({ + onEnter, + onExit, + onUserTurnCompleted, + sttNode, + llmNode, + ttsNode, + realtimeAudioOutputNode, + ...taskOptions + }: AgentTaskCreateOptions) { + super({ + ...taskOptions, + id: taskOptions.id ?? 'default_agent', + }); + + this.hookAdapter = new AgentHookAdapter( + { + onEnter, + onExit, + onUserTurnCompleted, + sttNode, + llmNode, + ttsNode, + realtimeAudioOutputNode, + }, + new AgentTaskHookContext(this), + ); + } + + override async onEnter(): Promise { + return this.hookAdapter.onEnter(() => super.onEnter()); + } + + override async onExit(): Promise { + return this.hookAdapter.onExit(() => super.onExit()); + } + + override async onUserTurnCompleted( + chatCtx: ChatContext, + newMessage: ChatMessage, + ): Promise { + return this.hookAdapter.onUserTurnCompleted(chatCtx, newMessage, () => + super.onUserTurnCompleted(chatCtx, newMessage), + ); + } + + override async sttNode( + audio: ReadableStream, + modelSettings: ModelSettings, + ): Promise | null> { + return this.hookAdapter.sttNode(audio, modelSettings, () => + super.sttNode(audio, modelSettings), + ); + } + + override async llmNode( + chatCtx: ChatContext, + toolCtx: ToolContext, + modelSettings: ModelSettings, + ): Promise | null> { + return this.hookAdapter.llmNode(chatCtx, toolCtx, modelSettings, () => + super.llmNode(chatCtx, toolCtx, modelSettings), + ); + } + + override async ttsNode( + text: ReadableStream, + modelSettings: ModelSettings, + ): Promise | null> { + return this.hookAdapter.ttsNode(text, modelSettings, () => + super.ttsNode(text, modelSettings), + ); + } + + override async realtimeAudioOutputNode( + audio: ReadableStream, + modelSettings: ModelSettings, + ): Promise | null> { + return this.hookAdapter.realtimeAudioOutputNode(audio, modelSettings, () => + super.realtimeAudioOutputNode(audio, modelSettings), + ); + } + } + + return new AgentTaskV2(options); +} + +class AgentHookAdapter> { + constructor( + private readonly hooks: AgentHooks, + private readonly context: ContextT, + ) {} + + async onEnter(fallback: () => Promise): Promise { + if (!this.hooks.onEnter) { + return fallback(); + } + + return this.hooks.onEnter(this.context); + } + + async onExit(fallback: () => Promise): Promise { + if (!this.hooks.onExit) { + return fallback(); + } + + return this.hooks.onExit(this.context); + } + + async onUserTurnCompleted( + chatCtx: ChatContext, + newMessage: ChatMessage, + fallback: () => Promise, + ): Promise { + if (!this.hooks.onUserTurnCompleted) { + return fallback(); + } + + return this.hooks.onUserTurnCompleted(this.context, chatCtx, newMessage); + } + + async sttNode( + audio: ReadableStream, + modelSettings: ModelSettings, + fallback: () => Promise | null>, + ): Promise | null> { + if (!this.hooks.sttNode) { + return fallback(); + } + + const result = await this.hooks.sttNode(this.context, readStream(audio), modelSettings); + return result === null ? null : toStream(result); + } + + async llmNode( + chatCtx: ChatContext, + toolCtx: ToolContext, + modelSettings: ModelSettings, + fallback: () => Promise | null>, + ): Promise | null> { + if (!this.hooks.llmNode) { + return fallback(); + } + + const result = await this.hooks.llmNode( + this.context, + chatCtx, + toolCtx as ToolContext, + modelSettings, + ); + return result === null ? null : toStream(result); + } + + async ttsNode( + text: ReadableStream, + modelSettings: ModelSettings, + fallback: () => Promise | null>, + ): Promise | null> { + if (!this.hooks.ttsNode) { + return fallback(); + } + + const result = await this.hooks.ttsNode(this.context, readStream(text), modelSettings); + return result === null ? null : toStream(result); + } + + async realtimeAudioOutputNode( + audio: ReadableStream, + modelSettings: ModelSettings, + fallback: () => Promise | null>, + ): Promise | null> { + if (!this.hooks.realtimeAudioOutputNode) { + return fallback(); + } + + const result = await this.hooks.realtimeAudioOutputNode( + this.context, + readStream(audio), + modelSettings, + ); + return result === null ? null : toStream(result); + } +} + +class AgentHookContext implements AgentContext { + constructor(readonly agent: Agent) {} + + get vad(): VAD | undefined { + return this.agent.vad; + } + + get stt(): STT | undefined { + return this.agent.stt; + } + + get llm(): LLM | RealtimeModel | undefined { + return this.agent.llm; + } + + get tts(): TTS | undefined { + return this.agent.tts; + } + + get useTtsAlignedTranscript(): boolean | undefined { + return this.agent.useTtsAlignedTranscript; + } + + get chatCtx(): ReadonlyChatContext { + return this.agent.chatCtx; + } + + get id(): string { + return this.agent.id; + } + + get instructions(): string | Instructions { + return this.agent.instructions; + } + + get toolCtx(): ToolContext { + return this.agent.toolCtx; + } + + get session(): AgentSession { + return this.agent.session; + } + + get turnHandling(): Partial | undefined { + return this.agent.turnHandling; + } + + get minConsecutiveSpeechDelay(): number | undefined { + return this.agent.minConsecutiveSpeechDelay; + } +} + +class AgentTaskHookContext + extends AgentHookContext + implements AgentTaskContext +{ + declare readonly agent: AgentTask; + + constructor(agent: AgentTask) { + super(agent); + } + + complete(result: ResultT | Error): void { + this.agent.complete(result); + } +} diff --git a/agents/src/voice/amd.test.ts b/agents/src/voice/amd.test.ts index 1b79ffa80..023d7a186 100644 --- a/agents/src/voice/amd.test.ts +++ b/agents/src/voice/amd.test.ts @@ -7,7 +7,7 @@ import type { ChatContext } from '../llm/chat_context.js'; import { FunctionCall } from '../llm/chat_context.js'; import type { ChatChunk } from '../llm/llm.js'; import { LLM, type LLMStream } from '../llm/llm.js'; -import type { ToolChoice, ToolContext } from '../llm/tool_context.js'; +import type { ToolChoice, ToolCtxInput } from '../llm/tool_context.js'; import type { SpeechEvent, SpeechStream } from '../stt/stt.js'; import { STT } from '../stt/stt.js'; import type { APIConnectOptions } from '../types.js'; @@ -48,7 +48,7 @@ class StaticLLM extends LLM { connOptions: _connOptions, }: { chatCtx: ChatContext; - toolCtx?: ToolContext; + toolCtx?: ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: ToolChoice; @@ -270,7 +270,7 @@ describe('AMD', () => { } chat({}: { chatCtx: ChatContext; - toolCtx?: ToolContext; + toolCtx?: ToolCtxInput; connOptions?: APIConnectOptions; }): LLMStream { return { @@ -547,7 +547,7 @@ describe('AMD', () => { label(): string { return 'postpone-llm'; } - chat({}: { chatCtx: ChatContext; toolCtx?: ToolContext }): LLMStream { + chat({}: { chatCtx: ChatContext; toolCtx?: ToolCtxInput }): LLMStream { callCount += 1; const isFirst = callCount === 1; return { diff --git a/agents/src/voice/amd.ts b/agents/src/voice/amd.ts index 848d6a511..be8042d78 100644 --- a/agents/src/voice/amd.ts +++ b/agents/src/voice/amd.ts @@ -14,8 +14,7 @@ import type { LLMModels, STTModels } from '../inference/index.js'; import { ChatContext } from '../llm/chat_context.js'; import type { FunctionCall } from '../llm/chat_context.js'; import { LLM, type LLMStream } from '../llm/llm.js'; -import { isFunctionTool, tool } from '../llm/tool_context.js'; -import type { ToolContext } from '../llm/tool_context.js'; +import { ToolContext, type ToolContextEntry, isFunctionTool, tool } from '../llm/tool_context.js'; import { log } from '../log.js'; import { STT, SpeechEventType, type SpeechStream } from '../stt/stt.js'; import { traceTypes, tracer } from '../telemetry/index.js'; @@ -1181,6 +1180,7 @@ export class AMD extends (EventEmitter as new () => TypedEmitter) const isStale = (): boolean => generation !== this.detectGeneration || this.settled; const savePrediction = tool({ + name: 'save_prediction', description: 'Save the AMD prediction to the verdict.', parameters: z.object({ label: z.enum([ @@ -1213,6 +1213,7 @@ export class AMD extends (EventEmitter as new () => TypedEmitter) }); const postponeTermination = tool({ + name: 'postpone_termination', description: 'Postpone the termination of the classification task. ' + 'Use when the transcript is ambiguous and more audio is expected.', @@ -1244,10 +1245,11 @@ export class AMD extends (EventEmitter as new () => TypedEmitter) }, }); - const toolCtx: ToolContext = { save_prediction: savePrediction }; + const toolList: ToolContextEntry[] = [savePrediction]; if (this.extensionCount < MAX_EXTENSIONS) { - toolCtx.postpone_termination = postponeTermination; + toolList.push(postponeTermination); } + const toolCtx = new ToolContext(toolList); const chatCtx = new ChatContext(); chatCtx.addMessage({ role: 'system', content: this.prompt }); @@ -1282,7 +1284,7 @@ export class AMD extends (EventEmitter as new () => TypedEmitter) // Execute tool calls (save_prediction populates `savedResult`, // postpone_termination mutates the silence timer and returns). for (const tc of toolCalls) { - const fnTool = toolCtx[tc.name]; + const fnTool = toolCtx.getFunctionTool(tc.name); if (!fnTool || !isFunctionTool(fnTool)) continue; let parsedArgs: unknown = {}; try { diff --git a/agents/src/voice/avatar/avatar_session.test.ts b/agents/src/voice/avatar/avatar_session.test.ts index e1c798c07..a4d3f6811 100644 --- a/agents/src/voice/avatar/avatar_session.test.ts +++ b/agents/src/voice/avatar/avatar_session.test.ts @@ -3,7 +3,6 @@ // SPDX-License-Identifier: Apache-2.0 import { afterEach, describe, expect, it, vi } from 'vitest'; import * as jobModule from '../../job.js'; -import * as logModule from '../../log.js'; import { AvatarSession } from './avatar_session.js'; describe('AvatarSession base', () => { @@ -46,30 +45,4 @@ describe('AvatarSession base', () => { await shutdownCallback?.(); expect(acloseSpy).toHaveBeenCalledTimes(1); }); - - it('warns when avatar is started after AgentSession.start()', async () => { - const warn = vi.fn(); - vi.spyOn(logModule, 'log').mockReturnValue({ - warn, - debug: vi.fn(), - } as unknown as ReturnType); - vi.spyOn(jobModule, 'getJobContext').mockReturnValue(undefined); - - const session = new AvatarSession(); - - await session.start( - { - ...mockAgentSession({ - _started: true, - output: { audio: { constructor: { name: 'MockAudioOutput' } } }, - }), - } as any, - mockRoom() as any, - ); - - expect(warn).toHaveBeenCalledWith( - { audioOutput: 'MockAudioOutput' }, - expect.stringContaining('AvatarSession.start() was called after AgentSession.start()'), - ); - }); }); diff --git a/agents/src/voice/avatar/avatar_session.ts b/agents/src/voice/avatar/avatar_session.ts index a2ad4bef1..4bd7bfee8 100644 --- a/agents/src/voice/avatar/avatar_session.ts +++ b/agents/src/voice/avatar/avatar_session.ts @@ -28,8 +28,6 @@ export type AvatarSessionCallbacks = { * first in their own `start()` method. The base: * - Registers {@link AvatarSession.aclose} as a job shutdown callback, so avatar resources * are released when the job shuts down. - * - Warns when the avatar session is started after {@link AgentSession.start} — in that - * case the existing audio output will be replaced by the avatar's. */ export class AvatarSession extends (EventEmitter as new () => TypedEmitter) { #logger = log(); @@ -64,16 +62,6 @@ export class AvatarSession extends (EventEmitter as new () => TypedEmitter { @@ -983,7 +988,7 @@ export function performToolExecutions({ // TODO(brian): assert other toolChoice values - const tool = toolCtx[toolCall.name]; + const tool = toolCtx.getFunctionTool(toolCall.name); if (!tool) { logger.warn( { diff --git a/agents/src/voice/generation_tools.test.ts b/agents/src/voice/generation_tools.test.ts index 2589fe7f4..0cd95e636 100644 --- a/agents/src/voice/generation_tools.test.ts +++ b/agents/src/voice/generation_tools.test.ts @@ -4,7 +4,7 @@ import { ReadableStream as NodeReadableStream } from 'stream/web'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { z } from 'zod'; -import { FunctionCall, type ToolContext, ToolError, tool } from '../llm/index.js'; +import { FunctionCall, ToolContext, ToolError, tool } from '../llm/index.js'; import { initializeLogger } from '../log.js'; import type { Task } from '../utils.js'; import { cancelAndWait, delay } from '../utils.js'; @@ -70,6 +70,7 @@ describe('Generation + Tool Execution', () => { // Tool that takes > 5 seconds let toolAborted = false; const getWeather = tool({ + name: 'getWeather', description: 'weather', parameters: z.object({ location: z.string() }), execute: async ({ location }, { abortSignal }) => { @@ -92,7 +93,7 @@ describe('Generation + Tool Execution', () => { const [execTask, toolOutput] = performToolExecutions({ session: {} as any, speechHandle: { id: 'speech_test', _itemAdded: () => {} } as any, - toolCtx: { getWeather } as any, + toolCtx: new ToolContext([getWeather]) as any, toolCallStream, controller: replyAbortController, onToolExecutionStarted: () => {}, @@ -123,6 +124,7 @@ describe('Generation + Tool Execution', () => { const replyAbortController = new AbortController(); const echo = tool({ + name: 'echo', description: 'echo', parameters: z.object({ msg: z.string() }), execute: async ({ msg }) => `echo: ${msg}`, @@ -138,7 +140,7 @@ describe('Generation + Tool Execution', () => { const [execTask, toolOutput] = performToolExecutions({ session: {} as any, speechHandle: { id: 'speech_test2', _itemAdded: () => {} } as any, - toolCtx: { echo } as any, + toolCtx: new ToolContext([echo]) as any, toolCallStream, controller: replyAbortController, }); @@ -154,6 +156,7 @@ describe('Generation + Tool Execution', () => { const replyAbortController = new AbortController(); const removeOrderItem = tool({ + name: 'removeOrderItem', description: 'remove order item', parameters: z.object({ orderId: z.array(z.string()) }), execute: async ({ orderId }) => orderId.join(','), @@ -170,7 +173,7 @@ describe('Generation + Tool Execution', () => { const [execTask, toolOutput] = performToolExecutions({ session: {} as AgentSession, speechHandle: { id: 'speech_repair', _itemAdded: () => {} } as unknown as SpeechHandle, - toolCtx: { removeOrderItem } as unknown as ToolContext, + toolCtx: new ToolContext([removeOrderItem]) as unknown as ToolContext, toolCallStream, controller: replyAbortController, }); @@ -189,6 +192,7 @@ describe('Generation + Tool Execution', () => { let aborted = false; const longOp = tool({ + name: 'longOp', description: 'longOp', parameters: z.object({ ms: z.number() }), execute: async ({ ms }, { abortSignal }) => { @@ -210,7 +214,7 @@ describe('Generation + Tool Execution', () => { const [execTask, toolOutput] = performToolExecutions({ session: {} as any, speechHandle: { id: 'speech_abort', _itemAdded: () => {} } as any, - toolCtx: { longOp } as any, + toolCtx: new ToolContext([longOp]) as any, toolCallStream, controller: replyAbortController, }); @@ -229,6 +233,7 @@ describe('Generation + Tool Execution', () => { const replyAbortController = new AbortController(); const echo = tool({ + name: 'echo', description: 'echo', parameters: z.object({ msg: z.string() }), execute: async ({ msg }) => `echo: ${msg}`, @@ -245,7 +250,7 @@ describe('Generation + Tool Execution', () => { const [execTask, toolOutput] = performToolExecutions({ session: {} as any, speechHandle: { id: 'speech_invalid', _itemAdded: () => {} } as any, - toolCtx: { echo } as any, + toolCtx: new ToolContext([echo]) as any, toolCallStream, controller: replyAbortController, }); @@ -267,6 +272,7 @@ describe('Generation + Tool Execution', () => { const replyAbortController = new AbortController(); const echo = tool({ + name: 'echo', description: 'echo', parameters: z.object({ msg: z.string() }), execute: async ({ msg }) => `echo: ${msg}`, @@ -283,7 +289,7 @@ describe('Generation + Tool Execution', () => { const [execTask, toolOutput] = performToolExecutions({ session: {} as any, speechHandle: { id: 'speech_bad_json', _itemAdded: () => {} } as any, - toolCtx: { echo } as any, + toolCtx: new ToolContext([echo]) as any, toolCallStream, controller: replyAbortController, }); @@ -304,6 +310,7 @@ describe('Generation + Tool Execution', () => { // The tool throws a regular Error whose message contains internals (db URL, // credentials) we must NOT forward to the LLM (and from there to end users). const sensitive = tool({ + name: 'sensitive', description: 'sensitive', parameters: z.object({}), execute: async () => { @@ -321,7 +328,7 @@ describe('Generation + Tool Execution', () => { const [execTask, toolOutput] = performToolExecutions({ session: {} as any, speechHandle: { id: 'speech_generic_err', _itemAdded: () => {} } as any, - toolCtx: { sensitive } as any, + toolCtx: new ToolContext([sensitive]) as any, toolCallStream, controller: replyAbortController, }); @@ -345,6 +352,7 @@ describe('Generation + Tool Execution', () => { // Tools that intend to give the LLM a corrective hint opt in by throwing // ToolError — its message is forwarded as-is. const checked = tool({ + name: 'checked', description: 'checked', parameters: z.object({ qty: z.number() }), execute: async ({ qty }) => { @@ -365,7 +373,7 @@ describe('Generation + Tool Execution', () => { const [execTask, toolOutput] = performToolExecutions({ session: {} as any, speechHandle: { id: 'speech_tool_error', _itemAdded: () => {} } as any, - toolCtx: { checked } as any, + toolCtx: new ToolContext([checked]) as any, toolCallStream, controller: replyAbortController, }); @@ -383,11 +391,13 @@ describe('Generation + Tool Execution', () => { const replyAbortController = new AbortController(); const sum = tool({ + name: 'sum', description: 'sum', parameters: z.object({ a: z.number(), b: z.number() }), execute: async ({ a, b }) => a + b, }); const upper = tool({ + name: 'upper', description: 'upper', parameters: z.object({ s: z.string() }), execute: async ({ s }) => s.toUpperCase(), @@ -408,7 +418,7 @@ describe('Generation + Tool Execution', () => { const [execTask, toolOutput] = performToolExecutions({ session: {} as any, speechHandle: { id: 'speech_multi', _itemAdded: () => {} } as any, - toolCtx: { sum, upper } as any, + toolCtx: new ToolContext([sum, upper]) as any, toolCallStream, controller: replyAbortController, }); diff --git a/agents/src/voice/generation_tts_timeout.test.ts b/agents/src/voice/generation_tts_timeout.test.ts index cbe636ebe..5cb3803f9 100644 --- a/agents/src/voice/generation_tts_timeout.test.ts +++ b/agents/src/voice/generation_tts_timeout.test.ts @@ -120,6 +120,7 @@ describe('TTS stream idle timeout', () => { vi.useFakeTimers(); + // Stray PLAYBACK_STARTED before this segment captures anything must be ignored. audioOutput.onPlaybackStarted(Date.now()); expect(audioOut.firstFrameFut.done).toBe(false); @@ -148,11 +149,13 @@ describe('TTS stream idle timeout', () => { const controller = new AbortController(); const [task, audioOut] = performAudioForwarding(stream, audioOutput, controller); + // Stray event before the loop captures anything must not skip resampling. audioOutput.onPlaybackStarted(Date.now()); await task.result; expect(audioOut.firstFrameFut.done).toBe(true); + // Every captured frame must match the output sample rate (i.e. was resampled). expect(audioOutput.capturedFrames.length).toBeGreaterThan(0); for (const f of audioOutput.capturedFrames) { expect(f.sampleRate).toBe(24000); diff --git a/agents/src/voice/index.ts b/agents/src/voice/index.ts index 5d181bdbb..9a5797290 100644 --- a/agents/src/voice/index.ts +++ b/agents/src/voice/index.ts @@ -1,7 +1,19 @@ // SPDX-FileCopyrightText: 2025 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -export { Agent, AgentTask, StopResponse, type AgentOptions, type ModelSettings } from './agent.js'; +export { + Agent, + AgentTask, + StopResponse, + type AgentContext, + type AgentCreateOptions, + type AgentHookNodeResult, + type AgentHooks, + type AgentOptions, + type AgentTaskContext, + type AgentTaskCreateOptions, + type ModelSettings, +} from './agent.js'; export * from './amd.js'; export { AgentSession, @@ -29,6 +41,8 @@ export { type PlaybackFinishedEvent, type PlaybackStartedEvent, type TimedString, + createTimedString, + isTimedString, } from './io.js'; export * from './report.js'; export * from './room_io/index.js'; diff --git a/agents/src/voice/io.ts b/agents/src/voice/io.ts index d2bed007f..168bc06bb 100644 --- a/agents/src/voice/io.ts +++ b/agents/src/voice/io.ts @@ -117,25 +117,44 @@ export abstract class AudioOutput extends EventEmitter { }; protected logger = log(); protected readonly capabilities: AudioOutputCapabilities; + protected _nextInChain?: AudioOutput; constructor( public sampleRate?: number, - protected readonly nextInChain?: AudioOutput, + nextInChain?: AudioOutput, capabilities: AudioOutputCapabilities = { pause: false }, ) { super(); this.capabilities = capabilities; + if ( + nextInChain !== undefined && + nextInChain.nextInChain === undefined && + !(nextInChain instanceof AudioSinkProxy) + ) { + nextInChain = new AudioSinkProxy(nextInChain); + } + + this._nextInChain = nextInChain; + if (this.nextInChain) { - this.nextInChain.on(AudioOutput.EVENT_PLAYBACK_STARTED, (ev: PlaybackStartedEvent) => - this.onPlaybackStarted(ev.createdAt), - ); - this.nextInChain.on(AudioOutput.EVENT_PLAYBACK_FINISHED, (ev: PlaybackFinishedEvent) => - this.onPlaybackFinished(ev), - ); + this.nextInChain.on(AudioOutput.EVENT_PLAYBACK_STARTED, this.forwardNextPlaybackStarted); + this.nextInChain.on(AudioOutput.EVENT_PLAYBACK_FINISHED, this.forwardNextPlaybackFinished); } } + get nextInChain(): AudioOutput | undefined { + return this._nextInChain; + } + + protected forwardNextPlaybackStarted = (ev: PlaybackStartedEvent) => { + this.onPlaybackStarted(ev.createdAt); + }; + + protected forwardNextPlaybackFinished = (ev: PlaybackFinishedEvent) => { + this.onPlaybackFinished(ev); + }; + /** * Whether this output and all outputs in the chain support pause/resume. */ @@ -242,6 +261,87 @@ export abstract class AudioOutput extends EventEmitter { } } +class AudioSinkProxy extends AudioOutput { + private attached: boolean = false; + private capturing: boolean = false; + private pushedDuration: number = 0; + + constructor(nextInChain: AudioOutput) { + super(undefined, undefined, { pause: true }); + this.setNextInChain(nextInChain); + } + + override get nextInChain(): AudioOutput { + if (this._nextInChain === undefined) { + throw new Error('AudioSinkProxy has no downstream sink'); + } + return this._nextInChain; + } + + override get canPause(): boolean { + return this.nextInChain.canPause; + } + + override onAttached(): void { + this.attached = true; + super.onAttached(); + } + + override onDetached(): void { + this.attached = false; + super.onDetached(); + } + + setNextInChain(sink: AudioOutput): void { + if (sink === this._nextInChain) return; + + const oldSink = this._nextInChain; + if (oldSink !== undefined) { + oldSink.off(AudioOutput.EVENT_PLAYBACK_STARTED, this.forwardNextPlaybackStarted); + oldSink.off(AudioOutput.EVENT_PLAYBACK_FINISHED, this.forwardNextPlaybackFinished); + if (this.pendingPlayoutSegments > 0) { + oldSink.clearBuffer(); + } + if (this.attached) { + oldSink.onDetached(); + } + } + + this._nextInChain = sink; + this.sampleRate = sink.sampleRate; + sink.on(AudioOutput.EVENT_PLAYBACK_STARTED, this.forwardNextPlaybackStarted); + sink.on(AudioOutput.EVENT_PLAYBACK_FINISHED, this.forwardNextPlaybackFinished); + if (this.attached) { + sink.onAttached(); + } + + if (oldSink !== undefined && this.pendingPlayoutSegments > 0 && !this.capturing) { + this.onPlaybackFinished({ playbackPosition: this.pushedDuration, interrupted: true }); + } + } + + override async captureFrame(frame: AudioFrame): Promise { + if (!this.capturing) { + this.capturing = true; + this.pushedDuration = 0; + } + + await super.captureFrame(frame); + await this.nextInChain.captureFrame(frame); + this.pushedDuration += frame.samplesPerChannel / frame.sampleRate; + } + + override flush(): void { + super.flush(); + this.nextInChain.flush(); + this.capturing = false; + } + + override clearBuffer(): void { + this.nextInChain.clearBuffer(); + } +} + export interface PlaybackFinishedEvent { /** How much of the audio was played back, in seconds */ playbackPosition: number; @@ -397,6 +497,18 @@ export class AgentOutput { } } + replaceAudioTail(sink: AudioOutput): void { + let current = this._audioSink; + while (current !== null && current !== undefined) { + if (current instanceof AudioSinkProxy) { + current.setNextInChain(sink); + return; + } + current = current.nextInChain ?? null; + } + this.audio = sink; + } + get transcription(): TextOutput | null { return this._transcriptionSink; } diff --git a/agents/src/voice/testing/fake_llm.ts b/agents/src/voice/testing/fake_llm.ts index ad3a1bf16..b6ba9b08a 100644 --- a/agents/src/voice/testing/fake_llm.ts +++ b/agents/src/voice/testing/fake_llm.ts @@ -4,7 +4,7 @@ import type { ChatContext } from '../../llm/chat_context.js'; import { FunctionCall } from '../../llm/chat_context.js'; import { LLMStream as BaseLLMStream, LLM, type LLMStream } from '../../llm/llm.js'; -import type { ToolChoice, ToolContext } from '../../llm/tool_context.js'; +import type { ToolChoice, ToolCtxInput } from '../../llm/tool_context.js'; import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../../types.js'; import { delay } from '../../utils.js'; @@ -42,7 +42,7 @@ export class FakeLLM extends LLM { connOptions = DEFAULT_API_CONNECT_OPTIONS, }: { chatCtx: ChatContext; - toolCtx?: ToolContext; + toolCtx?: ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: ToolChoice; @@ -65,7 +65,7 @@ class FakeLLMStream extends BaseLLMStream { constructor( fake: FakeLLM, - params: { chatCtx: ChatContext; toolCtx?: ToolContext; connOptions: APIConnectOptions }, + params: { chatCtx: ChatContext; toolCtx?: ToolCtxInput; connOptions: APIConnectOptions }, ) { super(fake, params); this.fake = fake; diff --git a/agents/src/voice/testing/run_result.ts b/agents/src/voice/testing/run_result.ts index 4ee0ccc56..0e60d03ce 100644 --- a/agents/src/voice/testing/run_result.ts +++ b/agents/src/voice/testing/run_result.ts @@ -817,6 +817,7 @@ export class MessageAssert extends EventAssert { // Create the check_intent tool const checkIntentTool = tool({ + name: 'check_intent', description: 'Determines whether the message correctly fulfills the given intent. ' + 'Returns success=true if the message satisfies the intent, false otherwise. ' + @@ -853,7 +854,7 @@ export class MessageAssert extends EventAssert { const stream = llm.chat({ chatCtx, - toolCtx: { check_intent: checkIntentTool }, + toolCtx: [checkIntentTool], toolChoice: { type: 'function', function: { name: 'check_intent' } }, extraKwargs: { temperature: 0 }, }); diff --git a/examples/src/background_audio.ts b/examples/src/background_audio.ts index fc718ac0c..8d5884be9 100644 --- a/examples/src/background_audio.ts +++ b/examples/src/background_audio.ts @@ -35,6 +35,7 @@ export default defineAgent({ logger.info('Connected to room'); const searchWeb = llm.tool({ + name: 'searchWeb', description: 'Search the web for information based on the given query. Always use this function whenever the user requests a web search', parameters: z.object({ @@ -49,9 +50,7 @@ export default defineAgent({ const agent = new voice.Agent({ instructions: 'You are a helpful assistant', - tools: { - searchWeb, - }, + tools: [searchWeb], }); const session = new voice.AgentSession({ diff --git a/examples/src/basic_agent.ts b/examples/src/basic_agent.ts index 95ecddb9a..21463556e 100644 --- a/examples/src/basic_agent.ts +++ b/examples/src/basic_agent.ts @@ -2,16 +2,18 @@ // // SPDX-License-Identifier: Apache-2.0 import { + Agent, + AgentSession, + AgentSessionEventTypes, type JobContext, type JobProcess, ServerOptions, cli, defineAgent, inference, - llm, log, - metrics, - voice, + logMetrics, + tool, } from '@livekit/agents'; import * as livekit from '@livekit/agents-plugin-livekit'; import * as silero from '@livekit/agents-plugin-silero'; @@ -24,11 +26,12 @@ export default defineAgent({ proc.userData.vad = await silero.VAD.load(); }, entry: async (ctx: JobContext) => { - const agent = new voice.Agent({ + const agent = Agent.create({ instructions: "You are a helpful assistant, you can hear the user's message and respond to it.", - tools: { - getWeather: llm.tool({ + tools: [ + tool({ + name: 'getWeather', description: 'Get the weather for a given location.', parameters: z.object({ location: z.string().describe('The location to get the weather for'), @@ -37,12 +40,12 @@ export default defineAgent({ return `The weather in ${location} is sunny.`; }, }), - }, + ], }); const logger = log(); - const session = new voice.AgentSession({ + const session = new AgentSession({ // VAD and turn detection are used to determine when the user is speaking and when the agent should respond // See more at https://docs.livekit.io/agents/build/turns vad: ctx.proc.userData.vad! as silero.VAD, @@ -103,8 +106,8 @@ export default defineAgent({ }); // Log metrics as they are emitted - session.on(voice.AgentSessionEventTypes.MetricsCollected, (ev) => { - metrics.logMetrics(ev.metrics); + session.on(AgentSessionEventTypes.MetricsCollected, (ev) => { + logMetrics(ev.metrics); }); // Log usage summary when job shuts down @@ -117,7 +120,7 @@ export default defineAgent({ ); }); - session.on(voice.AgentSessionEventTypes.OverlappingSpeech, (ev) => { + session.on(AgentSessionEventTypes.OverlappingSpeech, (ev) => { logger.warn({ type: ev.type, isInterruption: ev.isInterruption }, 'user overlapping speech'); }); diff --git a/examples/src/basic_agent_task.ts b/examples/src/basic_agent_task.ts index aacbeee5c..68a049694 100644 --- a/examples/src/basic_agent_task.ts +++ b/examples/src/basic_agent_task.ts @@ -2,11 +2,14 @@ // // SPDX-License-Identifier: Apache-2.0 import { + Agent, + AgentTask, type JobContext, type JobProcess, ServerOptions, cli, defineAgent, + handoff, inference, llm, voice, @@ -16,97 +19,101 @@ import * as silero from '@livekit/agents-plugin-silero'; import { fileURLToPath } from 'node:url'; import { z } from 'zod'; -class InfoTask extends voice.AgentTask { - constructor(private info: string) { - super({ - instructions: `Collect the user's information. around ${info}. Once you have the information, call the saveUserInfo tool to save the information to the database IMMEDIATELY. DO NOT have chitchat with the user, just collect the information and call the saveUserInfo tool.`, - tts: 'elevenlabs/eleven_turbo_v2_5', - tools: { - saveUserInfo: llm.tool({ - description: `Save the user's ${info} to database`, - parameters: z.object({ - [info]: z.string(), - }), - execute: async (args) => { - this.complete(args[info] as string); - return `Thanks, collected ${info} successfully: ${args[info]}`; - }, +function createInfoTask(info: string): AgentTask { + const task = AgentTask.create({ + instructions: `Collect the user's information. around ${info}. Once you have the information, call the saveUserInfo tool to save the information to the database IMMEDIATELY. DO NOT have chitchat with the user, just collect the information and call the saveUserInfo tool.`, + tts: 'elevenlabs/eleven_turbo_v2_5', + tools: [ + llm.tool({ + name: 'saveUserInfo', + description: `Save the user's ${info} to database`, + parameters: z.object({ + [info]: z.string(), }), - }, - }); - } + execute: async (args) => { + task.complete(args[info] as string); + return `Thanks, collected ${info} successfully: ${args[info]}`; + }, + }), + ], + onEnter: (ctx) => { + ctx.session.generateReply({ + userInput: `Ask the user for their ${info}`, + }); + }, + }); - async onEnter() { - this.session.generateReply({ - userInput: `Ask the user for their ${this.info}`, - }); - } + return task; } -class SurveyAgent extends voice.Agent { - constructor() { - super({ - instructions: - 'You orchestrate a short intro survey. Speak naturally and keep the interaction brief.', - tools: { - collectUserInfo: llm.tool({ - description: 'Call this when user want to provide some information to you', - parameters: z.object({ - key: z - .string() - .describe( - 'The key of the information to collect, e.g. "name" or "role" should be no space and underscore separated', - ), - }), - execute: async ({ key }) => { - const value = await new InfoTask(key).run(); - return `Collected ${key} successfully: ${value}`; - }, +function createWeatherAgent() { + return Agent.create({ + instructions: + 'You are a weather agent. You are responsible for providing the weather information to the user.', + tts: 'deepgram/aura-2', + tools: [ + llm.tool({ + name: 'getWeather', + description: 'Get the weather for a given location', + parameters: z.object({ + location: z.string().describe('The location to get the weather for'), }), - transferToWeatherAgent: llm.tool({ - description: 'Call this immediately after user want to know the weather', - execute: async () => { - const agent = new voice.Agent({ - instructions: - 'You are a weather agent. You are responsible for providing the weather information to the user.', - tts: 'deepgram/aura-2', - tools: { - getWeather: llm.tool({ - description: 'Get the weather for a given location', - parameters: z.object({ - location: z.string().describe('The location to get the weather for'), - }), - execute: async ({ location }) => { - return `The weather in ${location} is sunny today.`; - }, - }), - finishWeatherConversation: llm.tool({ - description: 'Call this when you want to finish the weather conversation', - execute: async () => { - return llm.handoff({ - agent: new SurveyAgent(), - returns: 'Transfer to survey agent successfully!', - }); - }, - }), - }, - }); + execute: async ({ location }) => { + return `The weather in ${location} is sunny today.`; + }, + }), + llm.tool({ + name: 'finishWeatherConversation', + description: 'Call this when you want to finish the weather conversation', + execute: async () => { + return llm.handoff({ + agent: createSurveyAgent(), + returns: 'Transfer to survey agent successfully!', + }); + }, + }), + ], + }); +} - return llm.handoff({ agent, returns: "Let's start the weather conversation!" }); - }, +function createSurveyAgent(): Agent { + return Agent.create({ + instructions: + 'You orchestrate a short intro survey. Speak naturally and keep the interaction brief.', + tools: [ + llm.tool({ + name: 'collectUserInfo', + description: 'Call this when user want to provide some information to you', + parameters: z.object({ + key: z + .string() + .describe( + 'The key of the information to collect, e.g. "name" or "role" should be no space and underscore separated', + ), }), - }, - }); - } - - async onEnter() { - const name = await new InfoTask('name').run(); - const role = await new InfoTask('role').run(); + execute: async ({ key }) => { + const value = await createInfoTask(key).run(); + return `Collected ${key} successfully: ${value}`; + }, + }), + llm.tool({ + name: 'transferToWeatherAgent', + description: 'Call this immediately after user want to know the weather', + execute: async () => { + const agent = createWeatherAgent(); + return handoff({ agent, returns: "Let's start the weather conversation!" }); + }, + }), + ], + onEnter: async (ctx) => { + const name = await createInfoTask('name').run(); + const role = await createInfoTask('role').run(); - await this.session.say( - `Great to meet you ${name}. I noted your role as ${role}. We can continue now.`, - ); - } + await ctx.session.say( + `Great to meet you ${name}. I noted your role as ${role}. We can continue now.`, + ); + }, + }); } export default defineAgent({ @@ -126,7 +133,7 @@ export default defineAgent({ await session.start({ room: ctx.room, - agent: new SurveyAgent(), + agent: createSurveyAgent(), }); }, }); diff --git a/examples/src/basic_task_group.ts b/examples/src/basic_task_group.ts index d40befe2a..3d8e04464 100644 --- a/examples/src/basic_task_group.ts +++ b/examples/src/basic_task_group.ts @@ -25,8 +25,9 @@ class CollectNameTask extends voice.AgentTask { instructions: 'Collect the user name from the latest user message. As soon as you have it, call save_name.', tts: taskTts, - tools: { - save_name: llm.tool({ + tools: [ + llm.tool({ + name: 'save_name', description: 'Save the user name.', parameters: z.object({ name: z.string().describe('The user name'), @@ -36,7 +37,7 @@ class CollectNameTask extends voice.AgentTask { return `Saved name: ${name}`; }, }), - }, + ], }); } @@ -54,8 +55,9 @@ class CollectEmailTask extends voice.AgentTask { instructions: 'Collect the user email from the latest user message. As soon as you have it, call save_email.', tts: taskTts, - tools: { - save_email: llm.tool({ + tools: [ + llm.tool({ + name: 'save_email', description: 'Save the user email.', parameters: z.object({ email: z.string().describe('The user email'), @@ -65,7 +67,7 @@ class CollectEmailTask extends voice.AgentTask { return `Saved email: ${email}`; }, }), - }, + ], }); } @@ -82,8 +84,9 @@ class TaskGroupDemoAgent extends voice.Agent { super({ instructions: 'You are onboarding assistant. When user asks to begin onboarding, call startOnboarding exactly once.', - tools: { - startOnboarding: llm.tool({ + tools: [ + llm.tool({ + name: 'startOnboarding', description: 'Start a two-step onboarding flow (name then email).', parameters: z.object({}), execute: async () => { @@ -107,7 +110,7 @@ class TaskGroupDemoAgent extends voice.Agent { return JSON.stringify(result.taskResults); }, }), - }, + ], }); } diff --git a/examples/src/basic_tool_call_agent.ts b/examples/src/basic_tool_call_agent.ts index 5642ef488..9a18de362 100644 --- a/examples/src/basic_tool_call_agent.ts +++ b/examples/src/basic_tool_call_agent.ts @@ -44,6 +44,7 @@ export default defineAgent({ }, entry: async (ctx: JobContext) => { const getWeather = llm.tool({ + name: 'getWeather', description: ' Called when the user asks about the weather.', parameters: z.object({ location: z.string().describe('The location to get the weather for'), @@ -55,6 +56,7 @@ export default defineAgent({ }); const toggleLight = llm.tool({ + name: 'toggleLight', description: 'Called when the user asks to turn on or off the light.', parameters: z.object({ room: roomNameSchema.describe('The room to turn the light in'), @@ -70,6 +72,7 @@ export default defineAgent({ }); const getNumber = llm.tool({ + name: 'getNumber', description: 'Called when the user wants to get a number value, None if user want a random value', parameters: z.object({ @@ -87,6 +90,7 @@ export default defineAgent({ }); const checkStoredNumber = llm.tool({ + name: 'checkStoredNumber', description: 'Called when the user wants to check the stored number.', execute: async (_, { ctx }: llm.ToolOptions) => { return `The stored number is ${ctx.userData.number}.`; @@ -94,6 +98,7 @@ export default defineAgent({ }); const updateStoredNumber = llm.tool({ + name: 'updateStoredNumber', description: 'Called when the user wants to update the stored number.', parameters: z.object({ number: z.number().describe('The number to update the stored number to'), @@ -106,31 +111,33 @@ export default defineAgent({ const routerAgent = new RouterAgent({ instructions: 'You are a helpful assistant.', - tools: { + tools: [ getWeather, toggleLight, - playGame: llm.tool({ + llm.tool({ + name: 'playGame', description: 'Called when the user wants to play a game (transfer user to a game agent).', execute: async (): Promise => { return llm.handoff({ agent: gameAgent, returns: 'The game is now playing.' }); }, }), - }, + ], }); const gameAgent = new GameAgent({ instructions: 'You are a game agent. You are playing a game with the user.', - tools: { + tools: [ getNumber, checkStoredNumber, updateStoredNumber, - finishGame: llm.tool({ + llm.tool({ + name: 'finishGame', description: 'Called when the user wants to finish the game.', execute: async () => { return llm.handoff({ agent: routerAgent, returns: 'The game is now finished.' }); }, }), - }, + ], }); const vad = ctx.proc.userData.vad! as silero.VAD; diff --git a/examples/src/basic_toolsets.ts b/examples/src/basic_toolsets.ts new file mode 100644 index 000000000..cb0cf2b6e --- /dev/null +++ b/examples/src/basic_toolsets.ts @@ -0,0 +1,180 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { + type JobContext, + type JobProcess, + ServerOptions, + cli, + defineAgent, + inference, + llm, + voice, +} from '@livekit/agents'; +import * as livekit from '@livekit/agents-plugin-livekit'; +import * as silero from '@livekit/agents-plugin-silero'; +import { BackgroundVoiceCancellation } from '@livekit/noise-cancellation-node'; +import { fileURLToPath } from 'node:url'; +import { z } from 'zod'; + +class InfoTask extends voice.AgentTask { + private key: string; + + constructor(key: string, sharedToolset: llm.Toolset) { + super({ + instructions: `Collect the user's ${key}. Once you have it, call saveUserInfo IMMEDIATELY. No chitchat.`, + tools: [ + sharedToolset, + llm.tool({ + name: 'saveUserInfo', + description: `Save the user's ${key} to the database`, + parameters: z.object({ + [key]: z.string(), + }), + execute: async (args) => { + this.complete(args[key] as string); + return `Thanks, collected ${key} successfully: ${args[key]}`; + }, + }), + ], + }); + this.key = key; + } + + async onEnter() { + this.session.generateReply({ userInput: `Ask the user for their ${this.key}` }); + } +} + +function makeWeatherAgent(returnHome: () => voice.Agent) { + const weatherToolset = new llm.Toolset({ + id: 'weather_tools', + tools: [ + llm.tool({ + name: 'getWeather', + description: 'Get the weather for a given location', + parameters: z.object({ location: z.string() }), + execute: async ({ location }) => `The weather in ${location} is sunny today.`, + }), + ], + }); + + return new voice.Agent({ + instructions: 'You are a weather agent. Provide weather information then hand back when done.', + tools: [ + weatherToolset, + llm.tool({ + name: 'finishWeatherConversation', + description: 'Call this when you want to finish the weather conversation', + execute: async () => { + return llm.handoff({ agent: returnHome(), returns: 'Transfer back to main agent.' }); + }, + }), + ], + }); +} + +class MainAgent extends voice.Agent { + private locationToolset: llm.Toolset; + + constructor(locationToolset: llm.Toolset) { + super({ + instructions: + 'You are a helpful assistant. Use the location toolset for weather/timezone. Use transferToWeather when the user asks about weather. Use swapToolset / reapplyTools to exercise updateTools.', + tools: [ + locationToolset, + llm.tool({ + name: 'transferToWeather', + description: 'Call this when the user wants to know the weather', + execute: async () => { + return llm.handoff({ + agent: makeWeatherAgent(() => new MainAgent(locationToolset)), + returns: "Let's switch to the weather agent.", + }); + }, + }), + llm.tool({ + name: 'swapToolset', + description: 'Replace the active toolset with a brand-new toolset (tests updateTools).', + execute: async () => { + const replacement = new llm.Toolset({ + id: 'location_tools_v2', + tools: [ + llm.tool({ + name: 'getWeather', + description: 'v2 weather', + parameters: z.object({ location: z.string() }), + execute: async ({ location }) => `v2: ${location} -> sunny`, + }), + ], + }); + await this.updateTools([replacement]); + return 'Swapped toolset.'; + }, + }), + llm.tool({ + name: 'reapplyTools', + description: 'Re-apply the current tool list unchanged (idempotent updateTools).', + execute: async () => { + await this.updateTools([...this.toolCtx.tools]); + return 'Re-applied the same tool list.'; + }, + }), + ], + }); + this.locationToolset = locationToolset; + } + + async onEnter() { + const name = await new InfoTask('name', this.locationToolset).run(); + await this.session.say( + `Got it, ${name}. Ask me about weather, or say "swap" / "reapply" to exercise updateTools.`, + ); + } +} + +export default defineAgent({ + prewarm: async (proc: JobProcess) => { + proc.userData.vad = await silero.VAD.load(); + }, + entry: async (ctx: JobContext) => { + const locationToolset = new llm.Toolset({ + id: 'location_tools', + tools: [ + llm.tool({ + name: 'getWeather', + description: 'Get the weather for a given location.', + parameters: z.object({ location: z.string() }), + execute: async ({ location }) => `The weather in ${location} is sunny.`, + }), + llm.tool({ + name: 'lookupTimezone', + description: 'Look up the timezone for a city or region.', + parameters: z.object({ location: z.string() }), + execute: async ({ location }) => `${location} is in the America/Los_Angeles timezone.`, + }), + ], + }); + + const session = new voice.AgentSession({ + vad: ctx.proc.userData.vad! as silero.VAD, + stt: new inference.STT({ model: 'deepgram/nova-3', language: 'en' }), + llm: new inference.LLM({ model: 'openai/gpt-4.1-mini' }), + tts: new inference.TTS({ + model: 'cartesia/sonic-3', + voice: '9626c31c-bec5-4cca-baa8-f8ba9e84c8bc', + }), + turnDetection: new livekit.turnDetector.MultilingualModel(), + }); + + await session.start({ + agent: new MainAgent(locationToolset), + room: ctx.room, + inputOptions: { noiseCancellation: BackgroundVoiceCancellation() }, + }); + + session.say('Hello! I will ask you a quick question, then we can chat.'); + }, +}); + +cli.runApp(new ServerOptions({ agent: fileURLToPath(import.meta.url) })); diff --git a/examples/src/comprehensive_test.ts b/examples/src/comprehensive_test.ts index 6e9fc8f07..fef64b9b9 100644 --- a/examples/src/comprehensive_test.ts +++ b/examples/src/comprehensive_test.ts @@ -76,8 +76,9 @@ class MainAgent extends voice.Agent { tts: ttsOptions['elevenlabs'](), llm: llmOptions['openai'](), turnDetection: eouOptions['multilingual'](), - tools: { - testAgent: llm.tool({ + tools: [ + llm.tool({ + name: 'testAgent', description: 'Called when user want to test an agent with STT, TTS, EOU, LLM, and optionally realtime LLM configuration', parameters: z.object({ @@ -102,7 +103,7 @@ class MainAgent extends voice.Agent { }); }, }), - }, + ], }); } @@ -159,8 +160,9 @@ class TestAgent extends voice.Agent { tts: tts, llm: realtimeModel ?? model, turnDetection: eou, - tools: { - testTool: llm.tool({ + tools: [ + llm.tool({ + name: 'testTool', description: "Testing agent's tool calling ability", parameters: z .object({ @@ -173,7 +175,8 @@ class TestAgent extends voice.Agent { }; }, }), - nextAgent: llm.tool({ + llm.tool({ + name: 'nextAgent', description: 'Called when user confirm current agent is working and want to proceed to next agent', parameters: z.object({ @@ -204,7 +207,7 @@ class TestAgent extends voice.Agent { }); }, }), - }, + ], }); this.sttChoice = sttChoice; diff --git a/examples/src/drive-thru/drivethru_agent.ts b/examples/src/drive-thru/drivethru_agent.ts index 9882f6fcd..e684f5c00 100644 --- a/examples/src/drive-thru/drivethru_agent.ts +++ b/examples/src/drive-thru/drivethru_agent.ts @@ -57,23 +57,24 @@ export class DriveThruAgent extends voice.Agent { super({ instructions, - tools: { - orderComboMeal: DriveThruAgent.buildComboOrderTool( + tools: [ + DriveThruAgent.buildComboOrderTool( userdata.comboItems, userdata.drinkItems, userdata.sauceItems, ), - orderHappyMeal: DriveThruAgent.buildHappyOrderTool( + DriveThruAgent.buildHappyOrderTool( userdata.happyItems, userdata.drinkItems, userdata.sauceItems, ), - orderRegularItem: DriveThruAgent.buildRegularOrderTool( + DriveThruAgent.buildRegularOrderTool( userdata.regularItems, userdata.drinkItems, userdata.sauceItems, ), - removeOrderItem: llm.tool({ + llm.tool({ + name: 'removeOrderItem', description: `Removes one or more items from the user's order using their \`orderId\`s. Useful when the user asks to cancel or delete existing items (e.g., "Remove the cheeseburger"). @@ -100,7 +101,8 @@ If the \`orderId\`s are unknown, call \`listOrderItems\` first to retrieve them. return 'Removed items:\n' + removedItems.map((item) => JSON.stringify(item)).join('\n'); }, }), - listOrderItems: llm.tool({ + llm.tool({ + name: 'listOrderItems', description: `Retrieves the current list of items in the user's order, including each item's internal \`orderId\`. Helpful when: @@ -120,7 +122,7 @@ Examples: return items.map((item) => JSON.stringify(item)).join('\n'); }, }), - }, + ], }); } @@ -134,6 +136,7 @@ Examples: const availableSauceIds = [...new Set(sauceItems.map((item) => item.id))]; return llm.tool({ + name: 'orderComboMeal', description: `Call this when the user orders a **Combo Meal**, like: "Number 4b with a large Sprite" or "I'll do a medium meal." Do not call this tool unless the user clearly refers to a known combo meal by name or number. @@ -222,6 +225,7 @@ If the user says just "a large meal," assume both drink and fries are that size. const availableSauceIds = [...new Set(sauceItems.map((item) => item.id))]; return llm.tool({ + name: 'orderHappyMeal', description: `Call this when the user orders a **Happy Meal**, typically for children. These meals come with a main item, a drink, and a sauce. The user must clearly specify a valid Happy Meal option (e.g., "Can I get a Happy Meal?"). @@ -299,6 +303,7 @@ Assume Small as default only if the user says "Happy Meal" and gives no size pre const availableIds = [...new Set(allItems.map((item) => item.id))]; return llm.tool({ + name: 'orderRegularItem', description: `Call this when the user orders **a single item on its own**, not as part of a Combo Meal or Happy Meal. The customer must provide clear and specific input. For example, item variants such as flavor must **always** be explicitly stated. diff --git a/examples/src/frontdesk/frontdesk_agent.ts b/examples/src/frontdesk/frontdesk_agent.ts index d5d2e1ab1..2fa60ba57 100644 --- a/examples/src/frontdesk/frontdesk_agent.ts +++ b/examples/src/frontdesk/frontdesk_agent.ts @@ -59,8 +59,9 @@ export class FrontDeskAgent extends voice.Agent { super({ instructions, - tools: { - scheduleAppointment: llm.tool({ + tools: [ + llm.tool({ + name: 'scheduleAppointment', description: 'Schedule an appointment at the given slot.', parameters: z.object({ slotId: z @@ -110,7 +111,8 @@ export class FrontDeskAgent extends voice.Agent { return `The appointment was successfully scheduled for ${formatted}.`; }, }), - listAvailableSlots: llm.tool({ + llm.tool({ + name: 'listAvailableSlots', description: `Return a plain-text list of available slots, one per line. - , , at () @@ -188,7 +190,7 @@ You must infer the appropriate range implicitly from the conversational context return lines.join('\n') || 'No slots available at the moment.'; }, }), - }, + ], }); this.tz = options.timezone; diff --git a/examples/src/gemini_realtime_agent.ts b/examples/src/gemini_realtime_agent.ts index 60cbb443e..300446778 100644 --- a/examples/src/gemini_realtime_agent.ts +++ b/examples/src/gemini_realtime_agent.ts @@ -46,6 +46,7 @@ type StoryData = { const roomNameSchema = z.enum(['bedroom', 'living room', 'kitchen', 'bathroom', 'office']); const getWeather = llm.tool({ + name: 'getWeather', description: 'Called when the user asks about the weather.', parameters: z.object({ location: z.string().describe('The location to get the weather for'), @@ -58,6 +59,7 @@ const getWeather = llm.tool({ }); const toggleLight = llm.tool({ + name: 'toggleLight', description: 'Called when the user asks to turn on or off the light.', parameters: z.object({ room: roomNameSchema.describe('The room to turn the light in'), @@ -75,28 +77,29 @@ class IntroAgent extends voice.Agent { }); } - static create() { + static createIntroAgent() { return new IntroAgent({ instructions: `You are a story teller. Your goal is to gather a few pieces of information from the user to make the story personalized and engaging. Ask the user for their name and where they are from.`, - tools: { - informationGathered: llm.tool({ + tools: [ + llm.tool({ + name: 'informationGathered', description: 'Called when the user has provided the information needed to make the story personalized and engaging.', parameters: z.object({ name: z.string().describe('The name of the user'), location: z.string().describe('The location of the user'), }), - execute: async ({ name, location }, { ctx }) => { + execute: async ({ name, location }, { ctx }: llm.ToolOptions) => { ctx.userData.name = name; ctx.userData.location = location; - const storyAgent = StoryAgent.create(name, location); + const storyAgent = StoryAgent.createStoryAgent(name, location); return llm.handoff({ agent: storyAgent, returns: "Let's start the story!" }); }, }), getWeather, toggleLight, - }, + ], }); } } @@ -106,7 +109,7 @@ class StoryAgent extends voice.Agent { this.session.generateReply(); } - static create(name: string, location: string) { + static createStoryAgent(name: string, location: string) { return new StoryAgent({ instructions: dedent` You are a storyteller. Use the user's information in order to make the story personalized. @@ -138,7 +141,7 @@ export default defineAgent({ }); await session.start({ - agent: IntroAgent.create(), + agent: IntroAgent.createIntroAgent(), room: ctx.room, }); diff --git a/examples/src/instructions_per_modality.ts b/examples/src/instructions_per_modality.ts index 71f2f3f04..643e3d433 100644 --- a/examples/src/instructions_per_modality.ts +++ b/examples/src/instructions_per_modality.ts @@ -57,8 +57,9 @@ class SchedulingAgent extends voice.Agent { super({ instructions, - tools: { - bookAppointment: llm.tool({ + tools: [ + llm.tool({ + name: 'bookAppointment', description: 'Book an appointment.', parameters: z.object({ date: z.string().describe('The date of the appointment in the format YYYY-MM-DD'), @@ -69,7 +70,7 @@ class SchedulingAgent extends voice.Agent { return `Appointment booked for ${date} at ${time}`; }, }), - }, + ], }); } diff --git a/examples/src/llm_fallback_adapter.ts b/examples/src/llm_fallback_adapter.ts index d053464dc..d86708548 100644 --- a/examples/src/llm_fallback_adapter.ts +++ b/examples/src/llm_fallback_adapter.ts @@ -71,8 +71,9 @@ export default defineAgent({ const agent = new voice.Agent({ instructions: 'You are a helpful assistant. Demonstrate that you are working by responding to user queries.', - tools: { - getWeather: llm.tool({ + tools: [ + llm.tool({ + name: 'getWeather', description: 'Get the weather for a given location.', parameters: z.object({ location: z.string().describe('The location to get the weather for'), @@ -81,7 +82,7 @@ export default defineAgent({ return `The weather in ${location} is sunny with a temperature of 72°F.`; }, }), - }, + ], }); const session = new voice.AgentSession({ diff --git a/examples/src/manual_shutdown.ts b/examples/src/manual_shutdown.ts index 96bedb901..edb21b87c 100644 --- a/examples/src/manual_shutdown.ts +++ b/examples/src/manual_shutdown.ts @@ -25,8 +25,9 @@ export default defineAgent({ const agent = new voice.Agent({ instructions: "You are a helpful assistant, you can hear the user's message and respond to it, end the call when the user asks you to.", - tools: { - getWeather: llm.tool({ + tools: [ + llm.tool({ + name: 'getWeather', description: 'Get the weather for a given location.', parameters: z.object({ location: z.string().describe('The location to get the weather for'), @@ -35,7 +36,8 @@ export default defineAgent({ return `The weather in ${location} is sunny.`; }, }), - endCall: llm.tool({ + llm.tool({ + name: 'endCall', description: 'End the call.', parameters: z.object({ reason: z @@ -56,7 +58,7 @@ export default defineAgent({ session.shutdown({ reason }); }, }), - }, + ], }); const session = new voice.AgentSession({ diff --git a/examples/src/multi_agent.ts b/examples/src/multi_agent.ts index 7f4819bed..cd22d299d 100644 --- a/examples/src/multi_agent.ts +++ b/examples/src/multi_agent.ts @@ -32,26 +32,27 @@ class IntroAgent extends voice.Agent { }); } - static create() { + static createIntroAgent() { return new IntroAgent({ instructions: `You are a story teller. Your goal is to gather a few pieces of information from the user to make the story personalized and engaging. Ask the user for their name and where they are from.`, - tools: { - informationGathered: llm.tool({ + tools: [ + llm.tool({ + name: 'informationGathered', description: 'Called when the user has provided the information needed to make the story personalized and engaging.', parameters: z.object({ name: z.string().describe('The name of the user'), location: z.string().describe('The location of the user'), }), - execute: async ({ name, location }, { ctx }) => { + execute: async ({ name, location }, { ctx }: llm.ToolOptions) => { ctx.userData.name = name; ctx.userData.location = location; - const storyAgent = StoryAgent.create(name, location); + const storyAgent = StoryAgent.createStoryAgent(name, location); return llm.handoff({ agent: storyAgent, returns: "Let's start the story!" }); }, }), - }, + ], }); } } @@ -61,7 +62,7 @@ class StoryAgent extends voice.Agent { this.session.generateReply(); } - static create(name: string, location: string) { + static createStoryAgent(name: string, location: string) { return new StoryAgent({ instructions: dedent` You are a storyteller. Use the user's information in order to make the story personalized. @@ -93,7 +94,7 @@ export default defineAgent({ }); await session.start({ - agent: IntroAgent.create(), + agent: IntroAgent.createIntroAgent(), room: ctx.room, }); diff --git a/examples/src/phonic_realtime_agent.ts b/examples/src/phonic_realtime_agent.ts index 35a038450..934485f5b 100644 --- a/examples/src/phonic_realtime_agent.ts +++ b/examples/src/phonic_realtime_agent.ts @@ -7,6 +7,7 @@ import { fileURLToPath } from 'node:url'; import { z } from 'zod'; const toggleLight = llm.tool({ + name: 'toggle_light', description: 'Toggle a light on or off. Available lights are A05, A06, A07, and A08.', parameters: z.object({ light_id: z.string().describe('The ID of the light to toggle'), @@ -23,9 +24,7 @@ export default defineAgent({ entry: async (ctx: JobContext) => { const agent = new voice.Agent({ instructions: 'You are a helpful voice AI assistant named Alex.', - tools: { - toggle_light: toggleLight, - }, + tools: [toggleLight], }); const session = new voice.AgentSession({ diff --git a/examples/src/raw_function_description.ts b/examples/src/raw_function_description.ts index 6548fd011..1e81c65d9 100644 --- a/examples/src/raw_function_description.ts +++ b/examples/src/raw_function_description.ts @@ -18,8 +18,9 @@ import { fileURLToPath } from 'node:url'; function createRawFunctionAgent() { return new voice.Agent({ instructions: 'You are a helpful assistant.', - tools: { - openGate: llm.tool({ + tools: [ + llm.tool({ + name: 'openGate', description: 'Opens a specified gate from a predefined set of access points.', parameters: { type: 'object', @@ -43,7 +44,7 @@ function createRawFunctionAgent() { return `The gate ${gateId} is now open.`; }, }), - }, + ], }); } diff --git a/examples/src/realtime_agent.ts b/examples/src/realtime_agent.ts index b30171776..647d5759b 100644 --- a/examples/src/realtime_agent.ts +++ b/examples/src/realtime_agent.ts @@ -24,6 +24,7 @@ export default defineAgent({ }, entry: async (ctx: JobContext) => { const getWeather = llm.tool({ + name: 'getWeather', description: ' Called when the user asks about the weather.', parameters: z.object({ location: z.string().describe('The location to get the weather for'), @@ -34,6 +35,7 @@ export default defineAgent({ }); const toggleLight = llm.tool({ + name: 'toggleLight', description: 'Called when the user asks to turn on or off the light.', parameters: z.object({ room: roomNameSchema.describe('The room to turn the light in'), @@ -65,10 +67,7 @@ export default defineAgent({ instructions: "You are a helpful assistant created by LiveKit, always speaking English, you can hear the user's message and respond to it.", chatCtx, - tools: { - getWeather, - toggleLight, - }, + tools: [getWeather, toggleLight], }); const session = new voice.AgentSession({ diff --git a/examples/src/realtime_with_tts.ts b/examples/src/realtime_with_tts.ts index d87db7853..88c450f06 100644 --- a/examples/src/realtime_with_tts.ts +++ b/examples/src/realtime_with_tts.ts @@ -26,6 +26,7 @@ export default defineAgent({ const logger = log(); const getWeather = llm.tool({ + name: 'getWeather', description: 'Called when the user asks about the weather.', parameters: z.object({ location: z.string().describe('The location to get the weather for'), @@ -38,9 +39,7 @@ export default defineAgent({ const agent = new voice.Agent({ instructions: 'You are a helpful assistant. Always speak in English.', - tools: { - getWeather, - }, + tools: [getWeather], }); const session = new voice.AgentSession({ diff --git a/examples/src/restaurant_agent.ts b/examples/src/restaurant_agent.ts index d9faaf9a5..4fb5281ba 100644 --- a/examples/src/restaurant_agent.ts +++ b/examples/src/restaurant_agent.ts @@ -79,6 +79,7 @@ function summarize({ } const updateName = llm.tool({ + name: 'updateName', description: 'Called when the user provides their name. Confirm the spelling with the user before calling the function.', parameters: z.object({ @@ -91,6 +92,7 @@ const updateName = llm.tool({ }); const updatePhone = llm.tool({ + name: 'updatePhone', description: 'Called when the user provides their phone number. Confirm the spelling with the user before calling the function.', parameters: z.object({ @@ -103,6 +105,7 @@ const updatePhone = llm.tool({ }); const toGreeter = llm.tool({ + name: 'toGreeter', description: 'Called when user asks any unrelated questions or requests any other services not in your job description.', execute: async (_, { ctx }: llm.ToolOptions) => { @@ -173,35 +176,37 @@ function createGreeterAgent(menu: string) { instructions: `You are a friendly restaurant receptionist. The menu is: ${menu}\nYour jobs are to greet the caller and understand if they want to make a reservation or order takeaway. Guide them to the right agent using tools.`, llm: new inference.LLM({ model: 'openai/gpt-4.1-mini' }), tts: new inference.TTS({ model: 'cartesia/sonic-3', voice: voices.greeter }), - tools: { - toReservation: llm.tool({ + tools: [ + llm.tool({ + name: 'toReservation', description: dedent` Called when user wants to make or update a reservation. This function handles transitioning to the reservation agent who will collect the necessary details like reservation time, customer name and phone number. `, - execute: async (_, { ctx }): Promise => { + execute: async (_, { ctx }: llm.ToolOptions): Promise => { return await greeter.transferToAgent({ name: 'reservation', ctx, }); }, }), - toTakeaway: llm.tool({ + llm.tool({ + name: 'toTakeaway', description: dedent` Called when the user wants to place a takeaway order. This includes handling orders for pickup, delivery, or when the user wants to proceed to checkout with their existing order. `, - execute: async (_, { ctx }): Promise => { + execute: async (_, { ctx }: llm.ToolOptions): Promise => { return await greeter.transferToAgent({ name: 'takeaway', ctx, }); }, }), - }, + ], }); return greeter; @@ -212,11 +217,12 @@ function createReservationAgent() { name: 'reservation', instructions: `You are a reservation agent at a restaurant. Your jobs are to ask for the reservation time, then customer's name, and phone number. Then confirm the reservation details with the customer.`, tts: new inference.TTS({ model: 'cartesia/sonic-3', voice: voices.reservation }), - tools: { + tools: [ updateName, updatePhone, toGreeter, - updateReservationTime: llm.tool({ + llm.tool({ + name: 'updateReservationTime', description: dedent` Called when the user provides their reservation time. Confirm the time with the user before calling the function. @@ -224,14 +230,18 @@ function createReservationAgent() { parameters: z.object({ time: z.string().describe('The reservation time'), }), - execute: async ({ time }, { ctx }) => { + execute: async ({ time }, { ctx }: llm.ToolOptions) => { ctx.userData.reservationTime = time; return `The reservation time is updated to ${time}`; }, }), - confirmReservation: llm.tool({ + llm.tool({ + name: 'confirmReservation', description: `Called when the user confirms the reservation.`, - execute: async (_, { ctx }): Promise => { + execute: async ( + _, + { ctx }: llm.ToolOptions, + ): Promise => { const userdata = ctx.userData; if (!userdata.customer.name || !userdata.customer.phone) { return 'Please provide your name and phone number first.'; @@ -245,7 +255,7 @@ function createReservationAgent() { }); }, }), - }, + ], }); return reservation; @@ -256,21 +266,26 @@ function createTakeawayAgent(menu: string) { name: 'takeaway', instructions: `Your are a takeaway agent that takes orders from the customer. Our menu is: ${menu}\nClarify special requests and confirm the order with the customer.`, tts: new inference.TTS({ model: 'cartesia/sonic-3', voice: voices.takeaway }), - tools: { + tools: [ toGreeter, - updateOrder: llm.tool({ + llm.tool({ + name: 'updateOrder', description: `Called when the user provides their order.`, parameters: z.object({ items: z.array(z.string()).describe('The items of the full order'), }), - execute: async ({ items }, { ctx }) => { + execute: async ({ items }, { ctx }: llm.ToolOptions) => { ctx.userData.order = items; return `The order is updated to ${items}`; }, }), - toCheckout: llm.tool({ + llm.tool({ + name: 'toCheckout', description: `Called when the user confirms the order.`, - execute: async (_, { ctx }): Promise => { + execute: async ( + _, + { ctx }: llm.ToolOptions, + ): Promise => { const userdata = ctx.userData; if (!userdata.order) { return 'No takeaway order found. Please make an order first.'; @@ -281,7 +296,7 @@ function createTakeawayAgent(menu: string) { }); }, }), - }, + ], }); return takeaway; @@ -292,21 +307,23 @@ function createCheckoutAgent(menu: string) { name: 'checkout', instructions: `You are a checkout agent at a restaurant. The menu is: ${menu}\nYour are responsible for confirming the expense of the order and then collecting customer's name, phone number and credit card information, including the card number, expiry date, and CVV step by step.`, tts: new inference.TTS({ model: 'cartesia/sonic-3', voice: voices.checkout }), - tools: { + tools: [ updateName, updatePhone, toGreeter, - confirmExpense: llm.tool({ + llm.tool({ + name: 'confirmExpense', description: `Called when the user confirms the expense.`, parameters: z.object({ expense: z.number().describe('The expense of the order'), }), - execute: async ({ expense }, { ctx }) => { + execute: async ({ expense }, { ctx }: llm.ToolOptions) => { ctx.userData.expense = expense; return `The expense is confirmed to be ${expense}`; }, }), - updateCreditCard: llm.tool({ + llm.tool({ + name: 'updateCreditCard', description: dedent` Called when the user provides their credit card number, expiry date, and CVV. Confirm the spelling with the user before calling the function. @@ -316,14 +333,18 @@ function createCheckoutAgent(menu: string) { expiry: z.string().describe('The expiry date of the credit card'), cvv: z.string().describe('The CVV of the credit card'), }), - execute: async ({ number, expiry, cvv }, { ctx }) => { + execute: async ({ number, expiry, cvv }, { ctx }: llm.ToolOptions) => { ctx.userData.creditCard = { number, expiry, cvv }; return `The credit card number is updated to ${number}`; }, }), - confirmCheckout: llm.tool({ + llm.tool({ + name: 'confirmCheckout', description: `Called when the user confirms the checkout.`, - execute: async (_, { ctx }): Promise => { + execute: async ( + _, + { ctx }: llm.ToolOptions, + ): Promise => { const userdata = ctx.userData; if (!userdata.expense) { return 'Please confirm the expense first.'; @@ -342,16 +363,17 @@ function createCheckoutAgent(menu: string) { }); }, }), - toTakeaway: llm.tool({ + llm.tool({ + name: 'toTakeaway', description: `Called when the user wants to update their order.`, - execute: async (_, { ctx }): Promise => { + execute: async (_, { ctx }: llm.ToolOptions): Promise => { return await checkout.transferToAgent({ name: 'takeaway', ctx, }); }, }), - }, + ], }); return checkout; diff --git a/examples/src/survey_agent.ts b/examples/src/survey_agent.ts index 8504fa1a7..918fd9859 100644 --- a/examples/src/survey_agent.ts +++ b/examples/src/survey_agent.ts @@ -76,6 +76,7 @@ async function writeCsvRow(path: string, data: Record): Promise function disqualifyTool() { return llm.tool({ + name: 'disqualify', description: 'End the interview if the candidate refuses to cooperate, provides inappropriate answers, or is not a fit.', parameters: z.object({ @@ -101,8 +102,9 @@ export class IntroTask extends voice.AgentTask { super({ instructions: 'You are Alex, an interviewer screening a software engineer candidate. Gather the candidate name and short self-introduction.', - tools: { - saveIntro: llm.tool({ + tools: [ + llm.tool({ + name: 'saveIntro', description: 'Save candidate name and intro notes.', parameters: z.object({ name: z.string().describe('Candidate name'), @@ -114,7 +116,7 @@ export class IntroTask extends voice.AgentTask { return `Saved intro for ${name}.`; }, }), - }, + ], }); } @@ -132,9 +134,10 @@ export class EmailTask extends voice.AgentTask { super({ instructions: 'Collect a valid email address. If the candidate refuses, call disqualify immediately.', - tools: { + tools: [ disqualify, - saveEmail: llm.tool({ + llm.tool({ + name: 'saveEmail', description: 'Save candidate email address.', parameters: z.object({ email: z.string().describe('Candidate email'), @@ -144,7 +147,7 @@ export class EmailTask extends voice.AgentTask { return `Saved email: ${email}`; }, }), - }, + ], }); } @@ -161,9 +164,10 @@ export class CommuteTask extends voice.AgentTask super({ instructions: 'Collect commute flexibility. The role expects office attendance three days per week.', - tools: { + tools: [ disqualify, - saveCommute: llm.tool({ + llm.tool({ + name: 'saveCommute', description: 'Save candidate commute information.', parameters: z.object({ canCommute: z.boolean().describe('Whether the candidate can commute to office'), @@ -176,7 +180,7 @@ export class CommuteTask extends voice.AgentTask return 'Saved commute flexibility.'; }, }), - }, + ], }); } @@ -194,9 +198,10 @@ export class ExperienceTask extends voice.AgentTask { super({ instructions: 'You are a survey interviewer for a software engineer screening. Be concise, professional, and natural. Call endScreening when the process is complete.', - tools: { - endScreening: llm.tool({ + tools: [ + llm.tool({ + name: 'endScreening', description: 'End interview and hang up.', execute: async (_, { ctx }: llm.ToolOptions) => { ctx.session.shutdown(); return 'Interview concluded.'; }, }), - }, + ], }); } diff --git a/examples/src/testing/agent_task.test.ts b/examples/src/testing/agent_task.test.ts index 163d232a1..38d2a85d9 100644 --- a/examples/src/testing/agent_task.test.ts +++ b/examples/src/testing/agent_task.test.ts @@ -148,8 +148,9 @@ describe('AgentTask examples', { timeout: 120_000 }, () => { super({ instructions: 'You are collecting a name and role. Extract both from user input and call recordIntro.', - tools: { - recordIntro: llm.tool({ + tools: [ + llm.tool({ + name: 'recordIntro', description: 'Record the name and role', parameters: z.object({ name: z.string().describe('User name'), @@ -160,7 +161,7 @@ describe('AgentTask examples', { timeout: 120_000 }, () => { return 'recorded'; }, }), - }, + ], }); } @@ -222,8 +223,9 @@ describe('AgentTask examples', { timeout: 120_000 }, () => { super({ instructions: 'When asked to capture email, ALWAYS call captureEmail exactly once, then respond briefly.', - tools: { - captureEmail: llm.tool({ + tools: [ + llm.tool({ + name: 'captureEmail', description: 'Capture an email by running a nested AgentTask.', parameters: z.object({}), execute: async () => { @@ -236,7 +238,7 @@ describe('AgentTask examples', { timeout: 120_000 }, () => { } }, }), - }, + ], }); } @@ -275,8 +277,9 @@ describe('AgentTask examples', { timeout: 120_000 }, () => { instructions: 'You are Alex, an interviewer. Extract the candidate name and a short intro from the latest user input. ' + 'Use the tool recordIntro exactly once when both are available.', - tools: { - recordIntro: llm.tool({ + tools: [ + llm.tool({ + name: 'recordIntro', description: 'Record candidate name and intro summary.', parameters: z.object({ name: z.string().describe('Candidate name'), @@ -288,7 +291,7 @@ describe('AgentTask examples', { timeout: 120_000 }, () => { return 'Intro recorded.'; }, }), - }, + ], }); } @@ -305,8 +308,9 @@ describe('AgentTask examples', { timeout: 120_000 }, () => { super({ instructions: 'When the user asks to run the intro task, ALWAYS call collectIntroWithTask exactly once.', - tools: { - collectIntroWithTask: llm.tool({ + tools: [ + llm.tool({ + name: 'collectIntroWithTask', description: 'Launch the IntroTask and return the captured intro details.', parameters: z.object({}), execute: async () => { @@ -316,7 +320,7 @@ describe('AgentTask examples', { timeout: 120_000 }, () => { return JSON.stringify(result); }, }), - }, + ], }); } } diff --git a/examples/src/testing/basic_task_group.test.ts b/examples/src/testing/basic_task_group.test.ts index 07931279b..fc90cccba 100644 --- a/examples/src/testing/basic_task_group.test.ts +++ b/examples/src/testing/basic_task_group.test.ts @@ -82,8 +82,9 @@ class CollectNameTask extends voice.AgentTask { super({ instructions: 'Collect the user name from the latest user message. As soon as you have it, call save_name.', - tools: { - save_name: llm.tool({ + tools: [ + llm.tool({ + name: 'save_name', description: 'Save the user name.', parameters: z.object({ name: z.string().describe('The user name') }), execute: async ({ name }) => { @@ -91,7 +92,7 @@ class CollectNameTask extends voice.AgentTask { return `Saved name: ${name}`; }, }), - }, + ], }); this.ready = ready; } @@ -108,8 +109,9 @@ class CollectEmailTask extends voice.AgentTask { super({ instructions: 'Collect the user email from the latest user message. As soon as you have it, call save_email.', - tools: { - save_email: llm.tool({ + tools: [ + llm.tool({ + name: 'save_email', description: 'Save the user email.', parameters: z.object({ email: z.string().describe('The user email') }), execute: async ({ email }) => { @@ -117,7 +119,7 @@ class CollectEmailTask extends voice.AgentTask { return `Saved email: ${email}`; }, }), - }, + ], }); this.ready = ready; } diff --git a/examples/src/testing/run_result.test.ts b/examples/src/testing/run_result.test.ts index 583cbaffa..4169dea93 100644 --- a/examples/src/testing/run_result.test.ts +++ b/examples/src/testing/run_result.test.ts @@ -45,8 +45,9 @@ Response rules: - After ordering, confirm what was added (e.g., "I've added the burger to your order"). - When asked about sizes, always ask for clarification if not specified. - Be friendly and proactive in suggesting next steps.`, - tools: { - getWeather: llm.tool({ + tools: [ + llm.tool({ + name: 'getWeather', description: 'Get the current weather for a location', parameters: z.object({ location: z.string().describe('The city name'), @@ -59,14 +60,16 @@ Response rules: }); }, }), - getCurrentTime: llm.tool({ + llm.tool({ + name: 'getCurrentTime', description: 'Get the current time', parameters: z.object({}), execute: async () => { return '3:00 PM'; }, }), - orderItem: llm.tool({ + llm.tool({ + name: 'orderItem', description: 'Add an item to the order', parameters: z.object({ itemId: z.string().describe('The menu item ID'), @@ -84,7 +87,8 @@ Response rules: }); }, }), - getOrderStatus: llm.tool({ + llm.tool({ + name: 'getOrderStatus', description: 'Get the current order status', parameters: z.object({}), execute: async () => { @@ -98,7 +102,8 @@ Response rules: }); }, }), - getMenuItems: llm.tool({ + llm.tool({ + name: 'getMenuItems', description: 'Get available menu items and prices', parameters: z.object({ category: z @@ -128,7 +133,7 @@ Response rules: return JSON.stringify(menu); }, }), - }, + ], }); } } diff --git a/examples/src/testing/task_group.test.ts b/examples/src/testing/task_group.test.ts index 3d5afff06..188d46e59 100644 --- a/examples/src/testing/task_group.test.ts +++ b/examples/src/testing/task_group.test.ts @@ -260,8 +260,9 @@ describe('TaskGroup', { timeout: 120_000 }, () => { super({ instructions: 'Extract the user name from the latest user message. Call recordName immediately.', - tools: { - recordName: llm.tool({ + tools: [ + llm.tool({ + name: 'recordName', description: 'Record the user name', parameters: z.object({ name: z.string().describe('The user name') }), execute: async ({ name }) => { @@ -269,7 +270,7 @@ describe('TaskGroup', { timeout: 120_000 }, () => { return 'recorded'; }, }), - }, + ], }); } @@ -283,8 +284,9 @@ describe('TaskGroup', { timeout: 120_000 }, () => { super({ instructions: 'Extract an email address from the latest user message. Call recordEmail immediately.', - tools: { - recordEmail: llm.tool({ + tools: [ + llm.tool({ + name: 'recordEmail', description: 'Record the user email', parameters: z.object({ email: z.string().describe('The email address') }), execute: async ({ email }) => { @@ -292,7 +294,7 @@ describe('TaskGroup', { timeout: 120_000 }, () => { return 'recorded'; }, }), - }, + ], }); } @@ -400,8 +402,9 @@ describe('TaskGroup', { timeout: 120_000 }, () => { super({ instructions: 'Extract the user favorite color from the latest message. Call recordColor immediately.', - tools: { - recordColor: llm.tool({ + tools: [ + llm.tool({ + name: 'recordColor', description: 'Record favorite color', parameters: z.object({ color: z.string() }), execute: async ({ color }) => { @@ -409,7 +412,7 @@ describe('TaskGroup', { timeout: 120_000 }, () => { return 'recorded'; }, }), - }, + ], }); } @@ -423,8 +426,9 @@ describe('TaskGroup', { timeout: 120_000 }, () => { super({ instructions: 'Extract the user favorite food from the latest message. Call recordFood immediately.', - tools: { - recordFood: llm.tool({ + tools: [ + llm.tool({ + name: 'recordFood', description: 'Record favorite food', parameters: z.object({ food: z.string() }), execute: async ({ food }) => { @@ -432,7 +436,7 @@ describe('TaskGroup', { timeout: 120_000 }, () => { return 'recorded'; }, }), - }, + ], }); } diff --git a/examples/src/tool_call_disfluency.ts b/examples/src/tool_call_disfluency.ts index 8f92183a8..7e8018c99 100644 --- a/examples/src/tool_call_disfluency.ts +++ b/examples/src/tool_call_disfluency.ts @@ -39,6 +39,7 @@ export default defineAgent({ const vad = ctx.proc.userData.vad! as silero.VAD; const getWeather = llm.tool({ + name: 'getWeather', description: ' Called when the user asks about the weather.', parameters: z.object({ location: z.string().describe('The location to get the weather for'), @@ -55,9 +56,7 @@ export default defineAgent({ const agent = new VoiceAgent({ instructions: "You are a helpful assistant, you can hear the user's message and respond to it.", - tools: { - getWeather, - }, + tools: [getWeather], }); const session = new voice.AgentSession({ diff --git a/examples/src/warm_transfer.ts b/examples/src/warm_transfer.ts index d2d56e7f1..0511d30ad 100644 --- a/examples/src/warm_transfer.ts +++ b/examples/src/warm_transfer.ts @@ -26,8 +26,9 @@ class SupportAgent extends voice.Agent { constructor() { super({ instructions: INSTRUCTIONS, - tools: { - transfer_to_human: llm.tool({ + tools: [ + llm.tool({ + name: 'transfer_to_human', description: `Called when the user asks to speak to a human agent. This will put the user on hold while the supervisor is connected. Ensure that the user has confirmed that they wanted to be transferred. Do not start transfer until the user has confirmed. @@ -86,7 +87,7 @@ Examples on when the tool should be called: } }, }), - }, + ], }); } diff --git a/examples/src/xai-realtime.ts b/examples/src/xai-realtime.ts index ad383a0a6..a5130a640 100644 --- a/examples/src/xai-realtime.ts +++ b/examples/src/xai-realtime.ts @@ -10,8 +10,9 @@ export default defineAgent({ entry: async (ctx: JobContext) => { const agent = new voice.Agent({ instructions: 'You are a helpful assistant. Keep your responses short and concise.', - tools: { - getWeather: llm.tool({ + tools: [ + llm.tool({ + name: 'getWeather', description: 'Get the weather for a given location.', parameters: z.object({ location: z.string().describe('The location to get the weather for'), @@ -20,7 +21,7 @@ export default defineAgent({ return `The weather in ${location} is sunny.`; }, }), - }, + ], }); const session = new voice.AgentSession({ diff --git a/plugins/anam/src/avatar.ts b/plugins/anam/src/avatar.ts index 9e2f40e62..05c8543d6 100644 --- a/plugins/anam/src/avatar.ts +++ b/plugins/anam/src/avatar.ts @@ -122,10 +122,12 @@ export class AvatarSession extends voice.AvatarSession { const started = await anam.startEngineSession({ sessionToken }); this.sessionId = started.sessionId; - agentSession.output.audio = new voice.DataStreamAudioOutput({ - room, - destinationIdentity: this.avatarIdentity, - waitRemoteTrack: TrackKind.KIND_VIDEO, - }); + agentSession.output.replaceAudioTail( + new voice.DataStreamAudioOutput({ + room, + destinationIdentity: this.avatarIdentity, + waitRemoteTrack: TrackKind.KIND_VIDEO, + }), + ); } } diff --git a/plugins/baseten/src/llm.ts b/plugins/baseten/src/llm.ts index 4039b7cac..179a09344 100644 --- a/plugins/baseten/src/llm.ts +++ b/plugins/baseten/src/llm.ts @@ -72,19 +72,20 @@ export class OpenAILLM extends llm.LLM { chat({ chatCtx, - toolCtx, + toolCtx: toolCtxInput, connOptions = DEFAULT_API_CONNECT_OPTIONS, parallelToolCalls, toolChoice, extraKwargs, }: { chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: llm.ToolChoice; extraKwargs?: Record; }): inference.LLMStream { + const toolCtx = llm.toToolContext(toolCtxInput); const extras: Record = { ...extraKwargs }; if (this.#opts.metadata) { @@ -125,7 +126,11 @@ export class OpenAILLM extends llm.LLM { parallelToolCalls = parallelToolCalls !== undefined ? parallelToolCalls : this.#opts.parallelToolCalls; - if (toolCtx && Object.keys(toolCtx).length > 0 && parallelToolCalls !== undefined) { + if ( + toolCtx && + Object.keys(toolCtx.functionTools).length > 0 && + parallelToolCalls !== undefined + ) { extras.parallel_tool_calls = parallelToolCalls; } diff --git a/plugins/bey/src/avatar.ts b/plugins/bey/src/avatar.ts index ce23bb31e..53d091216 100644 --- a/plugins/bey/src/avatar.ts +++ b/plugins/bey/src/avatar.ts @@ -204,11 +204,13 @@ export class AvatarSession extends voice.AvatarSession { this.#logger.debug('starting avatar session'); await this.startAgent(livekitUrl, livekitToken); - agentSession.output.audio = new voice.DataStreamAudioOutput({ - room, - destinationIdentity: this.avatarIdentity, - waitRemoteTrack: TrackKind.KIND_VIDEO, - }); + agentSession.output.replaceAudioTail( + new voice.DataStreamAudioOutput({ + room, + destinationIdentity: this.avatarIdentity, + waitRemoteTrack: TrackKind.KIND_VIDEO, + }), + ); } private async startAgent(livekitUrl: string, livekitToken: string): Promise { diff --git a/plugins/cerebras/src/llm.test.ts b/plugins/cerebras/src/llm.test.ts index 3a0a3ca8b..dda5d7b3c 100644 --- a/plugins/cerebras/src/llm.test.ts +++ b/plugins/cerebras/src/llm.test.ts @@ -88,8 +88,9 @@ class WeatherAgent extends voice.Agent { constructor() { super({ instructions: 'You are a helpful assistant.', - tools: { - get_weather: llm.tool({ + tools: [ + llm.tool({ + name: 'get_weather', description: 'Get the current weather for a location.', parameters: z.object({ location: z.string().describe('The city name'), @@ -98,7 +99,7 @@ class WeatherAgent extends voice.Agent { return `The weather in ${location} is sunny, 72°F.`; }, }), - }, + ], }); } } diff --git a/plugins/did/src/avatar.ts b/plugins/did/src/avatar.ts index 7906cc47f..d1610a1e5 100644 --- a/plugins/did/src/avatar.ts +++ b/plugins/did/src/avatar.ts @@ -159,11 +159,13 @@ export class AvatarSession extends voice.AvatarSession { audioConfig: { sample_rate: sampleRate }, }); - agentSession.output.audio = new voice.DataStreamAudioOutput({ - room, - destinationIdentity: this.avatarIdentity, - sampleRate, - waitRemoteTrack: TrackKind.KIND_VIDEO, - }); + agentSession.output.replaceAudioTail( + new voice.DataStreamAudioOutput({ + room, + destinationIdentity: this.avatarIdentity, + sampleRate, + waitRemoteTrack: TrackKind.KIND_VIDEO, + }), + ); } } diff --git a/plugins/elevenlabs/src/stt.test.ts b/plugins/elevenlabs/src/stt.test.ts index c0349641a..918c321dc 100644 --- a/plugins/elevenlabs/src/stt.test.ts +++ b/plugins/elevenlabs/src/stt.test.ts @@ -280,7 +280,7 @@ describe('ElevenLabs STT', () => { expect(url.pathname).toBe('/speech-to-text/realtime'); expect(url.searchParams.get('model_id')).toBe('scribe_v2_realtime'); expect(url.searchParams.get('audio_format')).toBe('pcm_16000'); - expect(url.searchParams.get('commit_strategy')).toBe('vad'); + expect(url.searchParams.get('commit_strategy')).toBe('manual'); expect(url.searchParams.get('include_language_detection')).toBe('true'); expect(receivedMessages[0]).toMatchObject({ message_type: 'input_audio_chunk', @@ -334,7 +334,6 @@ describe('ElevenLabs STT', () => { words: [{ text: 'kept', start: 0, end: 0.2 }], }), ); - ws.send(JSON.stringify({ message_type: 'committed_transcript_with_timestamps', text: '' })); }); }); @@ -364,6 +363,7 @@ describe('ElevenLabs STT', () => { const url = new URL(`ws://127.0.0.1${requestUrl}`); expect(url.searchParams.get('language_code')).toBe('en'); + expect(url.searchParams.get('commit_strategy')).toBe('vad'); expect(url.searchParams.get('include_language_detection')).toBeNull(); expect(url.searchParams.get('include_timestamps')).toBe('true'); expect(url.searchParams.get('vad_silence_threshold_secs')).toBe('0.5'); @@ -393,14 +393,18 @@ describe('ElevenLabs STT', () => { const stream = eleven.stream(); await waitUntil(() => urls.length === 1); - eleven.updateOptions({ serverVad: null }); + eleven.updateOptions({ serverVad: { vadSilenceThresholdSecs: 0.5 } }); await waitUntil(() => urls.length === 2, 2000); + eleven.updateOptions({ serverVad: null }); + await waitUntil(() => urls.length === 3, 2000); stream.close(); const first = new URL(`ws://127.0.0.1${urls[0]}`); const second = new URL(`ws://127.0.0.1${urls[1]}`); - expect(first.searchParams.get('commit_strategy')).toBe('vad'); - expect(second.searchParams.get('commit_strategy')).toBe('manual'); + const third = new URL(`ws://127.0.0.1${urls[2]}`); + expect(first.searchParams.get('commit_strategy')).toBe('manual'); + expect(second.searchParams.get('commit_strategy')).toBe('vad'); + expect(third.searchParams.get('commit_strategy')).toBe('manual'); } finally { await closeWebSocketServer(wss); } diff --git a/plugins/elevenlabs/src/stt.ts b/plugins/elevenlabs/src/stt.ts index a205a29b7..5ff10488b 100644 --- a/plugins/elevenlabs/src/stt.ts +++ b/plugins/elevenlabs/src/stt.ts @@ -330,17 +330,16 @@ export class STT extends stt.STT { language?: string, abortSignal?: AbortSignal, ): Promise { - if (language !== undefined) { - this.#opts.languageCode = normalizeLanguage(language); - } + const languageCode = + language !== undefined ? normalizeLanguage(language) : this.#opts.languageCode; const wavBytes = createWav(mergeFrames(buffer)); const form = new FormData(); form.append('file', new Blob([new Uint8Array(wavBytes)], { type: 'audio/x-wav' }), 'audio.wav'); form.append('model_id', this.#opts.modelId); form.append('tag_audio_events', String(this.#opts.tagAudioEvents)); - if (this.#opts.languageCode) { - form.append('language_code', this.#opts.languageCode); + if (languageCode) { + form.append('language_code', languageCode); } if (this.#opts.keyterms !== undefined) { for (const keyterm of this.#opts.keyterms) { @@ -369,7 +368,7 @@ export class STT extends stt.STT { const startTime = words.length > 0 ? Math.min(...words.map((word) => word.start ?? 0)) : 0; const endTime = words.length > 0 ? Math.max(...words.map((word) => word.end ?? 0)) : 0; const normalizedLanguage = normalizeLanguage( - responseJson.language_code ?? this.#opts.languageCode ?? '', + responseJson.language_code ?? languageCode ?? '', ); return this.#transcriptionToSpeechEvent( @@ -650,7 +649,8 @@ export class SpeechStream extends stt.SpeechStream { } async #connectWs(): Promise { - const commitStrategy = this.#opts.serverVad === null ? 'manual' : 'vad'; + const serverVad = this.#opts.serverVad; + const commitStrategy = serverVad === undefined || serverVad === null ? 'manual' : 'vad'; const params = [ `model_id=${this.#opts.modelId}`, `audio_format=pcm_${this.#opts.sampleRate}`, @@ -661,30 +661,21 @@ export class SpeechStream extends stt.SpeechStream { params.push('include_language_detection=true'); } - if (this.#opts.serverVad) { + if (serverVad !== undefined && serverVad !== null) { if ( - this.#opts.serverVad.vadSilenceThresholdSecs !== undefined && - this.#opts.serverVad.vadSilenceThresholdSecs !== null + serverVad.vadSilenceThresholdSecs !== undefined && + serverVad.vadSilenceThresholdSecs !== null ) { - params.push(`vad_silence_threshold_secs=${this.#opts.serverVad.vadSilenceThresholdSecs}`); + params.push(`vad_silence_threshold_secs=${serverVad.vadSilenceThresholdSecs}`); } - if ( - this.#opts.serverVad.vadThreshold !== undefined && - this.#opts.serverVad.vadThreshold !== null - ) { - params.push(`vad_threshold=${this.#opts.serverVad.vadThreshold}`); + if (serverVad.vadThreshold !== undefined && serverVad.vadThreshold !== null) { + params.push(`vad_threshold=${serverVad.vadThreshold}`); } - if ( - this.#opts.serverVad.minSpeechDurationMs !== undefined && - this.#opts.serverVad.minSpeechDurationMs !== null - ) { - params.push(`min_speech_duration_ms=${this.#opts.serverVad.minSpeechDurationMs}`); + if (serverVad.minSpeechDurationMs !== undefined && serverVad.minSpeechDurationMs !== null) { + params.push(`min_speech_duration_ms=${serverVad.minSpeechDurationMs}`); } - if ( - this.#opts.serverVad.minSilenceDurationMs !== undefined && - this.#opts.serverVad.minSilenceDurationMs !== null - ) { - params.push(`min_silence_duration_ms=${this.#opts.serverVad.minSilenceDurationMs}`); + if (serverVad.minSilenceDurationMs !== undefined && serverVad.minSilenceDurationMs !== null) { + params.push(`min_silence_duration_ms=${serverVad.minSilenceDurationMs}`); } } @@ -804,6 +795,10 @@ export class SpeechStream extends stt.SpeechStream { type: stt.SpeechEventType.FINAL_TRANSCRIPT, alternatives: [speechData], }); + if (this.#opts.serverVad !== undefined && this.#opts.serverVad !== null) { + this.queue.put({ type: stt.SpeechEventType.END_OF_SPEECH }); + this.#speaking = false; + } } else if (this.#speaking) { this.queue.put({ type: stt.SpeechEventType.END_OF_SPEECH }); this.#speaking = false; diff --git a/plugins/google/src/aiplatform_llm.ts b/plugins/google/src/aiplatform_llm.ts index 9217ee150..19e530eed 100644 --- a/plugins/google/src/aiplatform_llm.ts +++ b/plugins/google/src/aiplatform_llm.ts @@ -165,19 +165,20 @@ export class AIPlatformLLM extends llm.LLM { chat({ chatCtx, - toolCtx, + toolCtx: toolCtxInput, connOptions = DEFAULT_API_CONNECT_OPTIONS, parallelToolCalls, toolChoice, extraKwargs, }: { chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: llm.ToolChoice; extraKwargs?: Record; }): inference.LLMStream { + const toolCtx = llm.toToolContext(toolCtxInput); const extras: Record = { ...extraKwargs }; if (this.#opts.temperature !== undefined) { @@ -201,7 +202,11 @@ export class AIPlatformLLM extends llm.LLM { parallelToolCalls = parallelToolCalls !== undefined ? parallelToolCalls : this.#opts.parallelToolCalls; - if (toolCtx && Object.keys(toolCtx).length > 0 && parallelToolCalls !== undefined) { + if ( + toolCtx && + Object.keys(toolCtx.functionTools).length > 0 && + parallelToolCalls !== undefined + ) { extras.parallel_tool_calls = parallelToolCalls; } diff --git a/plugins/google/src/index.ts b/plugins/google/src/index.ts index 0afb408fc..8e786ae5b 100644 --- a/plugins/google/src/index.ts +++ b/plugins/google/src/index.ts @@ -15,6 +15,7 @@ export { export { beta }; export { LLM, LLMStream, type LLMOptions } from './llm.js'; export * from './models.js'; +export * from './tools.js'; export { realtime }; class GooglePlugin extends Plugin { diff --git a/plugins/google/src/llm.ts b/plugins/google/src/llm.ts index 4e8e52ef3..2d73d1c52 100644 --- a/plugins/google/src/llm.ts +++ b/plugins/google/src/llm.ts @@ -13,7 +13,7 @@ import { } from '@livekit/agents'; import type { ChatModels } from './models.js'; import type { LLMTools } from './tools.js'; -import { toFunctionDeclarations } from './utils.js'; +import { toToolsConfig } from './utils.js'; interface GoogleFormatData { systemMessages: string[] | null; @@ -189,20 +189,21 @@ export class LLM extends llm.LLM { chat({ chatCtx, - toolCtx, + toolCtx: toolCtxInput, connOptions = DEFAULT_API_CONNECT_OPTIONS, toolChoice, extraKwargs, geminiTools, }: { chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: llm.ToolChoice; extraKwargs?: Record; geminiTools?: LLMTools; }): LLMStream { + const toolCtx = llm.toToolContext(toolCtxInput); const extras: GenerateContentConfig = { ...extraKwargs } as GenerateContentConfig; if (this.#opts.httpOptions !== undefined && extras.httpOptions === undefined) { @@ -358,11 +359,11 @@ export class LLMStream extends llm.LLMStream { parts: turn.parts as types.Part[], })); - const functionDeclarations = this.toolCtx ? toFunctionDeclarations(this.toolCtx) : undefined; - const tools = - functionDeclarations && functionDeclarations.length > 0 - ? [{ functionDeclarations }] - : undefined; + const tools = toToolsConfig({ + toolCtx: this.toolCtx, + geminiTools: this.#geminiTools, + onlySingleType: true, + }); let systemInstruction: types.Content | undefined = undefined; if (extraData.systemMessages && extraData.systemMessages.length > 0) { diff --git a/plugins/google/src/realtime/realtime_api.ts b/plugins/google/src/realtime/realtime_api.ts index 01f37208a..d12ca3b92 100644 --- a/plugins/google/src/realtime/realtime_api.ts +++ b/plugins/google/src/realtime/realtime_api.ts @@ -33,7 +33,7 @@ import { import { Mutex } from '@livekit/mutex'; import { AudioFrame, AudioResampler, type VideoFrame } from '@livekit/rtc-node'; import { type LLMTools } from '../tools.js'; -import { toFunctionDeclarations } from '../utils.js'; +import { toToolsConfig } from '../utils.js'; import type * as api_proto from './api_proto.js'; import type { LiveAPIModels, Voice } from './api_proto.js'; @@ -70,13 +70,6 @@ export interface InputTranscription { transcript: string; } -/** - * Helper function to check if two sets are equal - */ -function setsEqual(a: Set, b: Set): boolean { - return a.size === b.size && [...a].every((x) => b.has(x)); -} - /** * Internal realtime options for Google Realtime API */ @@ -451,11 +444,10 @@ export class RealtimeModel extends llm.RealtimeModel { * supporting both text and audio modalities with function calling capabilities. */ export class RealtimeSession extends llm.RealtimeSession { - private _tools: llm.ToolContext = {}; + private _tools: llm.ToolContext = llm.ToolContext.empty(); private _chatCtx = llm.ChatContext.empty(); private options: RealtimeOptions; - private geminiDeclarations: types.FunctionDeclaration[] = []; private messageChannel = new Queue(); private inputResampler?: AudioResampler; private inputResamplerInputRate?: number; @@ -764,15 +756,12 @@ export class RealtimeSession extends llm.RealtimeSession { } async updateTools(tools: llm.ToolContext): Promise { - const newDeclarations = toFunctionDeclarations(tools); - const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name)); - const newToolNames = new Set(newDeclarations.map((f) => f.name)); - - if (!setsEqual(currentToolNames, newToolNames)) { - this.geminiDeclarations = newDeclarations; - this._tools = tools; - this.markRestartNeeded(); + if (this._tools.equals(tools)) { + return; } + + this._tools = tools; + this.markRestartNeeded(); } get chatCtx(): llm.ChatContext { @@ -780,7 +769,7 @@ export class RealtimeSession extends llm.RealtimeSession { } get tools(): llm.ToolContext { - return { ...this._tools }; + return this._tools.copy(); } get manualActivityDetection(): boolean { @@ -1452,21 +1441,11 @@ export class RealtimeSession extends llm.RealtimeSession { }, languageCode: opts.language, }, - tools: - this.geminiDeclarations.length > 0 || this.options.geminiTools - ? [ - { - functionDeclarations: - this.options.toolBehavior !== undefined - ? this.geminiDeclarations.map((d) => ({ - ...d, - behavior: this.options.toolBehavior, - })) - : this.geminiDeclarations, - ...this.options.geminiTools, - }, - ] - : undefined, + tools: toToolsConfig({ + toolCtx: this._tools, + geminiTools: this.options.geminiTools, + toolBehavior: this.options.toolBehavior, + }), inputAudioTranscription: opts.inputAudioTranscription, outputAudioTranscription: opts.outputAudioTranscription, sessionResumption: this.sessionResumptionHandle diff --git a/plugins/google/src/tools.ts b/plugins/google/src/tools.ts index 90cd9cc7c..b864e98a6 100644 --- a/plugins/google/src/tools.ts +++ b/plugins/google/src/tools.ts @@ -1,6 +1,100 @@ // SPDX-FileCopyrightText: 2025 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import type { Tool } from '@google/genai'; +import type * as types from '@google/genai'; +import { llm } from '@livekit/agents'; -export type LLMTools = Omit; +export type LLMTools = Omit; + +export abstract class GeminiTool extends llm.ProviderTool { + abstract toToolConfig(): types.Tool; +} + +export class GoogleSearch extends GeminiTool { + constructor(public readonly options: types.GoogleSearch = {}) { + super({ id: 'gemini_google_search' }); + } + + toToolConfig(): types.Tool { + return { googleSearch: this.options }; + } +} + +export class GoogleMaps extends GeminiTool { + constructor(public readonly options: types.GoogleMaps = {}) { + super({ id: 'gemini_google_maps' }); + } + + toToolConfig(): types.Tool { + return { googleMaps: this.options }; + } +} + +export class URLContext extends GeminiTool { + constructor() { + super({ id: 'gemini_url_context' }); + } + + toToolConfig(): types.Tool { + return { urlContext: {} }; + } +} + +export interface FileSearchOptions extends types.FileSearch { + fileSearchStoreNames: string[]; +} + +export class FileSearch extends GeminiTool { + constructor(public readonly options: FileSearchOptions) { + super({ id: 'gemini_file_search' }); + } + + toToolConfig(): types.Tool { + return { fileSearch: this.options }; + } +} + +export class ToolCodeExecution extends GeminiTool { + constructor() { + super({ id: 'gemini_code_execution' }); + } + + toToolConfig(): types.Tool { + return { codeExecution: {} }; + } +} + +export interface VertexRAGRetrievalOptions { + ragResources: string[]; + similarityTopK?: number; + vectorDistanceThreshold?: number; +} + +export class VertexRAGRetrieval extends GeminiTool { + readonly ragResources: string[]; + readonly similarityTopK: number; + readonly vectorDistanceThreshold?: number; + + constructor({ + ragResources, + similarityTopK = 3, + vectorDistanceThreshold, + }: VertexRAGRetrievalOptions) { + super({ id: 'gemini_vertex_rag_retrieval' }); + this.ragResources = ragResources; + this.similarityTopK = similarityTopK; + this.vectorDistanceThreshold = vectorDistanceThreshold; + } + + toToolConfig(): types.Tool { + return { + retrieval: { + vertexRagStore: { + ragResources: this.ragResources.map((ragCorpus) => ({ ragCorpus })), + similarityTopK: this.similarityTopK, + vectorDistanceThreshold: this.vectorDistanceThreshold, + }, + }, + }; + } +} diff --git a/plugins/google/src/utils.ts b/plugins/google/src/utils.ts index 3bb2ba0b1..c9fa471e1 100644 --- a/plugins/google/src/utils.ts +++ b/plugins/google/src/utils.ts @@ -1,9 +1,11 @@ // SPDX-FileCopyrightText: 2025 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 +import type * as types from '@google/genai'; import type { FunctionDeclaration, Schema } from '@google/genai'; import { llm } from '@livekit/agents'; import type { JSONSchema7 } from 'json-schema'; +import { GeminiTool, type LLMTools } from './tools.js'; /** * JSON Schema v7 @@ -139,6 +141,8 @@ function isEmptyObjectSchema(jsonSchema: JSONSchema7Definition): boolean { export function toFunctionDeclarations(toolCtx: llm.ToolContext): FunctionDeclaration[] { const functionDeclarations: FunctionDeclaration[] = []; + // Provider tools are not supported by the Gemini schema; `sortedToolEntries` yields only + // function tools (sorted by name), so they are skipped here. for (const [name, tool] of llm.sortedToolEntries(toolCtx)) { const { description, parameters } = tool; const jsonSchema = llm.toJsonSchema(parameters, false); @@ -155,3 +159,57 @@ export function toFunctionDeclarations(toolCtx: llm.ToolContext): FunctionDeclar return functionDeclarations; } + +export function toToolsConfig({ + toolCtx, + geminiTools, + toolBehavior, + onlySingleType = false, +}: { + toolCtx?: llm.ToolContext; + geminiTools?: LLMTools; + toolBehavior?: types.Behavior; + onlySingleType?: boolean; +}): types.Tool[] | undefined { + const tools: types.Tool[] = []; + const providerTools: types.Tool[] = []; + + if (toolCtx) { + const functionDeclarations = toFunctionDeclarations(toolCtx); + if (functionDeclarations.length > 0) { + tools.push({ + functionDeclarations: + toolBehavior !== undefined + ? functionDeclarations.map((declaration) => ({ + ...declaration, + behavior: toolBehavior, + })) + : functionDeclarations, + }); + } + } + + if (geminiTools !== undefined) { + providerTools.push(geminiTools); + } + + if (toolCtx) { + for (const tool of toolCtx.providerTools) { + if (tool instanceof GeminiTool) { + providerTools.push(tool.toToolConfig()); + } + } + } + + if (tools.length > 0 && providerTools.length > 0) { + throw new Error('Gemini does not support mixing function tools and provider tools'); + } + + if (onlySingleType && tools.length > 0) { + return tools; + } + + tools.push(...providerTools); + + return tools.length > 0 ? tools : undefined; +} diff --git a/plugins/lemonslice/src/avatar.ts b/plugins/lemonslice/src/avatar.ts index e4d6a970b..9861d0b4d 100644 --- a/plugins/lemonslice/src/avatar.ts +++ b/plugins/lemonslice/src/avatar.ts @@ -252,13 +252,15 @@ export class AvatarSession extends voice.AvatarSession { this.#logger.debug('starting avatar session'); const sessionId = await this.startAgent(livekitUrl, livekitToken); - agentSession.output.audio = new voice.DataStreamAudioOutput({ - room, - destinationIdentity: this.avatarIdentity, - sampleRate: SAMPLE_RATE, - waitRemoteTrack: TrackKind.KIND_VIDEO, - waitPlaybackStart: true, - }); + agentSession.output.replaceAudioTail( + new voice.DataStreamAudioOutput({ + room, + destinationIdentity: this.avatarIdentity, + sampleRate: SAMPLE_RATE, + waitRemoteTrack: TrackKind.KIND_VIDEO, + waitPlaybackStart: true, + }), + ); return sessionId; } diff --git a/plugins/liveavatar/src/avatar.ts b/plugins/liveavatar/src/avatar.ts index ce0575d3b..ff88af749 100644 --- a/plugins/liveavatar/src/avatar.ts +++ b/plugins/liveavatar/src/avatar.ts @@ -238,7 +238,7 @@ export class AvatarSession extends voice.AvatarSession { this.audioBuffer.on('clear_buffer', (ev: voice.QueueAudioOutputClearEvent) => this.onClearBuffer(ev), ); - agentSession.output.audio = this.audioBuffer; + agentSession.output.replaceAudioTail(this.audioBuffer); // Spawn the main task with an attached error handler so a websocket open or // protocol failure does not surface as an unhandled rejection. The main task diff --git a/plugins/mistralai/src/llm.ts b/plugins/mistralai/src/llm.ts index 97afbc6ff..cbafbeace 100644 --- a/plugins/mistralai/src/llm.ts +++ b/plugins/mistralai/src/llm.ts @@ -123,7 +123,7 @@ export class LLM extends llm.LLM { extraKwargs, }: { chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: llm.ToolChoice; @@ -187,7 +187,7 @@ export class LLMStream extends llm.LLMStream { client: Mistral; opts: LLMOpts; chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions: APIConnectOptions; extraKwargs: Record; }, @@ -211,7 +211,9 @@ export class LLMStream extends llm.LLMStream { // eslint-disable-next-line @typescript-eslint/no-explicit-any const toolsList: any[] = []; - if (this.toolCtx && Object.keys(this.toolCtx).length > 0) { + // Provider tools are not supported by the Mistral schema; `sortedToolEntries` yields only + // function tools (sorted by name), so they are skipped here. + if (this.toolCtx) { for (const [name, func] of llm.sortedToolEntries(this.toolCtx)) { toolsList.push({ type: 'function' as const, diff --git a/plugins/openai/src/index.ts b/plugins/openai/src/index.ts index ccffdcb3f..6a5d9cb7c 100644 --- a/plugins/openai/src/index.ts +++ b/plugins/openai/src/index.ts @@ -5,6 +5,7 @@ import { Plugin } from '@livekit/agents'; export { LLM, LLMStream, type LLMOptions } from './llm.js'; export * from './models.js'; +export * from './tools.js'; export * as realtime from './realtime/index.js'; export * as responses from './responses/index.js'; export { STT, type STTOptions } from './stt.js'; diff --git a/plugins/openai/src/llm.ts b/plugins/openai/src/llm.ts index 50e025b94..8639f92a5 100644 --- a/plugins/openai/src/llm.ts +++ b/plugins/openai/src/llm.ts @@ -469,19 +469,20 @@ export class LLM extends llm.LLM { chat({ chatCtx, - toolCtx, + toolCtx: toolCtxInput, connOptions = DEFAULT_API_CONNECT_OPTIONS, parallelToolCalls, toolChoice, extraKwargs, }: { chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: llm.ToolChoice; extraKwargs?: Record; }): LLMStream { + const toolCtx = llm.toToolContext(toolCtxInput); const extras: Record = { ...extraKwargs }; if (this.#opts.metadata) { @@ -514,7 +515,11 @@ export class LLM extends llm.LLM { parallelToolCalls = parallelToolCalls !== undefined ? parallelToolCalls : this.#opts.parallelToolCalls; - if (toolCtx && Object.keys(toolCtx).length > 0 && parallelToolCalls !== undefined) { + if ( + toolCtx && + Object.keys(toolCtx.functionTools).length > 0 && + parallelToolCalls !== undefined + ) { extras.parallel_tool_calls = parallelToolCalls; } diff --git a/plugins/openai/src/realtime/realtime_model.test.ts b/plugins/openai/src/realtime/realtime_model.test.ts index 6080cdbd6..d52d1090a 100644 --- a/plugins/openai/src/realtime/realtime_model.test.ts +++ b/plugins/openai/src/realtime/realtime_model.test.ts @@ -13,9 +13,15 @@ import { type RealtimeSessionInternals = { generateReply: RealtimeSession['generateReply']; + updateInstructions: RealtimeSession['updateInstructions']; responseCreatedFutures: Record; sendEvent: ReturnType; textModeRecoveryRetries: number; + instructions?: string; + _options: { + isAzure?: boolean; + apiVersion?: string; + }; }; type ResponseDoneSessionInternals = { @@ -36,6 +42,7 @@ function createSessionForTest(): RealtimeSessionInternals { session.responseCreatedFutures = {}; session.sendEvent = vi.fn(); session.textModeRecoveryRetries = 0; + session._options = {}; return session; } @@ -49,6 +56,37 @@ function stubTaskRuntime(): void { } describe('RealtimeSession.generateReply', () => { + it('preserves session instructions when generating with per-response instructions', async () => { + const session = createSessionForTest(); + await session.updateInstructions('Your name is Kelly. Always respond in English.'); + + const abortController = new AbortController(); + const promise = session.generateReply('Tell the user what your name is.', { + signal: abortController.signal, + }); + abortController.abort(); + + await expect(promise).rejects.toThrow('generateReply aborted'); + expect(session.instructions).toBe('Your name is Kelly. Always respond in English.'); + expect(session.sendEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'session.update', + session: expect.objectContaining({ + instructions: 'Your name is Kelly. Always respond in English.', + }), + }), + ); + expect(session.sendEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'response.create', + response: expect.objectContaining({ + instructions: + 'Your name is Kelly. Always respond in English.\nTell the user what your name is.', + }), + }), + ); + }); + it('cancels an in-flight response when aborted before response.created', async () => { const session = createSessionForTest(); const abortController = new AbortController(); diff --git a/plugins/openai/src/realtime/realtime_model.ts b/plugins/openai/src/realtime/realtime_model.ts index 1f1c4cf7c..e50ea5d3a 100644 --- a/plugins/openai/src/realtime/realtime_model.ts +++ b/plugins/openai/src/realtime/realtime_model.ts @@ -425,7 +425,7 @@ export function processBaseURL({ * - openai_client_event_queued: expose the raw client events sent to the OpenAI Realtime API */ export class RealtimeSession extends llm.RealtimeSession { - private _tools: llm.ToolContext = {}; + private _tools: llm.ToolContext = llm.ToolContext.empty(); private remoteChatCtx: llm.RemoteChatContext = new llm.RemoteChatContext(); private messageChannel = new Queue(); private inputResampler?: AudioResampler; @@ -552,7 +552,7 @@ export class RealtimeSession extends llm.RealtimeSession { } get tools() { - return { ...this._tools } as llm.ToolContext; + return this._tools.copy(); } async updateChatCtx(_chatCtx: llm.ChatContext): Promise { @@ -720,13 +720,12 @@ export class RealtimeSession extends llm.RealtimeSession { // TODO(brian): these logics below are noops I think, leaving it here to keep // parity with the python but we should remove them later const retainedToolNames = new Set(ev.session.tools.map((tool) => tool.name)); - const retainedTools = Object.fromEntries( - Object.entries(_tools).filter( - ([name, tool]) => llm.isFunctionTool(tool) && retainedToolNames.has(name), - ), + // Keep provider tools and Toolsets as-is; only drop function tools the server didn't accept. + const retainedEntries = _tools.tools.filter( + (entry) => !llm.isFunctionTool(entry) || retainedToolNames.has(entry.name), ); - this._tools = retainedTools as llm.ToolContext; + this._tools = new llm.ToolContext(retainedEntries); unlock(); } @@ -734,26 +733,26 @@ export class RealtimeSession extends llm.RealtimeSession { private createToolsUpdateEvent(_tools: llm.ToolContext): api_proto.SessionUpdateEvent { const oaiTools: api_proto.Tool[] = []; - for (const [name, tool] of Object.entries(_tools)) { - if (!llm.isFunctionTool(tool)) { - this.#logger.error({ name, tool }, "OpenAI Realtime API doesn't support this tool type"); - continue; - } + for (const t of _tools.flatten()) { + // TODO: support provider tools in the Realtime session-update schema. + if (!llm.isFunctionTool(t)) continue; - const { parameters: toolParameters, description } = tool; try { const parameters = llm.toJsonSchema( - toolParameters, + t.parameters, ) as unknown as api_proto.Tool['parameters']; oaiTools.push({ - name, - description, + name: t.name, + description: t.description, parameters: parameters, type: 'function', }); } catch (e) { - this.#logger.error({ name, tool }, "OpenAI Realtime API doesn't support this tool type"); + this.#logger.error( + { name: t.name, tool: t }, + "OpenAI Realtime API doesn't support this tool type", + ); continue; } } @@ -843,7 +842,18 @@ export class RealtimeSession extends llm.RealtimeSession { instructions?: string, options: { signal?: AbortSignal } = {}, ): Promise { - const handle = this.createResponse({ instructions, userInitiated: true }); + // In OpenAI realtime, the session-level instructions are completely replaced by the + // per-response instructions for this response. Prepend the session instructions so they + // are preserved (parity with the Python implementation). + let responseInstructions = instructions; + if (instructions && this.instructions) { + responseInstructions = `${this.instructions}\n${instructions}`; + } + + const handle = this.createResponse({ + instructions: responseInstructions, + userInitiated: true, + }); this.textModeRecoveryRetries = 0; const onAbort = () => { @@ -1050,7 +1060,7 @@ export class RealtimeSession extends llm.RealtimeSession { events.push(this.createSessionUpdateEvent()); // tools - if (Object.keys(this._tools).length > 0) { + if (Object.keys(this._tools.functionTools).length > 0) { events.push(this.createToolsUpdateEvent(this._tools)); } diff --git a/plugins/openai/src/responses/llm.ts b/plugins/openai/src/responses/llm.ts index 219651d83..aaf68a5ed 100644 --- a/plugins/openai/src/responses/llm.ts +++ b/plugins/openai/src/responses/llm.ts @@ -13,6 +13,7 @@ import { } from '@livekit/agents'; import OpenAI from 'openai'; import type { ChatModels } from '../models.js'; +import { toResponsesTools } from '../tool_utils.js'; import { WSLLM } from '../ws/llm.js'; export interface LLMOptions { @@ -77,25 +78,30 @@ class ResponsesHttpLLM extends llm.LLM { override chat({ chatCtx, - toolCtx, + toolCtx: toolCtxInput, connOptions = DEFAULT_API_CONNECT_OPTIONS, parallelToolCalls, toolChoice, extraKwargs, }: { chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: llm.ToolChoice; extraKwargs?: Record; }): ResponsesHttpLLMStream { + const toolCtx = llm.toToolContext(toolCtxInput); const modelOptions: Record = { ...(extraKwargs || {}) }; parallelToolCalls = parallelToolCalls !== undefined ? parallelToolCalls : this.#opts.parallelToolCalls; - if (toolCtx && Object.keys(toolCtx).length > 0 && parallelToolCalls !== undefined) { + if ( + toolCtx && + Object.keys(toolCtx.functionTools).length > 0 && + parallelToolCalls !== undefined + ) { modelOptions.parallel_tool_calls = parallelToolCalls; } @@ -181,25 +187,9 @@ class ResponsesHttpLLMStream extends llm.LLMStream { 'openai.responses', )) as OpenAI.Responses.ResponseInputItem[]; + // TODO: support provider tools in the Responses schema. const tools = this.toolCtx - ? llm.sortedToolEntries(this.toolCtx).map(([name, func]) => { - const oaiParams = { - type: 'function' as const, - name: name, - description: func.description, - parameters: llm.toJsonSchema( - func.parameters, - true, - this.strictToolSchema, - ) as unknown as OpenAI.Responses.FunctionTool['parameters'], - } as OpenAI.Responses.FunctionTool; - - if (this.strictToolSchema) { - oaiParams.strict = true; - } - - return oaiParams; - }) + ? toResponsesTools(this.toolCtx, this.strictToolSchema) : undefined; const requestOptions: Record = { ...this.modelOptions }; @@ -430,7 +420,7 @@ export class LLM extends llm.LLM { extraKwargs, }: { chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: llm.ToolChoice; diff --git a/plugins/openai/src/tool_utils.test.ts b/plugins/openai/src/tool_utils.test.ts new file mode 100644 index 000000000..ce1922f52 --- /dev/null +++ b/plugins/openai/src/tool_utils.test.ts @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { llm } from '@livekit/agents'; +import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { toResponsesTools } from './tool_utils.js'; +import { CodeInterpreter, FileSearch, WebSearch } from './tools.js'; + +describe('toResponsesTools', () => { + it('serializes function tools', () => { + const fn = llm.tool({ + name: 'lookup_weather', + description: 'Look up weather', + parameters: z.object({ city: z.string() }), + execute: async () => 'sunny', + }); + + expect(toResponsesTools(new llm.ToolContext([fn]), true)).toEqual([ + { + type: 'function', + name: 'lookup_weather', + description: 'Look up weather', + parameters: { + $schema: 'http://json-schema.org/draft-07/schema#', + type: 'object', + properties: { city: { type: 'string' } }, + required: ['city'], + additionalProperties: false, + }, + strict: true, + }, + ]); + }); + + it('serializes OpenAI provider tools', () => { + const tools = toResponsesTools( + new llm.ToolContext([ + new WebSearch({ + filters: { allowed_domains: ['docs.livekit.io'] }, + searchContextSize: 'low', + userLocation: { type: 'approximate', country: 'US' }, + }), + new FileSearch({ + vectorStoreIds: ['vs_123'], + maxNumResults: 3, + rankingOptions: { ranker: 'auto' }, + }), + new CodeInterpreter({ container: { type: 'auto', file_ids: ['file_123'] } }), + ]), + false, + ); + + expect(tools).toEqual([ + { + type: 'web_search', + search_context_size: 'low', + filters: { allowed_domains: ['docs.livekit.io'] }, + user_location: { type: 'approximate', country: 'US' }, + }, + { + type: 'file_search', + vector_store_ids: ['vs_123'], + max_num_results: 3, + ranking_options: { ranker: 'auto' }, + }, + { type: 'code_interpreter', container: { type: 'auto', file_ids: ['file_123'] } }, + ]); + }); + + it('omits the code interpreter container when unset', () => { + expect(toResponsesTools(new llm.ToolContext([new CodeInterpreter()]), false)).toEqual([ + { type: 'code_interpreter' }, + ]); + }); + + it('ignores non-OpenAI provider tools', () => { + class OtherProviderTool extends llm.ProviderTool {} + + expect( + toResponsesTools(new llm.ToolContext([new OtherProviderTool({ id: 'other' })]), false), + ).toBeUndefined(); + }); +}); diff --git a/plugins/openai/src/tool_utils.ts b/plugins/openai/src/tool_utils.ts new file mode 100644 index 000000000..a0442b496 --- /dev/null +++ b/plugins/openai/src/tool_utils.ts @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { llm } from '@livekit/agents'; +import type OpenAI from 'openai'; +import { OpenAITool } from './tools.js'; + +export function toResponsesTools( + toolCtx: llm.ToolContext, + strictToolSchema: boolean, +): OpenAI.Responses.Tool[] | undefined { + // Function tools are emitted first, sorted by name for deterministic payloads; provider + // tools follow in registration order. + const functionTools = llm.sortedToolEntries(toolCtx).map(([name, tool]) => { + const oaiParams = { + type: 'function' as const, + name, + description: tool.description, + parameters: llm.toJsonSchema( + tool.parameters, + true, + strictToolSchema, + ) as unknown as OpenAI.Responses.FunctionTool['parameters'], + } as OpenAI.Responses.FunctionTool; + + if (strictToolSchema) { + oaiParams.strict = true; + } + + return oaiParams; + }); + + const providerTools = toolCtx + .flatten() + .filter((tool) => !llm.isFunctionTool(tool)) + .map((tool) => + tool instanceof OpenAITool + ? (tool.toToolConfig() as unknown as OpenAI.Responses.Tool) + : undefined, + ) + .filter((tool): tool is OpenAI.Responses.Tool => tool !== undefined); + + const tools = [...functionTools, ...providerTools]; + + return tools.length > 0 ? tools : undefined; +} diff --git a/plugins/openai/src/tools.ts b/plugins/openai/src/tools.ts new file mode 100644 index 000000000..1ce779376 --- /dev/null +++ b/plugins/openai/src/tools.ts @@ -0,0 +1,166 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { llm } from '@livekit/agents'; +import type OpenAI from 'openai'; + +/** Base class for OpenAI Responses API provider tools. */ +export abstract class OpenAITool extends llm.ProviderTool { + /** Convert this provider tool to the OpenAI Responses API tool configuration. */ + abstract toToolConfig(): Record; +} + +/** + * High-level guidance for the amount of context window space to use for web search. + * OpenAI defaults this to `medium`. + */ +export type WebSearchContextSize = 'low' | 'medium' | 'high'; + +/** Options for the OpenAI web search tool. */ +export interface WebSearchOptions { + /** + * Filters for the search, such as allowed domains. If not provided, all domains are allowed. + */ + filters?: OpenAI.Responses.WebSearchTool['filters']; + + /** + * Amount of context window space to use for the search. Defaults to `medium`. + */ + searchContextSize?: WebSearchContextSize | null; + + /** Approximate location of the user, such as city, region, country, or timezone. */ + userLocation?: OpenAI.Responses.WebSearchTool['user_location']; +} + +/** + * Search the Internet for sources related to the prompt. + * + * @see https://platform.openai.com/docs/guides/tools-web-search + */ +export class WebSearch extends OpenAITool { + /** Filters for the search, such as allowed domains. */ + readonly filters: OpenAI.Responses.WebSearchTool['filters'] | undefined; + + /** Amount of context window space to use for the search. */ + readonly searchContextSize: WebSearchContextSize | null; + + /** Approximate location of the user. */ + readonly userLocation: OpenAI.Responses.WebSearchTool['user_location'] | undefined; + + constructor({ filters, searchContextSize = 'medium', userLocation }: WebSearchOptions = {}) { + super({ id: 'openai_web_search' }); + this.filters = filters; + this.searchContextSize = searchContextSize; + this.userLocation = userLocation; + } + + toToolConfig(): Record { + const result: Record = { + type: 'web_search', + search_context_size: this.searchContextSize, + }; + if (this.userLocation !== undefined) { + result.user_location = this.userLocation; + } + if (this.filters !== undefined) { + result.filters = this.filters; + } + return result; + } +} + +/** Options for the OpenAI file search tool. */ +export interface FileSearchOptions { + /** IDs of the vector stores to search. */ + vectorStoreIds?: string[]; + + /** Filter to apply to file search results. */ + filters?: OpenAI.Responses.FileSearchTool['filters']; + + /** Maximum number of results to return. This should be between 1 and 50 inclusive. */ + maxNumResults?: number; + + /** Ranking options for search, including ranker and score threshold. */ + rankingOptions?: OpenAI.Responses.FileSearchTool.RankingOptions; +} + +/** + * Search for relevant content from uploaded files. + * + * @see https://platform.openai.com/docs/guides/tools-file-search + */ +export class FileSearch extends OpenAITool { + /** IDs of the vector stores to search. */ + readonly vectorStoreIds: string[]; + + /** Filter to apply to file search results. */ + readonly filters: OpenAI.Responses.FileSearchTool['filters'] | undefined; + + /** Maximum number of results to return. */ + readonly maxNumResults: number | undefined; + + /** Ranking options for search. */ + readonly rankingOptions: OpenAI.Responses.FileSearchTool.RankingOptions | undefined; + + constructor({ + vectorStoreIds = [], + filters, + maxNumResults, + rankingOptions, + }: FileSearchOptions = {}) { + super({ id: 'openai_file_search' }); + this.vectorStoreIds = [...vectorStoreIds]; + this.filters = filters; + this.maxNumResults = maxNumResults; + this.rankingOptions = rankingOptions; + } + + toToolConfig(): Record { + const result: Record = { + type: 'file_search', + vector_store_ids: this.vectorStoreIds, + }; + if (this.filters !== undefined) { + result.filters = this.filters; + } + if (this.maxNumResults !== undefined) { + result.max_num_results = this.maxNumResults; + } + if (this.rankingOptions !== undefined) { + result.ranking_options = this.rankingOptions; + } + return result; + } +} + +/** Options for the OpenAI code interpreter tool. */ +export interface CodeInterpreterOptions { + /** + * Code interpreter container. Can be a container ID or an object that specifies uploaded file IDs + * to make available to the code. + */ + container?: OpenAI.Responses.Tool.CodeInterpreter['container'] | null; +} + +/** + * Run Python code to help generate a response to a prompt. + * + * @see https://platform.openai.com/docs/guides/tools-code-interpreter + */ +export class CodeInterpreter extends OpenAITool { + /** Code interpreter container ID or configuration. */ + readonly container: OpenAI.Responses.Tool.CodeInterpreter['container'] | null; + + constructor({ container = null }: CodeInterpreterOptions = {}) { + super({ id: 'openai_code_interpreter' }); + this.container = container; + } + + toToolConfig(): Record { + const result: Record = { type: 'code_interpreter' }; + if (this.container !== null) { + result.container = this.container; + } + return result; + } +} diff --git a/plugins/openai/src/ws/llm.ts b/plugins/openai/src/ws/llm.ts index 646853b7a..b9886ba24 100644 --- a/plugins/openai/src/ws/llm.ts +++ b/plugins/openai/src/ws/llm.ts @@ -15,6 +15,7 @@ import { import type OpenAI from 'openai'; import { WebSocket } from 'ws'; import type { ChatModels } from '../models.js'; +import { toResponsesTools } from '../tool_utils.js'; import type { WsOutputItemDoneEvent, WsOutputTextDeltaEvent, @@ -253,24 +254,29 @@ export class WSLLM extends llm.LLM { chat({ chatCtx, - toolCtx, + toolCtx: toolCtxInput, connOptions = DEFAULT_API_CONNECT_OPTIONS, parallelToolCalls, toolChoice, extraKwargs, }: { chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: llm.ToolChoice; extraKwargs?: Record; }): WSLLMStream { + const toolCtx = llm.toToolContext(toolCtxInput); const modelOptions: Record = { ...(extraKwargs ?? {}) }; parallelToolCalls = parallelToolCalls !== undefined ? parallelToolCalls : this.#opts.parallelToolCalls; - if (toolCtx && Object.keys(toolCtx).length > 0 && parallelToolCalls !== undefined) { + if ( + toolCtx && + Object.keys(toolCtx.functionTools).length > 0 && + parallelToolCalls !== undefined + ) { modelOptions.parallel_tool_calls = parallelToolCalls; } @@ -446,26 +452,7 @@ export class WSLLMStream extends llm.LLMStream { 'openai.responses', )) as OpenAI.Responses.ResponseInputItem[]; - const tools = this.toolCtx - ? llm.sortedToolEntries(this.toolCtx).map(([name, func]) => { - const oaiParams = { - type: 'function' as const, - name, - description: func.description, - parameters: llm.toJsonSchema( - func.parameters, - true, - this.#strictToolSchema, - ) as unknown as OpenAI.Responses.FunctionTool['parameters'], - } as OpenAI.Responses.FunctionTool; - - if (this.#strictToolSchema) { - oaiParams.strict = true; - } - - return oaiParams; - }) - : undefined; + const tools = this.toolCtx ? toResponsesTools(this.toolCtx, this.#strictToolSchema) : undefined; const requestOptions: Record = { ...this.#modelOptions }; if (!tools) { diff --git a/plugins/phonic/src/realtime/realtime_model.ts b/plugins/phonic/src/realtime/realtime_model.ts index aa2a28bf2..0ebf6a825 100644 --- a/plugins/phonic/src/realtime/realtime_model.ts +++ b/plugins/phonic/src/realtime/realtime_model.ts @@ -239,7 +239,7 @@ interface GenerationState { * Realtime session for Phonic (https://docs.phonic.co/) */ export class RealtimeSession extends llm.RealtimeSession { - private _tools: llm.ToolContext = {}; + private _tools: llm.ToolContext = llm.ToolContext.empty(); private _chatCtx = llm.ChatContext.empty(); private options: RealtimeModelOptions; @@ -290,7 +290,7 @@ export class RealtimeSession extends llm.RealtimeSession { } get tools(): llm.ToolContext { - return { ...this._tools }; + return this._tools.copy(); } async updateInstructions(instructions: string): Promise { @@ -367,23 +367,23 @@ export class RealtimeSession extends llm.RealtimeSession { return; } - this._tools = { ...tools }; - this.toolDefinitions = Object.entries(tools) - .filter(([_, tool]) => llm.isFunctionTool(tool)) - .map(([name, tool]) => ({ - type: 'custom_websocket', + this._tools = tools.copy(); + // TODO: support provider tools in the Phonic schema. + this.toolDefinitions = tools + .flatten() + .filter(llm.isFunctionTool) + .map((t) => ({ + type: 'custom_websocket' as const, tool_schema: { - type: 'function', + type: 'function' as const, function: { - name, - description: tool.description, - parameters: llm.toJsonSchema(tool.parameters), + name: t.name, + description: t.description, + parameters: llm.toJsonSchema(t.parameters), strict: true, }, }, tool_call_output_timeout_ms: TOOL_CALL_OUTPUT_TIMEOUT_MS, - // Tool chaining and tool calls during speech are not supported at this time - // for ease of implementation within the RealtimeSession generations framework wait_for_speech_before_tool_call: true, allow_tool_chaining: false, })); @@ -405,17 +405,19 @@ export class RealtimeSession extends llm.RealtimeSession { this.options.instructions = instructions; } if (tools !== undefined) { - this._tools = { ...tools }; - this.toolDefinitions = Object.entries(tools) - .filter(([, tool]) => llm.isFunctionTool(tool)) - .map(([name, tool]) => ({ - type: 'custom_websocket', + this._tools = tools.copy(); + // TODO: support provider tools in the Phonic schema. + this.toolDefinitions = tools + .flatten() + .filter(llm.isFunctionTool) + .map((t) => ({ + type: 'custom_websocket' as const, tool_schema: { - type: 'function', + type: 'function' as const, function: { - name, - description: tool.description, - parameters: llm.toJsonSchema(tool.parameters), + name: t.name, + description: t.description, + parameters: llm.toJsonSchema(t.parameters), strict: true, }, }, diff --git a/plugins/runway/src/avatar.ts b/plugins/runway/src/avatar.ts index c1a23c228..5361a49df 100644 --- a/plugins/runway/src/avatar.ts +++ b/plugins/runway/src/avatar.ts @@ -155,12 +155,14 @@ export class AvatarSession extends voice.AvatarSession { void this.ensureEndSessionPromise(); }); - agentSession.output.audio = new voice.DataStreamAudioOutput({ - room, - destinationIdentity: this.avatarIdentity, - waitRemoteTrack: TrackKind.KIND_VIDEO, - sampleRate: SAMPLE_RATE, - }); + agentSession.output.replaceAudioTail( + new voice.DataStreamAudioOutput({ + room, + destinationIdentity: this.avatarIdentity, + waitRemoteTrack: TrackKind.KIND_VIDEO, + sampleRate: SAMPLE_RATE, + }), + ); } private async createSession( diff --git a/plugins/tavus/src/avatar.ts b/plugins/tavus/src/avatar.ts index 1dd19ec03..28f47f9fc 100644 --- a/plugins/tavus/src/avatar.ts +++ b/plugins/tavus/src/avatar.ts @@ -149,11 +149,13 @@ export class AvatarSession extends voice.AvatarSession { properties: { livekit_ws_url: livekitUrl, livekit_room_token: livekitToken }, }); - agentSession.output.audio = new voice.DataStreamAudioOutput({ - room, - destinationIdentity: this.avatarIdentity, - sampleRate: SAMPLE_RATE, - waitRemoteTrack: TrackKind.KIND_VIDEO, - }); + agentSession.output.replaceAudioTail( + new voice.DataStreamAudioOutput({ + room, + destinationIdentity: this.avatarIdentity, + sampleRate: SAMPLE_RATE, + waitRemoteTrack: TrackKind.KIND_VIDEO, + }), + ); } } diff --git a/plugins/test/src/llm.ts b/plugins/test/src/llm.ts index 8d85c75c3..1f912654a 100644 --- a/plugins/test/src/llm.ts +++ b/plugins/test/src/llm.ts @@ -5,8 +5,9 @@ import { initializeLogger, llm as llmlib } from '@livekit/agents'; import { describe, expect, it } from 'vitest'; import { z } from 'zod/v4'; -const toolCtx: llmlib.ToolContext = { - getWeather: llmlib.tool({ +const toolCtx = new llmlib.ToolContext([ + llmlib.tool({ + name: 'getWeather', description: 'Get the current weather in a given location', parameters: z.object({ location: z.string().describe('The city and state, e.g. San Francisco, CA'), @@ -14,14 +15,16 @@ const toolCtx: llmlib.ToolContext = { }), execute: async () => {}, }), - playMusic: llmlib.tool({ + llmlib.tool({ + name: 'playMusic', description: 'Play music', parameters: z.object({ name: z.string().describe('The artist and name of the song'), }), execute: async () => {}, }), - toggleLight: llmlib.tool({ + llmlib.tool({ + name: 'toggleLight', description: 'Turn on/off the lights in a room', parameters: z.object({ name: z.string().describe('The room to control'), @@ -31,7 +34,8 @@ const toolCtx: llmlib.ToolContext = { await new Promise((resolve) => setTimeout(resolve, 60_000)); }, }), - selectCurrencies: llmlib.tool({ + llmlib.tool({ + name: 'selectCurrencies', description: 'Currencies of a specific area', parameters: z.object({ currencies: z @@ -40,7 +44,8 @@ const toolCtx: llmlib.ToolContext = { }), execute: async () => {}, }), - updateUserInfo: llmlib.tool({ + llmlib.tool({ + name: 'updateUserInfo', description: 'Update user info.', parameters: z.object({ email: z.string().optional().describe("User's email address"), @@ -49,18 +54,20 @@ const toolCtx: llmlib.ToolContext = { }), execute: async () => {}, }), - simulateFailure: llmlib.tool({ + llmlib.tool({ + name: 'simulateFailure', description: 'Simulate a failure', parameters: z.object({}), execute: async () => { throw new Error('Simulated failure'); }, }), -}; +]); // Tool context for strict mode - uses nullable() instead of optional() -const toolCtxStrict: llmlib.ToolContext = { - getWeather: llmlib.tool({ +const toolCtxStrict = new llmlib.ToolContext([ + llmlib.tool({ + name: 'getWeather', description: 'Get the current weather in a given location', parameters: z.object({ location: z.string().describe('The city and state, e.g. San Francisco, CA'), @@ -68,14 +75,16 @@ const toolCtxStrict: llmlib.ToolContext = { }), execute: async () => {}, }), - playMusic: llmlib.tool({ + llmlib.tool({ + name: 'playMusic', description: 'Play music', parameters: z.object({ name: z.string().describe('The artist and name of the song'), }), execute: async () => {}, }), - toggleLight: llmlib.tool({ + llmlib.tool({ + name: 'toggleLight', description: 'Turn on/off the lights in a room', parameters: z.object({ name: z.string().describe('The room to control'), @@ -85,7 +94,8 @@ const toolCtxStrict: llmlib.ToolContext = { await new Promise((resolve) => setTimeout(resolve, 60_000)); }, }), - selectCurrencies: llmlib.tool({ + llmlib.tool({ + name: 'selectCurrencies', description: 'Currencies of a specific area', parameters: z.object({ currencies: z @@ -94,7 +104,8 @@ const toolCtxStrict: llmlib.ToolContext = { }), execute: async () => {}, }), - updateUserInfo: llmlib.tool({ + llmlib.tool({ + name: 'updateUserInfo', description: 'Update user info.', parameters: z.object({ email: z.string().nullable().describe("User's email address"), @@ -103,14 +114,15 @@ const toolCtxStrict: llmlib.ToolContext = { }), execute: async () => {}, }), - simulateFailure: llmlib.tool({ + llmlib.tool({ + name: 'simulateFailure', description: 'Simulate a failure', parameters: z.object({}), execute: async () => { throw new Error('Simulated failure'); }, }), -}; +]); export const llm = async (llm: llmlib.LLM, skipOptionalArgs: boolean) => { initializeLogger({ pretty: false }); @@ -188,6 +200,57 @@ export const llm = async (llm: llmlib.LLM, skipOptionalArgs: boolean) => { expect(JSON.parse(calls[0]!.args).address).toBeUndefined(); }); }); + + describe('toolset', async () => { + const buildToolsetContext = () => { + const weatherToolset = new llmlib.Toolset({ + id: 'weather_toolset', + tools: [ + llmlib.tool({ + name: 'getWeather', + description: 'Get the current weather in a given location', + parameters: z.object({ + location: z.string().describe('The city and state, e.g. San Francisco, CA'), + unit: z.enum(['celsius', 'fahrenheit']).describe('The temperature unit to use'), + }), + execute: async () => {}, + }), + ], + }); + + const directTool = llmlib.tool({ + name: 'playMusic', + description: 'Play music', + parameters: z.object({ + name: z.string().describe('The artist and name of the song'), + }), + execute: async () => {}, + }); + + return new llmlib.ToolContext([weatherToolset, directTool]); + }; + + it('should call a function tool that lives inside a Toolset', async () => { + const ctx = buildToolsetContext(); + const calls = await requestFncCall( + llm, + "What's the weather in San Francisco, in Celsius?", + ctx, + ); + + expect(calls.length).toStrictEqual(1); + expect(calls[0]!.name).toStrictEqual('getWeather'); + expect(JSON.parse(calls[0]!.args).unit).toStrictEqual('celsius'); + }); + + it('should expose direct tools alongside Toolset tools', async () => { + const ctx = buildToolsetContext(); + const calls = await requestFncCall(llm, 'Play the song "Bohemian Rhapsody" by Queen.', ctx); + + expect(calls.length).toStrictEqual(1); + expect(calls[0]!.name).toStrictEqual('playMusic'); + }); + }); }); }; @@ -315,7 +378,7 @@ const executeCalls = async (calls: llmlib.FunctionCall[]) => { const results: llmlib.FunctionCallOutput[] = []; for (const call of calls) { - const tool = toolCtx[call.name]; + const tool = toolCtx.getFunctionTool(call.name); if (!tool) { throw new Error(`Tool ${call.name} not found`); } diff --git a/plugins/trugen/src/avatar.ts b/plugins/trugen/src/avatar.ts index 71276e012..11620a75b 100644 --- a/plugins/trugen/src/avatar.ts +++ b/plugins/trugen/src/avatar.ts @@ -167,11 +167,13 @@ export class AvatarSession extends voice.AvatarSession { this.#logger.debug('starting avatar session'); await this.startAgent(livekitUrl, livekitToken); - agentSession.output.audio = new voice.DataStreamAudioOutput({ - room, - destinationIdentity: this.avatarIdentity, - waitRemoteTrack: TrackKind.KIND_VIDEO, - }); + agentSession.output.replaceAudioTail( + new voice.DataStreamAudioOutput({ + room, + destinationIdentity: this.avatarIdentity, + waitRemoteTrack: TrackKind.KIND_VIDEO, + }), + ); } private async startAgent(livekitUrl: string, livekitToken: string): Promise {