From 244d8bc4dc0954ca9d94813d8113592f974d3eca Mon Sep 17 00:00:00 2001 From: Brian Yin Date: Fri, 29 May 2026 05:43:58 +0800 Subject: [PATCH 01/15] Refactor ToolContext to parity class taking a list of Tool | Toolset (#1517) --- .changeset/list-syntax-toolcontext.md | 5 + agents/src/beta/workflows/task_group.ts | 6 +- agents/src/inference/llm.ts | 57 +++--- agents/src/llm/chat_context.test.ts | 29 +++ agents/src/llm/chat_context.ts | 2 +- agents/src/llm/fallback_adapter.test.ts | 6 +- agents/src/llm/fallback_adapter.ts | 6 +- agents/src/llm/index.ts | 7 +- agents/src/llm/llm.ts | 18 +- agents/src/llm/tool_context.test.ts | 177 ++++++++++++++++- agents/src/llm/tool_context.ts | 185 ++++++++++++++++-- agents/src/llm/tool_context.type.test.ts | 4 + agents/src/llm/utils.ts | 4 +- agents/src/voice/agent.test.ts | 29 ++- agents/src/voice/agent.ts | 19 +- agents/src/voice/agent_activity.test.ts | 6 +- agents/src/voice/agent_activity.ts | 45 +++-- agents/src/voice/amd.test.ts | 8 +- agents/src/voice/amd.ts | 12 +- agents/src/voice/generation.ts | 7 +- agents/src/voice/generation_tools.test.ts | 18 +- agents/src/voice/remote_session.ts | 2 +- agents/src/voice/testing/fake_llm.ts | 6 +- agents/src/voice/testing/run_result.ts | 3 +- examples/src/background_audio.ts | 5 +- examples/src/basic_agent.ts | 7 +- examples/src/basic_agent_task.ts | 27 +-- examples/src/basic_task_group.ts | 21 +- examples/src/basic_tool_call_agent.ts | 19 +- examples/src/comprehensive_test.ts | 17 +- examples/src/drive-thru/drivethru_agent.ts | 19 +- examples/src/frontdesk/frontdesk_agent.ts | 10 +- examples/src/gemini_realtime_agent.ts | 11 +- examples/src/instructions_per_modality.ts | 7 +- examples/src/llm_fallback_adapter.ts | 7 +- examples/src/manual_shutdown.ts | 10 +- examples/src/multi_agent.ts | 9 +- examples/src/phonic_realtime_agent.ts | 5 +- examples/src/raw_function_description.ts | 7 +- examples/src/realtime_agent.ts | 7 +- examples/src/realtime_with_tts.ts | 5 +- examples/src/restaurant_agent.ts | 78 +++++--- examples/src/survey_agent.ts | 49 +++-- examples/src/testing/agent_task.test.ts | 28 +-- examples/src/testing/basic_task_group.test.ts | 14 +- examples/src/testing/run_result.test.ts | 19 +- examples/src/testing/task_group.test.ts | 28 +-- examples/src/tool_call_disfluency.ts | 5 +- examples/src/xai-realtime.ts | 7 +- plugins/baseten/src/llm.ts | 11 +- plugins/cerebras/src/llm.test.ts | 7 +- .../google/src/beta/realtime/realtime_api.ts | 4 +- plugins/google/src/llm.ts | 7 +- plugins/google/src/utils.ts | 2 +- plugins/mistralai/src/llm.ts | 8 +- plugins/openai/src/llm.ts | 11 +- plugins/openai/src/realtime/realtime_model.ts | 23 +-- plugins/openai/src/responses/llm.ts | 15 +- plugins/openai/src/ws/llm.ts | 13 +- plugins/phonic/src/realtime/realtime_model.ts | 72 ++++--- plugins/test/src/llm.ts | 46 +++-- 61 files changed, 901 insertions(+), 400 deletions(-) create mode 100644 .changeset/list-syntax-toolcontext.md diff --git a/.changeset/list-syntax-toolcontext.md b/.changeset/list-syntax-toolcontext.md new file mode 100644 index 000000000..0a4e0bfa0 --- /dev/null +++ b/.changeset/list-syntax-toolcontext.md @@ -0,0 +1,5 @@ +--- +"@livekit/agents": minor +--- + +**BREAKING**: `Agent({ tools })` and `agent.updateTools()` now accept a flat list `(FunctionTool | ProviderDefinedTool)[]` instead of a `Record` map, and `llm.tool({ ... })` requires a `name` field. `ToolContext` is now a Python-parity class with `functionTools` / `providerTools` / `toolsets` accessors, plus `flatten()`, `hasTool(name)`, `getFunctionTool(name)`, `updateTools()`, `copy()`, and `equals()`. To match the Python reference, registering two **different** function-tool instances under the same `name` now throws `duplicate function name: ` instead of silently overriding the earlier entry; passing the **same instance** twice is a no-op. `agent.toolCtx` returns a defensive copy so callers can no longer mutate the agent's internal state. `LLM.chat({ toolCtx })` accepts either a `ToolContext` instance or a raw `(FunctionTool | ProviderDefinedTool)[]` array (`ToolCtxInput`) and normalizes it internally, so callers don't have to construct a `ToolContext` themselves. Stateful `Toolset` containers are not part of this release — the `toolsets` accessor currently returns an empty list and `TODO`s in `tool_context.ts` mark every site where Python's Toolset support will plug in later. diff --git a/agents/src/beta/workflows/task_group.ts b/agents/src/beta/workflows/task_group.ts index 8c96790dd..add6655fb 100644 --- a/agents/src/beta/workflows/task_group.ts +++ b/agents/src/beta/workflows/task_group.ts @@ -84,10 +84,7 @@ export class TaskGroup extends AgentTask { const outOfScopeTool = this.buildOutOfScopeTool(taskId); if (outOfScopeTool) { - await this._currentTask.updateTools({ - ...this._currentTask.toolCtx, - out_of_scope: outOfScopeTool, - }); + await this._currentTask.updateTools([...this._currentTask.toolCtx.tools, outOfScopeTool]); } try { @@ -190,6 +187,7 @@ export class TaskGroup extends AgentTask { const visitedTasks = this._visitedTasks; return tool({ + name: 'out_of_scope', description, flags: ToolFlag.IGNORE_ON_ENTER, parameters: z.object({ diff --git a/agents/src/inference/llm.ts b/agents/src/inference/llm.ts index 3434e496c..87679389f 100644 --- a/agents/src/inference/llm.ts +++ b/agents/src/inference/llm.ts @@ -249,7 +249,7 @@ export class LLM extends llm.LLM { chat({ chatCtx, - toolCtx, + toolCtx: toolCtxInput, connOptions = DEFAULT_API_CONNECT_OPTIONS, parallelToolCalls, toolChoice, @@ -258,7 +258,7 @@ export class LLM extends llm.LLM { extraKwargs, }: { chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: llm.ToolChoice; @@ -266,6 +266,7 @@ export class LLM extends llm.LLM { // TODO(AJS-270): Add responseFormat parameter extraKwargs?: Record; }): LLMStream { + const toolCtx = llm.toToolContext(toolCtxInput); let modelOptions: Record = { ...(extraKwargs || {}) }; parallelToolCalls = @@ -273,7 +274,11 @@ export class LLM extends llm.LLM { ? parallelToolCalls : this.opts.modelOptions.parallel_tool_calls; - if (toolCtx && Object.keys(toolCtx).length > 0 && parallelToolCalls !== undefined) { + if ( + toolCtx && + Object.keys(toolCtx.functionTools).length > 0 && + parallelToolCalls !== undefined + ) { modelOptions.parallel_tool_calls = parallelToolCalls; } @@ -379,26 +384,32 @@ export class LLMStream extends llm.LLMStream { )) as OpenAI.ChatCompletionMessageParam[]; const tools = this.toolCtx - ? Object.entries(this.toolCtx).map(([name, func]) => { - const oaiParams = { - type: 'function' as const, - function: { - name, - description: func.description, - parameters: llm.toJsonSchema( - func.parameters, - true, - this.strictToolSchema, - ) as unknown as OpenAI.Chat.Completions.ChatCompletionFunctionTool['function']['parameters'], - } as OpenAI.Chat.Completions.ChatCompletionFunctionTool['function'], - }; - - if (this.strictToolSchema) { - oaiParams.function.strict = true; - } - - return oaiParams; - }) + ? this.toolCtx + .flatten() + .map((t) => { + if (llm.isFunctionTool(t)) { + const oaiParams = { + type: 'function' as const, + function: { + name: t.name, + description: t.description, + parameters: llm.toJsonSchema( + t.parameters, + true, + this.strictToolSchema, + ) as unknown as OpenAI.Chat.Completions.ChatCompletionFunctionTool['function']['parameters'], + } as OpenAI.Chat.Completions.ChatCompletionFunctionTool['function'], + }; + if (this.strictToolSchema) { + oaiParams.function.strict = true; + } + return oaiParams; + } + // Provider-defined tools are not yet supported by the inference adapter; skip them + // here rather than emitting a malformed tool definition. See AJS-112. + return undefined; + }) + .filter((t): t is NonNullable => t !== undefined) : undefined; const requestOptions: Record = dropUnsupportedParams( diff --git a/agents/src/llm/chat_context.test.ts b/agents/src/llm/chat_context.test.ts index 849469e55..350dcd1cd 100644 --- a/agents/src/llm/chat_context.test.ts +++ b/agents/src/llm/chat_context.test.ts @@ -19,6 +19,7 @@ import { isInstructions, renderInstructions, } from './chat_context.js'; +import { ToolContext, tool } from './tool_context.js'; initializeLogger({ pretty: false, level: 'error' }); @@ -1479,3 +1480,31 @@ extra`; expect((baseCtx.items[0]! as ChatMessage).content[0]).toBe(instr); }); }); + +describe('ChatContext.copy with toolCtx filter', () => { + it('drops function calls / outputs whose tool is not in the supplied ToolContext', () => { + const known = tool({ name: 'known', description: 'k', execute: async () => 'ok' }); + const ctx = new ChatContext([ + ChatMessage.create({ role: 'user', content: ['hello'] }), + FunctionCall.create({ callId: 'c1', name: 'known', args: '{}' }), + FunctionCallOutput.create({ callId: 'c1', name: 'known', output: 'done', isError: false }), + FunctionCall.create({ callId: 'c2', name: 'removed', args: '{}' }), + FunctionCallOutput.create({ callId: 'c2', name: 'removed', output: 'x', isError: false }), + ]); + + const filtered = ctx.copy({ toolCtx: new ToolContext([known]) }); + const types = filtered.items.map((i) => `${i.type}:${'name' in i ? i.name : ''}`); + expect(types).toEqual(['message:', 'function_call:known', 'function_call_output:known']); + }); + + it('keeps provider-tool calls when the ToolContext holds a matching provider tool id', () => { + const provider = tool({ id: 'code_runner', config: {} }); + const ctx = new ChatContext([ + FunctionCall.create({ callId: 'p1', name: 'code_runner', args: '{}' }), + FunctionCall.create({ callId: 'p2', name: 'other', args: '{}' }), + ]); + + const filtered = ctx.copy({ toolCtx: new ToolContext([provider]) }); + expect(filtered.items.map((i) => ('name' in i ? i.name : ''))).toEqual(['code_runner']); + }); +}); diff --git a/agents/src/llm/chat_context.ts b/agents/src/llm/chat_context.ts index 743e7efb8..34cf39200 100644 --- a/agents/src/llm/chat_context.ts +++ b/agents/src/llm/chat_context.ts @@ -835,7 +835,7 @@ export class ChatContext { continue; } - if (toolCtx !== undefined && isToolCallOrOutput(item) && toolCtx[item.name] === undefined) { + if (toolCtx !== undefined && isToolCallOrOutput(item) && !toolCtx.hasTool(item.name)) { continue; } diff --git a/agents/src/llm/fallback_adapter.test.ts b/agents/src/llm/fallback_adapter.test.ts index a9747c885..d466c9306 100644 --- a/agents/src/llm/fallback_adapter.test.ts +++ b/agents/src/llm/fallback_adapter.test.ts @@ -9,7 +9,7 @@ import { delay } from '../utils.js'; import type { ChatContext } from './chat_context.js'; import { FallbackAdapter } from './fallback_adapter.js'; import { type ChatChunk, LLM, LLMStream } from './llm.js'; -import type { ToolChoice, ToolContext } from './tool_context.js'; +import type { ToolChoice, ToolCtxInput } from './tool_context.js'; class MockLLMStream extends LLMStream { public myLLM: LLM; @@ -18,7 +18,7 @@ class MockLLMStream extends LLMStream { llm: LLM, opts: { chatCtx: ChatContext; - toolCtx?: ToolContext; + toolCtx?: ToolCtxInput; connOptions: APIConnectOptions; }, private shouldFail: boolean = false, @@ -64,7 +64,7 @@ class MockLLM extends LLM { chat(opts: { chatCtx: ChatContext; - toolCtx?: ToolContext; + toolCtx?: ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: ToolChoice; diff --git a/agents/src/llm/fallback_adapter.ts b/agents/src/llm/fallback_adapter.ts index 128c2392c..27d87d2a0 100644 --- a/agents/src/llm/fallback_adapter.ts +++ b/agents/src/llm/fallback_adapter.ts @@ -8,7 +8,7 @@ import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js import type { ChatContext } from './chat_context.js'; import type { ChatChunk } from './llm.js'; import { LLM, LLMStream } from './llm.js'; -import type { ToolChoice, ToolContext } from './tool_context.js'; +import type { ToolChoice, ToolCtxInput } from './tool_context.js'; /** * Default connection options for FallbackAdapter. @@ -113,7 +113,7 @@ export class FallbackAdapter extends LLM { chat(opts: { chatCtx: ChatContext; - toolCtx?: ToolContext; + toolCtx?: ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: ToolChoice; @@ -159,7 +159,7 @@ class FallbackLLMStream extends LLMStream { adapter: FallbackAdapter, opts: { chatCtx: ChatContext; - toolCtx?: ToolContext; + toolCtx?: ToolCtxInput; connOptions: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: ToolChoice; diff --git a/agents/src/llm/index.ts b/agents/src/llm/index.ts index 950296c9c..4837f2cb9 100644 --- a/agents/src/llm/index.ts +++ b/agents/src/llm/index.ts @@ -4,15 +4,20 @@ export { handoff, isFunctionTool, + isProviderDefinedTool, + isTool, tool, + ToolContext, ToolError, ToolFlag, + toToolContext, type AgentHandoff, type FunctionTool, type ProviderDefinedTool, type Tool, type ToolChoice, - type ToolContext, + type ToolContextEntry, + type ToolCtxInput, type ToolOptions, type ToolType, } from './tool_context.js'; diff --git a/agents/src/llm/llm.ts b/agents/src/llm/llm.ts index 553541939..0c05bbb2d 100644 --- a/agents/src/llm/llm.ts +++ b/agents/src/llm/llm.ts @@ -11,7 +11,12 @@ import { recordException, traceTypes, tracer } from '../telemetry/index.js'; import { type APIConnectOptions, intervalForRetry } from '../types.js'; import { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js'; import { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js'; -import type { ToolChoice, ToolContext } from './tool_context.js'; +import { + type ToolChoice, + type ToolContext, + type ToolCtxInput, + toToolContext, +} from './tool_context.js'; export interface ChoiceDelta { role: ChatRole; @@ -91,7 +96,12 @@ export abstract class LLM extends (EventEmitter as new () => TypedEmitter { connOptions, }: { chatCtx: ChatContext; - toolCtx?: ToolContext; + toolCtx?: ToolCtxInput; connOptions: APIConnectOptions; }, ) { this.#llm = llm; this.#chatCtx = chatCtx; - this.#toolCtx = toolCtx; + this.#toolCtx = toToolContext(toolCtx); this._connOptions = connOptions; this.monitorMetrics(); this.abortController.signal.addEventListener('abort', () => { diff --git a/agents/src/llm/tool_context.test.ts b/agents/src/llm/tool_context.test.ts index 183c7ff2a..4d9cdb83d 100644 --- a/agents/src/llm/tool_context.test.ts +++ b/agents/src/llm/tool_context.test.ts @@ -5,7 +5,7 @@ import { describe, expect, it } from 'vitest'; import { z } from 'zod'; import * as z3 from 'zod/v3'; import * as z4 from 'zod/v4'; -import { type ToolOptions, tool } from './tool_context.js'; +import { ToolContext, type ToolOptions, tool } from './tool_context.js'; import { createToolOptions, oaiParams } from './utils.js'; describe('Tool Context', () => { @@ -77,6 +77,7 @@ describe('Tool Context', () => { describe('tool', () => { it('should create and execute a basic core tool', async () => { const getWeather = tool({ + name: 'getWeather', description: 'Get the weather for a given location', parameters: z.object({ location: z.string(), @@ -95,6 +96,7 @@ describe('Tool Context', () => { it('should properly type a callable function', async () => { const testFunction = tool({ + name: 'testFunction', description: 'Test function', parameters: z.object({ name: z.string().describe('The user name'), @@ -114,6 +116,7 @@ describe('Tool Context', () => { it('should handle async execution', async () => { const testFunction = tool({ + name: 'asyncTestFunction', description: 'Async test function', parameters: z.object({ delay: z.number().describe('Delay in milliseconds'), @@ -157,6 +160,7 @@ describe('Tool Context', () => { describe('optional parameters', () => { it('should create a tool without parameters', async () => { const simpleAction = tool({ + name: 'simpleAction', description: 'Perform a simple action', execute: async () => { return 'Action performed'; @@ -175,6 +179,7 @@ describe('Tool Context', () => { it('should support .optional() fields in tool parameters', async () => { const weatherTool = tool({ + name: 'weatherTool', description: 'Get weather information', parameters: z.object({ location: z.string().describe('The city or location').optional(), @@ -205,6 +210,7 @@ describe('Tool Context', () => { it('should handle tools with context but no parameters', async () => { const greetUser = tool({ + name: 'greetUser', description: 'Greet the current user', execute: async (_, { ctx }: ToolOptions<{ username: string }>) => { return `Hello, ${ctx.userData.username}!`; @@ -217,6 +223,7 @@ describe('Tool Context', () => { it('should create a tool that accesses tool call id without parameters', async () => { const getCallId = tool({ + name: 'getCallId', description: 'Get the current tool call ID', execute: async (_, { toolCallId }) => { return `Tool call ID: ${toolCallId}`; @@ -231,6 +238,7 @@ describe('Tool Context', () => { describe('Zod v3 and v4 compatibility', () => { it('should work with Zod v3 schemas', async () => { const v3Tool = tool({ + name: 'v3Tool', description: 'A tool using Zod v3 schema', parameters: z3.object({ name: z3.string(), @@ -250,6 +258,7 @@ describe('Tool Context', () => { it('should work with Zod v4 schemas', async () => { const v4Tool = tool({ + name: 'v4Tool', description: 'A tool using Zod v4 schema', parameters: z4.object({ name: z4.string(), @@ -269,6 +278,7 @@ describe('Tool Context', () => { it('should handle v4 schemas with optional fields', async () => { const v4Tool = tool({ + name: 'v4OptionalTool', description: 'Tool with optional field using v4', parameters: z4.object({ required: z4.string(), @@ -291,6 +301,7 @@ describe('Tool Context', () => { it('should handle v4 enum schemas', async () => { const v4Tool = tool({ + name: 'v4EnumTool', description: 'Tool with enum using v4', parameters: z4.object({ color: z4.enum(['red', 'blue', 'green']), @@ -306,6 +317,7 @@ describe('Tool Context', () => { it('should handle v4 array schemas', async () => { const v4Tool = tool({ + name: 'v4ArrayTool', description: 'Tool with array using v4', parameters: z4.object({ tags: z4.array(z4.string()), @@ -324,6 +336,7 @@ describe('Tool Context', () => { it('should handle v4 nested object schemas', async () => { const v4Tool = tool({ + name: 'v4NestedTool', description: 'Tool with nested object using v4', parameters: z4.object({ user: z4.object({ @@ -405,3 +418,165 @@ describe('Tool Context', () => { }); }); }); + +describe('tool() name requirement', () => { + it('throws when name is missing', () => { + expect(() => + // @ts-expect-error - name is required + tool({ + description: 'no name', + execute: async () => 'x', + }), + ).toThrow('requires a non-empty name'); + }); + + it('throws when name is empty', () => { + expect(() => + tool({ + name: '', + description: 'empty name', + execute: async () => 'x', + }), + ).toThrow('requires a non-empty name'); + }); + + it('stores the name on the returned function tool', () => { + const t = tool({ + name: 'doStuff', + description: 'd', + execute: async () => 'x', + }); + expect(t.name).toBe('doStuff'); + }); +}); + +describe('ToolContext', () => { + const makeFn = (name: string) => + tool({ + name, + description: `${name} tool`, + execute: async () => name, + }); + + it('empty() returns an empty context', () => { + const ctx = ToolContext.empty(); + expect(ctx.functionTools).toEqual({}); + expect(ctx.providerTools).toEqual([]); + expect(ctx.toolsets).toEqual([]); + expect(ctx.flatten()).toEqual([]); + }); + + it('indexes function tools by name and supports lookup', () => { + const a = makeFn('a'); + const b = makeFn('b'); + const ctx = new ToolContext([a, b]); + + expect(ctx.functionTools).toEqual({ a, b }); + expect(ctx.getFunctionTool('a')).toBe(a); + expect(ctx.getFunctionTool('b')).toBe(b); + expect(ctx.getFunctionTool('missing')).toBeUndefined(); + }); + + it('throws on duplicate function names with different instances', () => { + // Matches Python's `if existing is not tool: raise ValueError(...)` — silently overriding + // a registered tool would mask a real bug at the caller (two distinct functions colliding + // on a single advertised name). + const a1 = makeFn('a'); + const a2 = makeFn('a'); + expect(() => new ToolContext([a1, a2])).toThrow('duplicate function name: a'); + }); + + it('silently skips the same function tool instance listed multiple times', () => { + // Matches Python's `return # same instance, skip` branch. Useful when a tool gets + // included both directly and via a future Toolset that re-exports it. + const a = makeFn('a'); + const ctx = new ToolContext([a, a]); + expect(ctx.getFunctionTool('a')).toBe(a); + expect(Object.keys(ctx.functionTools)).toEqual(['a']); + }); + + it('separates provider tools from function tools', () => { + const fnA = makeFn('a'); + const provider = tool({ id: 'code', config: { language: 'python' } }); + const ctx = new ToolContext([fnA, provider]); + + expect(ctx.functionTools).toEqual({ a: fnA }); + expect(ctx.providerTools).toEqual([provider]); + expect(ctx.flatten()).toEqual([fnA, provider]); + }); + + it('updateTools replaces the entire context', () => { + const a = makeFn('a'); + const b = makeFn('b'); + const ctx = new ToolContext([a]); + ctx.updateTools([b]); + expect(ctx.getFunctionTool('a')).toBeUndefined(); + expect(ctx.getFunctionTool('b')).toBe(b); + }); + + it('copy() yields an independent context with the same tools', () => { + const a = makeFn('a'); + const ctx = new ToolContext([a]); + const dup = ctx.copy(); + + expect(dup.getFunctionTool('a')).toBe(a); + dup.updateTools([]); + expect(ctx.getFunctionTool('a')).toBe(a); + expect(dup.getFunctionTool('a')).toBeUndefined(); + }); + + it('equals() compares function tool maps and provider lists by identity', () => { + const a = makeFn('a'); + const b = makeFn('b'); + const c = makeFn('c'); + + expect(new ToolContext([a, b]).equals(new ToolContext([a, b]))).toBe(true); + expect(new ToolContext([a, b]).equals(new ToolContext([a]))).toBe(false); + expect(new ToolContext([a, b]).equals(new ToolContext([a, c]))).toBe(false); + }); + + it('equals() is reflexive', () => { + const a = makeFn('a'); + const provider = tool({ id: 'code', config: { language: 'python' } }); + const ctx = new ToolContext([a, provider]); + expect(ctx.equals(ctx)).toBe(true); + }); + + it('equals() treats provider tool order as insignificant', () => { + // Matches Python's `set(id(t) for t in self._provider_tools)` comparison: two contexts + // that hold the same provider-tool identities in different order are still equal so + // realtime-session / preemptive-generation reuse fast paths are not invalidated. + const a = makeFn('a'); + const p1 = tool({ id: 'code', config: { language: 'python' } }); + const p2 = tool({ id: 'browser', config: {} }); + expect(new ToolContext([a, p1, p2]).equals(new ToolContext([a, p2, p1]))).toBe(true); + }); + + it('equals() supports contexts with only provider tools', () => { + const p1 = tool({ id: 'code', config: {} }); + const p2 = tool({ id: 'browser', config: {} }); + expect(new ToolContext([p1, p2]).equals(new ToolContext([p1, p2]))).toBe(true); + const p3 = tool({ id: 'code', config: {} }); // distinct identity, same id + expect(new ToolContext([p1]).equals(new ToolContext([p3]))).toBe(false); + }); + + it('hasTool() matches function tools by name and provider tools by id', () => { + const a = makeFn('a'); + const provider = tool({ id: 'code_runner', config: {} }); + const ctx = new ToolContext([a, provider]); + + expect(ctx.hasTool('a')).toBe(true); + expect(ctx.hasTool('code_runner')).toBe(true); + expect(ctx.hasTool('missing')).toBe(false); + }); + + it('flatten() returns function tools in insertion order followed by provider tools', () => { + // Matches Python's `flatten()`: list(self._fnc_tools_map.values()) + self._provider_tools. + const a = makeFn('a'); + const b = makeFn('b'); + const provider = tool({ id: 'code', config: {} }); + const ctx = new ToolContext([b, provider, a]); + + expect(ctx.flatten()).toEqual([b, a, provider]); + }); +}); diff --git a/agents/src/llm/tool_context.ts b/agents/src/llm/tool_context.ts index ca2888167..df714d57a 100644 --- a/agents/src/llm/tool_context.ts +++ b/agents/src/llm/tool_context.ts @@ -167,6 +167,12 @@ export interface FunctionTool< > extends Tool { type: 'function'; + /** + * The name of the tool. Used to identify it inside a `ToolContext` and exposed to the LLM + * as the function name to call. + */ + name: string; + /** * The description of the tool. Will be used by the language model to decide whether to use the tool. */ @@ -190,38 +196,168 @@ export interface FunctionTool< [FUNCTION_TOOL_SYMBOL]: true; } -// TODO(AJS-112): support provider-defined tools in the future) -export type ToolContext = { - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Generic tool registry needs to accept any parameter/result types - [name: string]: FunctionTool; -}; +/** + * Convenience input shape accepted by APIs that want to take a list of tools directly without + * forcing callers to wrap them in `new ToolContext(...)`. + */ +export type ToolCtxInput = + | ToolContext + | readonly ToolContextEntry[]; + +export function toToolContext( + input: ToolCtxInput, +): ToolContext; +export function toToolContext( + input: ToolCtxInput | undefined, +): ToolContext | undefined; +export function toToolContext( + input: ToolCtxInput | undefined, +): ToolContext | undefined { + if (input === undefined) return undefined; + return input instanceof ToolContext ? input : new ToolContext(input); +} -export function isSameToolContext(ctx1: ToolContext, ctx2: ToolContext): boolean { - const toolNames = new Set(Object.keys(ctx1)); - const toolNames2 = new Set(Object.keys(ctx2)); +//TODO: toolset - accept stateful `Toolset` containers alongside `FunctionTool` / +// eslint-disable-next-line @typescript-eslint/no-explicit-any -- ToolContext entries accept any function-tool parameter/result types +export type ToolContextEntry = + // eslint-disable-next-line @typescript-eslint/no-explicit-any + FunctionTool | ProviderDefinedTool; + +export class ToolContext { + // TODO: toolset - widen entries to `FunctionTool | ProviderDefinedTool | Toolset` once Toolset + // lands so this stays heterogeneous like Python's `Sequence[Tool | Toolset]`. + private _tools: ToolContextEntry[] = []; + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- ToolContext stores generic function tools + private _functionToolsMap: Map> = new Map(); + private _providerTools: ProviderDefinedTool[] = []; + // TODO: toolset - populate when Toolset support is supported. + // so the `toolsets` getter and `equals` toolset-identity check stay byte-compatible with the + private _toolSets: unknown[] = []; + + // TODO: toolset - widen `tools` to `Sequence` once Toolset lands. + constructor(tools: readonly ToolContextEntry[] = []) { + this.updateTools(tools); + } - if (toolNames.size !== toolNames2.size) { - return false; + static empty(): ToolContext { + return new ToolContext([]); } - for (const name of toolNames) { - if (!toolNames2.has(name)) { - return false; + /** A copy of all function tools in the tool context, including those in tool sets. */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + get functionTools(): Record> { + return Object.fromEntries(this._functionToolsMap); + } + + /** A copy of all provider tools in the tool context, including those in tool sets. */ + get providerTools(): ProviderDefinedTool[] { + return this._providerTools; + } + + /** + * A copy of all tool sets in the tool context. + * + * TODO: toolset - wire up once Toolset is ported. + */ + get toolsets(): unknown[] { + return this._toolSets; + } + + /** + * A copy of the raw tool list this context was constructed with. + */ + get tools(): readonly ToolContextEntry[] { + return [...this._tools]; + } + + /** Flatten the tool context to a list of tools. */ + flatten(): Tool[] { + return [...this._functionToolsMap.values(), ...this._providerTools]; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Generic registry over any parameter/result types + getFunctionTool(name: string): FunctionTool | undefined { + return this._functionToolsMap.get(name); + } + + hasTool(name: string): boolean { + if (this._functionToolsMap.has(name)) { + return true; } + return this._providerTools.some((tool) => tool.id === name); + } - const tool1 = ctx1[name]; - const tool2 = ctx2[name]; + // TODO: toolset - widen `tools` to `Sequence` once Toolset lands. + updateTools(tools: readonly ToolContextEntry[]): void { + this._tools = [...tools]; + this._functionToolsMap = new Map(); + this._providerTools = []; + this._toolSets = []; + + // Mirrors Python's recursive `add_tool` (minus Toolset flattening, which is TODO). + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts any tool shape + const addTool = (tool: any): void => { + if (isProviderDefinedTool(tool)) { + this._providerTools.push(tool); + return; + } + + if (isFunctionTool(tool)) { + const existing = this._functionToolsMap.get(tool.name); + if (existing !== undefined) { + if (existing !== tool) { + throw new Error(`duplicate function name: ${tool.name}`); + } + return; // same instance, skip + } + this._functionToolsMap.set(tool.name, tool); + return; + } + + // TODO: toolset - if (tool instanceof Toolset) { for (const t of tool.tools) addTool(t); + // this._toolSets.push(tool); return; } + + throw new Error(`unknown tool type: ${typeof tool}`); + }; - if (!tool1 || !tool2) { - return false; + // TODO: toolset - Python also chains `find_function_tools(self)` here so subclasses can + // declare tools as class members. JS doesn't use that decorator pattern, so we only walk + // the explicit input list. + for (const tool of tools) { + addTool(tool); } + } - if (tool1.description !== tool2.description) { + copy(): ToolContext { + return new ToolContext([...this._tools]); + } + + equals(other: ToolContext): boolean { + if (this._functionToolsMap.size !== other._functionToolsMap.size) { + return false; + } + for (const [name, tool] of this._functionToolsMap) { + if (other._functionToolsMap.get(name) !== tool) { + return false; + } + } + if (this._providerTools.length !== other._providerTools.length) { return false; } + // Provider tools compare as identity sets to match Python's `set(id(t) for t in ...)` + // semantics — order is not significant. + const otherProviderIds = new Set(other._providerTools); + for (const tool of this._providerTools) { + if (!otherProviderIds.has(tool)) { + return false; + } + } + // TODO: toolset - once Toolset lands, also compare `_toolSets` as identity sets per Python + // self_tool_set_ids = {id(ts) for ts in self._tool_sets} + // other_tool_set_ids = {id(ts) for ts in other._tool_sets} + // if self_tool_set_ids != other_tool_set_ids: return False + return true; } - - return true; } export function isSameToolChoice(choice1: ToolChoice | null, choice2: ToolChoice | null): boolean { @@ -248,11 +384,13 @@ export function tool< UserData = UnknownUserData, Result = unknown, >({ + name, description, parameters, execute, flags, }: { + name: string; description: string; parameters: Schema; execute: ToolExecuteFunction, UserData, Result>; @@ -263,10 +401,12 @@ export function tool< * Create a function tool without parameters. */ export function tool({ + name, description, execute, flags, }: { + name: string; description: string; parameters?: never; execute: ToolExecuteFunction, UserData, Result>; @@ -290,6 +430,10 @@ export function tool({ // eslint-disable-next-line @typescript-eslint/no-explicit-any export function tool(tool: any): any { if (tool.execute !== undefined) { + if (typeof tool.name !== 'string' || tool.name.length === 0) { + throw new Error('tool({ name, ... }) requires a non-empty name'); + } + // Default parameters to z.object({}) if not provided const parameters = tool.parameters ?? z.object({}); @@ -305,6 +449,7 @@ export function tool(tool: any): any { return { type: 'function', + name: tool.name, description: tool.description, parameters, execute: tool.execute, diff --git a/agents/src/llm/tool_context.type.test.ts b/agents/src/llm/tool_context.type.test.ts index 27cf9fe55..187f95e7b 100644 --- a/agents/src/llm/tool_context.type.test.ts +++ b/agents/src/llm/tool_context.type.test.ts @@ -8,6 +8,7 @@ import { type FunctionTool, type ProviderDefinedTool, type ToolOptions, tool } f describe('tool type inference', () => { it('should infer argument type from zod schema', () => { const toolType = tool({ + name: 'test', description: 'test', parameters: z.object({ number: z.number() }), execute: async () => 'test' as const, @@ -29,6 +30,7 @@ describe('tool type inference', () => { it('should infer run context type', () => { const toolType = tool({ + name: 'test', description: 'test', parameters: z.object({ number: z.number() }), execute: async ({ number }, { ctx }: ToolOptions<{ name: string }>) => { @@ -91,6 +93,7 @@ describe('tool type inference', () => { it('should infer empty object type when parameters are omitted', () => { const toolType = tool({ + name: 'simpleAction', description: 'Simple action without parameters', execute: async () => 'done' as const, }); @@ -100,6 +103,7 @@ describe('tool type inference', () => { it('should infer correct types with context but no parameters', () => { const toolType = tool({ + name: 'actionWithCtx', description: 'Action with context', execute: async (args, { ctx }: ToolOptions<{ userId: number }>) => { expectTypeOf(args).toEqualTypeOf>(); diff --git a/agents/src/llm/utils.ts b/agents/src/llm/utils.ts index 0271deb2a..e8e2c4cab 100644 --- a/agents/src/llm/utils.ts +++ b/agents/src/llm/utils.ts @@ -171,7 +171,7 @@ export const oaiBuildFunctionInfo = ( toolName: string, rawArgs: string, ): FunctionCall => { - const tool = toolCtx[toolName]; + const tool = toolCtx.getFunctionTool(toolName); if (!tool) { throw new Error(`AI tool ${toolName} not found`); } @@ -187,7 +187,7 @@ export async function executeToolCall( toolCall: FunctionCall, toolCtx: ToolContext, ): Promise { - const tool = toolCtx[toolCall.name]!; + const tool = toolCtx.getFunctionTool(toolCall.name)!; let args: object | undefined; let params: object | undefined; diff --git a/agents/src/voice/agent.test.ts b/agents/src/voice/agent.test.ts index 8dd83fee3..f2afa7c42 100644 --- a/agents/src/voice/agent.test.ts +++ b/agents/src/voice/agent.test.ts @@ -29,12 +29,14 @@ describe('Agent', () => { // Create mock tools using the tool function const mockTool1 = tool({ + name: 'getTool1', description: 'First test tool', parameters: z.object({}), execute: async () => 'tool1 result', }); const mockTool2 = tool({ + name: 'getTool2', description: 'Second test tool', parameters: z.object({ input: z.string().describe('Input parameter'), @@ -44,17 +46,14 @@ describe('Agent', () => { const agent = new Agent({ instructions, - tools: { - getTool1: mockTool1, - getTool2: mockTool2, - }, + tools: [mockTool1, mockTool2], }); expect(agent).toBeDefined(); expect(agent.instructions).toBe(instructions); // Assert tools are set correctly - const agentTools = agent.toolCtx; + const agentTools = agent.toolCtx.functionTools; expect(Object.keys(agentTools)).toHaveLength(2); expect(agentTools).toHaveProperty('getTool1'); expect(agentTools).toHaveProperty('getTool2'); @@ -64,27 +63,21 @@ describe('Agent', () => { expect(agentTools.getTool2?.description).toBe('Second test tool'); }); - it('should return a copy of tools, not the original reference', () => { + it('toolCtx returns a defensive copy that exposes the same tools', () => { const instructions = 'You are a helpful assistant'; const mockTool = tool({ + name: 'testTool', description: 'Test tool', parameters: z.object({}), execute: async () => 'result', }); - const tools = { testTool: mockTool }; - const agent = new Agent({ instructions, tools }); - - const tools1 = agent.toolCtx; - const tools2 = agent.toolCtx; - - // Should return different object references - expect(tools1).not.toBe(tools2); - expect(tools1).not.toBe(tools); + const agent = new Agent({ instructions, tools: [mockTool] }); - // Should contain the same set of tools - expect(tools1).toEqual(tools2); - expect(tools1).toEqual(tools); + // Each call returns a fresh ToolContext so external mutation can't escape into the agent's + // internal state. + expect(agent.toolCtx).not.toBe(agent.toolCtx); + expect(agent.toolCtx.getFunctionTool('testTool')).toBe(mockTool); }); it('should require AgentTask to run inside task context', async () => { diff --git a/agents/src/voice/agent.ts b/agents/src/voice/agent.ts index 890d7ea7d..3145de2b1 100644 --- a/agents/src/voice/agent.ts +++ b/agents/src/voice/agent.ts @@ -20,7 +20,8 @@ import { LLM, RealtimeModel, type ToolChoice, - type ToolContext, + ToolContext, + type ToolContextEntry, } from '../llm/index.js'; import { log } from '../log.js'; import type { STT, SpeechEvent } from '../stt/index.js'; @@ -119,7 +120,7 @@ export interface AgentOptions { id?: string; instructions: string | Instructions; chatCtx?: ChatContext; - tools?: ToolContext; + tools?: readonly ToolContextEntry[]; stt?: STT | STTModelString; vad?: VAD; llm?: LLM | RealtimeModel | LLMModels; @@ -157,7 +158,7 @@ export class Agent { _instructions: string | Instructions; /** @internal */ - _tools?: ToolContext; + _toolCtx: ToolContext; constructor({ id, @@ -190,10 +191,10 @@ export class Agent { } this._instructions = instructions; - this._tools = { ...tools }; + this._toolCtx = new ToolContext(tools ?? []); this._chatCtx = chatCtx ? chatCtx.copy({ - toolCtx: this._tools, + toolCtx: this._toolCtx, }) : ChatContext.empty(); @@ -269,7 +270,7 @@ export class Agent { } get toolCtx(): ToolContext { - return { ...this._tools }; + return this._toolCtx.copy(); } get session(): AgentSession { @@ -345,10 +346,10 @@ export class Agent { } // TODO(parity): Add when AgentConfigUpdate is ported to ChatContext. - async updateTools(tools: ToolContext): Promise { + async updateTools(tools: readonly ToolContextEntry[]): Promise { if (!this._agentActivity) { - this._tools = { ...tools }; - this._chatCtx = this._chatCtx.copy({ toolCtx: this._tools }); + this._toolCtx = new ToolContext(tools); + this._chatCtx = this._chatCtx.copy({ toolCtx: this._toolCtx }); return; } diff --git a/agents/src/voice/agent_activity.test.ts b/agents/src/voice/agent_activity.test.ts index 03ddd2dd4..186b4c904 100644 --- a/agents/src/voice/agent_activity.test.ts +++ b/agents/src/voice/agent_activity.test.ts @@ -18,6 +18,7 @@ import { Heap } from 'heap-js'; import { describe, expect, it, vi } from 'vitest'; import type { ChatContext } from '../llm/chat_context.js'; import { LLM, type LLMStream } from '../llm/llm.js'; +import { ToolContext } from '../llm/tool_context.js'; import { Future } from '../utils.js'; import { AgentActivity } from './agent_activity.js'; import type { PreemptiveGenerationInfo } from './audio_recognition.js'; @@ -270,15 +271,16 @@ function buildPreemptiveRunner(opts: Partial = {}) { const fakeChatCtx = { copy: () => fakeChatCtx } as unknown as ChatContext; + const emptyToolCtx = ToolContext.empty(); const fakeActivity = { _preemptiveGenerationCount: 0, _preemptiveGeneration: undefined, _currentSpeech: undefined as SpeechHandle | undefined, schedulingPaused: false, llm: new FakePreemptiveLLM(), - tools: {}, + tools: emptyToolCtx, toolChoice: null, - agent: { chatCtx: fakeChatCtx }, + agent: { chatCtx: fakeChatCtx, _toolCtx: emptyToolCtx }, agentSession: { sessionOptions: { turnHandling: { preemptiveGeneration: preemptiveOpts }, diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index e53f39b14..3068b408d 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -36,11 +36,12 @@ import { type RealtimeModelError, type RealtimeSession, type ToolChoice, - type ToolContext, + ToolContext, + type ToolContextEntry, ToolFlag, } from '../llm/index.js'; import type { LLMError } from '../llm/llm.js'; -import { isSameToolChoice, isSameToolContext } from '../llm/tool_context.js'; +import { isSameToolChoice } from '../llm/tool_context.js'; import { log } from '../log.js'; import type { EOUMetrics, @@ -484,7 +485,12 @@ export class AgentActivity implements RecognitionHooks { } } - const initialTools = Object.keys(this.tools); + // Surface every tool the agent advertises at start — function tools by name and provider + // tools by id. + const initialTools = [ + ...Object.keys(this.agent._toolCtx.functionTools), + ...this.agent._toolCtx.providerTools.map((t) => t.id), + ]; if (runOnEnter && (this.agent.instructions || initialTools.length > 0)) { const initialConfig = new AgentConfigUpdate({ instructions: this.agent.instructions, @@ -609,7 +615,8 @@ export class AgentActivity implements RecognitionHooks { // tools update is supported or tools are the same reusable = reusable && - (capabilities.midSessionToolsUpdate || isSameToolContext(this.tools, newActivity.tools)); + (capabilities.midSessionToolsUpdate || + this.agent._toolCtx.equals(newActivity.agent._toolCtx)); if (reusable) { // detach: remove event listeners but don't close the session @@ -759,13 +766,14 @@ export class AgentActivity implements RecognitionHooks { } } - async updateTools(tools: ToolContext): Promise { - const oldToolNames = new Set(Object.keys(this.tools)); - const newToolNames = new Set(Object.keys(tools)); + async updateTools(tools: readonly ToolContextEntry[]): Promise { + const oldToolNames = new Set(Object.keys(this.agent._toolCtx.functionTools)); + const newToolCtx = new ToolContext(tools); + const newToolNames = new Set(Object.keys(newToolCtx.functionTools)); const toolsAdded = [...newToolNames].filter((name) => !oldToolNames.has(name)); const toolsRemoved = [...oldToolNames].filter((name) => !newToolNames.has(name)); - this.agent._tools = { ...tools }; + this.agent._toolCtx = newToolCtx; if (toolsAdded.length > 0 || toolsRemoved.length > 0) { const configUpdate = new AgentConfigUpdate({ @@ -777,12 +785,12 @@ export class AgentActivity implements RecognitionHooks { } if (this.realtimeSession) { - await this.realtimeSession.updateTools(tools); + await this.realtimeSession.updateTools(newToolCtx); } if (this.llm instanceof LLM) { // for realtime LLM, we assume the server will remove unvalid tool messages - await this.updateChatCtx(this.agent._chatCtx.copy({ toolCtx: tools })); + await this.updateChatCtx(this.agent._chatCtx.copy({ toolCtx: newToolCtx })); } } @@ -1421,7 +1429,7 @@ export class AgentActivity implements RecognitionHooks { userMessage, info, chatCtx: chatCtx.copy(), - tools: { ...this.tools }, + tools: this.agent._toolCtx.copy(), toolChoice: this.toolChoice, createdAt: Date.now(), }; @@ -1725,11 +1733,14 @@ export class AgentActivity implements RecognitionHooks { const shouldFilterTools = onEnterData?.agent === this.agent && onEnterData?.session === this.agentSession; - const tools = shouldFilterTools - ? Object.fromEntries( - Object.entries(this.agent.toolCtx).filter( - ([, fnTool]) => !(fnTool.flags & ToolFlag.IGNORE_ON_ENTER), - ), + const tools: ToolContext = shouldFilterTools + ? new ToolContext( + this.agent.toolCtx.tools.filter((t) => { + if (t.type === 'function') { + return !(t.flags & ToolFlag.IGNORE_ON_ENTER); + } + return true; + }), ) : this.agent.toolCtx; @@ -1912,7 +1923,7 @@ export class AgentActivity implements RecognitionHooks { if ( preemptive.info.newTranscript === userMessage?.textContent && preemptive.chatCtx.isEquivalent(chatCtx) && - isSameToolContext(preemptive.tools, this.tools) && + preemptive.tools.equals(this.agent._toolCtx) && isSameToolChoice(preemptive.toolChoice, this.toolChoice) ) { speechHandle = preemptive.speechHandle; diff --git a/agents/src/voice/amd.test.ts b/agents/src/voice/amd.test.ts index 8f8acd822..197bee11a 100644 --- a/agents/src/voice/amd.test.ts +++ b/agents/src/voice/amd.test.ts @@ -7,7 +7,7 @@ import type { ChatContext } from '../llm/chat_context.js'; import { FunctionCall } from '../llm/chat_context.js'; import type { ChatChunk } from '../llm/llm.js'; import { LLM, type LLMStream } from '../llm/llm.js'; -import type { ToolChoice, ToolContext } from '../llm/tool_context.js'; +import type { ToolChoice, ToolCtxInput } from '../llm/tool_context.js'; import type { SpeechEvent, SpeechStream } from '../stt/stt.js'; import { STT } from '../stt/stt.js'; import type { APIConnectOptions } from '../types.js'; @@ -30,7 +30,7 @@ class StaticLLM extends LLM { connOptions: _connOptions, }: { chatCtx: ChatContext; - toolCtx?: ToolContext; + toolCtx?: ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: ToolChoice; @@ -194,7 +194,7 @@ describe('AMD', () => { } chat({}: { chatCtx: ChatContext; - toolCtx?: ToolContext; + toolCtx?: ToolCtxInput; connOptions?: APIConnectOptions; }): LLMStream { return { @@ -538,7 +538,7 @@ describe('AMD', () => { label(): string { return 'postpone-llm'; } - chat({}: { chatCtx: ChatContext; toolCtx?: ToolContext }): LLMStream { + chat({}: { chatCtx: ChatContext; toolCtx?: ToolCtxInput }): LLMStream { callCount += 1; const isFirst = callCount === 1; return { diff --git a/agents/src/voice/amd.ts b/agents/src/voice/amd.ts index c1af28457..eeb1fd053 100644 --- a/agents/src/voice/amd.ts +++ b/agents/src/voice/amd.ts @@ -13,8 +13,7 @@ import type { LLMModels, STTModels } from '../inference/index.js'; import { ChatContext } from '../llm/chat_context.js'; import type { FunctionCall } from '../llm/chat_context.js'; import { LLM, type LLMStream } from '../llm/llm.js'; -import { isFunctionTool, tool } from '../llm/tool_context.js'; -import type { ToolContext } from '../llm/tool_context.js'; +import { ToolContext, type ToolContextEntry, isFunctionTool, tool } from '../llm/tool_context.js'; import { log } from '../log.js'; import { STT, SpeechEventType, type SpeechStream } from '../stt/stt.js'; import { traceTypes, tracer } from '../telemetry/index.js'; @@ -934,6 +933,7 @@ export class AMD extends (EventEmitter as new () => TypedEmitter) const isStale = (): boolean => generation !== this.detectGeneration || this.settled; const savePrediction = tool({ + name: 'save_prediction', description: 'Save the AMD prediction to the verdict.', parameters: z.object({ label: z.enum([ @@ -966,6 +966,7 @@ export class AMD extends (EventEmitter as new () => TypedEmitter) }); const postponeTermination = tool({ + name: 'postpone_termination', description: 'Postpone the termination of the classification task. ' + 'Use when the transcript is ambiguous and more audio is expected.', @@ -996,10 +997,11 @@ export class AMD extends (EventEmitter as new () => TypedEmitter) }, }); - const toolCtx: ToolContext = { save_prediction: savePrediction }; + const toolList: ToolContextEntry[] = [savePrediction]; if (this.extensionCount < MAX_EXTENSIONS) { - toolCtx.postpone_termination = postponeTermination; + toolList.push(postponeTermination); } + const toolCtx = new ToolContext(toolList); const chatCtx = new ChatContext(); chatCtx.addMessage({ role: 'system', content: this.prompt }); @@ -1035,7 +1037,7 @@ export class AMD extends (EventEmitter as new () => TypedEmitter) // Execute tool calls (save_prediction populates `savedResult`, // postpone_termination mutates the silence timer and returns). for (const tc of toolCalls) { - const fnTool = toolCtx[tc.name]; + const fnTool = toolCtx.getFunctionTool(tc.name); if (!fnTool || !isFunctionTool(fnTool)) continue; let parsedArgs: unknown = {}; try { diff --git a/agents/src/voice/generation.ts b/agents/src/voice/generation.ts index 3d938abdc..8bce7c198 100644 --- a/agents/src/voice/generation.ts +++ b/agents/src/voice/generation.ts @@ -485,7 +485,10 @@ export function performLLMInference( traceTypes.ATTR_CHAT_CTX, JSON.stringify(chatCtx.toJSON({ excludeTimestamp: false })), ); - span.setAttribute(traceTypes.ATTR_FUNCTION_TOOLS, JSON.stringify(Object.keys(toolCtx))); + span.setAttribute( + traceTypes.ATTR_FUNCTION_TOOLS, + JSON.stringify(Object.keys(toolCtx.functionTools)), + ); if (model) { span.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, model); @@ -992,7 +995,7 @@ export function performToolExecutions({ // TODO(brian): assert other toolChoice values - const tool = toolCtx[toolCall.name]; + const tool = toolCtx.getFunctionTool(toolCall.name); if (!tool) { logger.warn( { diff --git a/agents/src/voice/generation_tools.test.ts b/agents/src/voice/generation_tools.test.ts index d53e12196..9b4f7d1df 100644 --- a/agents/src/voice/generation_tools.test.ts +++ b/agents/src/voice/generation_tools.test.ts @@ -4,7 +4,7 @@ import { ReadableStream as NodeReadableStream } from 'stream/web'; import { describe, expect, it } from 'vitest'; import { z } from 'zod'; -import { FunctionCall, tool } from '../llm/index.js'; +import { FunctionCall, ToolContext, tool } from '../llm/index.js'; import { initializeLogger } from '../log.js'; import type { Task } from '../utils.js'; import { cancelAndWait, delay } from '../utils.js'; @@ -63,6 +63,7 @@ describe('Generation + Tool Execution', () => { // Tool that takes > 5 seconds let toolAborted = false; const getWeather = tool({ + name: 'getWeather', description: 'weather', parameters: z.object({ location: z.string() }), execute: async ({ location }, { abortSignal }) => { @@ -87,7 +88,7 @@ describe('Generation + Tool Execution', () => { const [execTask, toolOutput] = performToolExecutions({ session: {} as any, speechHandle: { id: 'speech_test', _itemAdded: () => {} } as any, - toolCtx: { getWeather } as any, + toolCtx: new ToolContext([getWeather]) as any, toolCallStream, controller: replyAbortController, onToolExecutionStarted: () => {}, @@ -115,6 +116,7 @@ describe('Generation + Tool Execution', () => { const replyAbortController = new AbortController(); const echo = tool({ + name: 'echo', description: 'echo', parameters: z.object({ msg: z.string() }), execute: async ({ msg }) => `echo: ${msg}`, @@ -130,7 +132,7 @@ describe('Generation + Tool Execution', () => { const [execTask, toolOutput] = performToolExecutions({ session: {} as any, speechHandle: { id: 'speech_test2', _itemAdded: () => {} } as any, - toolCtx: { echo } as any, + toolCtx: new ToolContext([echo]) as any, toolCallStream, controller: replyAbortController, }); @@ -147,6 +149,7 @@ describe('Generation + Tool Execution', () => { let aborted = false; const longOp = tool({ + name: 'longOp', description: 'longOp', parameters: z.object({ ms: z.number() }), execute: async ({ ms }, { abortSignal }) => { @@ -170,7 +173,7 @@ describe('Generation + Tool Execution', () => { const [execTask, toolOutput] = performToolExecutions({ session: {} as any, speechHandle: { id: 'speech_abort', _itemAdded: () => {} } as any, - toolCtx: { longOp } as any, + toolCtx: new ToolContext([longOp]) as any, toolCallStream, controller: replyAbortController, }); @@ -189,6 +192,7 @@ describe('Generation + Tool Execution', () => { const replyAbortController = new AbortController(); const echo = tool({ + name: 'echo', description: 'echo', parameters: z.object({ msg: z.string() }), execute: async ({ msg }) => `echo: ${msg}`, @@ -205,7 +209,7 @@ describe('Generation + Tool Execution', () => { const [execTask, toolOutput] = performToolExecutions({ session: {} as any, speechHandle: { id: 'speech_invalid', _itemAdded: () => {} } as any, - toolCtx: { echo } as any, + toolCtx: new ToolContext([echo]) as any, toolCallStream, controller: replyAbortController, }); @@ -220,11 +224,13 @@ describe('Generation + Tool Execution', () => { const replyAbortController = new AbortController(); const sum = tool({ + name: 'sum', description: 'sum', parameters: z.object({ a: z.number(), b: z.number() }), execute: async ({ a, b }) => a + b, }); const upper = tool({ + name: 'upper', description: 'upper', parameters: z.object({ s: z.string() }), execute: async ({ s }) => s.toUpperCase(), @@ -245,7 +251,7 @@ describe('Generation + Tool Execution', () => { const [execTask, toolOutput] = performToolExecutions({ session: {} as any, speechHandle: { id: 'speech_multi', _itemAdded: () => {} } as any, - toolCtx: { sum, upper } as any, + toolCtx: new ToolContext([sum, upper]) as any, toolCallStream, controller: replyAbortController, }); diff --git a/agents/src/voice/remote_session.ts b/agents/src/voice/remote_session.ts index f970b064e..ee1a3dab8 100644 --- a/agents/src/voice/remote_session.ts +++ b/agents/src/voice/remote_session.ts @@ -471,7 +471,7 @@ function sessionUsageToProto(usage: AgentSessionUsage): pb.AgentSessionUsage { function toolNames(toolCtx: ToolContext | undefined): string[] { if (!toolCtx) return []; - return Object.keys(toolCtx); + return Object.keys(toolCtx.functionTools); } function protoSerializeOptions(opts: { diff --git a/agents/src/voice/testing/fake_llm.ts b/agents/src/voice/testing/fake_llm.ts index ad3a1bf16..b6ba9b08a 100644 --- a/agents/src/voice/testing/fake_llm.ts +++ b/agents/src/voice/testing/fake_llm.ts @@ -4,7 +4,7 @@ import type { ChatContext } from '../../llm/chat_context.js'; import { FunctionCall } from '../../llm/chat_context.js'; import { LLMStream as BaseLLMStream, LLM, type LLMStream } from '../../llm/llm.js'; -import type { ToolChoice, ToolContext } from '../../llm/tool_context.js'; +import type { ToolChoice, ToolCtxInput } from '../../llm/tool_context.js'; import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../../types.js'; import { delay } from '../../utils.js'; @@ -42,7 +42,7 @@ export class FakeLLM extends LLM { connOptions = DEFAULT_API_CONNECT_OPTIONS, }: { chatCtx: ChatContext; - toolCtx?: ToolContext; + toolCtx?: ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: ToolChoice; @@ -65,7 +65,7 @@ class FakeLLMStream extends BaseLLMStream { constructor( fake: FakeLLM, - params: { chatCtx: ChatContext; toolCtx?: ToolContext; connOptions: APIConnectOptions }, + params: { chatCtx: ChatContext; toolCtx?: ToolCtxInput; connOptions: APIConnectOptions }, ) { super(fake, params); this.fake = fake; diff --git a/agents/src/voice/testing/run_result.ts b/agents/src/voice/testing/run_result.ts index 4ee0ccc56..0e60d03ce 100644 --- a/agents/src/voice/testing/run_result.ts +++ b/agents/src/voice/testing/run_result.ts @@ -817,6 +817,7 @@ export class MessageAssert extends EventAssert { // Create the check_intent tool const checkIntentTool = tool({ + name: 'check_intent', description: 'Determines whether the message correctly fulfills the given intent. ' + 'Returns success=true if the message satisfies the intent, false otherwise. ' + @@ -853,7 +854,7 @@ export class MessageAssert extends EventAssert { const stream = llm.chat({ chatCtx, - toolCtx: { check_intent: checkIntentTool }, + toolCtx: [checkIntentTool], toolChoice: { type: 'function', function: { name: 'check_intent' } }, extraKwargs: { temperature: 0 }, }); diff --git a/examples/src/background_audio.ts b/examples/src/background_audio.ts index fc718ac0c..8d5884be9 100644 --- a/examples/src/background_audio.ts +++ b/examples/src/background_audio.ts @@ -35,6 +35,7 @@ export default defineAgent({ logger.info('Connected to room'); const searchWeb = llm.tool({ + name: 'searchWeb', description: 'Search the web for information based on the given query. Always use this function whenever the user requests a web search', parameters: z.object({ @@ -49,9 +50,7 @@ export default defineAgent({ const agent = new voice.Agent({ instructions: 'You are a helpful assistant', - tools: { - searchWeb, - }, + tools: [searchWeb], }); const session = new voice.AgentSession({ diff --git a/examples/src/basic_agent.ts b/examples/src/basic_agent.ts index 95ecddb9a..79e79808a 100644 --- a/examples/src/basic_agent.ts +++ b/examples/src/basic_agent.ts @@ -27,8 +27,9 @@ export default defineAgent({ const agent = new voice.Agent({ instructions: "You are a helpful assistant, you can hear the user's message and respond to it.", - tools: { - getWeather: llm.tool({ + tools: [ + llm.tool({ + name: 'getWeather', description: 'Get the weather for a given location.', parameters: z.object({ location: z.string().describe('The location to get the weather for'), @@ -37,7 +38,7 @@ export default defineAgent({ return `The weather in ${location} is sunny.`; }, }), - }, + ], }); const logger = log(); diff --git a/examples/src/basic_agent_task.ts b/examples/src/basic_agent_task.ts index aacbeee5c..c450a81bb 100644 --- a/examples/src/basic_agent_task.ts +++ b/examples/src/basic_agent_task.ts @@ -21,8 +21,9 @@ class InfoTask extends voice.AgentTask { super({ instructions: `Collect the user's information. around ${info}. Once you have the information, call the saveUserInfo tool to save the information to the database IMMEDIATELY. DO NOT have chitchat with the user, just collect the information and call the saveUserInfo tool.`, tts: 'elevenlabs/eleven_turbo_v2_5', - tools: { - saveUserInfo: llm.tool({ + tools: [ + llm.tool({ + name: 'saveUserInfo', description: `Save the user's ${info} to database`, parameters: z.object({ [info]: z.string(), @@ -32,7 +33,7 @@ class InfoTask extends voice.AgentTask { return `Thanks, collected ${info} successfully: ${args[info]}`; }, }), - }, + ], }); } @@ -48,8 +49,9 @@ class SurveyAgent extends voice.Agent { super({ instructions: 'You orchestrate a short intro survey. Speak naturally and keep the interaction brief.', - tools: { - collectUserInfo: llm.tool({ + tools: [ + llm.tool({ + name: 'collectUserInfo', description: 'Call this when user want to provide some information to you', parameters: z.object({ key: z @@ -63,15 +65,17 @@ class SurveyAgent extends voice.Agent { return `Collected ${key} successfully: ${value}`; }, }), - transferToWeatherAgent: llm.tool({ + llm.tool({ + name: 'transferToWeatherAgent', description: 'Call this immediately after user want to know the weather', execute: async () => { const agent = new voice.Agent({ instructions: 'You are a weather agent. You are responsible for providing the weather information to the user.', tts: 'deepgram/aura-2', - tools: { - getWeather: llm.tool({ + tools: [ + llm.tool({ + name: 'getWeather', description: 'Get the weather for a given location', parameters: z.object({ location: z.string().describe('The location to get the weather for'), @@ -80,7 +84,8 @@ class SurveyAgent extends voice.Agent { return `The weather in ${location} is sunny today.`; }, }), - finishWeatherConversation: llm.tool({ + llm.tool({ + name: 'finishWeatherConversation', description: 'Call this when you want to finish the weather conversation', execute: async () => { return llm.handoff({ @@ -89,13 +94,13 @@ class SurveyAgent extends voice.Agent { }); }, }), - }, + ], }); return llm.handoff({ agent, returns: "Let's start the weather conversation!" }); }, }), - }, + ], }); } diff --git a/examples/src/basic_task_group.ts b/examples/src/basic_task_group.ts index d40befe2a..3d8e04464 100644 --- a/examples/src/basic_task_group.ts +++ b/examples/src/basic_task_group.ts @@ -25,8 +25,9 @@ class CollectNameTask extends voice.AgentTask { instructions: 'Collect the user name from the latest user message. As soon as you have it, call save_name.', tts: taskTts, - tools: { - save_name: llm.tool({ + tools: [ + llm.tool({ + name: 'save_name', description: 'Save the user name.', parameters: z.object({ name: z.string().describe('The user name'), @@ -36,7 +37,7 @@ class CollectNameTask extends voice.AgentTask { return `Saved name: ${name}`; }, }), - }, + ], }); } @@ -54,8 +55,9 @@ class CollectEmailTask extends voice.AgentTask { instructions: 'Collect the user email from the latest user message. As soon as you have it, call save_email.', tts: taskTts, - tools: { - save_email: llm.tool({ + tools: [ + llm.tool({ + name: 'save_email', description: 'Save the user email.', parameters: z.object({ email: z.string().describe('The user email'), @@ -65,7 +67,7 @@ class CollectEmailTask extends voice.AgentTask { return `Saved email: ${email}`; }, }), - }, + ], }); } @@ -82,8 +84,9 @@ class TaskGroupDemoAgent extends voice.Agent { super({ instructions: 'You are onboarding assistant. When user asks to begin onboarding, call startOnboarding exactly once.', - tools: { - startOnboarding: llm.tool({ + tools: [ + llm.tool({ + name: 'startOnboarding', description: 'Start a two-step onboarding flow (name then email).', parameters: z.object({}), execute: async () => { @@ -107,7 +110,7 @@ class TaskGroupDemoAgent extends voice.Agent { return JSON.stringify(result.taskResults); }, }), - }, + ], }); } diff --git a/examples/src/basic_tool_call_agent.ts b/examples/src/basic_tool_call_agent.ts index 5642ef488..9a18de362 100644 --- a/examples/src/basic_tool_call_agent.ts +++ b/examples/src/basic_tool_call_agent.ts @@ -44,6 +44,7 @@ export default defineAgent({ }, entry: async (ctx: JobContext) => { const getWeather = llm.tool({ + name: 'getWeather', description: ' Called when the user asks about the weather.', parameters: z.object({ location: z.string().describe('The location to get the weather for'), @@ -55,6 +56,7 @@ export default defineAgent({ }); const toggleLight = llm.tool({ + name: 'toggleLight', description: 'Called when the user asks to turn on or off the light.', parameters: z.object({ room: roomNameSchema.describe('The room to turn the light in'), @@ -70,6 +72,7 @@ export default defineAgent({ }); const getNumber = llm.tool({ + name: 'getNumber', description: 'Called when the user wants to get a number value, None if user want a random value', parameters: z.object({ @@ -87,6 +90,7 @@ export default defineAgent({ }); const checkStoredNumber = llm.tool({ + name: 'checkStoredNumber', description: 'Called when the user wants to check the stored number.', execute: async (_, { ctx }: llm.ToolOptions) => { return `The stored number is ${ctx.userData.number}.`; @@ -94,6 +98,7 @@ export default defineAgent({ }); const updateStoredNumber = llm.tool({ + name: 'updateStoredNumber', description: 'Called when the user wants to update the stored number.', parameters: z.object({ number: z.number().describe('The number to update the stored number to'), @@ -106,31 +111,33 @@ export default defineAgent({ const routerAgent = new RouterAgent({ instructions: 'You are a helpful assistant.', - tools: { + tools: [ getWeather, toggleLight, - playGame: llm.tool({ + llm.tool({ + name: 'playGame', description: 'Called when the user wants to play a game (transfer user to a game agent).', execute: async (): Promise => { return llm.handoff({ agent: gameAgent, returns: 'The game is now playing.' }); }, }), - }, + ], }); const gameAgent = new GameAgent({ instructions: 'You are a game agent. You are playing a game with the user.', - tools: { + tools: [ getNumber, checkStoredNumber, updateStoredNumber, - finishGame: llm.tool({ + llm.tool({ + name: 'finishGame', description: 'Called when the user wants to finish the game.', execute: async () => { return llm.handoff({ agent: routerAgent, returns: 'The game is now finished.' }); }, }), - }, + ], }); const vad = ctx.proc.userData.vad! as silero.VAD; diff --git a/examples/src/comprehensive_test.ts b/examples/src/comprehensive_test.ts index ddebfc4a7..f3b4974f0 100644 --- a/examples/src/comprehensive_test.ts +++ b/examples/src/comprehensive_test.ts @@ -76,8 +76,9 @@ class MainAgent extends voice.Agent { tts: ttsOptions['elevenlabs'](), llm: llmOptions['openai'](), turnDetection: eouOptions['multilingual'](), - tools: { - testAgent: llm.tool({ + tools: [ + llm.tool({ + name: 'testAgent', description: 'Called when user want to test an agent with STT, TTS, EOU, LLM, and optionally realtime LLM configuration', parameters: z.object({ @@ -102,7 +103,7 @@ class MainAgent extends voice.Agent { }); }, }), - }, + ], }); } @@ -159,8 +160,9 @@ class TestAgent extends voice.Agent { tts: tts, llm: realtimeModel ?? model, turnDetection: eou, - tools: { - testTool: llm.tool({ + tools: [ + llm.tool({ + name: 'testTool', description: "Testing agent's tool calling ability", parameters: z .object({ @@ -173,7 +175,8 @@ class TestAgent extends voice.Agent { }; }, }), - nextAgent: llm.tool({ + llm.tool({ + name: 'nextAgent', description: 'Called when user confirm current agent is working and want to proceed to next agent', parameters: z.object({ @@ -204,7 +207,7 @@ class TestAgent extends voice.Agent { }); }, }), - }, + ], }); this.sttChoice = sttChoice; diff --git a/examples/src/drive-thru/drivethru_agent.ts b/examples/src/drive-thru/drivethru_agent.ts index 9882f6fcd..e684f5c00 100644 --- a/examples/src/drive-thru/drivethru_agent.ts +++ b/examples/src/drive-thru/drivethru_agent.ts @@ -57,23 +57,24 @@ export class DriveThruAgent extends voice.Agent { super({ instructions, - tools: { - orderComboMeal: DriveThruAgent.buildComboOrderTool( + tools: [ + DriveThruAgent.buildComboOrderTool( userdata.comboItems, userdata.drinkItems, userdata.sauceItems, ), - orderHappyMeal: DriveThruAgent.buildHappyOrderTool( + DriveThruAgent.buildHappyOrderTool( userdata.happyItems, userdata.drinkItems, userdata.sauceItems, ), - orderRegularItem: DriveThruAgent.buildRegularOrderTool( + DriveThruAgent.buildRegularOrderTool( userdata.regularItems, userdata.drinkItems, userdata.sauceItems, ), - removeOrderItem: llm.tool({ + llm.tool({ + name: 'removeOrderItem', description: `Removes one or more items from the user's order using their \`orderId\`s. Useful when the user asks to cancel or delete existing items (e.g., "Remove the cheeseburger"). @@ -100,7 +101,8 @@ If the \`orderId\`s are unknown, call \`listOrderItems\` first to retrieve them. return 'Removed items:\n' + removedItems.map((item) => JSON.stringify(item)).join('\n'); }, }), - listOrderItems: llm.tool({ + llm.tool({ + name: 'listOrderItems', description: `Retrieves the current list of items in the user's order, including each item's internal \`orderId\`. Helpful when: @@ -120,7 +122,7 @@ Examples: return items.map((item) => JSON.stringify(item)).join('\n'); }, }), - }, + ], }); } @@ -134,6 +136,7 @@ Examples: const availableSauceIds = [...new Set(sauceItems.map((item) => item.id))]; return llm.tool({ + name: 'orderComboMeal', description: `Call this when the user orders a **Combo Meal**, like: "Number 4b with a large Sprite" or "I'll do a medium meal." Do not call this tool unless the user clearly refers to a known combo meal by name or number. @@ -222,6 +225,7 @@ If the user says just "a large meal," assume both drink and fries are that size. const availableSauceIds = [...new Set(sauceItems.map((item) => item.id))]; return llm.tool({ + name: 'orderHappyMeal', description: `Call this when the user orders a **Happy Meal**, typically for children. These meals come with a main item, a drink, and a sauce. The user must clearly specify a valid Happy Meal option (e.g., "Can I get a Happy Meal?"). @@ -299,6 +303,7 @@ Assume Small as default only if the user says "Happy Meal" and gives no size pre const availableIds = [...new Set(allItems.map((item) => item.id))]; return llm.tool({ + name: 'orderRegularItem', description: `Call this when the user orders **a single item on its own**, not as part of a Combo Meal or Happy Meal. The customer must provide clear and specific input. For example, item variants such as flavor must **always** be explicitly stated. diff --git a/examples/src/frontdesk/frontdesk_agent.ts b/examples/src/frontdesk/frontdesk_agent.ts index d5d2e1ab1..2fa60ba57 100644 --- a/examples/src/frontdesk/frontdesk_agent.ts +++ b/examples/src/frontdesk/frontdesk_agent.ts @@ -59,8 +59,9 @@ export class FrontDeskAgent extends voice.Agent { super({ instructions, - tools: { - scheduleAppointment: llm.tool({ + tools: [ + llm.tool({ + name: 'scheduleAppointment', description: 'Schedule an appointment at the given slot.', parameters: z.object({ slotId: z @@ -110,7 +111,8 @@ export class FrontDeskAgent extends voice.Agent { return `The appointment was successfully scheduled for ${formatted}.`; }, }), - listAvailableSlots: llm.tool({ + llm.tool({ + name: 'listAvailableSlots', description: `Return a plain-text list of available slots, one per line. - , , at () @@ -188,7 +190,7 @@ You must infer the appropriate range implicitly from the conversational context return lines.join('\n') || 'No slots available at the moment.'; }, }), - }, + ], }); this.tz = options.timezone; diff --git a/examples/src/gemini_realtime_agent.ts b/examples/src/gemini_realtime_agent.ts index dd22db51f..d70368352 100644 --- a/examples/src/gemini_realtime_agent.ts +++ b/examples/src/gemini_realtime_agent.ts @@ -47,6 +47,7 @@ type StoryData = { const roomNameSchema = z.enum(['bedroom', 'living room', 'kitchen', 'bathroom', 'office']); const getWeather = llm.tool({ + name: 'getWeather', description: 'Called when the user asks about the weather.', parameters: z.object({ location: z.string().describe('The location to get the weather for'), @@ -59,6 +60,7 @@ const getWeather = llm.tool({ }); const toggleLight = llm.tool({ + name: 'toggleLight', description: 'Called when the user asks to turn on or off the light.', parameters: z.object({ room: roomNameSchema.describe('The room to turn the light in'), @@ -79,15 +81,16 @@ class IntroAgent extends voice.Agent { static create() { return new IntroAgent({ instructions: `You are a story teller. Your goal is to gather a few pieces of information from the user to make the story personalized and engaging. Ask the user for their name and where they are from.`, - tools: { - informationGathered: llm.tool({ + tools: [ + llm.tool({ + name: 'informationGathered', description: 'Called when the user has provided the information needed to make the story personalized and engaging.', parameters: z.object({ name: z.string().describe('The name of the user'), location: z.string().describe('The location of the user'), }), - execute: async ({ name, location }, { ctx }) => { + execute: async ({ name, location }, { ctx }: llm.ToolOptions) => { ctx.userData.name = name; ctx.userData.location = location; @@ -97,7 +100,7 @@ class IntroAgent extends voice.Agent { }), getWeather, toggleLight, - }, + ], }); } } diff --git a/examples/src/instructions_per_modality.ts b/examples/src/instructions_per_modality.ts index 71f2f3f04..643e3d433 100644 --- a/examples/src/instructions_per_modality.ts +++ b/examples/src/instructions_per_modality.ts @@ -57,8 +57,9 @@ class SchedulingAgent extends voice.Agent { super({ instructions, - tools: { - bookAppointment: llm.tool({ + tools: [ + llm.tool({ + name: 'bookAppointment', description: 'Book an appointment.', parameters: z.object({ date: z.string().describe('The date of the appointment in the format YYYY-MM-DD'), @@ -69,7 +70,7 @@ class SchedulingAgent extends voice.Agent { return `Appointment booked for ${date} at ${time}`; }, }), - }, + ], }); } diff --git a/examples/src/llm_fallback_adapter.ts b/examples/src/llm_fallback_adapter.ts index d053464dc..d86708548 100644 --- a/examples/src/llm_fallback_adapter.ts +++ b/examples/src/llm_fallback_adapter.ts @@ -71,8 +71,9 @@ export default defineAgent({ const agent = new voice.Agent({ instructions: 'You are a helpful assistant. Demonstrate that you are working by responding to user queries.', - tools: { - getWeather: llm.tool({ + tools: [ + llm.tool({ + name: 'getWeather', description: 'Get the weather for a given location.', parameters: z.object({ location: z.string().describe('The location to get the weather for'), @@ -81,7 +82,7 @@ export default defineAgent({ return `The weather in ${location} is sunny with a temperature of 72°F.`; }, }), - }, + ], }); const session = new voice.AgentSession({ diff --git a/examples/src/manual_shutdown.ts b/examples/src/manual_shutdown.ts index 96bedb901..edb21b87c 100644 --- a/examples/src/manual_shutdown.ts +++ b/examples/src/manual_shutdown.ts @@ -25,8 +25,9 @@ export default defineAgent({ const agent = new voice.Agent({ instructions: "You are a helpful assistant, you can hear the user's message and respond to it, end the call when the user asks you to.", - tools: { - getWeather: llm.tool({ + tools: [ + llm.tool({ + name: 'getWeather', description: 'Get the weather for a given location.', parameters: z.object({ location: z.string().describe('The location to get the weather for'), @@ -35,7 +36,8 @@ export default defineAgent({ return `The weather in ${location} is sunny.`; }, }), - endCall: llm.tool({ + llm.tool({ + name: 'endCall', description: 'End the call.', parameters: z.object({ reason: z @@ -56,7 +58,7 @@ export default defineAgent({ session.shutdown({ reason }); }, }), - }, + ], }); const session = new voice.AgentSession({ diff --git a/examples/src/multi_agent.ts b/examples/src/multi_agent.ts index 7f4819bed..12d87377d 100644 --- a/examples/src/multi_agent.ts +++ b/examples/src/multi_agent.ts @@ -35,15 +35,16 @@ class IntroAgent extends voice.Agent { static create() { return new IntroAgent({ instructions: `You are a story teller. Your goal is to gather a few pieces of information from the user to make the story personalized and engaging. Ask the user for their name and where they are from.`, - tools: { - informationGathered: llm.tool({ + tools: [ + llm.tool({ + name: 'informationGathered', description: 'Called when the user has provided the information needed to make the story personalized and engaging.', parameters: z.object({ name: z.string().describe('The name of the user'), location: z.string().describe('The location of the user'), }), - execute: async ({ name, location }, { ctx }) => { + execute: async ({ name, location }, { ctx }: llm.ToolOptions) => { ctx.userData.name = name; ctx.userData.location = location; @@ -51,7 +52,7 @@ class IntroAgent extends voice.Agent { return llm.handoff({ agent: storyAgent, returns: "Let's start the story!" }); }, }), - }, + ], }); } } diff --git a/examples/src/phonic_realtime_agent.ts b/examples/src/phonic_realtime_agent.ts index 35a038450..934485f5b 100644 --- a/examples/src/phonic_realtime_agent.ts +++ b/examples/src/phonic_realtime_agent.ts @@ -7,6 +7,7 @@ import { fileURLToPath } from 'node:url'; import { z } from 'zod'; const toggleLight = llm.tool({ + name: 'toggle_light', description: 'Toggle a light on or off. Available lights are A05, A06, A07, and A08.', parameters: z.object({ light_id: z.string().describe('The ID of the light to toggle'), @@ -23,9 +24,7 @@ export default defineAgent({ entry: async (ctx: JobContext) => { const agent = new voice.Agent({ instructions: 'You are a helpful voice AI assistant named Alex.', - tools: { - toggle_light: toggleLight, - }, + tools: [toggleLight], }); const session = new voice.AgentSession({ diff --git a/examples/src/raw_function_description.ts b/examples/src/raw_function_description.ts index 6548fd011..1e81c65d9 100644 --- a/examples/src/raw_function_description.ts +++ b/examples/src/raw_function_description.ts @@ -18,8 +18,9 @@ import { fileURLToPath } from 'node:url'; function createRawFunctionAgent() { return new voice.Agent({ instructions: 'You are a helpful assistant.', - tools: { - openGate: llm.tool({ + tools: [ + llm.tool({ + name: 'openGate', description: 'Opens a specified gate from a predefined set of access points.', parameters: { type: 'object', @@ -43,7 +44,7 @@ function createRawFunctionAgent() { return `The gate ${gateId} is now open.`; }, }), - }, + ], }); } diff --git a/examples/src/realtime_agent.ts b/examples/src/realtime_agent.ts index b30171776..647d5759b 100644 --- a/examples/src/realtime_agent.ts +++ b/examples/src/realtime_agent.ts @@ -24,6 +24,7 @@ export default defineAgent({ }, entry: async (ctx: JobContext) => { const getWeather = llm.tool({ + name: 'getWeather', description: ' Called when the user asks about the weather.', parameters: z.object({ location: z.string().describe('The location to get the weather for'), @@ -34,6 +35,7 @@ export default defineAgent({ }); const toggleLight = llm.tool({ + name: 'toggleLight', description: 'Called when the user asks to turn on or off the light.', parameters: z.object({ room: roomNameSchema.describe('The room to turn the light in'), @@ -65,10 +67,7 @@ export default defineAgent({ instructions: "You are a helpful assistant created by LiveKit, always speaking English, you can hear the user's message and respond to it.", chatCtx, - tools: { - getWeather, - toggleLight, - }, + tools: [getWeather, toggleLight], }); const session = new voice.AgentSession({ diff --git a/examples/src/realtime_with_tts.ts b/examples/src/realtime_with_tts.ts index d87db7853..88c450f06 100644 --- a/examples/src/realtime_with_tts.ts +++ b/examples/src/realtime_with_tts.ts @@ -26,6 +26,7 @@ export default defineAgent({ const logger = log(); const getWeather = llm.tool({ + name: 'getWeather', description: 'Called when the user asks about the weather.', parameters: z.object({ location: z.string().describe('The location to get the weather for'), @@ -38,9 +39,7 @@ export default defineAgent({ const agent = new voice.Agent({ instructions: 'You are a helpful assistant. Always speak in English.', - tools: { - getWeather, - }, + tools: [getWeather], }); const session = new voice.AgentSession({ diff --git a/examples/src/restaurant_agent.ts b/examples/src/restaurant_agent.ts index d9faaf9a5..4fb5281ba 100644 --- a/examples/src/restaurant_agent.ts +++ b/examples/src/restaurant_agent.ts @@ -79,6 +79,7 @@ function summarize({ } const updateName = llm.tool({ + name: 'updateName', description: 'Called when the user provides their name. Confirm the spelling with the user before calling the function.', parameters: z.object({ @@ -91,6 +92,7 @@ const updateName = llm.tool({ }); const updatePhone = llm.tool({ + name: 'updatePhone', description: 'Called when the user provides their phone number. Confirm the spelling with the user before calling the function.', parameters: z.object({ @@ -103,6 +105,7 @@ const updatePhone = llm.tool({ }); const toGreeter = llm.tool({ + name: 'toGreeter', description: 'Called when user asks any unrelated questions or requests any other services not in your job description.', execute: async (_, { ctx }: llm.ToolOptions) => { @@ -173,35 +176,37 @@ function createGreeterAgent(menu: string) { instructions: `You are a friendly restaurant receptionist. The menu is: ${menu}\nYour jobs are to greet the caller and understand if they want to make a reservation or order takeaway. Guide them to the right agent using tools.`, llm: new inference.LLM({ model: 'openai/gpt-4.1-mini' }), tts: new inference.TTS({ model: 'cartesia/sonic-3', voice: voices.greeter }), - tools: { - toReservation: llm.tool({ + tools: [ + llm.tool({ + name: 'toReservation', description: dedent` Called when user wants to make or update a reservation. This function handles transitioning to the reservation agent who will collect the necessary details like reservation time, customer name and phone number. `, - execute: async (_, { ctx }): Promise => { + execute: async (_, { ctx }: llm.ToolOptions): Promise => { return await greeter.transferToAgent({ name: 'reservation', ctx, }); }, }), - toTakeaway: llm.tool({ + llm.tool({ + name: 'toTakeaway', description: dedent` Called when the user wants to place a takeaway order. This includes handling orders for pickup, delivery, or when the user wants to proceed to checkout with their existing order. `, - execute: async (_, { ctx }): Promise => { + execute: async (_, { ctx }: llm.ToolOptions): Promise => { return await greeter.transferToAgent({ name: 'takeaway', ctx, }); }, }), - }, + ], }); return greeter; @@ -212,11 +217,12 @@ function createReservationAgent() { name: 'reservation', instructions: `You are a reservation agent at a restaurant. Your jobs are to ask for the reservation time, then customer's name, and phone number. Then confirm the reservation details with the customer.`, tts: new inference.TTS({ model: 'cartesia/sonic-3', voice: voices.reservation }), - tools: { + tools: [ updateName, updatePhone, toGreeter, - updateReservationTime: llm.tool({ + llm.tool({ + name: 'updateReservationTime', description: dedent` Called when the user provides their reservation time. Confirm the time with the user before calling the function. @@ -224,14 +230,18 @@ function createReservationAgent() { parameters: z.object({ time: z.string().describe('The reservation time'), }), - execute: async ({ time }, { ctx }) => { + execute: async ({ time }, { ctx }: llm.ToolOptions) => { ctx.userData.reservationTime = time; return `The reservation time is updated to ${time}`; }, }), - confirmReservation: llm.tool({ + llm.tool({ + name: 'confirmReservation', description: `Called when the user confirms the reservation.`, - execute: async (_, { ctx }): Promise => { + execute: async ( + _, + { ctx }: llm.ToolOptions, + ): Promise => { const userdata = ctx.userData; if (!userdata.customer.name || !userdata.customer.phone) { return 'Please provide your name and phone number first.'; @@ -245,7 +255,7 @@ function createReservationAgent() { }); }, }), - }, + ], }); return reservation; @@ -256,21 +266,26 @@ function createTakeawayAgent(menu: string) { name: 'takeaway', instructions: `Your are a takeaway agent that takes orders from the customer. Our menu is: ${menu}\nClarify special requests and confirm the order with the customer.`, tts: new inference.TTS({ model: 'cartesia/sonic-3', voice: voices.takeaway }), - tools: { + tools: [ toGreeter, - updateOrder: llm.tool({ + llm.tool({ + name: 'updateOrder', description: `Called when the user provides their order.`, parameters: z.object({ items: z.array(z.string()).describe('The items of the full order'), }), - execute: async ({ items }, { ctx }) => { + execute: async ({ items }, { ctx }: llm.ToolOptions) => { ctx.userData.order = items; return `The order is updated to ${items}`; }, }), - toCheckout: llm.tool({ + llm.tool({ + name: 'toCheckout', description: `Called when the user confirms the order.`, - execute: async (_, { ctx }): Promise => { + execute: async ( + _, + { ctx }: llm.ToolOptions, + ): Promise => { const userdata = ctx.userData; if (!userdata.order) { return 'No takeaway order found. Please make an order first.'; @@ -281,7 +296,7 @@ function createTakeawayAgent(menu: string) { }); }, }), - }, + ], }); return takeaway; @@ -292,21 +307,23 @@ function createCheckoutAgent(menu: string) { name: 'checkout', instructions: `You are a checkout agent at a restaurant. The menu is: ${menu}\nYour are responsible for confirming the expense of the order and then collecting customer's name, phone number and credit card information, including the card number, expiry date, and CVV step by step.`, tts: new inference.TTS({ model: 'cartesia/sonic-3', voice: voices.checkout }), - tools: { + tools: [ updateName, updatePhone, toGreeter, - confirmExpense: llm.tool({ + llm.tool({ + name: 'confirmExpense', description: `Called when the user confirms the expense.`, parameters: z.object({ expense: z.number().describe('The expense of the order'), }), - execute: async ({ expense }, { ctx }) => { + execute: async ({ expense }, { ctx }: llm.ToolOptions) => { ctx.userData.expense = expense; return `The expense is confirmed to be ${expense}`; }, }), - updateCreditCard: llm.tool({ + llm.tool({ + name: 'updateCreditCard', description: dedent` Called when the user provides their credit card number, expiry date, and CVV. Confirm the spelling with the user before calling the function. @@ -316,14 +333,18 @@ function createCheckoutAgent(menu: string) { expiry: z.string().describe('The expiry date of the credit card'), cvv: z.string().describe('The CVV of the credit card'), }), - execute: async ({ number, expiry, cvv }, { ctx }) => { + execute: async ({ number, expiry, cvv }, { ctx }: llm.ToolOptions) => { ctx.userData.creditCard = { number, expiry, cvv }; return `The credit card number is updated to ${number}`; }, }), - confirmCheckout: llm.tool({ + llm.tool({ + name: 'confirmCheckout', description: `Called when the user confirms the checkout.`, - execute: async (_, { ctx }): Promise => { + execute: async ( + _, + { ctx }: llm.ToolOptions, + ): Promise => { const userdata = ctx.userData; if (!userdata.expense) { return 'Please confirm the expense first.'; @@ -342,16 +363,17 @@ function createCheckoutAgent(menu: string) { }); }, }), - toTakeaway: llm.tool({ + llm.tool({ + name: 'toTakeaway', description: `Called when the user wants to update their order.`, - execute: async (_, { ctx }): Promise => { + execute: async (_, { ctx }: llm.ToolOptions): Promise => { return await checkout.transferToAgent({ name: 'takeaway', ctx, }); }, }), - }, + ], }); return checkout; diff --git a/examples/src/survey_agent.ts b/examples/src/survey_agent.ts index 8504fa1a7..918fd9859 100644 --- a/examples/src/survey_agent.ts +++ b/examples/src/survey_agent.ts @@ -76,6 +76,7 @@ async function writeCsvRow(path: string, data: Record): Promise function disqualifyTool() { return llm.tool({ + name: 'disqualify', description: 'End the interview if the candidate refuses to cooperate, provides inappropriate answers, or is not a fit.', parameters: z.object({ @@ -101,8 +102,9 @@ export class IntroTask extends voice.AgentTask { super({ instructions: 'You are Alex, an interviewer screening a software engineer candidate. Gather the candidate name and short self-introduction.', - tools: { - saveIntro: llm.tool({ + tools: [ + llm.tool({ + name: 'saveIntro', description: 'Save candidate name and intro notes.', parameters: z.object({ name: z.string().describe('Candidate name'), @@ -114,7 +116,7 @@ export class IntroTask extends voice.AgentTask { return `Saved intro for ${name}.`; }, }), - }, + ], }); } @@ -132,9 +134,10 @@ export class EmailTask extends voice.AgentTask { super({ instructions: 'Collect a valid email address. If the candidate refuses, call disqualify immediately.', - tools: { + tools: [ disqualify, - saveEmail: llm.tool({ + llm.tool({ + name: 'saveEmail', description: 'Save candidate email address.', parameters: z.object({ email: z.string().describe('Candidate email'), @@ -144,7 +147,7 @@ export class EmailTask extends voice.AgentTask { return `Saved email: ${email}`; }, }), - }, + ], }); } @@ -161,9 +164,10 @@ export class CommuteTask extends voice.AgentTask super({ instructions: 'Collect commute flexibility. The role expects office attendance three days per week.', - tools: { + tools: [ disqualify, - saveCommute: llm.tool({ + llm.tool({ + name: 'saveCommute', description: 'Save candidate commute information.', parameters: z.object({ canCommute: z.boolean().describe('Whether the candidate can commute to office'), @@ -176,7 +180,7 @@ export class CommuteTask extends voice.AgentTask return 'Saved commute flexibility.'; }, }), - }, + ], }); } @@ -194,9 +198,10 @@ export class ExperienceTask extends voice.AgentTask { super({ instructions: 'You are a survey interviewer for a software engineer screening. Be concise, professional, and natural. Call endScreening when the process is complete.', - tools: { - endScreening: llm.tool({ + tools: [ + llm.tool({ + name: 'endScreening', description: 'End interview and hang up.', execute: async (_, { ctx }: llm.ToolOptions) => { ctx.session.shutdown(); return 'Interview concluded.'; }, }), - }, + ], }); } diff --git a/examples/src/testing/agent_task.test.ts b/examples/src/testing/agent_task.test.ts index 163d232a1..38d2a85d9 100644 --- a/examples/src/testing/agent_task.test.ts +++ b/examples/src/testing/agent_task.test.ts @@ -148,8 +148,9 @@ describe('AgentTask examples', { timeout: 120_000 }, () => { super({ instructions: 'You are collecting a name and role. Extract both from user input and call recordIntro.', - tools: { - recordIntro: llm.tool({ + tools: [ + llm.tool({ + name: 'recordIntro', description: 'Record the name and role', parameters: z.object({ name: z.string().describe('User name'), @@ -160,7 +161,7 @@ describe('AgentTask examples', { timeout: 120_000 }, () => { return 'recorded'; }, }), - }, + ], }); } @@ -222,8 +223,9 @@ describe('AgentTask examples', { timeout: 120_000 }, () => { super({ instructions: 'When asked to capture email, ALWAYS call captureEmail exactly once, then respond briefly.', - tools: { - captureEmail: llm.tool({ + tools: [ + llm.tool({ + name: 'captureEmail', description: 'Capture an email by running a nested AgentTask.', parameters: z.object({}), execute: async () => { @@ -236,7 +238,7 @@ describe('AgentTask examples', { timeout: 120_000 }, () => { } }, }), - }, + ], }); } @@ -275,8 +277,9 @@ describe('AgentTask examples', { timeout: 120_000 }, () => { instructions: 'You are Alex, an interviewer. Extract the candidate name and a short intro from the latest user input. ' + 'Use the tool recordIntro exactly once when both are available.', - tools: { - recordIntro: llm.tool({ + tools: [ + llm.tool({ + name: 'recordIntro', description: 'Record candidate name and intro summary.', parameters: z.object({ name: z.string().describe('Candidate name'), @@ -288,7 +291,7 @@ describe('AgentTask examples', { timeout: 120_000 }, () => { return 'Intro recorded.'; }, }), - }, + ], }); } @@ -305,8 +308,9 @@ describe('AgentTask examples', { timeout: 120_000 }, () => { super({ instructions: 'When the user asks to run the intro task, ALWAYS call collectIntroWithTask exactly once.', - tools: { - collectIntroWithTask: llm.tool({ + tools: [ + llm.tool({ + name: 'collectIntroWithTask', description: 'Launch the IntroTask and return the captured intro details.', parameters: z.object({}), execute: async () => { @@ -316,7 +320,7 @@ describe('AgentTask examples', { timeout: 120_000 }, () => { return JSON.stringify(result); }, }), - }, + ], }); } } diff --git a/examples/src/testing/basic_task_group.test.ts b/examples/src/testing/basic_task_group.test.ts index afbc7400e..5201a6c7c 100644 --- a/examples/src/testing/basic_task_group.test.ts +++ b/examples/src/testing/basic_task_group.test.ts @@ -82,8 +82,9 @@ class CollectNameTask extends voice.AgentTask { super({ instructions: 'Collect the user name from the latest user message. As soon as you have it, call save_name.', - tools: { - save_name: llm.tool({ + tools: [ + llm.tool({ + name: 'save_name', description: 'Save the user name.', parameters: z.object({ name: z.string().describe('The user name') }), execute: async ({ name }) => { @@ -91,7 +92,7 @@ class CollectNameTask extends voice.AgentTask { return `Saved name: ${name}`; }, }), - }, + ], }); this.ready = ready; } @@ -108,8 +109,9 @@ class CollectEmailTask extends voice.AgentTask { super({ instructions: 'Collect the user email from the latest user message. As soon as you have it, call save_email.', - tools: { - save_email: llm.tool({ + tools: [ + llm.tool({ + name: 'save_email', description: 'Save the user email.', parameters: z.object({ email: z.string().describe('The user email') }), execute: async ({ email }) => { @@ -117,7 +119,7 @@ class CollectEmailTask extends voice.AgentTask { return `Saved email: ${email}`; }, }), - }, + ], }); this.ready = ready; } diff --git a/examples/src/testing/run_result.test.ts b/examples/src/testing/run_result.test.ts index 583cbaffa..4169dea93 100644 --- a/examples/src/testing/run_result.test.ts +++ b/examples/src/testing/run_result.test.ts @@ -45,8 +45,9 @@ Response rules: - After ordering, confirm what was added (e.g., "I've added the burger to your order"). - When asked about sizes, always ask for clarification if not specified. - Be friendly and proactive in suggesting next steps.`, - tools: { - getWeather: llm.tool({ + tools: [ + llm.tool({ + name: 'getWeather', description: 'Get the current weather for a location', parameters: z.object({ location: z.string().describe('The city name'), @@ -59,14 +60,16 @@ Response rules: }); }, }), - getCurrentTime: llm.tool({ + llm.tool({ + name: 'getCurrentTime', description: 'Get the current time', parameters: z.object({}), execute: async () => { return '3:00 PM'; }, }), - orderItem: llm.tool({ + llm.tool({ + name: 'orderItem', description: 'Add an item to the order', parameters: z.object({ itemId: z.string().describe('The menu item ID'), @@ -84,7 +87,8 @@ Response rules: }); }, }), - getOrderStatus: llm.tool({ + llm.tool({ + name: 'getOrderStatus', description: 'Get the current order status', parameters: z.object({}), execute: async () => { @@ -98,7 +102,8 @@ Response rules: }); }, }), - getMenuItems: llm.tool({ + llm.tool({ + name: 'getMenuItems', description: 'Get available menu items and prices', parameters: z.object({ category: z @@ -128,7 +133,7 @@ Response rules: return JSON.stringify(menu); }, }), - }, + ], }); } } diff --git a/examples/src/testing/task_group.test.ts b/examples/src/testing/task_group.test.ts index 3d5afff06..188d46e59 100644 --- a/examples/src/testing/task_group.test.ts +++ b/examples/src/testing/task_group.test.ts @@ -260,8 +260,9 @@ describe('TaskGroup', { timeout: 120_000 }, () => { super({ instructions: 'Extract the user name from the latest user message. Call recordName immediately.', - tools: { - recordName: llm.tool({ + tools: [ + llm.tool({ + name: 'recordName', description: 'Record the user name', parameters: z.object({ name: z.string().describe('The user name') }), execute: async ({ name }) => { @@ -269,7 +270,7 @@ describe('TaskGroup', { timeout: 120_000 }, () => { return 'recorded'; }, }), - }, + ], }); } @@ -283,8 +284,9 @@ describe('TaskGroup', { timeout: 120_000 }, () => { super({ instructions: 'Extract an email address from the latest user message. Call recordEmail immediately.', - tools: { - recordEmail: llm.tool({ + tools: [ + llm.tool({ + name: 'recordEmail', description: 'Record the user email', parameters: z.object({ email: z.string().describe('The email address') }), execute: async ({ email }) => { @@ -292,7 +294,7 @@ describe('TaskGroup', { timeout: 120_000 }, () => { return 'recorded'; }, }), - }, + ], }); } @@ -400,8 +402,9 @@ describe('TaskGroup', { timeout: 120_000 }, () => { super({ instructions: 'Extract the user favorite color from the latest message. Call recordColor immediately.', - tools: { - recordColor: llm.tool({ + tools: [ + llm.tool({ + name: 'recordColor', description: 'Record favorite color', parameters: z.object({ color: z.string() }), execute: async ({ color }) => { @@ -409,7 +412,7 @@ describe('TaskGroup', { timeout: 120_000 }, () => { return 'recorded'; }, }), - }, + ], }); } @@ -423,8 +426,9 @@ describe('TaskGroup', { timeout: 120_000 }, () => { super({ instructions: 'Extract the user favorite food from the latest message. Call recordFood immediately.', - tools: { - recordFood: llm.tool({ + tools: [ + llm.tool({ + name: 'recordFood', description: 'Record favorite food', parameters: z.object({ food: z.string() }), execute: async ({ food }) => { @@ -432,7 +436,7 @@ describe('TaskGroup', { timeout: 120_000 }, () => { return 'recorded'; }, }), - }, + ], }); } diff --git a/examples/src/tool_call_disfluency.ts b/examples/src/tool_call_disfluency.ts index 8f92183a8..7e8018c99 100644 --- a/examples/src/tool_call_disfluency.ts +++ b/examples/src/tool_call_disfluency.ts @@ -39,6 +39,7 @@ export default defineAgent({ const vad = ctx.proc.userData.vad! as silero.VAD; const getWeather = llm.tool({ + name: 'getWeather', description: ' Called when the user asks about the weather.', parameters: z.object({ location: z.string().describe('The location to get the weather for'), @@ -55,9 +56,7 @@ export default defineAgent({ const agent = new VoiceAgent({ instructions: "You are a helpful assistant, you can hear the user's message and respond to it.", - tools: { - getWeather, - }, + tools: [getWeather], }); const session = new voice.AgentSession({ diff --git a/examples/src/xai-realtime.ts b/examples/src/xai-realtime.ts index ad383a0a6..a5130a640 100644 --- a/examples/src/xai-realtime.ts +++ b/examples/src/xai-realtime.ts @@ -10,8 +10,9 @@ export default defineAgent({ entry: async (ctx: JobContext) => { const agent = new voice.Agent({ instructions: 'You are a helpful assistant. Keep your responses short and concise.', - tools: { - getWeather: llm.tool({ + tools: [ + llm.tool({ + name: 'getWeather', description: 'Get the weather for a given location.', parameters: z.object({ location: z.string().describe('The location to get the weather for'), @@ -20,7 +21,7 @@ export default defineAgent({ return `The weather in ${location} is sunny.`; }, }), - }, + ], }); const session = new voice.AgentSession({ diff --git a/plugins/baseten/src/llm.ts b/plugins/baseten/src/llm.ts index 4039b7cac..179a09344 100644 --- a/plugins/baseten/src/llm.ts +++ b/plugins/baseten/src/llm.ts @@ -72,19 +72,20 @@ export class OpenAILLM extends llm.LLM { chat({ chatCtx, - toolCtx, + toolCtx: toolCtxInput, connOptions = DEFAULT_API_CONNECT_OPTIONS, parallelToolCalls, toolChoice, extraKwargs, }: { chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: llm.ToolChoice; extraKwargs?: Record; }): inference.LLMStream { + const toolCtx = llm.toToolContext(toolCtxInput); const extras: Record = { ...extraKwargs }; if (this.#opts.metadata) { @@ -125,7 +126,11 @@ export class OpenAILLM extends llm.LLM { parallelToolCalls = parallelToolCalls !== undefined ? parallelToolCalls : this.#opts.parallelToolCalls; - if (toolCtx && Object.keys(toolCtx).length > 0 && parallelToolCalls !== undefined) { + if ( + toolCtx && + Object.keys(toolCtx.functionTools).length > 0 && + parallelToolCalls !== undefined + ) { extras.parallel_tool_calls = parallelToolCalls; } diff --git a/plugins/cerebras/src/llm.test.ts b/plugins/cerebras/src/llm.test.ts index 3a0a3ca8b..dda5d7b3c 100644 --- a/plugins/cerebras/src/llm.test.ts +++ b/plugins/cerebras/src/llm.test.ts @@ -88,8 +88,9 @@ class WeatherAgent extends voice.Agent { constructor() { super({ instructions: 'You are a helpful assistant.', - tools: { - get_weather: llm.tool({ + tools: [ + llm.tool({ + name: 'get_weather', description: 'Get the current weather for a location.', parameters: z.object({ location: z.string().describe('The city name'), @@ -98,7 +99,7 @@ class WeatherAgent extends voice.Agent { return `The weather in ${location} is sunny, 72°F.`; }, }), - }, + ], }); } } diff --git a/plugins/google/src/beta/realtime/realtime_api.ts b/plugins/google/src/beta/realtime/realtime_api.ts index 8b9ada6eb..66bc6a7f9 100644 --- a/plugins/google/src/beta/realtime/realtime_api.ts +++ b/plugins/google/src/beta/realtime/realtime_api.ts @@ -451,7 +451,7 @@ export class RealtimeModel extends llm.RealtimeModel { * supporting both text and audio modalities with function calling capabilities. */ export class RealtimeSession extends llm.RealtimeSession { - private _tools: llm.ToolContext = {}; + private _tools: llm.ToolContext = llm.ToolContext.empty(); private _chatCtx = llm.ChatContext.empty(); private options: RealtimeOptions; @@ -780,7 +780,7 @@ export class RealtimeSession extends llm.RealtimeSession { } get tools(): llm.ToolContext { - return { ...this._tools }; + return this._tools.copy(); } get manualActivityDetection(): boolean { diff --git a/plugins/google/src/llm.ts b/plugins/google/src/llm.ts index 302b679af..e452b70d2 100644 --- a/plugins/google/src/llm.ts +++ b/plugins/google/src/llm.ts @@ -189,20 +189,21 @@ export class LLM extends llm.LLM { chat({ chatCtx, - toolCtx, + toolCtx: toolCtxInput, connOptions = DEFAULT_API_CONNECT_OPTIONS, toolChoice, extraKwargs, geminiTools, }: { chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: llm.ToolChoice; extraKwargs?: Record; geminiTools?: LLMTools; }): LLMStream { + const toolCtx = llm.toToolContext(toolCtxInput); const extras: GenerateContentConfig = { ...extraKwargs } as GenerateContentConfig; toolChoice = toolChoice !== undefined ? toolChoice : this.#opts.toolChoice; @@ -218,7 +219,7 @@ export class LLM extends llm.LLM { }, }; } else if (toolChoice === 'required') { - const toolNames = Object.entries(toolCtx || {}).map(([name]) => name); + const toolNames = Object.keys(toolCtx?.functionTools ?? {}); geminiToolConfig = { functionCallingConfig: { mode: FunctionCallingConfigMode.ANY, diff --git a/plugins/google/src/utils.ts b/plugins/google/src/utils.ts index 732ae0c3d..5548c076e 100644 --- a/plugins/google/src/utils.ts +++ b/plugins/google/src/utils.ts @@ -139,7 +139,7 @@ function isEmptyObjectSchema(jsonSchema: JSONSchema7Definition): boolean { export function toFunctionDeclarations(toolCtx: llm.ToolContext): FunctionDeclaration[] { const functionDeclarations: FunctionDeclaration[] = []; - for (const [name, tool] of Object.entries(toolCtx)) { + for (const [name, tool] of Object.entries(toolCtx.functionTools)) { const { description, parameters } = tool; const jsonSchema = llm.toJsonSchema(parameters, false); diff --git a/plugins/mistralai/src/llm.ts b/plugins/mistralai/src/llm.ts index f0c07f7a2..f6685b042 100644 --- a/plugins/mistralai/src/llm.ts +++ b/plugins/mistralai/src/llm.ts @@ -123,7 +123,7 @@ export class LLM extends llm.LLM { extraKwargs, }: { chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: llm.ToolChoice; @@ -187,7 +187,7 @@ export class LLMStream extends llm.LLMStream { client: Mistral; opts: LLMOpts; chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions: APIConnectOptions; extraKwargs: Record; }, @@ -211,8 +211,8 @@ export class LLMStream extends llm.LLMStream { // eslint-disable-next-line @typescript-eslint/no-explicit-any const toolsList: any[] = []; - if (this.toolCtx && Object.keys(this.toolCtx).length > 0) { - for (const [name, func] of Object.entries(this.toolCtx)) { + if (this.toolCtx && Object.keys(this.toolCtx.functionTools).length > 0) { + for (const [name, func] of Object.entries(this.toolCtx.functionTools)) { toolsList.push({ type: 'function' as const, function: { diff --git a/plugins/openai/src/llm.ts b/plugins/openai/src/llm.ts index e96551011..a358f5abd 100644 --- a/plugins/openai/src/llm.ts +++ b/plugins/openai/src/llm.ts @@ -468,19 +468,20 @@ export class LLM extends llm.LLM { chat({ chatCtx, - toolCtx, + toolCtx: toolCtxInput, connOptions = DEFAULT_API_CONNECT_OPTIONS, parallelToolCalls, toolChoice, extraKwargs, }: { chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: llm.ToolChoice; extraKwargs?: Record; }): LLMStream { + const toolCtx = llm.toToolContext(toolCtxInput); const extras: Record = { ...extraKwargs }; if (this.#opts.metadata) { @@ -509,7 +510,11 @@ export class LLM extends llm.LLM { parallelToolCalls = parallelToolCalls !== undefined ? parallelToolCalls : this.#opts.parallelToolCalls; - if (toolCtx && Object.keys(toolCtx).length > 0 && parallelToolCalls !== undefined) { + if ( + toolCtx && + Object.keys(toolCtx.functionTools).length > 0 && + parallelToolCalls !== undefined + ) { extras.parallel_tool_calls = parallelToolCalls; } diff --git a/plugins/openai/src/realtime/realtime_model.ts b/plugins/openai/src/realtime/realtime_model.ts index 3a6a82095..8481e0f47 100644 --- a/plugins/openai/src/realtime/realtime_model.ts +++ b/plugins/openai/src/realtime/realtime_model.ts @@ -413,7 +413,7 @@ function processBaseURL({ * - openai_client_event_queued: expose the raw client events sent to the OpenAI Realtime API */ export class RealtimeSession extends llm.RealtimeSession { - private _tools: llm.ToolContext = {}; + private _tools: llm.ToolContext = llm.ToolContext.empty(); private remoteChatCtx: llm.RemoteChatContext = new llm.RemoteChatContext(); private messageChannel = new Queue(); private inputResampler?: AudioResampler; @@ -536,7 +536,7 @@ export class RealtimeSession extends llm.RealtimeSession { } get tools() { - return { ...this._tools } as llm.ToolContext; + return this._tools.copy(); } async updateChatCtx(_chatCtx: llm.ChatContext): Promise { @@ -698,13 +698,11 @@ export class RealtimeSession extends llm.RealtimeSession { // TODO(brian): these logics below are noops I think, leaving it here to keep // parity with the python but we should remove them later const retainedToolNames = new Set(ev.session.tools.map((tool) => tool.name)); - const retainedTools = Object.fromEntries( - Object.entries(_tools).filter( - ([name, tool]) => llm.isFunctionTool(tool) && retainedToolNames.has(name), - ), - ); + const retainedTools = Object.entries(_tools.functionTools) + .filter(([name]) => retainedToolNames.has(name)) + .map(([, tool]) => tool); - this._tools = retainedTools as llm.ToolContext; + this._tools = new llm.ToolContext(retainedTools); unlock(); } @@ -712,12 +710,7 @@ export class RealtimeSession extends llm.RealtimeSession { private createToolsUpdateEvent(_tools: llm.ToolContext): api_proto.SessionUpdateEvent { const oaiTools: api_proto.Tool[] = []; - for (const [name, tool] of Object.entries(_tools)) { - if (!llm.isFunctionTool(tool)) { - this.#logger.error({ name, tool }, "OpenAI Realtime API doesn't support this tool type"); - continue; - } - + for (const [name, tool] of Object.entries(_tools.functionTools)) { const { parameters: toolParameters, description } = tool; try { const parameters = llm.toJsonSchema( @@ -998,7 +991,7 @@ export class RealtimeSession extends llm.RealtimeSession { events.push(this.createSessionUpdateEvent()); // tools - if (Object.keys(this._tools).length > 0) { + if (Object.keys(this._tools.functionTools).length > 0) { events.push(this.createToolsUpdateEvent(this._tools)); } diff --git a/plugins/openai/src/responses/llm.ts b/plugins/openai/src/responses/llm.ts index 494a05c87..9a255d046 100644 --- a/plugins/openai/src/responses/llm.ts +++ b/plugins/openai/src/responses/llm.ts @@ -77,25 +77,30 @@ class ResponsesHttpLLM extends llm.LLM { override chat({ chatCtx, - toolCtx, + toolCtx: toolCtxInput, connOptions = DEFAULT_API_CONNECT_OPTIONS, parallelToolCalls, toolChoice, extraKwargs, }: { chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: llm.ToolChoice; extraKwargs?: Record; }): ResponsesHttpLLMStream { + const toolCtx = llm.toToolContext(toolCtxInput); const modelOptions: Record = { ...(extraKwargs || {}) }; parallelToolCalls = parallelToolCalls !== undefined ? parallelToolCalls : this.#opts.parallelToolCalls; - if (toolCtx && Object.keys(toolCtx).length > 0 && parallelToolCalls !== undefined) { + if ( + toolCtx && + Object.keys(toolCtx.functionTools).length > 0 && + parallelToolCalls !== undefined + ) { modelOptions.parallel_tool_calls = parallelToolCalls; } @@ -182,7 +187,7 @@ class ResponsesHttpLLMStream extends llm.LLMStream { )) as OpenAI.Responses.ResponseInputItem[]; const tools = this.toolCtx - ? Object.entries(this.toolCtx).map(([name, func]) => { + ? Object.entries(this.toolCtx.functionTools).map(([name, func]) => { const oaiParams = { type: 'function' as const, name: name, @@ -417,7 +422,7 @@ export class LLM extends llm.LLM { extraKwargs, }: { chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: llm.ToolChoice; diff --git a/plugins/openai/src/ws/llm.ts b/plugins/openai/src/ws/llm.ts index 1dd6483c6..d22d7a753 100644 --- a/plugins/openai/src/ws/llm.ts +++ b/plugins/openai/src/ws/llm.ts @@ -231,24 +231,29 @@ export class WSLLM extends llm.LLM { chat({ chatCtx, - toolCtx, + toolCtx: toolCtxInput, connOptions = DEFAULT_API_CONNECT_OPTIONS, parallelToolCalls, toolChoice, extraKwargs, }: { chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: llm.ToolChoice; extraKwargs?: Record; }): WSLLMStream { + const toolCtx = llm.toToolContext(toolCtxInput); const modelOptions: Record = { ...(extraKwargs ?? {}) }; parallelToolCalls = parallelToolCalls !== undefined ? parallelToolCalls : this.#opts.parallelToolCalls; - if (toolCtx && Object.keys(toolCtx).length > 0 && parallelToolCalls !== undefined) { + if ( + toolCtx && + Object.keys(toolCtx.functionTools).length > 0 && + parallelToolCalls !== undefined + ) { modelOptions.parallel_tool_calls = parallelToolCalls; } @@ -425,7 +430,7 @@ export class WSLLMStream extends llm.LLMStream { )) as OpenAI.Responses.ResponseInputItem[]; const tools = this.toolCtx - ? Object.entries(this.toolCtx).map(([name, func]) => { + ? Object.entries(this.toolCtx.functionTools).map(([name, func]) => { const oaiParams = { type: 'function' as const, name, diff --git a/plugins/phonic/src/realtime/realtime_model.ts b/plugins/phonic/src/realtime/realtime_model.ts index 665c5c5ba..09933b580 100644 --- a/plugins/phonic/src/realtime/realtime_model.ts +++ b/plugins/phonic/src/realtime/realtime_model.ts @@ -239,7 +239,7 @@ interface GenerationState { * Realtime session for Phonic (https://docs.phonic.co/) */ export class RealtimeSession extends llm.RealtimeSession { - private _tools: llm.ToolContext = {}; + private _tools: llm.ToolContext = llm.ToolContext.empty(); private _chatCtx = llm.ChatContext.empty(); private options: RealtimeModelOptions; @@ -290,7 +290,7 @@ export class RealtimeSession extends llm.RealtimeSession { } get tools(): llm.ToolContext { - return { ...this._tools }; + return this._tools.copy(); } async updateInstructions(instructions: string): Promise { @@ -367,26 +367,24 @@ export class RealtimeSession extends llm.RealtimeSession { return; } - this._tools = { ...tools }; - this.toolDefinitions = Object.entries(tools) - .filter(([_, tool]) => llm.isFunctionTool(tool)) - .map(([name, tool]) => ({ - type: 'custom_websocket', - tool_schema: { - type: 'function', - function: { - name, - description: tool.description, - parameters: llm.toJsonSchema(tool.parameters), - strict: true, - }, + this._tools = tools.copy(); + this.toolDefinitions = Object.entries(tools.functionTools).map(([name, tool]) => ({ + type: 'custom_websocket', + tool_schema: { + type: 'function', + function: { + name, + description: tool.description, + parameters: llm.toJsonSchema(tool.parameters), + strict: true, }, - tool_call_output_timeout_ms: TOOL_CALL_OUTPUT_TIMEOUT_MS, - // Tool chaining and tool calls during speech are not supported at this time - // for ease of implementation within the RealtimeSession generations framework - wait_for_speech_before_tool_call: true, - allow_tool_chaining: false, - })); + }, + tool_call_output_timeout_ms: TOOL_CALL_OUTPUT_TIMEOUT_MS, + // Tool chaining and tool calls during speech are not supported at this time + // for ease of implementation within the RealtimeSession generations framework + wait_for_speech_before_tool_call: true, + allow_tool_chaining: false, + })); this.toolsReady.resolve(); } @@ -405,24 +403,22 @@ export class RealtimeSession extends llm.RealtimeSession { this.options.instructions = instructions; } if (tools !== undefined) { - this._tools = { ...tools }; - this.toolDefinitions = Object.entries(tools) - .filter(([, tool]) => llm.isFunctionTool(tool)) - .map(([name, tool]) => ({ - type: 'custom_websocket', - tool_schema: { - type: 'function', - function: { - name, - description: tool.description, - parameters: llm.toJsonSchema(tool.parameters), - strict: true, - }, + this._tools = tools.copy(); + this.toolDefinitions = Object.entries(tools.functionTools).map(([name, tool]) => ({ + type: 'custom_websocket', + tool_schema: { + type: 'function', + function: { + name, + description: tool.description, + parameters: llm.toJsonSchema(tool.parameters), + strict: true, }, - tool_call_output_timeout_ms: TOOL_CALL_OUTPUT_TIMEOUT_MS, - wait_for_speech_before_tool_call: true, - allow_tool_chaining: false, - })); + }, + tool_call_output_timeout_ms: TOOL_CALL_OUTPUT_TIMEOUT_MS, + wait_for_speech_before_tool_call: true, + allow_tool_chaining: false, + })); } if (chatCtx !== undefined) { this._chatCtx = chatCtx.copy(); diff --git a/plugins/test/src/llm.ts b/plugins/test/src/llm.ts index 8d85c75c3..534dce2df 100644 --- a/plugins/test/src/llm.ts +++ b/plugins/test/src/llm.ts @@ -5,8 +5,9 @@ import { initializeLogger, llm as llmlib } from '@livekit/agents'; import { describe, expect, it } from 'vitest'; import { z } from 'zod/v4'; -const toolCtx: llmlib.ToolContext = { - getWeather: llmlib.tool({ +const toolCtx = new llmlib.ToolContext([ + llmlib.tool({ + name: 'getWeather', description: 'Get the current weather in a given location', parameters: z.object({ location: z.string().describe('The city and state, e.g. San Francisco, CA'), @@ -14,14 +15,16 @@ const toolCtx: llmlib.ToolContext = { }), execute: async () => {}, }), - playMusic: llmlib.tool({ + llmlib.tool({ + name: 'playMusic', description: 'Play music', parameters: z.object({ name: z.string().describe('The artist and name of the song'), }), execute: async () => {}, }), - toggleLight: llmlib.tool({ + llmlib.tool({ + name: 'toggleLight', description: 'Turn on/off the lights in a room', parameters: z.object({ name: z.string().describe('The room to control'), @@ -31,7 +34,8 @@ const toolCtx: llmlib.ToolContext = { await new Promise((resolve) => setTimeout(resolve, 60_000)); }, }), - selectCurrencies: llmlib.tool({ + llmlib.tool({ + name: 'selectCurrencies', description: 'Currencies of a specific area', parameters: z.object({ currencies: z @@ -40,7 +44,8 @@ const toolCtx: llmlib.ToolContext = { }), execute: async () => {}, }), - updateUserInfo: llmlib.tool({ + llmlib.tool({ + name: 'updateUserInfo', description: 'Update user info.', parameters: z.object({ email: z.string().optional().describe("User's email address"), @@ -49,18 +54,20 @@ const toolCtx: llmlib.ToolContext = { }), execute: async () => {}, }), - simulateFailure: llmlib.tool({ + llmlib.tool({ + name: 'simulateFailure', description: 'Simulate a failure', parameters: z.object({}), execute: async () => { throw new Error('Simulated failure'); }, }), -}; +]); // Tool context for strict mode - uses nullable() instead of optional() -const toolCtxStrict: llmlib.ToolContext = { - getWeather: llmlib.tool({ +const toolCtxStrict = new llmlib.ToolContext([ + llmlib.tool({ + name: 'getWeather', description: 'Get the current weather in a given location', parameters: z.object({ location: z.string().describe('The city and state, e.g. San Francisco, CA'), @@ -68,14 +75,16 @@ const toolCtxStrict: llmlib.ToolContext = { }), execute: async () => {}, }), - playMusic: llmlib.tool({ + llmlib.tool({ + name: 'playMusic', description: 'Play music', parameters: z.object({ name: z.string().describe('The artist and name of the song'), }), execute: async () => {}, }), - toggleLight: llmlib.tool({ + llmlib.tool({ + name: 'toggleLight', description: 'Turn on/off the lights in a room', parameters: z.object({ name: z.string().describe('The room to control'), @@ -85,7 +94,8 @@ const toolCtxStrict: llmlib.ToolContext = { await new Promise((resolve) => setTimeout(resolve, 60_000)); }, }), - selectCurrencies: llmlib.tool({ + llmlib.tool({ + name: 'selectCurrencies', description: 'Currencies of a specific area', parameters: z.object({ currencies: z @@ -94,7 +104,8 @@ const toolCtxStrict: llmlib.ToolContext = { }), execute: async () => {}, }), - updateUserInfo: llmlib.tool({ + llmlib.tool({ + name: 'updateUserInfo', description: 'Update user info.', parameters: z.object({ email: z.string().nullable().describe("User's email address"), @@ -103,14 +114,15 @@ const toolCtxStrict: llmlib.ToolContext = { }), execute: async () => {}, }), - simulateFailure: llmlib.tool({ + llmlib.tool({ + name: 'simulateFailure', description: 'Simulate a failure', parameters: z.object({}), execute: async () => { throw new Error('Simulated failure'); }, }), -}; +]); export const llm = async (llm: llmlib.LLM, skipOptionalArgs: boolean) => { initializeLogger({ pretty: false }); @@ -315,7 +327,7 @@ const executeCalls = async (calls: llmlib.FunctionCall[]) => { const results: llmlib.FunctionCallOutput[] = []; for (const call of calls) { - const tool = toolCtx[call.name]; + const tool = toolCtx.getFunctionTool(call.name); if (!tool) { throw new Error(`Tool ${call.name} not found`); } From 6b85bfea91af00a17a80860a674aff456ba5c9df Mon Sep 17 00:00:00 2001 From: Brian Yin Date: Tue, 2 Jun 2026 01:23:21 +0800 Subject: [PATCH 02/15] feat(agents): add Toolset support to ToolContext and AgentActivity (#1525) Co-authored-by: rosetta-livekit-bot[bot] <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Co-authored-by: u9g --- .changeset/cold-avocados-behave.md | 5 + .changeset/gemini-provider-tools.md | 5 + .changeset/list-syntax-toolcontext.md | 15 +- .changeset/openai-provider-tools.md | 5 + .changeset/quick-meals-breathe.md | 5 + agents/src/generator.test.ts | 19 + agents/src/generator.ts | 23 +- agents/src/index.test.ts | 23 + agents/src/index.ts | 6 +- agents/src/llm/chat_context.test.ts | 5 +- agents/src/llm/index.ts | 9 +- agents/src/llm/llm.ts | 2 +- agents/src/llm/tool_context.test.ts | 163 +++++- agents/src/llm/tool_context.ts | 320 ++++++++---- agents/src/llm/tool_context.type.test.ts | 21 +- agents/src/utils.test.ts | 79 +++ agents/src/utils.ts | 52 +- agents/src/voice/agent.test.ts | 356 ++++++++++++- agents/src/voice/agent.ts | 25 + agents/src/voice/agent_activity.ts | 59 ++- agents/src/voice/agent_v2.ts | 484 ++++++++++++++++++ agents/src/voice/index.ts | 8 + examples/src/basic_agent.ts | 20 +- examples/src/basic_agent_task.ts | 180 +++---- examples/src/basic_toolsets.ts | 180 +++++++ examples/src/gemini_realtime_agent.ts | 8 +- examples/src/multi_agent.ts | 8 +- .../google/src/beta/realtime/realtime_api.ts | 43 +- plugins/google/src/index.ts | 1 + plugins/google/src/llm.ts | 12 +- plugins/google/src/tools.ts | 98 +++- plugins/google/src/utils.ts | 62 ++- plugins/mistralai/src/llm.ts | 12 +- plugins/openai/src/index.ts | 1 + plugins/openai/src/realtime/realtime_model.ts | 26 +- plugins/openai/src/responses/llm.ts | 20 +- plugins/openai/src/tool_utils.test.ts | 84 +++ plugins/openai/src/tool_utils.ts | 43 ++ plugins/openai/src/tools.ts | 166 ++++++ plugins/openai/src/ws/llm.ts | 22 +- plugins/phonic/src/realtime/realtime_model.ts | 66 +-- plugins/test/src/llm.ts | 51 ++ 42 files changed, 2399 insertions(+), 393 deletions(-) create mode 100644 .changeset/cold-avocados-behave.md create mode 100644 .changeset/gemini-provider-tools.md create mode 100644 .changeset/openai-provider-tools.md create mode 100644 .changeset/quick-meals-breathe.md create mode 100644 agents/src/generator.test.ts create mode 100644 agents/src/index.test.ts create mode 100644 agents/src/voice/agent_v2.ts create mode 100644 examples/src/basic_toolsets.ts create mode 100644 plugins/openai/src/tool_utils.test.ts create mode 100644 plugins/openai/src/tool_utils.ts create mode 100644 plugins/openai/src/tools.ts diff --git a/.changeset/cold-avocados-behave.md b/.changeset/cold-avocados-behave.md new file mode 100644 index 000000000..a2eef8a7b --- /dev/null +++ b/.changeset/cold-avocados-behave.md @@ -0,0 +1,5 @@ +--- +"@livekit/agents": patch +--- + +Add Agent.create method diff --git a/.changeset/gemini-provider-tools.md b/.changeset/gemini-provider-tools.md new file mode 100644 index 000000000..3b1093432 --- /dev/null +++ b/.changeset/gemini-provider-tools.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-google': minor +--- + +Add Gemini provider tools for Google Search, Google Maps, URL context, File Search, code execution, and Vertex RAG retrieval, and serialize them from `ToolContext` for Google LLM and realtime sessions. diff --git a/.changeset/list-syntax-toolcontext.md b/.changeset/list-syntax-toolcontext.md index 0a4e0bfa0..5e854a813 100644 --- a/.changeset/list-syntax-toolcontext.md +++ b/.changeset/list-syntax-toolcontext.md @@ -1,5 +1,16 @@ --- -"@livekit/agents": minor +'@livekit/agents': minor --- -**BREAKING**: `Agent({ tools })` and `agent.updateTools()` now accept a flat list `(FunctionTool | ProviderDefinedTool)[]` instead of a `Record` map, and `llm.tool({ ... })` requires a `name` field. `ToolContext` is now a Python-parity class with `functionTools` / `providerTools` / `toolsets` accessors, plus `flatten()`, `hasTool(name)`, `getFunctionTool(name)`, `updateTools()`, `copy()`, and `equals()`. To match the Python reference, registering two **different** function-tool instances under the same `name` now throws `duplicate function name: ` instead of silently overriding the earlier entry; passing the **same instance** twice is a no-op. `agent.toolCtx` returns a defensive copy so callers can no longer mutate the agent's internal state. `LLM.chat({ toolCtx })` accepts either a `ToolContext` instance or a raw `(FunctionTool | ProviderDefinedTool)[]` array (`ToolCtxInput`) and normalizes it internally, so callers don't have to construct a `ToolContext` themselves. Stateful `Toolset` containers are not part of this release — the `toolsets` accessor currently returns an empty list and `TODO`s in `tool_context.ts` mark every site where Python's Toolset support will plug in later. +**BREAKING**: `Agent({ tools })` and `agent.updateTools()` now accept a flat list `(FunctionTool | ProviderTool | Toolset)[]` instead of a `Record` map, and `llm.tool({ ... })` requires a `name` field. `ToolContext` is now a Python-parity class with `functionTools` / `providerTools` / `toolsets` accessors, plus `flatten()`, `hasTool(id)`, `getFunctionTool(id)`, `updateTools()`, `copy()`, and `equals()`. To match the Python reference, registering two **different** function-tool instances under the same `name` now throws `duplicate function name: ` instead of silently overriding the earlier entry; passing the **same instance** twice is a no-op. `agent.toolCtx` returns a defensive copy so callers can no longer mutate the agent's internal state. `LLM.chat({ toolCtx })` accepts either a `ToolContext` instance or a raw `(FunctionTool | ProviderTool | Toolset)[]` array (`ToolCtxInput`) and normalizes it internally, so callers don't have to construct a `ToolContext` themselves. + +Tools also expose an `id: string` field on the base `Tool` interface (parity with Python's `Tool.id` property): for `FunctionTool` it mirrors `name`, for `ProviderTool` it is the provider tool id. `ToolContext` keys and equality now use `tool.id` consistently. + +**BREAKING**: Provider tools are now modeled to match Python's `ProviderTool`: + +- `ProviderDefinedTool` is renamed to `ProviderTool`, and `isProviderDefinedTool` is renamed to `isProviderTool`. +- `ProviderTool` is now an **abstract class** (Python parity). Plugins must subclass it (`class WebSearch extends ProviderTool { ... }`) to attach provider-specific fields and serializers; bare `new ProviderTool(...)` is rejected at compile time. +- The `tool({ id })` factory overload is removed; `tool({ ... })` only creates function tools now. Construct provider tools by instantiating a `ProviderTool` subclass. +- The `ToolType` literal for provider tools is renamed from `'provider-defined'` to `'provider'`. + +`Toolset` now carries a `TOOLSET_SYMBOL` marker and is detected via a new `isToolset()` guard (consistent with `isFunctionTool` / `isProviderTool`). Existing `instanceof Toolset` checks still work, but symbol-based detection is preferred for cross-realm safety. diff --git a/.changeset/openai-provider-tools.md b/.changeset/openai-provider-tools.md new file mode 100644 index 000000000..8e793a935 --- /dev/null +++ b/.changeset/openai-provider-tools.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-openai': minor +--- + +Add OpenAI Responses provider tools for web search, file search, and code interpreter. diff --git a/.changeset/quick-meals-breathe.md b/.changeset/quick-meals-breathe.md new file mode 100644 index 000000000..d32233b93 --- /dev/null +++ b/.changeset/quick-meals-breathe.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Adds base `Toolset` support: a stateful container for a group of tools with `setup()` / `aclose()` lifecycle hooks. Toolsets can be passed directly into `Agent({ tools: [...] })` alongside individual function tools; their tools are flattened into the agent's `ToolContext` and the runtime drives `setup()` on activity start, `aclose()` on close, and a setup/close diff when `agent.updateTools()` adds or removes Toolsets mid-session. Per-toolset `setup()` errors are logged but do not abort the activity. The `IGNORE_ON_ENTER` flag is also respected for function tools nested inside a Toolset. Every LLM and realtime plugin tool builder iterates `ToolContext.flatten()` so toolset-contributed tools are correctly advertised. Also exports `ToolCalledEvent` / `ToolCompletedEvent` payload types. diff --git a/agents/src/generator.test.ts b/agents/src/generator.test.ts new file mode 100644 index 000000000..a92ff9e58 --- /dev/null +++ b/agents/src/generator.test.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { describe, expect, it } from 'vitest'; +import { defineAgent, isAgent } from './generator.js'; + +describe('generator', () => { + it('marks definitions created with defineAgent as agents', () => { + const agent = defineAgent({ + entry: async () => {}, + }); + + expect(isAgent(agent)).toBe(true); + }); + + it('does not treat unmarked structural objects as agents', () => { + expect(isAgent({ entry: async () => {} })).toBe(false); + }); +}); diff --git a/agents/src/generator.ts b/agents/src/generator.ts index b7e7ad93f..e9bb09075 100644 --- a/agents/src/generator.ts +++ b/agents/src/generator.ts @@ -3,24 +3,22 @@ // SPDX-License-Identifier: Apache-2.0 import type { JobContext, JobProcess } from './job.js'; +export const AGENT_DEFINITION_SYMBOL = Symbol.for('livekit.agents.AgentDefinition'); + /** @see {@link defineAgent} */ -export interface Agent> { +export interface AgentDefinition> { entry: (ctx: JobContext) => Promise; prewarm?: (proc: JobProcess) => unknown; } +export type Agent> = AgentDefinition; + /** Helper to check if an object is an agent before running it. * * @internal */ -export function isAgent(obj: unknown): obj is Agent { - return ( - typeof obj === 'object' && - obj !== null && - 'entry' in obj && - typeof (obj as Agent).entry === 'function' && - (('prewarm' in obj && typeof (obj as Agent).prewarm === 'function') || !('prewarm' in obj)) - ); +export function isAgent(obj: unknown): obj is AgentDefinition { + return typeof obj === 'object' && obj !== null && AGENT_DEFINITION_SYMBOL in obj; } /** @@ -34,7 +32,10 @@ export function isAgent(obj: unknown): obj is Agent { * ``` */ export function defineAgent>( - agent: Agent, -): Agent { + agent: AgentDefinition, +): AgentDefinition { + Object.defineProperty(agent, AGENT_DEFINITION_SYMBOL, { + value: true, + }); return agent; } diff --git a/agents/src/index.test.ts b/agents/src/index.test.ts new file mode 100644 index 000000000..40b7ade44 --- /dev/null +++ b/agents/src/index.test.ts @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { describe, expect, it } from 'vitest'; +import { + Agent, + AgentSession, + ChatContext, + ModelUsageCollector, + logMetrics, + tool, +} from './index.js'; + +describe('index exports', () => { + it('exports voice, llm, and metrics APIs directly from the package root', () => { + expect(Agent).toBeDefined(); + expect(AgentSession).toBeDefined(); + expect(ChatContext).toBeDefined(); + expect(tool).toBeDefined(); + expect(ModelUsageCollector).toBeDefined(); + expect(logMetrics).toBeDefined(); + }); +}); diff --git a/agents/src/index.ts b/agents/src/index.ts index 07e4b45da..9f1519bf3 100644 --- a/agents/src/index.ts +++ b/agents/src/index.ts @@ -14,14 +14,16 @@ export * from './audio.js'; export * as beta from './beta/index.js'; export * as cli from './cli.js'; export * from './connection_pool.js'; -export * from './generator.js'; +export { defineAgent, isAgent, type AgentDefinition } from './generator.js'; export * as inference from './inference/index.js'; export * from './inference_runner.js'; export * as ipc from './ipc/index.js'; export * from './job.js'; export * from './language.js'; +export * from './llm/index.js'; export * as llm from './llm/index.js'; export * from './log.js'; +export * from './metrics/index.js'; export * as metrics from './metrics/index.js'; export * from './plugin.js'; export * as stream from './stream/index.js'; @@ -34,6 +36,6 @@ export * from './types.js'; export * from './utils.js'; export * from './vad.js'; export * from './version.js'; +export * from './voice/index.js'; export * as voice from './voice/index.js'; -export { createTimedString, isTimedString, type TimedString } from './voice/io.js'; export * from './worker.js'; diff --git a/agents/src/llm/chat_context.test.ts b/agents/src/llm/chat_context.test.ts index 350dcd1cd..101a9fe46 100644 --- a/agents/src/llm/chat_context.test.ts +++ b/agents/src/llm/chat_context.test.ts @@ -19,7 +19,7 @@ import { isInstructions, renderInstructions, } from './chat_context.js'; -import { ToolContext, tool } from './tool_context.js'; +import { ProviderTool, ToolContext, tool } from './tool_context.js'; initializeLogger({ pretty: false, level: 'error' }); @@ -1498,7 +1498,8 @@ describe('ChatContext.copy with toolCtx filter', () => { }); it('keeps provider-tool calls when the ToolContext holds a matching provider tool id', () => { - const provider = tool({ id: 'code_runner', config: {} }); + class CodeRunner extends ProviderTool {} + const provider = new CodeRunner({ id: 'code_runner' }); const ctx = new ChatContext([ FunctionCall.create({ callId: 'p1', name: 'code_runner', args: '{}' }), FunctionCall.create({ callId: 'p2', name: 'other', args: '{}' }), diff --git a/agents/src/llm/index.ts b/agents/src/llm/index.ts index 4837f2cb9..098a84fca 100644 --- a/agents/src/llm/index.ts +++ b/agents/src/llm/index.ts @@ -4,21 +4,26 @@ export { handoff, isFunctionTool, - isProviderDefinedTool, + isProviderTool, isTool, + isToolset, + ProviderTool, tool, ToolContext, ToolError, ToolFlag, + Toolset, toToolContext, type AgentHandoff, type FunctionTool, - type ProviderDefinedTool, type Tool, + type ToolCalledEvent, type ToolChoice, + type ToolCompletedEvent, type ToolContextEntry, type ToolCtxInput, type ToolOptions, + type ToolsetCreateOptions, type ToolType, } from './tool_context.js'; diff --git a/agents/src/llm/llm.ts b/agents/src/llm/llm.ts index 0c05bbb2d..828e134e9 100644 --- a/agents/src/llm/llm.ts +++ b/agents/src/llm/llm.ts @@ -98,7 +98,7 @@ export abstract class LLM extends (EventEmitter as new () => TypedEmitter { @@ -448,8 +455,20 @@ describe('tool() name requirement', () => { }); expect(t.name).toBe('doStuff'); }); + + it('exposes id mirroring the function tool name', () => { + const t = tool({ + name: 'doStuff', + description: 'd', + execute: async () => 'x', + }); + expect(t.id).toBe('doStuff'); + expect(t.id).toBe(t.name); + }); }); +class TestProviderTool extends ProviderTool {} + describe('ToolContext', () => { const makeFn = (name: string) => tool({ @@ -497,7 +516,7 @@ describe('ToolContext', () => { it('separates provider tools from function tools', () => { const fnA = makeFn('a'); - const provider = tool({ id: 'code', config: { language: 'python' } }); + const provider = new TestProviderTool({ id: 'code' }); const ctx = new ToolContext([fnA, provider]); expect(ctx.functionTools).toEqual({ a: fnA }); @@ -537,7 +556,7 @@ describe('ToolContext', () => { it('equals() is reflexive', () => { const a = makeFn('a'); - const provider = tool({ id: 'code', config: { language: 'python' } }); + const provider = new TestProviderTool({ id: 'code' }); const ctx = new ToolContext([a, provider]); expect(ctx.equals(ctx)).toBe(true); }); @@ -547,22 +566,22 @@ describe('ToolContext', () => { // that hold the same provider-tool identities in different order are still equal so // realtime-session / preemptive-generation reuse fast paths are not invalidated. const a = makeFn('a'); - const p1 = tool({ id: 'code', config: { language: 'python' } }); - const p2 = tool({ id: 'browser', config: {} }); + const p1 = new TestProviderTool({ id: 'code' }); + const p2 = new TestProviderTool({ id: 'browser' }); expect(new ToolContext([a, p1, p2]).equals(new ToolContext([a, p2, p1]))).toBe(true); }); it('equals() supports contexts with only provider tools', () => { - const p1 = tool({ id: 'code', config: {} }); - const p2 = tool({ id: 'browser', config: {} }); + const p1 = new TestProviderTool({ id: 'code' }); + const p2 = new TestProviderTool({ id: 'browser' }); expect(new ToolContext([p1, p2]).equals(new ToolContext([p1, p2]))).toBe(true); - const p3 = tool({ id: 'code', config: {} }); // distinct identity, same id + const p3 = new TestProviderTool({ id: 'code' }); // distinct identity, same id expect(new ToolContext([p1]).equals(new ToolContext([p3]))).toBe(false); }); it('hasTool() matches function tools by name and provider tools by id', () => { const a = makeFn('a'); - const provider = tool({ id: 'code_runner', config: {} }); + const provider = new TestProviderTool({ id: 'code_runner' }); const ctx = new ToolContext([a, provider]); expect(ctx.hasTool('a')).toBe(true); @@ -574,9 +593,133 @@ describe('ToolContext', () => { // Matches Python's `flatten()`: list(self._fnc_tools_map.values()) + self._provider_tools. const a = makeFn('a'); const b = makeFn('b'); - const provider = tool({ id: 'code', config: {} }); + const provider = new TestProviderTool({ id: 'code' }); const ctx = new ToolContext([b, provider, a]); expect(ctx.flatten()).toEqual([b, a, provider]); }); }); + +describe('Toolset', () => { + const makeFn = (name: string) => + tool({ + name, + description: `${name} tool`, + execute: async () => name, + }); + + it('exposes its id and the tools it was constructed with', () => { + const a = makeFn('a'); + const b = makeFn('b'); + const ts = new Toolset({ id: 'set1', tools: [a, b] }); + + expect(ts.id).toBe('set1'); + expect(ts.tools).toEqual([a, b]); + }); + + it('default setup and aclose are no-ops', async () => { + const ts = new Toolset({ id: 'noop', tools: [] }); + await expect(ts.setup()).resolves.toBeUndefined(); + await expect(ts.aclose()).resolves.toBeUndefined(); + }); + + it('lets subclasses override lifecycle hooks', async () => { + const events: string[] = []; + class Recording extends Toolset { + override async setup(): Promise { + events.push(`setup:${this.id}`); + } + override async aclose(): Promise { + events.push(`close:${this.id}`); + } + } + + const ts = new Recording({ id: 'rec', tools: [] }); + await ts.setup(); + await ts.aclose(); + expect(events).toEqual(['setup:rec', 'close:rec']); + }); + + it('Toolset.create() composes lifecycle callbacks without subclassing', async () => { + const a = makeFn('a'); + const events: string[] = []; + const ts = Toolset.create({ + id: 'composed', + tools: [a], + setup: async () => { + events.push('setup'); + }, + aclose: async () => { + events.push('close'); + }, + }); + + expect(ts).toBeInstanceOf(Toolset); + expect(ts.id).toBe('composed'); + expect(ts.tools).toEqual([a]); + + await ts.setup(); + await ts.aclose(); + expect(events).toEqual(['setup', 'close']); + }); + + it('Toolset.create() defaults setup and aclose to no-ops when callbacks are omitted', async () => { + const ts = Toolset.create({ id: 'bare', tools: [] }); + await expect(ts.setup()).resolves.toBeUndefined(); + await expect(ts.aclose()).resolves.toBeUndefined(); + }); + + it('Toolset.create() accepts a tools thunk, re-evaluated on every access (dynamic)', () => { + const a = makeFn('a'); + const b = makeFn('b'); + const current: Tool[] = [a]; + let calls = 0; + const ts = Toolset.create({ + id: 'dynamic', + tools: () => { + calls += 1; + return current; + }, + }); + // Each access re-invokes the thunk so the toolset reflects the current source-of-truth. + expect(ts.tools).toEqual([a]); + expect(calls).toBe(1); + current.push(b); + expect(ts.tools).toEqual([a, b]); + expect(calls).toBe(2); + }); + + it('is flattened into a ToolContext: function tools merged, toolset tracked', () => { + const a = makeFn('a'); + const b = makeFn('b'); + const ts = new Toolset({ id: 'set', tools: [a, b] }); + const direct = makeFn('direct'); + + const ctx = new ToolContext([direct, ts]); + + expect(Object.keys(ctx.functionTools).sort()).toEqual(['a', 'b', 'direct']); + expect(ctx.toolsets).toEqual([ts]); + }); + + it('throws when a Toolset contributes a duplicate function name', () => { + // Mirrors Python's `add_tool`: a name collision between top-level and toolset-contributed + // tools is an error, not silent overwrite. + const a1 = makeFn('a'); + const a2 = makeFn('a'); + const ts = new Toolset({ id: 'collides', tools: [a2] }); + + expect(() => new ToolContext([a1, ts])).toThrow(/duplicate function name: a/); + }); + + it('equals() compares toolsets as identity sets, not by order', () => { + // Matches Python's `{id(ts) for ts in self._tool_sets}` semantics. + const ts1 = new Toolset({ id: 'one', tools: [] }); + const ts2 = new Toolset({ id: 'two', tools: [] }); + + expect(new ToolContext([ts1, ts2]).equals(new ToolContext([ts2, ts1]))).toBe(true); + + const ts3 = new Toolset({ id: 'three', tools: [] }); + expect(new ToolContext([ts1, ts2]).equals(new ToolContext([ts1, ts3]))).toBe(false); + expect(new ToolContext([ts1]).equals(new ToolContext([ts1, ts2]))).toBe(false); + }); +}); diff --git a/agents/src/llm/tool_context.ts b/agents/src/llm/tool_context.ts index df714d57a..284d5d7cf 100644 --- a/agents/src/llm/tool_context.ts +++ b/agents/src/llm/tool_context.ts @@ -12,7 +12,8 @@ import { isZodObjectSchema, isZodSchema } from './zod-utils.js'; const TOOL_SYMBOL = Symbol('tool'); const FUNCTION_TOOL_SYMBOL = Symbol('function_tool'); -const PROVIDER_DEFINED_TOOL_SYMBOL = Symbol('provider_defined_tool'); +const PROVIDER_TOOL_SYMBOL = Symbol('provider_tool'); +const TOOLSET_SYMBOL = Symbol('toolset'); const TOOL_ERROR_SYMBOL = Symbol('tool_error'); const HANDOFF_SYMBOL = Symbol('handoff'); @@ -57,7 +58,7 @@ export type InferToolInput = T extends { _output: infer O } ? O : any; // eslint-disable-line @typescript-eslint/no-explicit-any -- Fallback type for JSON Schema objects without type inference -export type ToolType = 'function' | 'provider-defined'; +export type ToolType = 'function' | 'provider'; export type ToolChoice = | 'auto' @@ -136,28 +137,32 @@ export type ToolExecuteFunction< export interface Tool { /** * The type of the tool. - * @internal Either user-defined core tool or provider-defined tool. + * @internal Either user-defined function tool or provider-side tool. */ type: ToolType; + /** + * Stable identifier used to key the tool inside a `ToolContext`. For function tools this + * mirrors `name`; for provider tools this is the provider tool id. + */ + id: string; + [TOOL_SYMBOL]: true; } -// TODO(AJS-112): support provider-defined tools -export interface ProviderDefinedTool extends Tool { - type: 'provider-defined'; +// TODO(AJS-112): support provider tools +export abstract class ProviderTool implements Tool { + readonly type = 'provider' as const; - /** - * The ID of the tool. - */ - id: string; + readonly id: string; - /** - * The configuration of the tool. - */ - config: Record; + readonly [TOOL_SYMBOL] = true as const; - [PROVIDER_DEFINED_TOOL_SYMBOL]: true; + readonly [PROVIDER_TOOL_SYMBOL] = true as const; + + constructor({ id }: { id: string }) { + this.id = id; + } } export interface FunctionTool< @@ -169,7 +174,7 @@ export interface FunctionTool< /** * The name of the tool. Used to identify it inside a `ToolContext` and exposed to the LLM - * as the function name to call. + * as the function name to call. Also surfaced as the inherited `Tool.id`. */ name: string; @@ -196,6 +201,125 @@ export interface FunctionTool< [FUNCTION_TOOL_SYMBOL]: true; } +export interface ToolCalledEvent { + ctx: RunContext; + arguments: Record; +} + +export interface ToolCompletedEvent { + ctx: RunContext; + output?: { type: 'output'; value: unknown } | { type: 'error'; value: Error }; +} + +/** + * A stateful collection of tools sharing a lifecycle. Tools registered through a `Toolset` are + * flattened into the surrounding `ToolContext`, while the `Toolset` itself is tracked so its + * `setup()` / `aclose()` hooks can be invoked by the agent runtime. + */ +export class Toolset { + readonly #id: string; + + readonly #tools: Tool[]; + + readonly [TOOLSET_SYMBOL] = true as const; + + constructor({ id, tools }: { id: string; tools: readonly Tool[] }) { + this.#id = id; + this.#tools = [...tools]; + } + + /** + * Compose a `Toolset` with inline `setup` / `aclose` hooks instead of subclassing. `tools` + * may also be a thunk that is re-evaluated on every `.tools` access, so the toolset can + * expose a dynamic list that changes after `setup()` runs. + * + * @example Static tool list with a shared backing resource + * ```ts + * function createPostgresToolset(connectionUrl: string): Toolset { + * const pool = new pg.Pool({ connectionString: connectionUrl }); + * return Toolset.create({ + * id: 'postgres', + * tools: [queryOrders, queryCustomers], + * setup: () => pool.connect(), + * aclose: () => pool.end(), + * }); + * } + * ``` + * + * @example Dynamic tool list + * ```ts + * function createMcpToolset(url: string): Toolset { + * const client = new MCPClient({ url }); + * return Toolset.create({ + * id: 'mcp_remote', + * tools: () => client.getTools(), + * setup: () => client.connect(), + * aclose: () => client.disconnect(), + * }); + * } + * ``` + */ + static create(options: ToolsetCreateOptions): Toolset { + return new ToolsetFactory(options); + } + + get id(): string { + return this.#id; + } + + get tools(): readonly Tool[] { + return this.#tools; + } + + async setup(): Promise {} + + async aclose(): Promise {} +} + +/** Options accepted by `Toolset.create()` — id + tools plus optional lifecycle hooks. */ +export interface ToolsetCreateOptions { + id: string; + /** + * Either a static list of tools, or a thunk re-evaluated on every `tools` access — useful + * when the underlying source (e.g. an MCP discovery loop) can produce a dynamic tool list. + */ + tools: readonly Tool[] | (() => readonly Tool[]); + /** Invoked when the toolset becomes active in an `AgentActivity`. */ + setup?: () => Promise; + /** Invoked when the toolset is being torn down. */ + aclose?: () => Promise; +} + +/** Backing implementation of `Toolset.create()`. Kept private so callers go through the factory. */ +class ToolsetFactory extends Toolset { + readonly #toolsSource: readonly Tool[] | (() => readonly Tool[]); + + readonly #setupFn?: () => Promise; + + readonly #acloseFn?: () => Promise; + + constructor({ id, tools, setup, aclose }: ToolsetCreateOptions) { + // Pass [] to super and override the `tools` getter so a thunk can be re-evaluated on + // every access (lets callers expose a dynamic tool list). + super({ id, tools: [] }); + this.#toolsSource = tools; + this.#setupFn = setup; + this.#acloseFn = aclose; + } + + override get tools(): readonly Tool[] { + return typeof this.#toolsSource === 'function' ? this.#toolsSource() : this.#toolsSource; + } + + override async setup(): Promise { + if (this.#setupFn) await this.#setupFn(); + } + + override async aclose(): Promise { + if (this.#acloseFn) await this.#acloseFn(); + } +} + /** * Convenience input shape accepted by APIs that want to take a list of tools directly without * forcing callers to wrap them in `new ToolContext(...)`. @@ -217,24 +341,18 @@ export function toToolContext( return input instanceof ToolContext ? input : new ToolContext(input); } -//TODO: toolset - accept stateful `Toolset` containers alongside `FunctionTool` / // eslint-disable-next-line @typescript-eslint/no-explicit-any -- ToolContext entries accept any function-tool parameter/result types export type ToolContextEntry = // eslint-disable-next-line @typescript-eslint/no-explicit-any - FunctionTool | ProviderDefinedTool; + FunctionTool | ProviderTool | Toolset; export class ToolContext { - // TODO: toolset - widen entries to `FunctionTool | ProviderDefinedTool | Toolset` once Toolset - // lands so this stays heterogeneous like Python's `Sequence[Tool | Toolset]`. private _tools: ToolContextEntry[] = []; // eslint-disable-next-line @typescript-eslint/no-explicit-any -- ToolContext stores generic function tools private _functionToolsMap: Map> = new Map(); - private _providerTools: ProviderDefinedTool[] = []; - // TODO: toolset - populate when Toolset support is supported. - // so the `toolsets` getter and `equals` toolset-identity check stay byte-compatible with the - private _toolSets: unknown[] = []; + private _providerTools: ProviderTool[] = []; + private _toolsets: Toolset[] = []; - // TODO: toolset - widen `tools` to `Sequence` once Toolset lands. constructor(tools: readonly ToolContextEntry[] = []) { this.updateTools(tools); } @@ -250,17 +368,13 @@ export class ToolContext { } /** A copy of all provider tools in the tool context, including those in tool sets. */ - get providerTools(): ProviderDefinedTool[] { + get providerTools(): ProviderTool[] { return this._providerTools; } - /** - * A copy of all tool sets in the tool context. - * - * TODO: toolset - wire up once Toolset is ported. - */ - get toolsets(): unknown[] { - return this._toolSets; + /** A copy of all toolsets registered in the context. */ + get toolsets(): readonly Toolset[] { + return [...this._toolsets]; } /** @@ -276,53 +390,53 @@ export class ToolContext { } // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Generic registry over any parameter/result types - getFunctionTool(name: string): FunctionTool | undefined { - return this._functionToolsMap.get(name); + getFunctionTool(id: string): FunctionTool | undefined { + return this._functionToolsMap.get(id); } - hasTool(name: string): boolean { - if (this._functionToolsMap.has(name)) { + hasTool(id: string): boolean { + if (this._functionToolsMap.has(id)) { return true; } - return this._providerTools.some((tool) => tool.id === name); + return this._providerTools.some((tool) => tool.id === id); } - // TODO: toolset - widen `tools` to `Sequence` once Toolset lands. updateTools(tools: readonly ToolContextEntry[]): void { this._tools = [...tools]; this._functionToolsMap = new Map(); this._providerTools = []; - this._toolSets = []; + this._toolsets = []; - // Mirrors Python's recursive `add_tool` (minus Toolset flattening, which is TODO). // eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts any tool shape const addTool = (tool: any): void => { - if (isProviderDefinedTool(tool)) { + if (isToolset(tool)) { + for (const inner of tool.tools) { + addTool(inner); + } + this._toolsets.push(tool); + return; + } + + if (isProviderTool(tool)) { this._providerTools.push(tool); return; } if (isFunctionTool(tool)) { - const existing = this._functionToolsMap.get(tool.name); + const existing = this._functionToolsMap.get(tool.id); if (existing !== undefined) { if (existing !== tool) { - throw new Error(`duplicate function name: ${tool.name}`); + throw new Error(`duplicate function name: ${tool.id}`); } return; // same instance, skip } - this._functionToolsMap.set(tool.name, tool); + this._functionToolsMap.set(tool.id, tool); return; } - // TODO: toolset - if (tool instanceof Toolset) { for (const t of tool.tools) addTool(t); - // this._toolSets.push(tool); return; } - throw new Error(`unknown tool type: ${typeof tool}`); }; - // TODO: toolset - Python also chains `find_function_tools(self)` here so subclasses can - // declare tools as class members. JS doesn't use that decorator pattern, so we only walk - // the explicit input list. for (const tool of tools) { addTool(tool); } @@ -336,14 +450,17 @@ export class ToolContext { if (this._functionToolsMap.size !== other._functionToolsMap.size) { return false; } - for (const [name, tool] of this._functionToolsMap) { - if (other._functionToolsMap.get(name) !== tool) { + + for (const [id, tool] of this._functionToolsMap) { + if (other._functionToolsMap.get(id) !== tool) { return false; } } + if (this._providerTools.length !== other._providerTools.length) { return false; } + // Provider tools compare as identity sets to match Python's `set(id(t) for t in ...)` // semantics — order is not significant. const otherProviderIds = new Set(other._providerTools); @@ -352,10 +469,17 @@ export class ToolContext { return false; } } - // TODO: toolset - once Toolset lands, also compare `_toolSets` as identity sets per Python - // self_tool_set_ids = {id(ts) for ts in self._tool_sets} - // other_tool_set_ids = {id(ts) for ts in other._tool_sets} - // if self_tool_set_ids != other_tool_set_ids: return False + + if (this._toolsets.length !== other._toolsets.length) { + return false; + } + + const otherToolsets = new Set(other._toolsets); + for (const ts of this._toolsets) { + if (!otherToolsets.has(ts)) { + return false; + } + } return true; } } @@ -413,63 +537,36 @@ export function tool({ flags?: number; }): FunctionTool, UserData, Result>; -/** - * Create a provider-defined tool. - * - * @param id - The ID of the tool. - * @param config - The configuration of the tool. - */ -export function tool({ - id, - config, -}: { - id: string; - config: Record; -}): ProviderDefinedTool; - // eslint-disable-next-line @typescript-eslint/no-explicit-any export function tool(tool: any): any { - if (tool.execute !== undefined) { - if (typeof tool.name !== 'string' || tool.name.length === 0) { - throw new Error('tool({ name, ... }) requires a non-empty name'); - } - - // Default parameters to z.object({}) if not provided - const parameters = tool.parameters ?? z.object({}); - - // if parameters is a Zod schema, ensure it's an object schema - if (isZodSchema(parameters) && !isZodObjectSchema(parameters)) { - throw new Error('Tool parameters must be a Zod object schema (z.object(...))'); - } + if (typeof tool.name !== 'string' || tool.name.length === 0) { + throw new Error('tool({ name, ... }) requires a non-empty name'); + } - // Ensure parameters is either a Zod schema or a plain object (JSON schema) - if (!isZodSchema(parameters) && !(typeof parameters === 'object')) { - throw new Error('Tool parameters must be a Zod object schema or a raw JSON schema'); - } + // Default parameters to z.object({}) if not provided + const parameters = tool.parameters ?? z.object({}); - return { - type: 'function', - name: tool.name, - description: tool.description, - parameters, - execute: tool.execute, - flags: tool.flags ?? ToolFlag.NONE, - [TOOL_SYMBOL]: true, - [FUNCTION_TOOL_SYMBOL]: true, - }; + // if parameters is a Zod schema, ensure it's an object schema + if (isZodSchema(parameters) && !isZodObjectSchema(parameters)) { + throw new Error('Tool parameters must be a Zod object schema (z.object(...))'); } - if (tool.config !== undefined && tool.id !== undefined) { - return { - type: 'provider-defined', - id: tool.id, - config: tool.config, - [TOOL_SYMBOL]: true, - [PROVIDER_DEFINED_TOOL_SYMBOL]: true, - }; + // Ensure parameters is either a Zod schema or a plain object (JSON schema) + if (!isZodSchema(parameters) && !(typeof parameters === 'object')) { + throw new Error('Tool parameters must be a Zod object schema or a raw JSON schema'); } - throw new Error('Invalid tool'); + return { + type: 'function', + id: tool.name, + name: tool.name, + description: tool.description, + parameters, + execute: tool.execute, + flags: tool.flags ?? ToolFlag.NONE, + [TOOL_SYMBOL]: true, + [FUNCTION_TOOL_SYMBOL]: true, + }; } // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -485,10 +582,15 @@ export function isFunctionTool(tool: any): tool is FunctionTool { } // eslint-disable-next-line @typescript-eslint/no-explicit-any -export function isProviderDefinedTool(tool: any): tool is ProviderDefinedTool { +export function isProviderTool(tool: any): tool is ProviderTool { const isTool = tool && tool[TOOL_SYMBOL] === true; - const isProviderDefinedTool = tool[PROVIDER_DEFINED_TOOL_SYMBOL] === true; - return isTool && isProviderDefinedTool; + const isProviderTool = tool[PROVIDER_TOOL_SYMBOL] === true; + return isTool && isProviderTool; +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function isToolset(value: any): value is Toolset { + return value && value[TOOLSET_SYMBOL] === true; } // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/agents/src/llm/tool_context.type.test.ts b/agents/src/llm/tool_context.type.test.ts index 187f95e7b..5d33124ad 100644 --- a/agents/src/llm/tool_context.type.test.ts +++ b/agents/src/llm/tool_context.type.test.ts @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, expectTypeOf, it } from 'vitest'; import { z } from 'zod'; -import { type FunctionTool, type ProviderDefinedTool, type ToolOptions, tool } from './index.js'; +import { type FunctionTool, ProviderTool, type ToolOptions, tool } from './index.js'; describe('tool type inference', () => { it('should infer argument type from zod schema', () => { @@ -17,15 +17,15 @@ describe('tool type inference', () => { expectTypeOf(toolType).toEqualTypeOf>(); }); - it('should infer provider defined tool type', () => { - const toolType = tool({ - id: 'code-interpreter', - config: { - language: 'python', - }, - }); + it('rejects direct instantiation of the abstract ProviderTool base', () => { + // @ts-expect-error - ProviderTool is abstract; plugins must subclass it. + new ProviderTool({ id: 'code-interpreter' }); - expectTypeOf(toolType).toEqualTypeOf(); + class CodeInterpreter extends ProviderTool {} + const providerTool = new CodeInterpreter({ id: 'code-interpreter' }); + expectTypeOf(providerTool).toMatchTypeOf(); + expect(providerTool.id).toBe('code-interpreter'); + expect(providerTool.type).toBe('provider'); }); it('should infer run context type', () => { @@ -45,7 +45,6 @@ describe('tool type inference', () => { it('should not accept primitive zod schemas', () => { expect(() => { - // @ts-expect-error - Testing that non-object schemas are rejected tool({ name: 'test', description: 'test', @@ -57,7 +56,6 @@ describe('tool type inference', () => { it('should not accept array schemas', () => { expect(() => { - // @ts-expect-error - Testing that array schemas are rejected tool({ name: 'test', description: 'test', @@ -69,7 +67,6 @@ describe('tool type inference', () => { it('should not accept union schemas', () => { expect(() => { - // @ts-expect-error - Testing that union schemas are rejected tool({ name: 'test', description: 'test', diff --git a/agents/src/utils.test.ts b/agents/src/utils.test.ts index df2fc91de..ddfe230cd 100644 --- a/agents/src/utils.test.ts +++ b/agents/src/utils.test.ts @@ -14,6 +14,7 @@ import { delay, isPending, resampleStream, + toStream, } from '../src/utils.js'; describe('utils', () => { @@ -32,6 +33,84 @@ describe('utils', () => { }); }); + describe('toStream', () => { + it('converts an async iterable into a ReadableStream', async () => { + async function* source() { + yield 1; + yield 2; + yield 3; + } + + const reader = toStream(source()).getReader(); + + await expect(reader.read()).resolves.toEqual({ done: false, value: 1 }); + await expect(reader.read()).resolves.toEqual({ done: false, value: 2 }); + await expect(reader.read()).resolves.toEqual({ done: false, value: 3 }); + await expect(reader.read()).resolves.toEqual({ done: true, value: undefined }); + }); + + it('propagates errors from the async iterable', async () => { + const expectedError = new Error('source failed'); + async function* source() { + yield 1; + throw expectedError; + } + + const reader = toStream(source()).getReader(); + + await expect(reader.read()).resolves.toEqual({ done: false, value: 1 }); + await expect(reader.read()).rejects.toBe(expectedError); + }); + + it('runs async iterable cleanup when the stream is canceled mid-stream', async () => { + let cleanupRan = false; + async function* source() { + try { + yield 1; + yield 2; + } finally { + cleanupRan = true; + } + } + + const reader = toStream(source()).getReader(); + + await expect(reader.read()).resolves.toEqual({ done: false, value: 1 }); + await reader.cancel('stop early'); + + expect(cleanupRan).toBe(true); + }); + + it('does not wait for a pending next value when canceled mid-stream', async () => { + let releaseNextValue: (() => void) | undefined; + let cleanupRan = false; + async function* source() { + try { + yield 1; + await new Promise((resolve) => { + releaseNextValue = resolve; + }); + yield 2; + } finally { + cleanupRan = true; + } + } + + const reader = toStream(source()).getReader(); + + await expect(reader.read()).resolves.toEqual({ done: false, value: 1 }); + const pendingRead = reader.read(); + await delay(1); + await expect( + Promise.race([reader.cancel('stop early'), delay(10).then(() => 'timeout')]), + ).resolves.not.toBe('timeout'); + releaseNextValue?.(); + await expect(pendingRead).resolves.toEqual({ done: true, value: undefined }); + + expect(cleanupRan).toBe(true); + }); + }); + describe('Task', () => { it('should execute task successfully and return result', async () => { const expectedResult = 'task completed'; diff --git a/agents/src/utils.ts b/agents/src/utils.ts index 6343c932f..2b3d651b2 100644 --- a/agents/src/utils.ts +++ b/agents/src/utils.ts @@ -13,8 +13,11 @@ import { type Throws, ThrowsPromise } from '@livekit/throws-transformer/throws'; import { AsyncLocalStorage } from 'node:async_hooks'; import { randomUUID } from 'node:crypto'; import { EventEmitter, once } from 'node:events'; -import type { ReadableStream } from 'node:stream/web'; -import { TransformStream, type TransformStreamDefaultController } from 'node:stream/web'; +import { + ReadableStream, + TransformStream, + type TransformStreamDefaultController, +} from 'node:stream/web'; import { log } from './log.js'; /** @@ -1127,15 +1130,21 @@ export async function* readStream( const abortPromise = waitForAbort(signal); while (true) { const result = await ThrowsPromise.race([reader.read(), abortPromise]); - if (!result) break; + if (!result) { + break; + } const { done, value } = result; - if (done) break; + if (done) { + break; + } yield value; } } else { while (true) { const { done, value } = await reader.read(); - if (done) break; + if (done) { + break; + } yield value; } } @@ -1148,6 +1157,39 @@ export async function* readStream( } } +export function toStream(iterable: AsyncIterable): ReadableStream { + let iterator: AsyncIterator | undefined; + let cancelled = false; + + return new ReadableStream({ + async start(controller) { + iterator = iterable[Symbol.asyncIterator](); + + try { + while (true) { + const { done, value } = await iterator.next(); + if (done || cancelled) { + break; + } + controller.enqueue(value); + } + + if (!cancelled) { + controller.close(); + } + } catch (error) { + if (!cancelled) { + controller.error(error); + } + } + }, + cancel(reason) { + cancelled = true; + void iterator?.return?.(reason).catch(() => {}); + }, + }); +} + export async function waitForAbort(signal: AbortSignal) { if (signal.aborted) { return; diff --git a/agents/src/voice/agent.test.ts b/agents/src/voice/agent.test.ts index f2afa7c42..0bfbf7df3 100644 --- a/agents/src/voice/agent.test.ts +++ b/agents/src/voice/agent.test.ts @@ -1,9 +1,11 @@ // SPDX-FileCopyrightText: 2025 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 +import type { AudioFrame } from '@livekit/rtc-node'; +import { ReadableStream } from 'node:stream/web'; import { describe, expect, it, vi } from 'vitest'; import { z } from 'zod'; -import { tool } from '../llm/index.js'; +import { ChatContext, ChatMessage, tool } from '../llm/index.js'; import { initializeLogger } from '../log.js'; import { Task } from '../utils.js'; import { Agent, AgentTask, _setActivityTaskInfo } from './agent.js'; @@ -15,6 +17,14 @@ vi.mock('ofetch', () => ({ ofetch: vi.fn() })); initializeLogger({ pretty: false, level: 'error' }); +async function collectReadableStream(stream: ReadableStream): Promise { + const chunks: T[] = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + return chunks; +} + describe('Agent', () => { it('should create agent with basic instructions', () => { const instructions = 'You are a helpful assistant'; @@ -80,6 +90,258 @@ describe('Agent', () => { expect(agent.toolCtx.getFunctionTool('testTool')).toBe(mockTool); }); + describe('create', () => { + it('preserves constructor options and base Agent default id', () => { + const mockTool = tool({ + name: 'testTool', + description: 'Test tool', + parameters: z.object({}), + execute: async () => 'result', + }); + + const agent = Agent.create({ + instructions: 'factory instructions', + tools: [mockTool], + }); + + expect(agent).toBeInstanceOf(Agent); + expect(agent.instructions).toBe('factory instructions'); + expect(agent.id).toBe('default_agent'); + expect(agent.toolCtx.getFunctionTool('testTool')).toBe(mockTool); + }); + + it('passes AgentContext to lifecycle hooks', async () => { + const calls: string[] = []; + const chatCtx = ChatContext.empty(); + const newMessage = ChatMessage.create({ role: 'user', content: ['hello'] }); + const agent = Agent.create({ + id: 'factory_agent', + instructions: 'factory instructions', + minConsecutiveSpeechDelay: 12, + ttsPronunciationMap: { LiveKit: 'live kit' }, + onEnter: (ctx) => { + expect(ctx.agent).toBe(agent); + expect(ctx.id).toBe(agent.id); + expect(ctx.instructions).toBe(agent.instructions); + expect(ctx.toolCtx.functionTools).toEqual(agent.toolCtx.functionTools); + expect(ctx.chatCtx.items).toEqual(agent.chatCtx.items); + expect(ctx.minConsecutiveSpeechDelay).toBe(agent.minConsecutiveSpeechDelay); + expect(ctx.ttsPronunciationMap).toBe(agent.ttsPronunciationMap); + calls.push('enter'); + }, + onExit: async (ctx) => { + expect(ctx.agent).toBe(agent); + calls.push('exit'); + }, + onUserTurnCompleted: (ctx, receivedChatCtx, receivedMessage) => { + expect(ctx.agent).toBe(agent); + expect(receivedChatCtx).toBe(chatCtx); + expect(receivedMessage).toBe(newMessage); + calls.push('turn'); + }, + }); + + await agent.onEnter(); + await agent.onExit(); + await agent.onUserTurnCompleted(chatCtx, newMessage); + + expect(calls).toEqual(['enter', 'exit', 'turn']); + }); + + it('adapts stream node hooks between ReadableStream and AsyncIterable', async () => { + const audioFrame = 'audio' as unknown as AudioFrame; + const agent = Agent.create({ + instructions: 'factory instructions', + async sttNode(ctx, audio) { + async function* stream() { + expect(ctx.agent).toBe(agent); + const frames: AudioFrame[] = []; + for await (const frame of audio) { + frames.push(frame); + } + expect(frames).toEqual([audioFrame]); + yield 'transcript'; + } + + return stream(); + }, + }); + const audio = new ReadableStream({ + start(controller) { + controller.enqueue(audioFrame); + controller.close(); + }, + }); + + const result = await agent.sttNode(audio, {}); + + expect(result).not.toBeNull(); + await expect(collectReadableStream(result!)).resolves.toEqual(['transcript']); + }); + + it('supports async generator stream node hooks', async () => { + const audioFrame = 'audio' as unknown as AudioFrame; + const outputFrame = 'output-audio' as unknown as AudioFrame; + const agent = Agent.create({ + instructions: 'factory instructions', + async *sttNode(ctx, audio) { + expect(ctx.agent).toBe(agent); + const frames: AudioFrame[] = []; + for await (const frame of audio) { + frames.push(frame); + } + expect(frames).toEqual([audioFrame]); + yield 'transcript'; + }, + async *llmNode(ctx, chatCtx, toolCtx) { + expect(ctx.agent).toBe(agent); + expect(chatCtx).toBeInstanceOf(ChatContext); + expect(toolCtx.equals(agent.toolCtx)).toBe(true); + yield 'llm-output'; + }, + async *ttsNode(ctx, text) { + expect(ctx.agent).toBe(agent); + const chunks: string[] = []; + for await (const chunk of text) { + chunks.push(chunk); + } + expect(chunks).toEqual(['hello']); + yield outputFrame; + }, + async *realtimeAudioOutputNode(ctx, audio) { + expect(ctx.agent).toBe(agent); + const frames: AudioFrame[] = []; + for await (const frame of audio) { + frames.push(frame); + } + expect(frames).toEqual([audioFrame]); + yield outputFrame; + }, + }); + const audio = new ReadableStream({ + start(controller) { + controller.enqueue(audioFrame); + controller.close(); + }, + }); + const text = new ReadableStream({ + start(controller) { + controller.enqueue('hello'); + controller.close(); + }, + }); + + const sttResult = await agent.sttNode(audio, {}); + const llmResult = await agent.llmNode(ChatContext.empty(), agent.toolCtx, {}); + const ttsResult = await agent.ttsNode(text, {}); + const realtimeResult = await agent.realtimeAudioOutputNode( + new ReadableStream({ + start(controller) { + controller.enqueue(audioFrame); + controller.close(); + }, + }), + {}, + ); + + expect(sttResult).not.toBeNull(); + expect(llmResult).not.toBeNull(); + expect(ttsResult).not.toBeNull(); + expect(realtimeResult).not.toBeNull(); + await expect(collectReadableStream(sttResult!)).resolves.toEqual(['transcript']); + await expect(collectReadableStream(llmResult!)).resolves.toEqual(['llm-output']); + await expect(collectReadableStream(ttsResult!)).resolves.toEqual([outputFrame]); + await expect(collectReadableStream(realtimeResult!)).resolves.toEqual([outputFrame]); + }); + + it('supports stream node hooks that return async iterables', async () => { + function asyncIterableOf(...items: T[]): AsyncIterable { + return { + async *[Symbol.asyncIterator]() { + for (const item of items) { + yield item; + } + }, + }; + } + + const audioFrame = 'audio' as unknown as AudioFrame; + const outputFrame = 'output-audio' as unknown as AudioFrame; + const agent = Agent.create({ + instructions: 'factory instructions', + sttNode(ctx) { + expect(ctx.agent).toBe(agent); + return asyncIterableOf('transcript'); + }, + llmNode(ctx) { + expect(ctx.agent).toBe(agent); + return asyncIterableOf('llm-output'); + }, + ttsNode(ctx) { + expect(ctx.agent).toBe(agent); + return asyncIterableOf(outputFrame); + }, + realtimeAudioOutputNode(ctx) { + expect(ctx.agent).toBe(agent); + return asyncIterableOf(outputFrame); + }, + }); + + const sttResult = await agent.sttNode( + new ReadableStream({ + start(controller) { + controller.enqueue(audioFrame); + controller.close(); + }, + }), + {}, + ); + const llmResult = await agent.llmNode(ChatContext.empty(), agent.toolCtx, {}); + const ttsResult = await agent.ttsNode( + new ReadableStream({ + start(controller) { + controller.enqueue('hello'); + controller.close(); + }, + }), + {}, + ); + const realtimeResult = await agent.realtimeAudioOutputNode( + new ReadableStream({ + start(controller) { + controller.enqueue(audioFrame); + controller.close(); + }, + }), + {}, + ); + + expect(sttResult).not.toBeNull(); + expect(llmResult).not.toBeNull(); + expect(ttsResult).not.toBeNull(); + expect(realtimeResult).not.toBeNull(); + await expect(collectReadableStream(sttResult!)).resolves.toEqual(['transcript']); + await expect(collectReadableStream(llmResult!)).resolves.toEqual(['llm-output']); + await expect(collectReadableStream(ttsResult!)).resolves.toEqual([outputFrame]); + await expect(collectReadableStream(realtimeResult!)).resolves.toEqual([outputFrame]); + }); + + it('falls back to existing defaults for missing hooks', async () => { + const audioFrame = 'audio' as unknown as AudioFrame; + const audio = new ReadableStream({ + start(controller) { + controller.enqueue(audioFrame); + controller.close(); + }, + }); + const agent = Agent.create({ instructions: 'factory instructions' }); + + const result = await agent.realtimeAudioOutputNode(audio, {}); + + expect(result).toBe(audio); + }); + }); + it('should require AgentTask to run inside task context', async () => { class TestTask extends AgentTask { constructor() { @@ -147,6 +409,94 @@ describe('Agent', () => { await expect(wrapper.result).resolves.toBe('ok'); }); + describe('AgentTask.create', () => { + it('exposes complete on hook context', async () => { + const task = AgentTask.create({ + instructions: 'factory task', + onEnter: (ctx) => { + expect(ctx.agent).toBe(task); + expect(ctx.id).toBe('default_agent'); + expect(ctx.instructions).toBe('factory task'); + ctx.complete('ok'); + }, + }); + const oldAgent = new Agent({ instructions: 'old agent' }); + const mockSession = { + currentAgent: oldAgent, + _globalRunState: undefined, + _updateActivity: async (agent: Agent) => { + if (agent === task) { + await agent.onEnter(); + } + }, + }; + const mockActivity = { + agent: oldAgent, + agentSession: mockSession, + _onEnterTask: undefined, + llm: undefined, + close: async () => {}, + }; + + const wrapper = Task.from(async () => { + const currentTask = Task.current(); + if (!currentTask) { + throw new Error('expected task context'); + } + _setActivityTaskInfo(currentTask, { inlineTask: true }); + return await agentActivityStorage.run(mockActivity as any, () => task.run()); + }); + + await expect(wrapper.result).resolves.toBe('ok'); + }); + + it('adapts stream node hooks between ReadableStream and AsyncIterable', async () => { + const audioFrame = 'audio' as unknown as AudioFrame; + const task = AgentTask.create({ + instructions: 'factory task', + async sttNode(ctx, audio) { + async function* stream() { + expect(ctx.agent).toBe(task); + const frames: AudioFrame[] = []; + for await (const frame of audio) { + frames.push(frame); + } + expect(frames).toEqual([audioFrame]); + yield 'transcript'; + } + + return stream(); + }, + }); + const audio = new ReadableStream({ + start(controller) { + controller.enqueue(audioFrame); + controller.close(); + }, + }); + + const result = await task.sttNode(audio, {}); + + expect(result).not.toBeNull(); + await expect(collectReadableStream(result!)).resolves.toEqual(['transcript']); + }); + + it('falls back to existing defaults for missing hooks', async () => { + const audioFrame = 'audio' as unknown as AudioFrame; + const audio = new ReadableStream({ + start(controller) { + controller.enqueue(audioFrame); + controller.close(); + }, + }); + const task = AgentTask.create({ instructions: 'factory task' }); + + const result = await task.realtimeAudioOutputNode(audio, {}); + + expect(result).toBe(audio); + }); + }); + it('should require AgentTask to run inside AgentActivity context', async () => { class TestTask extends AgentTask { constructor() { @@ -235,6 +585,7 @@ describe('Agent', () => { turnHandling: { endpointing: { minDelay: 999 }, interruption: {}, + preemptiveGeneration: {}, turnDetection: 'vad', }, allowInterruptions: false, @@ -251,6 +602,7 @@ describe('Agent', () => { turnHandling: { endpointing: { minDelay: 999, maxDelay: 4000 }, interruption: { enabled: true }, + preemptiveGeneration: {}, turnDetection: 'vad', }, allowInterruptions: false, @@ -268,6 +620,7 @@ describe('Agent', () => { turnHandling: { interruption: { mode: 'adaptive' }, endpointing: {}, + preemptiveGeneration: {}, turnDetection: undefined, }, }); @@ -280,6 +633,7 @@ describe('Agent', () => { turnHandling: { endpointing: { minDelay: 111, maxDelay: 222 }, interruption: { enabled: false }, + preemptiveGeneration: {}, turnDetection: 'manual', }, }); diff --git a/agents/src/voice/agent.ts b/agents/src/voice/agent.ts index 3145de2b1..4072543ef 100644 --- a/agents/src/voice/agent.ts +++ b/agents/src/voice/agent.ts @@ -34,11 +34,26 @@ import { Future, Task } from '../utils.js'; import type { VAD } from '../vad.js'; import { type AgentActivity, agentActivityStorage } from './agent_activity.js'; import type { AgentSession, TurnDetectionMode } from './agent_session.js'; +import { + type AgentCreateOptions, + type AgentTaskCreateOptions, + createAgentTaskV2, + createAgentV2, +} from './agent_v2.js'; import type { TimedString } from './io.js'; import type { SpeechHandle } from './speech_handle.js'; import type { TurnHandlingOptions } from './turn_config/turn_handling.js'; import { migrateTurnHandling } from './turn_config/utils.js'; +export type { + AgentContext, + AgentCreateOptions, + AgentHookNodeResult, + AgentHooks, + AgentTaskContext, + AgentTaskCreateOptions, +} from './agent_v2.js'; + // speechHandle identifies which SpeechHandle owns the current tool call, enabling // SpeechHandle.waitForPlayout() to distinguish self-wait (deadlock) from waiting // on a different handle scheduled inside the tool. @@ -160,6 +175,10 @@ export class Agent { /** @internal */ _toolCtx: ToolContext; + static create(options: AgentCreateOptions): Agent { + return createAgentV2(Agent, options); + } + constructor({ id, instructions, @@ -660,6 +679,12 @@ export class AgentTask extends Agent( + options: AgentTaskCreateOptions, + ): AgentTask { + return createAgentTaskV2(AgentTask, options); + } + constructor(options: AgentTaskOptions) { const { preserveFunctionCallHistory = false, ...rest } = options; super(rest); diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 3068b408d..793809a3e 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -35,10 +35,14 @@ import { RealtimeModel, type RealtimeModelError, type RealtimeSession, + type Tool, type ToolChoice, ToolContext, type ToolContextEntry, ToolFlag, + Toolset, + isFunctionTool, + isToolset, } from '../llm/index.js'; import type { LLMError } from '../llm/llm.js'; import { isSameToolChoice } from '../llm/tool_context.js'; @@ -215,6 +219,7 @@ export class AgentActivity implements RecognitionHooks { private toolChoice: ToolChoice | null = null; private _preemptiveGeneration?: PreemptiveGeneration; private _preemptiveGenerationCount = 0; + private _toolsetsSetup = false; private interruptionDetector?: AdaptiveInterruptionDetector; private isInterruptionDetectionEnabled: boolean; private isInterruptionByAudioActivityEnabled: boolean; @@ -421,6 +426,8 @@ export class AgentActivity implements RecognitionHooks { this.agent._agentActivity = this; + await this.setupToolsets(); + if (this.llm instanceof RealtimeModel) { const rtReused = reuseResources?.rtSession !== undefined; @@ -767,13 +774,20 @@ export class AgentActivity implements RecognitionHooks { } async updateTools(tools: readonly ToolContextEntry[]): Promise { - const oldToolNames = new Set(Object.keys(this.agent._toolCtx.functionTools)); + const oldToolCtx = this.agent._toolCtx; + const oldToolNames = new Set(Object.keys(oldToolCtx.functionTools)); + const oldToolsets = oldToolCtx.toolsets; const newToolCtx = new ToolContext(tools); const newToolNames = new Set(Object.keys(newToolCtx.functionTools)); + const newToolsets = newToolCtx.toolsets; const toolsAdded = [...newToolNames].filter((name) => !oldToolNames.has(name)); const toolsRemoved = [...oldToolNames].filter((name) => !newToolNames.has(name)); + const addedToolsets = newToolsets.filter((ts) => !oldToolsets.includes(ts)); + const removedToolsets = oldToolsets.filter((ts) => !newToolsets.includes(ts)); + await this.setupToolsetList(addedToolsets); this.agent._toolCtx = newToolCtx; + await this.closeToolsetList(removedToolsets); if (toolsAdded.length > 0 || toolsRemoved.length > 0) { const configUpdate = new AgentConfigUpdate({ @@ -1735,11 +1749,13 @@ export class AgentActivity implements RecognitionHooks { const tools: ToolContext = shouldFilterTools ? new ToolContext( - this.agent.toolCtx.tools.filter((t) => { - if (t.type === 'function') { - return !(t.flags & ToolFlag.IGNORE_ON_ENTER); + this.agent.toolCtx.tools.flatMap((t): ToolContextEntry[] => { + const keepFn = (fn: Tool): boolean => + !isFunctionTool(fn) || !(fn.flags & ToolFlag.IGNORE_ON_ENTER); + if (isToolset(t)) { + return t.tools.filter(keepFn) as ToolContextEntry[]; } - return true; + return keepFn(t) ? [t] : []; }), ) : this.agent.toolCtx; @@ -3728,9 +3744,42 @@ export class AgentActivity implements RecognitionHooks { this.realtimeSpans?.clear(); await this.realtimeSession?.close(); await this.audioRecognition?.close(); + await this.closeToolsets(); this.realtimeSession = undefined; this.audioRecognition = undefined; } + + private async setupToolsets(): Promise { + // Guard against resume() re-entering _startSession on an activity whose toolsets are + // already initialized. + if (this._toolsetsSetup) return; + this._toolsetsSetup = true; + await this.setupToolsetList(this.agent.toolCtx.toolsets); + } + + private async closeToolsets(): Promise { + if (!this._toolsetsSetup) return; + this._toolsetsSetup = false; + await this.closeToolsetList(this.agent.toolCtx.toolsets); + } + + private async setupToolsetList(toolsets: readonly Toolset[]): Promise { + const outputs = await Promise.allSettled(toolsets.map((ts) => ts.setup())); + for (const output of outputs) { + if (output.status === 'rejected') { + this.logger.error({ error: output.reason }, 'error setting up toolset'); + } + } + } + + private async closeToolsetList(toolsets: readonly Toolset[]): Promise { + const outputs = await Promise.allSettled(toolsets.map((ts) => ts.aclose())); + for (const output of outputs) { + if (output.status === 'rejected') { + this.logger.error({ error: output.reason }, 'error closing toolset'); + } + } + } } function toOaiToolChoice(toolChoice: ToolChoice | null): ToolChoice | undefined { diff --git a/agents/src/voice/agent_v2.ts b/agents/src/voice/agent_v2.ts new file mode 100644 index 000000000..6e368e2d3 --- /dev/null +++ b/agents/src/voice/agent_v2.ts @@ -0,0 +1,484 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { AudioFrame } from '@livekit/rtc-node'; +import type { ReadableStream } from 'node:stream/web'; +import type { Instructions, ReadonlyChatContext } from '../llm/chat_context.js'; +import type { + ChatChunk, + ChatContext, + ChatMessage, + LLM, + RealtimeModel, + ToolContext, +} from '../llm/index.js'; +import type { STT, SpeechEvent } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import { readStream, toStream } from '../utils.js'; +import type { VAD } from '../vad.js'; +import type { + Agent, + AgentOptions, + AgentTask, + AgentTaskOptions, + ModelSettings, + TTSPronunciationMap, +} from './agent.js'; +import type { AgentSession } from './agent_session.js'; +import type { TurnHandlingOptions } from './turn_config/turn_handling.js'; + +/** Context passed to hooks created with `Agent.create()`. */ +export interface AgentContext { + /** The agent instance currently running the hook. */ + agent: Agent; + /** Voice activity detector configured for the agent. */ + vad: VAD | undefined; + /** Speech-to-text model configured for the agent. */ + stt: STT | undefined; + /** LLM or realtime model configured for the agent. */ + llm: LLM | RealtimeModel | undefined; + /** Text-to-speech model configured for the agent. */ + tts: TTS | undefined; + /** Whether TTS-aligned transcripts are enabled for the agent. */ + useTtsAlignedTranscript: boolean | undefined; + /** Pronunciation replacements applied before TTS synthesis. */ + ttsPronunciationMap: TTSPronunciationMap | undefined; + /** Readonly view of the agent's current chat context. */ + chatCtx: ReadonlyChatContext; + /** Agent identifier. */ + id: string; + /** Agent instructions. */ + instructions: string | Instructions; + /** Copy of the agent tool context. */ + toolCtx: ToolContext; + /** Current session for the agent. */ + session: AgentSession; + /** Agent-level turn handling configuration. */ + turnHandling: Partial | undefined; + /** Minimum delay between consecutive speech. */ + minConsecutiveSpeechDelay: number | undefined; +} + +/** Return type for stream hooks. Returning `null` stops that pipeline node. */ +export type AgentHookNodeResult = AsyncIterable | Promise | null> | null; + +export interface AgentHooks< + UserData, + ContextT extends AgentContext = AgentContext, +> { + /** Called when the agent becomes active in a session. */ + onEnter?: (ctx: ContextT) => Promise | void; + /** Called when the agent is leaving the active session. */ + onExit?: (ctx: ContextT) => Promise | void; + /** Called after the user's turn has been committed to the chat context. */ + onUserTurnCompleted?: ( + ctx: ContextT, + chatCtx: ChatContext, + newMessage: ChatMessage, + ) => Promise | void; + /** Transforms incoming audio into speech events or transcript text for the agent. */ + sttNode?: ( + ctx: ContextT, + audio: AsyncIterable, + modelSettings: ModelSettings, + ) => AgentHookNodeResult; + /** Produces LLM chunks or text from the current chat and tool context. */ + llmNode?: ( + ctx: ContextT, + chatCtx: ChatContext, + toolCtx: ToolContext, + modelSettings: ModelSettings, + ) => AgentHookNodeResult; + /** Synthesizes agent text into audio frames for playout. */ + ttsNode?: ( + ctx: ContextT, + text: AsyncIterable, + modelSettings: ModelSettings, + ) => AgentHookNodeResult; + /** Processes realtime model audio before it is sent to the agent output. */ + realtimeAudioOutputNode?: ( + ctx: ContextT, + audio: AsyncIterable, + modelSettings: ModelSettings, + ) => AgentHookNodeResult; +} + +export interface AgentCreateOptions + extends AgentOptions, + AgentHooks {} + +/** Context passed to hooks created with `AgentTask.create()`. */ +export interface AgentTaskContext + extends AgentContext { + /** The task instance currently running the hook. */ + agent: AgentTask; + /** Complete the task with either a result or an error. */ + complete(result: ResultT | Error): void; +} + +export interface AgentTaskCreateOptions + extends AgentTaskOptions, + AgentHooks> {} + +// agent.ts passes these runtime base classes in to avoid a circular runtime import. +type AgentCtor = new (options: AgentOptions) => Agent; + +type AgentTaskCtor = new ( + options: AgentTaskOptions, +) => AgentTask; + +export function createAgentV2( + AgentBase: AgentCtor, + options: AgentCreateOptions, +): Agent { + class AgentV2 extends AgentBase { + private readonly hookAdapter: AgentHookAdapter>; + + constructor({ + onEnter, + onExit, + onUserTurnCompleted, + sttNode, + llmNode, + ttsNode, + realtimeAudioOutputNode, + ...agentOptions + }: AgentCreateOptions) { + super({ + ...agentOptions, + id: agentOptions.id ?? 'default_agent', + }); + + this.hookAdapter = new AgentHookAdapter( + { + onEnter, + onExit, + onUserTurnCompleted, + sttNode, + llmNode, + ttsNode, + realtimeAudioOutputNode, + }, + new AgentHookContext(this), + ); + } + + override async onEnter(): Promise { + return this.hookAdapter.onEnter(() => super.onEnter()); + } + + override async onExit(): Promise { + return this.hookAdapter.onExit(() => super.onExit()); + } + + override async onUserTurnCompleted( + chatCtx: ChatContext, + newMessage: ChatMessage, + ): Promise { + return this.hookAdapter.onUserTurnCompleted(chatCtx, newMessage, () => + super.onUserTurnCompleted(chatCtx, newMessage), + ); + } + + override async sttNode( + audio: ReadableStream, + modelSettings: ModelSettings, + ): Promise | null> { + return this.hookAdapter.sttNode(audio, modelSettings, () => + super.sttNode(audio, modelSettings), + ); + } + + override async llmNode( + chatCtx: ChatContext, + toolCtx: ToolContext, + modelSettings: ModelSettings, + ): Promise | null> { + return this.hookAdapter.llmNode(chatCtx, toolCtx, modelSettings, () => + super.llmNode(chatCtx, toolCtx, modelSettings), + ); + } + + override async ttsNode( + text: ReadableStream, + modelSettings: ModelSettings, + ): Promise | null> { + return this.hookAdapter.ttsNode(text, modelSettings, () => + super.ttsNode(text, modelSettings), + ); + } + + override async realtimeAudioOutputNode( + audio: ReadableStream, + modelSettings: ModelSettings, + ): Promise | null> { + return this.hookAdapter.realtimeAudioOutputNode(audio, modelSettings, () => + super.realtimeAudioOutputNode(audio, modelSettings), + ); + } + } + + return new AgentV2(options); +} + +export function createAgentTaskV2( + AgentTaskBase: AgentTaskCtor, + options: AgentTaskCreateOptions, +): AgentTask { + class AgentTaskV2 extends AgentTaskBase { + private readonly hookAdapter: AgentHookAdapter>; + + constructor({ + onEnter, + onExit, + onUserTurnCompleted, + sttNode, + llmNode, + ttsNode, + realtimeAudioOutputNode, + ...taskOptions + }: AgentTaskCreateOptions) { + super({ + ...taskOptions, + id: taskOptions.id ?? 'default_agent', + }); + + this.hookAdapter = new AgentHookAdapter( + { + onEnter, + onExit, + onUserTurnCompleted, + sttNode, + llmNode, + ttsNode, + realtimeAudioOutputNode, + }, + new AgentTaskHookContext(this), + ); + } + + override async onEnter(): Promise { + return this.hookAdapter.onEnter(() => super.onEnter()); + } + + override async onExit(): Promise { + return this.hookAdapter.onExit(() => super.onExit()); + } + + override async onUserTurnCompleted( + chatCtx: ChatContext, + newMessage: ChatMessage, + ): Promise { + return this.hookAdapter.onUserTurnCompleted(chatCtx, newMessage, () => + super.onUserTurnCompleted(chatCtx, newMessage), + ); + } + + override async sttNode( + audio: ReadableStream, + modelSettings: ModelSettings, + ): Promise | null> { + return this.hookAdapter.sttNode(audio, modelSettings, () => + super.sttNode(audio, modelSettings), + ); + } + + override async llmNode( + chatCtx: ChatContext, + toolCtx: ToolContext, + modelSettings: ModelSettings, + ): Promise | null> { + return this.hookAdapter.llmNode(chatCtx, toolCtx, modelSettings, () => + super.llmNode(chatCtx, toolCtx, modelSettings), + ); + } + + override async ttsNode( + text: ReadableStream, + modelSettings: ModelSettings, + ): Promise | null> { + return this.hookAdapter.ttsNode(text, modelSettings, () => + super.ttsNode(text, modelSettings), + ); + } + + override async realtimeAudioOutputNode( + audio: ReadableStream, + modelSettings: ModelSettings, + ): Promise | null> { + return this.hookAdapter.realtimeAudioOutputNode(audio, modelSettings, () => + super.realtimeAudioOutputNode(audio, modelSettings), + ); + } + } + + return new AgentTaskV2(options); +} + +class AgentHookAdapter> { + constructor( + private readonly hooks: AgentHooks, + private readonly context: ContextT, + ) {} + + async onEnter(fallback: () => Promise): Promise { + if (!this.hooks.onEnter) { + return fallback(); + } + + return this.hooks.onEnter(this.context); + } + + async onExit(fallback: () => Promise): Promise { + if (!this.hooks.onExit) { + return fallback(); + } + + return this.hooks.onExit(this.context); + } + + async onUserTurnCompleted( + chatCtx: ChatContext, + newMessage: ChatMessage, + fallback: () => Promise, + ): Promise { + if (!this.hooks.onUserTurnCompleted) { + return fallback(); + } + + return this.hooks.onUserTurnCompleted(this.context, chatCtx, newMessage); + } + + async sttNode( + audio: ReadableStream, + modelSettings: ModelSettings, + fallback: () => Promise | null>, + ): Promise | null> { + if (!this.hooks.sttNode) { + return fallback(); + } + + const result = await this.hooks.sttNode(this.context, readStream(audio), modelSettings); + return result === null ? null : toStream(result); + } + + async llmNode( + chatCtx: ChatContext, + toolCtx: ToolContext, + modelSettings: ModelSettings, + fallback: () => Promise | null>, + ): Promise | null> { + if (!this.hooks.llmNode) { + return fallback(); + } + + const result = await this.hooks.llmNode( + this.context, + chatCtx, + toolCtx as ToolContext, + modelSettings, + ); + return result === null ? null : toStream(result); + } + + async ttsNode( + text: ReadableStream, + modelSettings: ModelSettings, + fallback: () => Promise | null>, + ): Promise | null> { + if (!this.hooks.ttsNode) { + return fallback(); + } + + const result = await this.hooks.ttsNode(this.context, readStream(text), modelSettings); + return result === null ? null : toStream(result); + } + + async realtimeAudioOutputNode( + audio: ReadableStream, + modelSettings: ModelSettings, + fallback: () => Promise | null>, + ): Promise | null> { + if (!this.hooks.realtimeAudioOutputNode) { + return fallback(); + } + + const result = await this.hooks.realtimeAudioOutputNode( + this.context, + readStream(audio), + modelSettings, + ); + return result === null ? null : toStream(result); + } +} + +class AgentHookContext implements AgentContext { + constructor(readonly agent: Agent) {} + + get vad(): VAD | undefined { + return this.agent.vad; + } + + get stt(): STT | undefined { + return this.agent.stt; + } + + get llm(): LLM | RealtimeModel | undefined { + return this.agent.llm; + } + + get tts(): TTS | undefined { + return this.agent.tts; + } + + get useTtsAlignedTranscript(): boolean | undefined { + return this.agent.useTtsAlignedTranscript; + } + + get ttsPronunciationMap(): TTSPronunciationMap | undefined { + return this.agent.ttsPronunciationMap; + } + + get chatCtx(): ReadonlyChatContext { + return this.agent.chatCtx; + } + + get id(): string { + return this.agent.id; + } + + get instructions(): string | Instructions { + return this.agent.instructions; + } + + get toolCtx(): ToolContext { + return this.agent.toolCtx; + } + + get session(): AgentSession { + return this.agent.session; + } + + get turnHandling(): Partial | undefined { + return this.agent.turnHandling; + } + + get minConsecutiveSpeechDelay(): number | undefined { + return this.agent.minConsecutiveSpeechDelay; + } +} + +class AgentTaskHookContext + extends AgentHookContext + implements AgentTaskContext +{ + declare readonly agent: AgentTask; + + constructor(agent: AgentTask) { + super(agent); + } + + complete(result: ResultT | Error): void { + this.agent.complete(result); + } +} diff --git a/agents/src/voice/index.ts b/agents/src/voice/index.ts index e8813e460..5833f95f2 100644 --- a/agents/src/voice/index.ts +++ b/agents/src/voice/index.ts @@ -5,7 +5,13 @@ export { Agent, AgentTask, StopResponse, + type AgentContext, + type AgentCreateOptions, + type AgentHookNodeResult, + type AgentHooks, type AgentOptions, + type AgentTaskContext, + type AgentTaskCreateOptions, type ModelSettings, type TTSPronunciationMap, } from './agent.js'; @@ -35,6 +41,8 @@ export { type PlaybackFinishedEvent, type PlaybackStartedEvent, type TimedString, + createTimedString, + isTimedString, } from './io.js'; export * from './report.js'; export * from './room_io/index.js'; diff --git a/examples/src/basic_agent.ts b/examples/src/basic_agent.ts index 79e79808a..21463556e 100644 --- a/examples/src/basic_agent.ts +++ b/examples/src/basic_agent.ts @@ -2,16 +2,18 @@ // // SPDX-License-Identifier: Apache-2.0 import { + Agent, + AgentSession, + AgentSessionEventTypes, type JobContext, type JobProcess, ServerOptions, cli, defineAgent, inference, - llm, log, - metrics, - voice, + logMetrics, + tool, } from '@livekit/agents'; import * as livekit from '@livekit/agents-plugin-livekit'; import * as silero from '@livekit/agents-plugin-silero'; @@ -24,11 +26,11 @@ export default defineAgent({ proc.userData.vad = await silero.VAD.load(); }, entry: async (ctx: JobContext) => { - const agent = new voice.Agent({ + const agent = Agent.create({ instructions: "You are a helpful assistant, you can hear the user's message and respond to it.", tools: [ - llm.tool({ + tool({ name: 'getWeather', description: 'Get the weather for a given location.', parameters: z.object({ @@ -43,7 +45,7 @@ export default defineAgent({ const logger = log(); - const session = new voice.AgentSession({ + const session = new AgentSession({ // VAD and turn detection are used to determine when the user is speaking and when the agent should respond // See more at https://docs.livekit.io/agents/build/turns vad: ctx.proc.userData.vad! as silero.VAD, @@ -104,8 +106,8 @@ export default defineAgent({ }); // Log metrics as they are emitted - session.on(voice.AgentSessionEventTypes.MetricsCollected, (ev) => { - metrics.logMetrics(ev.metrics); + session.on(AgentSessionEventTypes.MetricsCollected, (ev) => { + logMetrics(ev.metrics); }); // Log usage summary when job shuts down @@ -118,7 +120,7 @@ export default defineAgent({ ); }); - session.on(voice.AgentSessionEventTypes.OverlappingSpeech, (ev) => { + session.on(AgentSessionEventTypes.OverlappingSpeech, (ev) => { logger.warn({ type: ev.type, isInterruption: ev.isInterruption }, 'user overlapping speech'); }); diff --git a/examples/src/basic_agent_task.ts b/examples/src/basic_agent_task.ts index c450a81bb..68a049694 100644 --- a/examples/src/basic_agent_task.ts +++ b/examples/src/basic_agent_task.ts @@ -2,11 +2,14 @@ // // SPDX-License-Identifier: Apache-2.0 import { + Agent, + AgentTask, type JobContext, type JobProcess, ServerOptions, cli, defineAgent, + handoff, inference, llm, voice, @@ -16,102 +19,101 @@ import * as silero from '@livekit/agents-plugin-silero'; import { fileURLToPath } from 'node:url'; import { z } from 'zod'; -class InfoTask extends voice.AgentTask { - constructor(private info: string) { - super({ - instructions: `Collect the user's information. around ${info}. Once you have the information, call the saveUserInfo tool to save the information to the database IMMEDIATELY. DO NOT have chitchat with the user, just collect the information and call the saveUserInfo tool.`, - tts: 'elevenlabs/eleven_turbo_v2_5', - tools: [ - llm.tool({ - name: 'saveUserInfo', - description: `Save the user's ${info} to database`, - parameters: z.object({ - [info]: z.string(), - }), - execute: async (args) => { - this.complete(args[info] as string); - return `Thanks, collected ${info} successfully: ${args[info]}`; - }, +function createInfoTask(info: string): AgentTask { + const task = AgentTask.create({ + instructions: `Collect the user's information. around ${info}. Once you have the information, call the saveUserInfo tool to save the information to the database IMMEDIATELY. DO NOT have chitchat with the user, just collect the information and call the saveUserInfo tool.`, + tts: 'elevenlabs/eleven_turbo_v2_5', + tools: [ + llm.tool({ + name: 'saveUserInfo', + description: `Save the user's ${info} to database`, + parameters: z.object({ + [info]: z.string(), }), - ], - }); - } + execute: async (args) => { + task.complete(args[info] as string); + return `Thanks, collected ${info} successfully: ${args[info]}`; + }, + }), + ], + onEnter: (ctx) => { + ctx.session.generateReply({ + userInput: `Ask the user for their ${info}`, + }); + }, + }); - async onEnter() { - this.session.generateReply({ - userInput: `Ask the user for their ${this.info}`, - }); - } + return task; } -class SurveyAgent extends voice.Agent { - constructor() { - super({ - instructions: - 'You orchestrate a short intro survey. Speak naturally and keep the interaction brief.', - tools: [ - llm.tool({ - name: 'collectUserInfo', - description: 'Call this when user want to provide some information to you', - parameters: z.object({ - key: z - .string() - .describe( - 'The key of the information to collect, e.g. "name" or "role" should be no space and underscore separated', - ), - }), - execute: async ({ key }) => { - const value = await new InfoTask(key).run(); - return `Collected ${key} successfully: ${value}`; - }, +function createWeatherAgent() { + return Agent.create({ + instructions: + 'You are a weather agent. You are responsible for providing the weather information to the user.', + tts: 'deepgram/aura-2', + tools: [ + llm.tool({ + name: 'getWeather', + description: 'Get the weather for a given location', + parameters: z.object({ + location: z.string().describe('The location to get the weather for'), }), - llm.tool({ - name: 'transferToWeatherAgent', - description: 'Call this immediately after user want to know the weather', - execute: async () => { - const agent = new voice.Agent({ - instructions: - 'You are a weather agent. You are responsible for providing the weather information to the user.', - tts: 'deepgram/aura-2', - tools: [ - llm.tool({ - name: 'getWeather', - description: 'Get the weather for a given location', - parameters: z.object({ - location: z.string().describe('The location to get the weather for'), - }), - execute: async ({ location }) => { - return `The weather in ${location} is sunny today.`; - }, - }), - llm.tool({ - name: 'finishWeatherConversation', - description: 'Call this when you want to finish the weather conversation', - execute: async () => { - return llm.handoff({ - agent: new SurveyAgent(), - returns: 'Transfer to survey agent successfully!', - }); - }, - }), - ], - }); + execute: async ({ location }) => { + return `The weather in ${location} is sunny today.`; + }, + }), + llm.tool({ + name: 'finishWeatherConversation', + description: 'Call this when you want to finish the weather conversation', + execute: async () => { + return llm.handoff({ + agent: createSurveyAgent(), + returns: 'Transfer to survey agent successfully!', + }); + }, + }), + ], + }); +} - return llm.handoff({ agent, returns: "Let's start the weather conversation!" }); - }, +function createSurveyAgent(): Agent { + return Agent.create({ + instructions: + 'You orchestrate a short intro survey. Speak naturally and keep the interaction brief.', + tools: [ + llm.tool({ + name: 'collectUserInfo', + description: 'Call this when user want to provide some information to you', + parameters: z.object({ + key: z + .string() + .describe( + 'The key of the information to collect, e.g. "name" or "role" should be no space and underscore separated', + ), }), - ], - }); - } - - async onEnter() { - const name = await new InfoTask('name').run(); - const role = await new InfoTask('role').run(); + execute: async ({ key }) => { + const value = await createInfoTask(key).run(); + return `Collected ${key} successfully: ${value}`; + }, + }), + llm.tool({ + name: 'transferToWeatherAgent', + description: 'Call this immediately after user want to know the weather', + execute: async () => { + const agent = createWeatherAgent(); + return handoff({ agent, returns: "Let's start the weather conversation!" }); + }, + }), + ], + onEnter: async (ctx) => { + const name = await createInfoTask('name').run(); + const role = await createInfoTask('role').run(); - await this.session.say( - `Great to meet you ${name}. I noted your role as ${role}. We can continue now.`, - ); - } + await ctx.session.say( + `Great to meet you ${name}. I noted your role as ${role}. We can continue now.`, + ); + }, + }); } export default defineAgent({ @@ -131,7 +133,7 @@ export default defineAgent({ await session.start({ room: ctx.room, - agent: new SurveyAgent(), + agent: createSurveyAgent(), }); }, }); diff --git a/examples/src/basic_toolsets.ts b/examples/src/basic_toolsets.ts new file mode 100644 index 000000000..cb0cf2b6e --- /dev/null +++ b/examples/src/basic_toolsets.ts @@ -0,0 +1,180 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { + type JobContext, + type JobProcess, + ServerOptions, + cli, + defineAgent, + inference, + llm, + voice, +} from '@livekit/agents'; +import * as livekit from '@livekit/agents-plugin-livekit'; +import * as silero from '@livekit/agents-plugin-silero'; +import { BackgroundVoiceCancellation } from '@livekit/noise-cancellation-node'; +import { fileURLToPath } from 'node:url'; +import { z } from 'zod'; + +class InfoTask extends voice.AgentTask { + private key: string; + + constructor(key: string, sharedToolset: llm.Toolset) { + super({ + instructions: `Collect the user's ${key}. Once you have it, call saveUserInfo IMMEDIATELY. No chitchat.`, + tools: [ + sharedToolset, + llm.tool({ + name: 'saveUserInfo', + description: `Save the user's ${key} to the database`, + parameters: z.object({ + [key]: z.string(), + }), + execute: async (args) => { + this.complete(args[key] as string); + return `Thanks, collected ${key} successfully: ${args[key]}`; + }, + }), + ], + }); + this.key = key; + } + + async onEnter() { + this.session.generateReply({ userInput: `Ask the user for their ${this.key}` }); + } +} + +function makeWeatherAgent(returnHome: () => voice.Agent) { + const weatherToolset = new llm.Toolset({ + id: 'weather_tools', + tools: [ + llm.tool({ + name: 'getWeather', + description: 'Get the weather for a given location', + parameters: z.object({ location: z.string() }), + execute: async ({ location }) => `The weather in ${location} is sunny today.`, + }), + ], + }); + + return new voice.Agent({ + instructions: 'You are a weather agent. Provide weather information then hand back when done.', + tools: [ + weatherToolset, + llm.tool({ + name: 'finishWeatherConversation', + description: 'Call this when you want to finish the weather conversation', + execute: async () => { + return llm.handoff({ agent: returnHome(), returns: 'Transfer back to main agent.' }); + }, + }), + ], + }); +} + +class MainAgent extends voice.Agent { + private locationToolset: llm.Toolset; + + constructor(locationToolset: llm.Toolset) { + super({ + instructions: + 'You are a helpful assistant. Use the location toolset for weather/timezone. Use transferToWeather when the user asks about weather. Use swapToolset / reapplyTools to exercise updateTools.', + tools: [ + locationToolset, + llm.tool({ + name: 'transferToWeather', + description: 'Call this when the user wants to know the weather', + execute: async () => { + return llm.handoff({ + agent: makeWeatherAgent(() => new MainAgent(locationToolset)), + returns: "Let's switch to the weather agent.", + }); + }, + }), + llm.tool({ + name: 'swapToolset', + description: 'Replace the active toolset with a brand-new toolset (tests updateTools).', + execute: async () => { + const replacement = new llm.Toolset({ + id: 'location_tools_v2', + tools: [ + llm.tool({ + name: 'getWeather', + description: 'v2 weather', + parameters: z.object({ location: z.string() }), + execute: async ({ location }) => `v2: ${location} -> sunny`, + }), + ], + }); + await this.updateTools([replacement]); + return 'Swapped toolset.'; + }, + }), + llm.tool({ + name: 'reapplyTools', + description: 'Re-apply the current tool list unchanged (idempotent updateTools).', + execute: async () => { + await this.updateTools([...this.toolCtx.tools]); + return 'Re-applied the same tool list.'; + }, + }), + ], + }); + this.locationToolset = locationToolset; + } + + async onEnter() { + const name = await new InfoTask('name', this.locationToolset).run(); + await this.session.say( + `Got it, ${name}. Ask me about weather, or say "swap" / "reapply" to exercise updateTools.`, + ); + } +} + +export default defineAgent({ + prewarm: async (proc: JobProcess) => { + proc.userData.vad = await silero.VAD.load(); + }, + entry: async (ctx: JobContext) => { + const locationToolset = new llm.Toolset({ + id: 'location_tools', + tools: [ + llm.tool({ + name: 'getWeather', + description: 'Get the weather for a given location.', + parameters: z.object({ location: z.string() }), + execute: async ({ location }) => `The weather in ${location} is sunny.`, + }), + llm.tool({ + name: 'lookupTimezone', + description: 'Look up the timezone for a city or region.', + parameters: z.object({ location: z.string() }), + execute: async ({ location }) => `${location} is in the America/Los_Angeles timezone.`, + }), + ], + }); + + const session = new voice.AgentSession({ + vad: ctx.proc.userData.vad! as silero.VAD, + stt: new inference.STT({ model: 'deepgram/nova-3', language: 'en' }), + llm: new inference.LLM({ model: 'openai/gpt-4.1-mini' }), + tts: new inference.TTS({ + model: 'cartesia/sonic-3', + voice: '9626c31c-bec5-4cca-baa8-f8ba9e84c8bc', + }), + turnDetection: new livekit.turnDetector.MultilingualModel(), + }); + + await session.start({ + agent: new MainAgent(locationToolset), + room: ctx.room, + inputOptions: { noiseCancellation: BackgroundVoiceCancellation() }, + }); + + session.say('Hello! I will ask you a quick question, then we can chat.'); + }, +}); + +cli.runApp(new ServerOptions({ agent: fileURLToPath(import.meta.url) })); diff --git a/examples/src/gemini_realtime_agent.ts b/examples/src/gemini_realtime_agent.ts index d70368352..93e1bb890 100644 --- a/examples/src/gemini_realtime_agent.ts +++ b/examples/src/gemini_realtime_agent.ts @@ -78,7 +78,7 @@ class IntroAgent extends voice.Agent { }); } - static create() { + static createIntroAgent() { return new IntroAgent({ instructions: `You are a story teller. Your goal is to gather a few pieces of information from the user to make the story personalized and engaging. Ask the user for their name and where they are from.`, tools: [ @@ -94,7 +94,7 @@ class IntroAgent extends voice.Agent { ctx.userData.name = name; ctx.userData.location = location; - const storyAgent = StoryAgent.create(name, location); + const storyAgent = StoryAgent.createStoryAgent(name, location); return llm.handoff({ agent: storyAgent, returns: "Let's start the story!" }); }, }), @@ -110,7 +110,7 @@ class StoryAgent extends voice.Agent { this.session.generateReply(); } - static create(name: string, location: string) { + static createStoryAgent(name: string, location: string) { return new StoryAgent({ instructions: dedent` You are a storyteller. Use the user's information in order to make the story personalized. @@ -142,7 +142,7 @@ export default defineAgent({ }); await session.start({ - agent: IntroAgent.create(), + agent: IntroAgent.createIntroAgent(), room: ctx.room, }); diff --git a/examples/src/multi_agent.ts b/examples/src/multi_agent.ts index 12d87377d..cd22d299d 100644 --- a/examples/src/multi_agent.ts +++ b/examples/src/multi_agent.ts @@ -32,7 +32,7 @@ class IntroAgent extends voice.Agent { }); } - static create() { + static createIntroAgent() { return new IntroAgent({ instructions: `You are a story teller. Your goal is to gather a few pieces of information from the user to make the story personalized and engaging. Ask the user for their name and where they are from.`, tools: [ @@ -48,7 +48,7 @@ class IntroAgent extends voice.Agent { ctx.userData.name = name; ctx.userData.location = location; - const storyAgent = StoryAgent.create(name, location); + const storyAgent = StoryAgent.createStoryAgent(name, location); return llm.handoff({ agent: storyAgent, returns: "Let's start the story!" }); }, }), @@ -62,7 +62,7 @@ class StoryAgent extends voice.Agent { this.session.generateReply(); } - static create(name: string, location: string) { + static createStoryAgent(name: string, location: string) { return new StoryAgent({ instructions: dedent` You are a storyteller. Use the user's information in order to make the story personalized. @@ -94,7 +94,7 @@ export default defineAgent({ }); await session.start({ - agent: IntroAgent.create(), + agent: IntroAgent.createIntroAgent(), room: ctx.room, }); diff --git a/plugins/google/src/beta/realtime/realtime_api.ts b/plugins/google/src/beta/realtime/realtime_api.ts index 66bc6a7f9..1d8365584 100644 --- a/plugins/google/src/beta/realtime/realtime_api.ts +++ b/plugins/google/src/beta/realtime/realtime_api.ts @@ -33,7 +33,7 @@ import { import { Mutex } from '@livekit/mutex'; import { AudioFrame, AudioResampler, type VideoFrame } from '@livekit/rtc-node'; import { type LLMTools } from '../../tools.js'; -import { toFunctionDeclarations } from '../../utils.js'; +import { toToolsConfig } from '../../utils.js'; import type * as api_proto from './api_proto.js'; import type { LiveAPIModels, Voice } from './api_proto.js'; @@ -70,13 +70,6 @@ export interface InputTranscription { transcript: string; } -/** - * Helper function to check if two sets are equal - */ -function setsEqual(a: Set, b: Set): boolean { - return a.size === b.size && [...a].every((x) => b.has(x)); -} - /** * Internal realtime options for Google Realtime API */ @@ -455,7 +448,6 @@ export class RealtimeSession extends llm.RealtimeSession { private _chatCtx = llm.ChatContext.empty(); private options: RealtimeOptions; - private geminiDeclarations: types.FunctionDeclaration[] = []; private messageChannel = new Queue(); private inputResampler?: AudioResampler; private inputResamplerInputRate?: number; @@ -764,15 +756,12 @@ export class RealtimeSession extends llm.RealtimeSession { } async updateTools(tools: llm.ToolContext): Promise { - const newDeclarations = toFunctionDeclarations(tools); - const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name)); - const newToolNames = new Set(newDeclarations.map((f) => f.name)); - - if (!setsEqual(currentToolNames, newToolNames)) { - this.geminiDeclarations = newDeclarations; - this._tools = tools; - this.markRestartNeeded(); + if (this._tools.equals(tools)) { + return; } + + this._tools = tools; + this.markRestartNeeded(); } get chatCtx(): llm.ChatContext { @@ -1424,21 +1413,11 @@ export class RealtimeSession extends llm.RealtimeSession { }, languageCode: opts.language, }, - tools: - this.geminiDeclarations.length > 0 || this.options.geminiTools - ? [ - { - functionDeclarations: - this.options.toolBehavior !== undefined - ? this.geminiDeclarations.map((d) => ({ - ...d, - behavior: this.options.toolBehavior, - })) - : this.geminiDeclarations, - ...this.options.geminiTools, - }, - ] - : undefined, + tools: toToolsConfig({ + toolCtx: this._tools, + geminiTools: this.options.geminiTools, + toolBehavior: this.options.toolBehavior, + }), inputAudioTranscription: opts.inputAudioTranscription, outputAudioTranscription: opts.outputAudioTranscription, sessionResumption: this.sessionResumptionHandle diff --git a/plugins/google/src/index.ts b/plugins/google/src/index.ts index fbafc1d66..326ca6270 100644 --- a/plugins/google/src/index.ts +++ b/plugins/google/src/index.ts @@ -6,6 +6,7 @@ import { Plugin } from '@livekit/agents'; export * as beta from './beta/index.js'; export { LLM, LLMStream, type LLMOptions } from './llm.js'; export * from './models.js'; +export * from './tools.js'; class GooglePlugin extends Plugin { constructor() { diff --git a/plugins/google/src/llm.ts b/plugins/google/src/llm.ts index e452b70d2..4958ccc6c 100644 --- a/plugins/google/src/llm.ts +++ b/plugins/google/src/llm.ts @@ -13,7 +13,7 @@ import { } from '@livekit/agents'; import type { ChatModels } from './models.js'; import type { LLMTools } from './tools.js'; -import { toFunctionDeclarations } from './utils.js'; +import { toToolsConfig } from './utils.js'; interface GoogleFormatData { systemMessages: string[] | null; @@ -355,11 +355,11 @@ export class LLMStream extends llm.LLMStream { parts: turn.parts as types.Part[], })); - const functionDeclarations = this.toolCtx ? toFunctionDeclarations(this.toolCtx) : undefined; - const tools = - functionDeclarations && functionDeclarations.length > 0 - ? [{ functionDeclarations }] - : undefined; + const tools = toToolsConfig({ + toolCtx: this.toolCtx, + geminiTools: this.#geminiTools, + onlySingleType: true, + }); let systemInstruction: types.Content | undefined = undefined; if (extraData.systemMessages && extraData.systemMessages.length > 0) { diff --git a/plugins/google/src/tools.ts b/plugins/google/src/tools.ts index 90cd9cc7c..b864e98a6 100644 --- a/plugins/google/src/tools.ts +++ b/plugins/google/src/tools.ts @@ -1,6 +1,100 @@ // SPDX-FileCopyrightText: 2025 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import type { Tool } from '@google/genai'; +import type * as types from '@google/genai'; +import { llm } from '@livekit/agents'; -export type LLMTools = Omit; +export type LLMTools = Omit; + +export abstract class GeminiTool extends llm.ProviderTool { + abstract toToolConfig(): types.Tool; +} + +export class GoogleSearch extends GeminiTool { + constructor(public readonly options: types.GoogleSearch = {}) { + super({ id: 'gemini_google_search' }); + } + + toToolConfig(): types.Tool { + return { googleSearch: this.options }; + } +} + +export class GoogleMaps extends GeminiTool { + constructor(public readonly options: types.GoogleMaps = {}) { + super({ id: 'gemini_google_maps' }); + } + + toToolConfig(): types.Tool { + return { googleMaps: this.options }; + } +} + +export class URLContext extends GeminiTool { + constructor() { + super({ id: 'gemini_url_context' }); + } + + toToolConfig(): types.Tool { + return { urlContext: {} }; + } +} + +export interface FileSearchOptions extends types.FileSearch { + fileSearchStoreNames: string[]; +} + +export class FileSearch extends GeminiTool { + constructor(public readonly options: FileSearchOptions) { + super({ id: 'gemini_file_search' }); + } + + toToolConfig(): types.Tool { + return { fileSearch: this.options }; + } +} + +export class ToolCodeExecution extends GeminiTool { + constructor() { + super({ id: 'gemini_code_execution' }); + } + + toToolConfig(): types.Tool { + return { codeExecution: {} }; + } +} + +export interface VertexRAGRetrievalOptions { + ragResources: string[]; + similarityTopK?: number; + vectorDistanceThreshold?: number; +} + +export class VertexRAGRetrieval extends GeminiTool { + readonly ragResources: string[]; + readonly similarityTopK: number; + readonly vectorDistanceThreshold?: number; + + constructor({ + ragResources, + similarityTopK = 3, + vectorDistanceThreshold, + }: VertexRAGRetrievalOptions) { + super({ id: 'gemini_vertex_rag_retrieval' }); + this.ragResources = ragResources; + this.similarityTopK = similarityTopK; + this.vectorDistanceThreshold = vectorDistanceThreshold; + } + + toToolConfig(): types.Tool { + return { + retrieval: { + vertexRagStore: { + ragResources: this.ragResources.map((ragCorpus) => ({ ragCorpus })), + similarityTopK: this.similarityTopK, + vectorDistanceThreshold: this.vectorDistanceThreshold, + }, + }, + }; + } +} diff --git a/plugins/google/src/utils.ts b/plugins/google/src/utils.ts index 5548c076e..30b5fcc89 100644 --- a/plugins/google/src/utils.ts +++ b/plugins/google/src/utils.ts @@ -1,9 +1,11 @@ // SPDX-FileCopyrightText: 2025 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 +import type * as types from '@google/genai'; import type { FunctionDeclaration, Schema } from '@google/genai'; import { llm } from '@livekit/agents'; import type { JSONSchema7 } from 'json-schema'; +import { GeminiTool, type LLMTools } from './tools.js'; /** * JSON Schema v7 @@ -139,8 +141,10 @@ function isEmptyObjectSchema(jsonSchema: JSONSchema7Definition): boolean { export function toFunctionDeclarations(toolCtx: llm.ToolContext): FunctionDeclaration[] { const functionDeclarations: FunctionDeclaration[] = []; - for (const [name, tool] of Object.entries(toolCtx.functionTools)) { - const { description, parameters } = tool; + for (const tool of toolCtx.flatten()) { + // TODO: support provider tools in the Gemini schema. + if (!llm.isFunctionTool(tool)) continue; + const { name, description, parameters } = tool; const jsonSchema = llm.toJsonSchema(parameters, false); // Create a deep copy to prevent the Google GenAI library from mutating the schema @@ -155,3 +159,57 @@ export function toFunctionDeclarations(toolCtx: llm.ToolContext): FunctionDeclar return functionDeclarations; } + +export function toToolsConfig({ + toolCtx, + geminiTools, + toolBehavior, + onlySingleType = false, +}: { + toolCtx?: llm.ToolContext; + geminiTools?: LLMTools; + toolBehavior?: types.Behavior; + onlySingleType?: boolean; +}): types.Tool[] | undefined { + const tools: types.Tool[] = []; + const providerTools: types.Tool[] = []; + + if (toolCtx) { + const functionDeclarations = toFunctionDeclarations(toolCtx); + if (functionDeclarations.length > 0) { + tools.push({ + functionDeclarations: + toolBehavior !== undefined + ? functionDeclarations.map((declaration) => ({ + ...declaration, + behavior: toolBehavior, + })) + : functionDeclarations, + }); + } + } + + if (geminiTools !== undefined) { + providerTools.push(geminiTools); + } + + if (toolCtx) { + for (const tool of toolCtx.providerTools) { + if (tool instanceof GeminiTool) { + providerTools.push(tool.toToolConfig()); + } + } + } + + if (tools.length > 0 && providerTools.length > 0) { + throw new Error('Gemini does not support mixing function tools and provider tools'); + } + + if (onlySingleType && tools.length > 0) { + return tools; + } + + tools.push(...providerTools); + + return tools.length > 0 ? tools : undefined; +} diff --git a/plugins/mistralai/src/llm.ts b/plugins/mistralai/src/llm.ts index f6685b042..901325f0a 100644 --- a/plugins/mistralai/src/llm.ts +++ b/plugins/mistralai/src/llm.ts @@ -211,14 +211,16 @@ export class LLMStream extends llm.LLMStream { // eslint-disable-next-line @typescript-eslint/no-explicit-any const toolsList: any[] = []; - if (this.toolCtx && Object.keys(this.toolCtx.functionTools).length > 0) { - for (const [name, func] of Object.entries(this.toolCtx.functionTools)) { + if (this.toolCtx) { + for (const t of this.toolCtx.flatten()) { + // TODO: support provider tools in the Mistral schema. + if (!llm.isFunctionTool(t)) continue; toolsList.push({ type: 'function' as const, function: { - name, - description: func.description, - parameters: llm.toJsonSchema(func.parameters, true, false), + name: t.name, + description: t.description, + parameters: llm.toJsonSchema(t.parameters, true, false), }, }); } diff --git a/plugins/openai/src/index.ts b/plugins/openai/src/index.ts index ccffdcb3f..6a5d9cb7c 100644 --- a/plugins/openai/src/index.ts +++ b/plugins/openai/src/index.ts @@ -5,6 +5,7 @@ import { Plugin } from '@livekit/agents'; export { LLM, LLMStream, type LLMOptions } from './llm.js'; export * from './models.js'; +export * from './tools.js'; export * as realtime from './realtime/index.js'; export * as responses from './responses/index.js'; export { STT, type STTOptions } from './stt.js'; diff --git a/plugins/openai/src/realtime/realtime_model.ts b/plugins/openai/src/realtime/realtime_model.ts index 8481e0f47..49a0cb506 100644 --- a/plugins/openai/src/realtime/realtime_model.ts +++ b/plugins/openai/src/realtime/realtime_model.ts @@ -698,11 +698,12 @@ export class RealtimeSession extends llm.RealtimeSession { // TODO(brian): these logics below are noops I think, leaving it here to keep // parity with the python but we should remove them later const retainedToolNames = new Set(ev.session.tools.map((tool) => tool.name)); - const retainedTools = Object.entries(_tools.functionTools) - .filter(([name]) => retainedToolNames.has(name)) - .map(([, tool]) => tool); + // Keep provider tools and Toolsets as-is; only drop function tools the server didn't accept. + const retainedEntries = _tools.tools.filter( + (entry) => !llm.isFunctionTool(entry) || retainedToolNames.has(entry.name), + ); - this._tools = new llm.ToolContext(retainedTools); + this._tools = new llm.ToolContext(retainedEntries); unlock(); } @@ -710,21 +711,26 @@ export class RealtimeSession extends llm.RealtimeSession { private createToolsUpdateEvent(_tools: llm.ToolContext): api_proto.SessionUpdateEvent { const oaiTools: api_proto.Tool[] = []; - for (const [name, tool] of Object.entries(_tools.functionTools)) { - const { parameters: toolParameters, description } = tool; + for (const t of _tools.flatten()) { + // TODO: support provider tools in the Realtime session-update schema. + if (!llm.isFunctionTool(t)) continue; + try { const parameters = llm.toJsonSchema( - toolParameters, + t.parameters, ) as unknown as api_proto.Tool['parameters']; oaiTools.push({ - name, - description, + name: t.name, + description: t.description, parameters: parameters, type: 'function', }); } catch (e) { - this.#logger.error({ name, tool }, "OpenAI Realtime API doesn't support this tool type"); + this.#logger.error( + { name: t.name, tool: t }, + "OpenAI Realtime API doesn't support this tool type", + ); continue; } } diff --git a/plugins/openai/src/responses/llm.ts b/plugins/openai/src/responses/llm.ts index 9a255d046..4a1dc9d92 100644 --- a/plugins/openai/src/responses/llm.ts +++ b/plugins/openai/src/responses/llm.ts @@ -13,6 +13,7 @@ import { } from '@livekit/agents'; import OpenAI from 'openai'; import type { ChatModels } from '../models.js'; +import { toResponsesTools } from '../tool_utils.js'; import { WSLLM } from '../ws/llm.js'; export interface LLMOptions { @@ -187,24 +188,7 @@ class ResponsesHttpLLMStream extends llm.LLMStream { )) as OpenAI.Responses.ResponseInputItem[]; const tools = this.toolCtx - ? Object.entries(this.toolCtx.functionTools).map(([name, func]) => { - const oaiParams = { - type: 'function' as const, - name: name, - description: func.description, - parameters: llm.toJsonSchema( - func.parameters, - true, - this.strictToolSchema, - ) as unknown as OpenAI.Responses.FunctionTool['parameters'], - } as OpenAI.Responses.FunctionTool; - - if (this.strictToolSchema) { - oaiParams.strict = true; - } - - return oaiParams; - }) + ? toResponsesTools(this.toolCtx, this.strictToolSchema) : undefined; const requestOptions: Record = { ...this.modelOptions }; diff --git a/plugins/openai/src/tool_utils.test.ts b/plugins/openai/src/tool_utils.test.ts new file mode 100644 index 000000000..ce1922f52 --- /dev/null +++ b/plugins/openai/src/tool_utils.test.ts @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { llm } from '@livekit/agents'; +import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { toResponsesTools } from './tool_utils.js'; +import { CodeInterpreter, FileSearch, WebSearch } from './tools.js'; + +describe('toResponsesTools', () => { + it('serializes function tools', () => { + const fn = llm.tool({ + name: 'lookup_weather', + description: 'Look up weather', + parameters: z.object({ city: z.string() }), + execute: async () => 'sunny', + }); + + expect(toResponsesTools(new llm.ToolContext([fn]), true)).toEqual([ + { + type: 'function', + name: 'lookup_weather', + description: 'Look up weather', + parameters: { + $schema: 'http://json-schema.org/draft-07/schema#', + type: 'object', + properties: { city: { type: 'string' } }, + required: ['city'], + additionalProperties: false, + }, + strict: true, + }, + ]); + }); + + it('serializes OpenAI provider tools', () => { + const tools = toResponsesTools( + new llm.ToolContext([ + new WebSearch({ + filters: { allowed_domains: ['docs.livekit.io'] }, + searchContextSize: 'low', + userLocation: { type: 'approximate', country: 'US' }, + }), + new FileSearch({ + vectorStoreIds: ['vs_123'], + maxNumResults: 3, + rankingOptions: { ranker: 'auto' }, + }), + new CodeInterpreter({ container: { type: 'auto', file_ids: ['file_123'] } }), + ]), + false, + ); + + expect(tools).toEqual([ + { + type: 'web_search', + search_context_size: 'low', + filters: { allowed_domains: ['docs.livekit.io'] }, + user_location: { type: 'approximate', country: 'US' }, + }, + { + type: 'file_search', + vector_store_ids: ['vs_123'], + max_num_results: 3, + ranking_options: { ranker: 'auto' }, + }, + { type: 'code_interpreter', container: { type: 'auto', file_ids: ['file_123'] } }, + ]); + }); + + it('omits the code interpreter container when unset', () => { + expect(toResponsesTools(new llm.ToolContext([new CodeInterpreter()]), false)).toEqual([ + { type: 'code_interpreter' }, + ]); + }); + + it('ignores non-OpenAI provider tools', () => { + class OtherProviderTool extends llm.ProviderTool {} + + expect( + toResponsesTools(new llm.ToolContext([new OtherProviderTool({ id: 'other' })]), false), + ).toBeUndefined(); + }); +}); diff --git a/plugins/openai/src/tool_utils.ts b/plugins/openai/src/tool_utils.ts new file mode 100644 index 000000000..1e2e6c709 --- /dev/null +++ b/plugins/openai/src/tool_utils.ts @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { llm } from '@livekit/agents'; +import type OpenAI from 'openai'; +import { OpenAITool } from './tools.js'; + +export function toResponsesTools( + toolCtx: llm.ToolContext, + strictToolSchema: boolean, +): OpenAI.Responses.Tool[] | undefined { + const tools = toolCtx + .flatten() + .map((tool) => { + if (llm.isFunctionTool(tool)) { + const oaiParams = { + type: 'function' as const, + name: tool.name, + description: tool.description, + parameters: llm.toJsonSchema( + tool.parameters, + true, + strictToolSchema, + ) as unknown as OpenAI.Responses.FunctionTool['parameters'], + } as OpenAI.Responses.FunctionTool; + + if (strictToolSchema) { + oaiParams.strict = true; + } + + return oaiParams; + } + + if (tool instanceof OpenAITool) { + return tool.toToolConfig() as unknown as OpenAI.Responses.Tool; + } + + return undefined; + }) + .filter((tool): tool is OpenAI.Responses.Tool => tool !== undefined); + + return tools.length > 0 ? tools : undefined; +} diff --git a/plugins/openai/src/tools.ts b/plugins/openai/src/tools.ts new file mode 100644 index 000000000..1ce779376 --- /dev/null +++ b/plugins/openai/src/tools.ts @@ -0,0 +1,166 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { llm } from '@livekit/agents'; +import type OpenAI from 'openai'; + +/** Base class for OpenAI Responses API provider tools. */ +export abstract class OpenAITool extends llm.ProviderTool { + /** Convert this provider tool to the OpenAI Responses API tool configuration. */ + abstract toToolConfig(): Record; +} + +/** + * High-level guidance for the amount of context window space to use for web search. + * OpenAI defaults this to `medium`. + */ +export type WebSearchContextSize = 'low' | 'medium' | 'high'; + +/** Options for the OpenAI web search tool. */ +export interface WebSearchOptions { + /** + * Filters for the search, such as allowed domains. If not provided, all domains are allowed. + */ + filters?: OpenAI.Responses.WebSearchTool['filters']; + + /** + * Amount of context window space to use for the search. Defaults to `medium`. + */ + searchContextSize?: WebSearchContextSize | null; + + /** Approximate location of the user, such as city, region, country, or timezone. */ + userLocation?: OpenAI.Responses.WebSearchTool['user_location']; +} + +/** + * Search the Internet for sources related to the prompt. + * + * @see https://platform.openai.com/docs/guides/tools-web-search + */ +export class WebSearch extends OpenAITool { + /** Filters for the search, such as allowed domains. */ + readonly filters: OpenAI.Responses.WebSearchTool['filters'] | undefined; + + /** Amount of context window space to use for the search. */ + readonly searchContextSize: WebSearchContextSize | null; + + /** Approximate location of the user. */ + readonly userLocation: OpenAI.Responses.WebSearchTool['user_location'] | undefined; + + constructor({ filters, searchContextSize = 'medium', userLocation }: WebSearchOptions = {}) { + super({ id: 'openai_web_search' }); + this.filters = filters; + this.searchContextSize = searchContextSize; + this.userLocation = userLocation; + } + + toToolConfig(): Record { + const result: Record = { + type: 'web_search', + search_context_size: this.searchContextSize, + }; + if (this.userLocation !== undefined) { + result.user_location = this.userLocation; + } + if (this.filters !== undefined) { + result.filters = this.filters; + } + return result; + } +} + +/** Options for the OpenAI file search tool. */ +export interface FileSearchOptions { + /** IDs of the vector stores to search. */ + vectorStoreIds?: string[]; + + /** Filter to apply to file search results. */ + filters?: OpenAI.Responses.FileSearchTool['filters']; + + /** Maximum number of results to return. This should be between 1 and 50 inclusive. */ + maxNumResults?: number; + + /** Ranking options for search, including ranker and score threshold. */ + rankingOptions?: OpenAI.Responses.FileSearchTool.RankingOptions; +} + +/** + * Search for relevant content from uploaded files. + * + * @see https://platform.openai.com/docs/guides/tools-file-search + */ +export class FileSearch extends OpenAITool { + /** IDs of the vector stores to search. */ + readonly vectorStoreIds: string[]; + + /** Filter to apply to file search results. */ + readonly filters: OpenAI.Responses.FileSearchTool['filters'] | undefined; + + /** Maximum number of results to return. */ + readonly maxNumResults: number | undefined; + + /** Ranking options for search. */ + readonly rankingOptions: OpenAI.Responses.FileSearchTool.RankingOptions | undefined; + + constructor({ + vectorStoreIds = [], + filters, + maxNumResults, + rankingOptions, + }: FileSearchOptions = {}) { + super({ id: 'openai_file_search' }); + this.vectorStoreIds = [...vectorStoreIds]; + this.filters = filters; + this.maxNumResults = maxNumResults; + this.rankingOptions = rankingOptions; + } + + toToolConfig(): Record { + const result: Record = { + type: 'file_search', + vector_store_ids: this.vectorStoreIds, + }; + if (this.filters !== undefined) { + result.filters = this.filters; + } + if (this.maxNumResults !== undefined) { + result.max_num_results = this.maxNumResults; + } + if (this.rankingOptions !== undefined) { + result.ranking_options = this.rankingOptions; + } + return result; + } +} + +/** Options for the OpenAI code interpreter tool. */ +export interface CodeInterpreterOptions { + /** + * Code interpreter container. Can be a container ID or an object that specifies uploaded file IDs + * to make available to the code. + */ + container?: OpenAI.Responses.Tool.CodeInterpreter['container'] | null; +} + +/** + * Run Python code to help generate a response to a prompt. + * + * @see https://platform.openai.com/docs/guides/tools-code-interpreter + */ +export class CodeInterpreter extends OpenAITool { + /** Code interpreter container ID or configuration. */ + readonly container: OpenAI.Responses.Tool.CodeInterpreter['container'] | null; + + constructor({ container = null }: CodeInterpreterOptions = {}) { + super({ id: 'openai_code_interpreter' }); + this.container = container; + } + + toToolConfig(): Record { + const result: Record = { type: 'code_interpreter' }; + if (this.container !== null) { + result.container = this.container; + } + return result; + } +} diff --git a/plugins/openai/src/ws/llm.ts b/plugins/openai/src/ws/llm.ts index d22d7a753..f75054387 100644 --- a/plugins/openai/src/ws/llm.ts +++ b/plugins/openai/src/ws/llm.ts @@ -15,6 +15,7 @@ import { import type OpenAI from 'openai'; import { WebSocket } from 'ws'; import type { ChatModels } from '../models.js'; +import { toResponsesTools } from '../tool_utils.js'; import type { WsOutputItemDoneEvent, WsOutputTextDeltaEvent, @@ -429,26 +430,7 @@ export class WSLLMStream extends llm.LLMStream { 'openai.responses', )) as OpenAI.Responses.ResponseInputItem[]; - const tools = this.toolCtx - ? Object.entries(this.toolCtx.functionTools).map(([name, func]) => { - const oaiParams = { - type: 'function' as const, - name, - description: func.description, - parameters: llm.toJsonSchema( - func.parameters, - true, - this.#strictToolSchema, - ) as unknown as OpenAI.Responses.FunctionTool['parameters'], - } as OpenAI.Responses.FunctionTool; - - if (this.#strictToolSchema) { - oaiParams.strict = true; - } - - return oaiParams; - }) - : undefined; + const tools = this.toolCtx ? toResponsesTools(this.toolCtx, this.#strictToolSchema) : undefined; const requestOptions: Record = { ...this.#modelOptions }; if (!tools) { diff --git a/plugins/phonic/src/realtime/realtime_model.ts b/plugins/phonic/src/realtime/realtime_model.ts index 09933b580..bec3906dd 100644 --- a/plugins/phonic/src/realtime/realtime_model.ts +++ b/plugins/phonic/src/realtime/realtime_model.ts @@ -368,23 +368,25 @@ export class RealtimeSession extends llm.RealtimeSession { } this._tools = tools.copy(); - this.toolDefinitions = Object.entries(tools.functionTools).map(([name, tool]) => ({ - type: 'custom_websocket', - tool_schema: { - type: 'function', - function: { - name, - description: tool.description, - parameters: llm.toJsonSchema(tool.parameters), - strict: true, + // TODO: support provider tools in the Phonic schema. + this.toolDefinitions = tools + .flatten() + .filter(llm.isFunctionTool) + .map((t) => ({ + type: 'custom_websocket' as const, + tool_schema: { + type: 'function' as const, + function: { + name: t.name, + description: t.description, + parameters: llm.toJsonSchema(t.parameters), + strict: true, + }, }, - }, - tool_call_output_timeout_ms: TOOL_CALL_OUTPUT_TIMEOUT_MS, - // Tool chaining and tool calls during speech are not supported at this time - // for ease of implementation within the RealtimeSession generations framework - wait_for_speech_before_tool_call: true, - allow_tool_chaining: false, - })); + tool_call_output_timeout_ms: TOOL_CALL_OUTPUT_TIMEOUT_MS, + wait_for_speech_before_tool_call: true, + allow_tool_chaining: false, + })); this.toolsReady.resolve(); } @@ -404,21 +406,25 @@ export class RealtimeSession extends llm.RealtimeSession { } if (tools !== undefined) { this._tools = tools.copy(); - this.toolDefinitions = Object.entries(tools.functionTools).map(([name, tool]) => ({ - type: 'custom_websocket', - tool_schema: { - type: 'function', - function: { - name, - description: tool.description, - parameters: llm.toJsonSchema(tool.parameters), - strict: true, + // TODO: support provider tools in the Phonic schema. + this.toolDefinitions = tools + .flatten() + .filter(llm.isFunctionTool) + .map((t) => ({ + type: 'custom_websocket' as const, + tool_schema: { + type: 'function' as const, + function: { + name: t.name, + description: t.description, + parameters: llm.toJsonSchema(t.parameters), + strict: true, + }, }, - }, - tool_call_output_timeout_ms: TOOL_CALL_OUTPUT_TIMEOUT_MS, - wait_for_speech_before_tool_call: true, - allow_tool_chaining: false, - })); + tool_call_output_timeout_ms: TOOL_CALL_OUTPUT_TIMEOUT_MS, + wait_for_speech_before_tool_call: true, + allow_tool_chaining: false, + })); } if (chatCtx !== undefined) { this._chatCtx = chatCtx.copy(); diff --git a/plugins/test/src/llm.ts b/plugins/test/src/llm.ts index 534dce2df..1f912654a 100644 --- a/plugins/test/src/llm.ts +++ b/plugins/test/src/llm.ts @@ -200,6 +200,57 @@ export const llm = async (llm: llmlib.LLM, skipOptionalArgs: boolean) => { expect(JSON.parse(calls[0]!.args).address).toBeUndefined(); }); }); + + describe('toolset', async () => { + const buildToolsetContext = () => { + const weatherToolset = new llmlib.Toolset({ + id: 'weather_toolset', + tools: [ + llmlib.tool({ + name: 'getWeather', + description: 'Get the current weather in a given location', + parameters: z.object({ + location: z.string().describe('The city and state, e.g. San Francisco, CA'), + unit: z.enum(['celsius', 'fahrenheit']).describe('The temperature unit to use'), + }), + execute: async () => {}, + }), + ], + }); + + const directTool = llmlib.tool({ + name: 'playMusic', + description: 'Play music', + parameters: z.object({ + name: z.string().describe('The artist and name of the song'), + }), + execute: async () => {}, + }); + + return new llmlib.ToolContext([weatherToolset, directTool]); + }; + + it('should call a function tool that lives inside a Toolset', async () => { + const ctx = buildToolsetContext(); + const calls = await requestFncCall( + llm, + "What's the weather in San Francisco, in Celsius?", + ctx, + ); + + expect(calls.length).toStrictEqual(1); + expect(calls[0]!.name).toStrictEqual('getWeather'); + expect(JSON.parse(calls[0]!.args).unit).toStrictEqual('celsius'); + }); + + it('should expose direct tools alongside Toolset tools', async () => { + const ctx = buildToolsetContext(); + const calls = await requestFncCall(llm, 'Play the song "Bohemian Rhapsody" by Queen.', ctx); + + expect(calls.length).toStrictEqual(1); + expect(calls[0]!.name).toStrictEqual('playMusic'); + }); + }); }); }; From a24fb0be51c96f1f8cee06e4cf04143ca774d232 Mon Sep 17 00:00:00 2001 From: Toubat Date: Mon, 8 Jun 2026 14:35:49 -0700 Subject: [PATCH 03/15] fix(agents): thread FlushSentinel through Agent.create llmNode types Agent.llmNode now returns ReadableStream, but the agent_v2 hook overrides and AgentHookAdapter still declared the narrower ChatChunk | string union, so passing super.llmNode as the fallback failed to type-check. Widen the override return types and the adapter's fallback/return signatures to include FlushSentinel. Co-authored-by: Cursor --- agents/src/voice/agent_v2.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/agents/src/voice/agent_v2.ts b/agents/src/voice/agent_v2.ts index 47df78b85..2f00402ce 100644 --- a/agents/src/voice/agent_v2.ts +++ b/agents/src/voice/agent_v2.ts @@ -14,6 +14,7 @@ import type { } from '../llm/index.js'; import type { STT, SpeechEvent } from '../stt/index.js'; import type { TTS } from '../tts/index.js'; +import type { FlushSentinel } from '../types.js'; import { readStream, toStream } from '../utils.js'; import type { VAD } from '../vad.js'; import type { Agent, AgentOptions, AgentTask, AgentTaskOptions, ModelSettings } from './agent.js'; @@ -184,7 +185,7 @@ export function createAgentV2( chatCtx: ChatContext, toolCtx: ToolContext, modelSettings: ModelSettings, - ): Promise | null> { + ): Promise | null> { return this.hookAdapter.llmNode(chatCtx, toolCtx, modelSettings, () => super.llmNode(chatCtx, toolCtx, modelSettings), ); @@ -278,7 +279,7 @@ export function createAgentTaskV2( chatCtx: ChatContext, toolCtx: ToolContext, modelSettings: ModelSettings, - ): Promise | null> { + ): Promise | null> { return this.hookAdapter.llmNode(chatCtx, toolCtx, modelSettings, () => super.llmNode(chatCtx, toolCtx, modelSettings), ); @@ -357,8 +358,8 @@ class AgentHookAdapter> { chatCtx: ChatContext, toolCtx: ToolContext, modelSettings: ModelSettings, - fallback: () => Promise | null>, - ): Promise | null> { + fallback: () => Promise | null>, + ): Promise | null> { if (!this.hooks.llmNode) { return fallback(); } From c8b56bd5d43e6f532ba40b69cb9de2ab31df07fb Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Mon, 8 Jun 2026 14:59:00 -0700 Subject: [PATCH 04/15] feat(agents): add beta end call tool (#1474) Co-authored-by: Brian Yin Co-authored-by: rosetta-livekit-bot[bot] <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Co-authored-by: u9g --- .changeset/port-end-call-tool.md | 5 + agents/src/beta/index.ts | 7 + agents/src/beta/tools/end_call.ts | 188 ++++++++++++++++++++++++ agents/src/beta/tools/index.ts | 10 ++ agents/src/llm/index.ts | 1 + agents/src/llm/tool_context.test.ts | 49 +++--- agents/src/llm/tool_context.ts | 79 ++++++---- agents/src/voice/agent_activity.test.ts | 97 +++++++++++- agents/src/voice/agent_activity.ts | 41 +++++- plugins/google/src/aiplatform_llm.ts | 5 +- plugins/openai/src/responses/llm.ts | 1 + 11 files changed, 420 insertions(+), 63 deletions(-) create mode 100644 .changeset/port-end-call-tool.md create mode 100644 agents/src/beta/tools/end_call.ts create mode 100644 agents/src/beta/tools/index.ts diff --git a/.changeset/port-end-call-tool.md b/.changeset/port-end-call-tool.md new file mode 100644 index 000000000..f1646f943 --- /dev/null +++ b/.changeset/port-end-call-tool.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': minor +--- + +Add beta EndCallTool for ending calls from agent tools diff --git a/agents/src/beta/index.ts b/agents/src/beta/index.ts index 98ac382e9..d5141be55 100644 --- a/agents/src/beta/index.ts +++ b/agents/src/beta/index.ts @@ -12,3 +12,10 @@ export { type WarmTransferTaskOptions, } from './workflows/index.js'; export { Instructions } from '../llm/index.js'; +export { + END_CALL_DESCRIPTION, + createEndCallTool, + type EndCallToolCalledEvent, + type EndCallToolCompletedEvent, + type EndCallToolOptions, +} from './tools/index.js'; diff --git a/agents/src/beta/tools/end_call.ts b/agents/src/beta/tools/end_call.ts new file mode 100644 index 000000000..34eb63f11 --- /dev/null +++ b/agents/src/beta/tools/end_call.ts @@ -0,0 +1,188 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { type EventEmitter, once } from 'node:events'; +import { setTimeout as waitFor } from 'node:timers/promises'; +import { getJobContext } from '../../job.js'; +import { + RealtimeModel, + type ToolCalledEvent, + type ToolCompletedEvent, + Toolset, + tool, +} from '../../llm/index.js'; +import { log } from '../../log.js'; +import type { AgentSession, AgentSessionCallbacks } from '../../voice/agent_session.js'; +import { AgentSessionEventTypes } from '../../voice/events.js'; +import type { UnknownUserData } from '../../voice/run_context.js'; + +/** How long to wait for the agent's goodbye reply to play out before forcing shutdown. */ +const END_CALL_REPLY_TIMEOUT = 5000; + +/** + * `events.once` typed against {@link AgentSessionCallbacks}, resolving to the event payload (e.g. + * `CloseEvent`) — every session event is single-arg — instead of the raw `any[]` tuple. The cast + * `events.once` forces is confined here. + * + * If `signal` aborts before the event fires, `events.once` rejects with an AbortError. Since these + * waits are fire-and-forget (or race losers), an uncaught rejection would surface as an unhandled + * rejection — so we absorb the abort and resolve to `undefined`, letting callers treat "aborted" + * as a normal (typed) outcome. Other rejections (e.g. an emitter `error`) still propagate. + */ +function onceEvent( + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- callbacks don't depend on UserData + session: AgentSession, + event: E, + options?: { signal?: AbortSignal }, +): Promise[0] | undefined> { + return ( + once(session as unknown as EventEmitter, event, options) as Promise< + Parameters + > + ).then( + ([payload]) => payload, + (err) => { + if (options?.signal?.aborted) return undefined; + throw err; + }, + ); +} + +export const END_CALL_DESCRIPTION = ` +Ends the current call and disconnects immediately. + +Call when: +- The user clearly indicates they are done (e.g., "that's all, bye"). + +Do not call when: +- The user asks to pause, hold, or transfer. +- Intent is unclear. + +This is the final action the agent can take. +Once called, no further interaction is possible with the user. +Don't generate any other text or response when the tool is called. +`; + +export type EndCallToolCalledEvent = ToolCalledEvent; + +export type EndCallToolCompletedEvent = ToolCompletedEvent; + +export type EndCallToolOptions = { + /** Additional description to add to the end call tool. */ + extraDescription?: string; + /** + * Whether to delete the room when the user ends the call. + * Deleting the room disconnects all remote users, including SIP callers. + */ + deleteRoom?: boolean; + /** Tool output to the LLM for generating the tool response. */ + endInstructions?: string | null; + /** Callback to call when the tool is called. */ + onToolCalled?: (event: EndCallToolCalledEvent) => Promise | void; + /** Callback to call when the tool is completed. */ + onToolCompleted?: (event: EndCallToolCompletedEvent) => Promise | void; +}; + +/** + * Allows the agent to end the call and disconnect from the room. + */ +export function createEndCallTool({ + extraDescription = '', + deleteRoom = true, + endInstructions = 'say goodbye to the user', + onToolCalled, + onToolCompleted, +}: EndCallToolOptions = {}): Toolset { + // For a realtime LLM that generates the goodbye reply itself, wait for that reply to play out + // (bounded by END_CALL_REPLY_TIMEOUT) before shutting down. `signal` is aborted when the call + // ends or the toolset is torn down, which cancels whichever of the two races is still pending. + const delayedSessionShutdown = async ( + session: AgentSession, + signal: AbortSignal, + ): Promise => { + const speech = onceEvent(session, AgentSessionEventTypes.SpeechCreated, { signal }).then( + (event) => event?.speechHandle, + ); + const timeout = waitFor(END_CALL_REPLY_TIMEOUT, 'timeout' as const, { signal }).catch( + () => undefined, + ); + + const winner = await Promise.race([speech, timeout]); + if (signal.aborted) return; // session already closed or toolset torn down + + if (winner === 'timeout') { + log().warn('tool reply timed out, shutting down session'); + session.shutdown(); + } else if (winner) { + await winner.waitForPlayout(); + session.shutdown(); + } + }; + + return Toolset.create({ + id: 'end_call', + tools: [ + tool({ + name: 'end_call', + description: `${END_CALL_DESCRIPTION}\n${extraDescription}`, + execute: async (_args, { ctx, abortSignal }) => { + log().debug('end_call tool called'); + const session = ctx.session; + const llm = session.currentAgent.getActivityOrThrow().llm; + + // Lifetime of this invocation: aborts when the session closes, and also when the tool + // call itself is aborted. All listeners/timers below are scoped to it. + const controller = new AbortController(); + const signal = abortSignal + ? AbortSignal.any([abortSignal, controller.signal]) + : controller.signal; + + void onceEvent(session, AgentSessionEventTypes.Close, { signal }).then((event) => { + if (!event) return; // signal aborted before close fired + controller.abort(); // stop the delayed-shutdown race + + const jobCtx = getJobContext(false); + if (!jobCtx) return; + + if (deleteRoom) { + jobCtx.addShutdownCallback(async () => { + log().info('deleting the room because the user ended the call'); + await jobCtx.deleteRoom(); + }); + } + + jobCtx.shutdown(String(event.reason)); + }); + + ctx.speechHandle.addDoneCallback(() => { + if (!(llm instanceof RealtimeModel) || !llm.capabilities.autoToolReplyGeneration) { + session.shutdown(); + return; + } + + void delayedSessionShutdown(session, signal).catch((error) => + log().error({ error }, 'error during delayed session shutdown'), + ); + }); + + if (onToolCalled) { + await onToolCalled({ ctx, arguments: {} }); + } + + const completedEvent = { + ctx, + output: + endInstructions === null + ? undefined + : ({ type: 'output', value: endInstructions } as const), + }; + if (onToolCompleted) { + await onToolCompleted(completedEvent); + } + + return endInstructions ?? undefined; + }, + }), + ], + }); +} diff --git a/agents/src/beta/tools/index.ts b/agents/src/beta/tools/index.ts new file mode 100644 index 000000000..8ef18b993 --- /dev/null +++ b/agents/src/beta/tools/index.ts @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +export { + END_CALL_DESCRIPTION, + createEndCallTool, + type EndCallToolCalledEvent, + type EndCallToolCompletedEvent, + type EndCallToolOptions, +} from './end_call.js'; diff --git a/agents/src/llm/index.ts b/agents/src/llm/index.ts index 03515c5ce..400849be4 100644 --- a/agents/src/llm/index.ts +++ b/agents/src/llm/index.ts @@ -25,6 +25,7 @@ export { type ToolContextEntry, type ToolCtxInput, type ToolOptions, + type ToolsetContext, type ToolsetCreateOptions, type ToolType, } from './tool_context.js'; diff --git a/agents/src/llm/tool_context.test.ts b/agents/src/llm/tool_context.test.ts index c42380478..a18946eb2 100644 --- a/agents/src/llm/tool_context.test.ts +++ b/agents/src/llm/tool_context.test.ts @@ -11,6 +11,7 @@ import { ToolContext, type ToolOptions, Toolset, + type ToolsetContext, tool, } from './tool_context.js'; import { createToolOptions, oaiParams } from './utils.js'; @@ -617,9 +618,13 @@ describe('Toolset', () => { expect(ts.tools).toEqual([a, b]); }); + const fakeToolsetContext = ( + updateTools: (tools: readonly Tool[]) => void = () => {}, + ): ToolsetContext => ({ updateTools }); + it('default setup and aclose are no-ops', async () => { const ts = new Toolset({ id: 'noop', tools: [] }); - await expect(ts.setup()).resolves.toBeUndefined(); + await expect(ts.setup(fakeToolsetContext())).resolves.toBeUndefined(); await expect(ts.aclose()).resolves.toBeUndefined(); }); @@ -635,20 +640,17 @@ describe('Toolset', () => { } const ts = new Recording({ id: 'rec', tools: [] }); - await ts.setup(); + await ts.setup(fakeToolsetContext()); await ts.aclose(); expect(events).toEqual(['setup:rec', 'close:rec']); }); - it('Toolset.create() composes lifecycle callbacks without subclassing', async () => { + it('Toolset.create() resolves a static tools list eagerly and composes aclose', async () => { const a = makeFn('a'); const events: string[] = []; const ts = Toolset.create({ id: 'composed', tools: [a], - setup: async () => { - events.push('setup'); - }, aclose: async () => { events.push('close'); }, @@ -656,37 +658,38 @@ describe('Toolset', () => { expect(ts).toBeInstanceOf(Toolset); expect(ts.id).toBe('composed'); - expect(ts.tools).toEqual([a]); + expect(ts.tools).toEqual([a]); // static tools available before activation - await ts.setup(); + await ts.setup(fakeToolsetContext()); await ts.aclose(); - expect(events).toEqual(['setup', 'close']); + expect(events).toEqual(['close']); }); - it('Toolset.create() defaults setup and aclose to no-ops when callbacks are omitted', async () => { + it('Toolset.create() defaults aclose to a no-op when omitted', async () => { const ts = Toolset.create({ id: 'bare', tools: [] }); - await expect(ts.setup()).resolves.toBeUndefined(); + await expect(ts.setup(fakeToolsetContext())).resolves.toBeUndefined(); await expect(ts.aclose()).resolves.toBeUndefined(); }); - it('Toolset.create() accepts a tools thunk, re-evaluated on every access (dynamic)', () => { + it('lets setup push tools after activation via ctx.updateTools', async () => { const a = makeFn('a'); const b = makeFn('b'); - const current: Tool[] = [a]; - let calls = 0; + let push!: (tools: readonly Tool[]) => void; const ts = Toolset.create({ - id: 'dynamic', - tools: () => { - calls += 1; - return current; + id: 'mcp', + setup: async ({ updateTools }) => { + push = updateTools; }, + tools: [], }); - // Each access re-invokes the thunk so the toolset reflects the current source-of-truth. - expect(ts.tools).toEqual([a]); - expect(calls).toBe(1); - current.push(b); + + // Mimic the runtime: ctx.updateTools writes the toolset's current tools. + await ts.setup(fakeToolsetContext((tools) => ts._setTools(tools))); + expect(ts.tools).toEqual([]); + + // A dynamic source (e.g. an MCP server) pushes its tools after connecting. + push([a, b]); expect(ts.tools).toEqual([a, b]); - expect(calls).toBe(2); }); it('is flattened into a ToolContext: function tools merged, toolset tracked', () => { diff --git a/agents/src/llm/tool_context.ts b/agents/src/llm/tool_context.ts index 85ed81223..4cc087352 100644 --- a/agents/src/llm/tool_context.ts +++ b/agents/src/llm/tool_context.ts @@ -211,6 +211,15 @@ export interface ToolCompletedEvent { output?: { type: 'output'; value: unknown } | { type: 'error'; value: Error }; } +/** Context passed to a {@link Toolset}'s `setup` hook when it activates. */ +export interface ToolsetContext { + /** + * Replace the toolset's tools. Useful for dynamic sources + * (e.g. an MCP server) whose tools are discovered after `setup` or change at runtime. + */ + updateTools(tools: readonly Tool[]): void; +} + /** * Function tools of a `ToolContext`, sorted by name for deterministic provider payloads. * Provider tools are intentionally excluded — callers that need them iterate `flatten()`. @@ -239,7 +248,7 @@ export function sortedToolNames(toolCtx: ToolContext | undefined): string[] { export class Toolset { readonly #id: string; - readonly #tools: Tool[]; + #tools: readonly Tool[]; readonly [TOOLSET_SYMBOL] = true as const; @@ -249,9 +258,10 @@ export class Toolset { } /** - * Compose a `Toolset` with inline `setup` / `aclose` hooks instead of subclassing. `tools` - * may also be a thunk that is re-evaluated on every `.tools` access, so the toolset can - * expose a dynamic list that changes after `setup()` runs. + * For when your tools share something that needs setup or cleanup, like a DB pool, an open MCP + * client, or listeners on a shared bus. `setup` runs once at activation, `aclose` once at + * teardown. If the tool list itself is dynamic (e.g. an MCP server), push it from `setup` via + * {@link ToolsetContext.updateTools}. * * @example Static tool list with a shared backing resource * ```ts @@ -260,20 +270,26 @@ export class Toolset { * return Toolset.create({ * id: 'postgres', * tools: [queryOrders, queryCustomers], - * setup: () => pool.connect(), * aclose: () => pool.end(), * }); * } * ``` * - * @example Dynamic tool list + * @example Dynamic tool list bound to an external source * ```ts * function createMcpToolset(url: string): Toolset { * const client = new MCPClient({ url }); * return Toolset.create({ * id: 'mcp_remote', - * tools: () => client.getTools(), - * setup: () => client.connect(), + * // setup connects and wires listeners that push the server's tools whenever they change; + * // the runtime re-advertises without re-running anything. + * setup: async ({ updateTools }) => { + * const sync = async () => updateTools(await client.listTools()); + * client.on('connect', sync); + * client.on('tool_list_changed', sync); + * await client.connect(); + * }, + * tools: [], * aclose: () => client.disconnect(), * }); * } @@ -291,48 +307,49 @@ export class Toolset { return this.#tools; } - async setup(): Promise {} + /** + * Replace the toolset's current tools. Backs {@link ToolsetContext.updateTools}; the runtime + * re-flattens and re-advertises after calling it. + * + * @internal + */ + _setTools(tools: readonly Tool[]): void { + this.#tools = [...tools]; + } + + async setup(_ctx: ToolsetContext): Promise {} async aclose(): Promise {} } -/** Options accepted by `Toolset.create()` — id + tools plus optional lifecycle hooks. */ +/** Options accepted by `Toolset.create()` — id + tools plus optional setup/teardown hooks. */ export interface ToolsetCreateOptions { id: string; /** - * Either a static list of tools, or a thunk re-evaluated on every `tools` access — useful - * when the underlying source (e.g. an MCP discovery loop) can produce a dynamic tool list. + * One-time async initialization run when the toolset activates — e.g. connecting to a server + * and wiring listeners. Push a changed tool list via {@link ToolsetContext.updateTools}. */ - tools: readonly Tool[] | (() => readonly Tool[]); - /** Invoked when the toolset becomes active in an `AgentActivity`. */ - setup?: () => Promise; - /** Invoked when the toolset is being torn down. */ + setup?: (ctx: ToolsetContext) => Promise; + /** The toolset's initial tools. */ + tools: readonly Tool[]; + /** Invoked when the toolset is being torn down. Release awaitable resources here. */ aclose?: () => Promise; } /** Backing implementation of `Toolset.create()`. Kept private so callers go through the factory. */ class ToolsetFactory extends Toolset { - readonly #toolsSource: readonly Tool[] | (() => readonly Tool[]); - - readonly #setupFn?: () => Promise; + readonly #setupFn?: (ctx: ToolsetContext) => Promise; readonly #acloseFn?: () => Promise; - constructor({ id, tools, setup, aclose }: ToolsetCreateOptions) { - // Pass [] to super and override the `tools` getter so a thunk can be re-evaluated on - // every access (lets callers expose a dynamic tool list). - super({ id, tools: [] }); - this.#toolsSource = tools; + constructor({ id, setup, tools, aclose }: ToolsetCreateOptions) { + super({ id, tools }); this.#setupFn = setup; this.#acloseFn = aclose; } - override get tools(): readonly Tool[] { - return typeof this.#toolsSource === 'function' ? this.#toolsSource() : this.#toolsSource; - } - - override async setup(): Promise { - if (this.#setupFn) await this.#setupFn(); + override async setup(ctx: ToolsetContext): Promise { + if (this.#setupFn) await this.#setupFn(ctx); } override async aclose(): Promise { @@ -389,7 +406,7 @@ export class ToolContext { /** A copy of all provider tools in the tool context, including those in tool sets. */ get providerTools(): ProviderTool[] { - return this._providerTools; + return [...this._providerTools]; } /** A copy of all toolsets registered in the context. */ diff --git a/agents/src/voice/agent_activity.test.ts b/agents/src/voice/agent_activity.test.ts index 2e76d9d71..da8bab4ba 100644 --- a/agents/src/voice/agent_activity.test.ts +++ b/agents/src/voice/agent_activity.test.ts @@ -16,9 +16,9 @@ */ import { Heap } from 'heap-js'; import { describe, expect, it, vi } from 'vitest'; -import { ChatContext } from '../llm/chat_context.js'; +import { AgentConfigUpdate, ChatContext } from '../llm/chat_context.js'; import { LLM, type LLMStream } from '../llm/llm.js'; -import { ToolContext } from '../llm/tool_context.js'; +import { type Tool, ToolContext, Toolset, tool } from '../llm/tool_context.js'; import { Future } from '../utils.js'; import { AgentActivity } from './agent_activity.js'; import type { PreemptiveGenerationInfo } from './audio_recognition.js'; @@ -409,3 +409,96 @@ describe('AgentActivity - onPreemptiveGeneration guards', () => { expect(cancelPreemptiveGeneration).not.toHaveBeenCalled(); }); }); + +/** + * Regression test for the dynamic-toolset push path. + * + * When an already-activated toolset swaps its tools at runtime (e.g. an MCP server pushes a new + * tool list via `ToolsetContext.updateTools`), `setupToolsetList`'s wiring must (1) invoke + * `onToolsetToolsChanged`, which now funnels through `updateTools`, and (2) record an + * `AgentConfigUpdate` in the agent chat context + session history so a non-realtime pipeline's + * chat context reflects the new tool set on the next turn. + */ +class FakeToolsetLLM extends LLM { + label(): string { + return 'fake.toolset.LLM'; + } + chat(): LLMStream { + throw new Error('not used in these tests'); + } +} + +describe('AgentActivity - onToolsetToolsChanged (dynamic toolset push)', () => { + const makeFn = (name: string) => + tool({ name, description: `${name} tool`, execute: async () => name }); + + function buildToolsetActivity(toolset: Toolset) { + const history = new ChatContext(); + const fakeActivity = { + _toolsetsSetup: true, + realtimeSession: undefined, + llm: new FakeToolsetLLM(), + agent: { + _toolCtx: new ToolContext([toolset]), + _chatCtx: new ChatContext(), + }, + agentSession: { history }, + updateChatCtx: vi.fn(async () => {}), + logger: { info() {}, debug() {}, warn() {}, error() {} }, + }; + Object.setPrototypeOf(fakeActivity, AgentActivity.prototype); + return { fakeActivity, history }; + } + + it('fires onToolsetToolsChanged on a dynamic push and records an AgentConfigUpdate', async () => { + const toolA = makeFn('toolA'); + const toolB = makeFn('toolB'); + + // Capture the wired ctx.updateTools the framework hands the toolset during setup. + let pushTools!: (tools: readonly Tool[]) => void; + const toolset = Toolset.create({ + id: 'dynamic', + tools: [toolA], + setup: async ({ updateTools }) => { + pushTools = updateTools; + }, + }); + + const { fakeActivity, history } = buildToolsetActivity(toolset); + + const changedSpy = vi.spyOn( + AgentActivity.prototype as unknown as Record<'onToolsetToolsChanged', () => Promise>, + 'onToolsetToolsChanged', + ); + + // Activate the toolset through the real path so it captures the push channel. + const setupToolsetList = (AgentActivity.prototype as Record) + .setupToolsetList as (this: unknown, toolsets: readonly Toolset[]) => Promise; + await setupToolsetList.call(fakeActivity, [toolset]); + + expect(changedSpy).not.toHaveBeenCalled(); + + // (1) A dynamic push swaps the toolset's tools — the wiring must invoke onToolsetToolsChanged. + pushTools([toolA, toolB]); + expect(changedSpy).toHaveBeenCalledTimes(1); + await changedSpy.mock.results[0]!.value; + + // (2) An AgentConfigUpdate naming the added tool lands in the session history. + const updates = history.items.filter( + (i): i is AgentConfigUpdate => i instanceof AgentConfigUpdate, + ); + expect(updates).toHaveLength(1); + expect(updates[0]!.toolsAdded).toContain('toolB'); + expect(updates[0]!.toolsRemoved ?? []).not.toContain('toolA'); + + // The refreshed tool context advertises the new tool to the next turn, and the non-realtime + // pipeline's chat context was refreshed via updateChatCtx. + expect(Object.keys(fakeActivity.agent._toolCtx.functionTools).sort()).toEqual([ + 'toolA', + 'toolB', + ]); + expect(fakeActivity.updateChatCtx).toHaveBeenCalledTimes(1); + + changedSpy.mockRestore(); + }); +}); diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index fd7f5c806..66c55a747 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -23,6 +23,7 @@ import { instructionsEqual, renderInstructions, } from '../llm/chat_context.js'; +import type { Toolset } from '../llm/index.js'; import { type ChatItem, type FunctionCall, @@ -41,7 +42,6 @@ import { ToolContext, type ToolContextEntry, ToolFlag, - Toolset, isFunctionTool, isToolset, } from '../llm/index.js'; @@ -786,14 +786,18 @@ export class AgentActivity implements RecognitionHooks { const oldToolNames = new Set(Object.keys(oldToolCtx.functionTools)); const oldToolsets = oldToolCtx.toolsets; const newToolCtx = new ToolContext(tools); - const newToolNames = new Set(Object.keys(newToolCtx.functionTools)); const newToolsets = newToolCtx.toolsets; - const toolsAdded = [...newToolNames].filter((name) => !oldToolNames.has(name)); - const toolsRemoved = [...oldToolNames].filter((name) => !newToolNames.has(name)); const addedToolsets = newToolsets.filter((ts) => !oldToolsets.includes(ts)); const removedToolsets = oldToolsets.filter((ts) => !newToolsets.includes(ts)); + // Resolve added factory toolsets before re-flattening, so their tools are included in the + // advertised set (newToolNames is computed below, after resolution). await this.setupToolsetList(addedToolsets); + newToolCtx.updateTools(newToolCtx.tools); + const newToolNames = new Set(Object.keys(newToolCtx.functionTools)); + const toolsAdded = [...newToolNames].filter((name) => !oldToolNames.has(name)); + const toolsRemoved = [...oldToolNames].filter((name) => !newToolNames.has(name)); + this.agent._toolCtx = newToolCtx; await this.closeToolsetList(removedToolsets); @@ -4185,6 +4189,8 @@ export class AgentActivity implements RecognitionHooks { if (this._toolsetsSetup) return; this._toolsetsSetup = true; await this.setupToolsetList(this.agent.toolCtx.toolsets); + // Re-flatten now that any factory toolsets have resolved their tools, so they're advertised. + this.agent._toolCtx.updateTools(this.agent._toolCtx.tools); } private async closeToolsets(): Promise { @@ -4193,8 +4199,33 @@ export class AgentActivity implements RecognitionHooks { await this.closeToolsetList(this.agent.toolCtx.toolsets); } + /** + * Refresh the agent's tool context after a dynamic toolset pushed a new tool list (via + * `ToolsetContext.updateTools`), routing through updateTools so history, chat context, and the + * realtime session stay in sync. The next LLM turn picks up the new tools automatically. + */ + private async onToolsetToolsChanged(): Promise { + if (!this._toolsetsSetup) return; + const current = this.agent._toolCtx; + if (new ToolContext(current.tools).equals(current)) return; + // Same toolset entries, so updateTools' setup/close steps are no-ops (no re-entrancy here). + await this.updateTools(current.tools); + } + private async setupToolsetList(toolsets: readonly Toolset[]): Promise { - const outputs = await Promise.allSettled(toolsets.map((ts) => ts.setup())); + const outputs = await Promise.allSettled( + toolsets.map((ts) => + ts.setup({ + // A dynamic toolset pushes a changed tool list here; re-flatten and re-advertise it. + updateTools: (tools) => { + ts._setTools(tools); + void this.onToolsetToolsChanged().catch((error) => + this.logger.error({ error }, 'error re-advertising toolset tools'), + ); + }, + }), + ), + ); for (const output of outputs) { if (output.status === 'rejected') { this.logger.error({ error: output.reason }, 'error setting up toolset'); diff --git a/plugins/google/src/aiplatform_llm.ts b/plugins/google/src/aiplatform_llm.ts index 8ac75d99c..19e530eed 100644 --- a/plugins/google/src/aiplatform_llm.ts +++ b/plugins/google/src/aiplatform_llm.ts @@ -165,19 +165,20 @@ export class AIPlatformLLM extends llm.LLM { chat({ chatCtx, - toolCtx, + toolCtx: toolCtxInput, connOptions = DEFAULT_API_CONNECT_OPTIONS, parallelToolCalls, toolChoice, extraKwargs, }: { chatCtx: llm.ChatContext; - toolCtx?: llm.ToolContext; + toolCtx?: llm.ToolCtxInput; connOptions?: APIConnectOptions; parallelToolCalls?: boolean; toolChoice?: llm.ToolChoice; extraKwargs?: Record; }): inference.LLMStream { + const toolCtx = llm.toToolContext(toolCtxInput); const extras: Record = { ...extraKwargs }; if (this.#opts.temperature !== undefined) { diff --git a/plugins/openai/src/responses/llm.ts b/plugins/openai/src/responses/llm.ts index 67fde0e91..aaf68a5ed 100644 --- a/plugins/openai/src/responses/llm.ts +++ b/plugins/openai/src/responses/llm.ts @@ -187,6 +187,7 @@ class ResponsesHttpLLMStream extends llm.LLMStream { 'openai.responses', )) as OpenAI.Responses.ResponseInputItem[]; + // TODO: support provider tools in the Responses schema. const tools = this.toolCtx ? toResponsesTools(this.toolCtx, this.strictToolSchema) : undefined; From f089da479135e750cb44e1fb4f4a288789e63f4e Mon Sep 17 00:00:00 2001 From: Toubat Date: Mon, 8 Jun 2026 16:20:39 -0700 Subject: [PATCH 05/15] arden end-call shutdown and tool guards Catch end-call close listener errors to avoid unhandled rejections during shutdown, and make public tool type guards return false for null inputs. --- agents/src/beta/tools/end_call.ts | 39 ++++++++++++----------------- agents/src/llm/tool_context.test.ts | 2 +- agents/src/llm/tool_context.ts | 16 +++++------- 3 files changed, 23 insertions(+), 34 deletions(-) diff --git a/agents/src/beta/tools/end_call.ts b/agents/src/beta/tools/end_call.ts index 34eb63f11..962deda36 100644 --- a/agents/src/beta/tools/end_call.ts +++ b/agents/src/beta/tools/end_call.ts @@ -19,16 +19,7 @@ import type { UnknownUserData } from '../../voice/run_context.js'; /** How long to wait for the agent's goodbye reply to play out before forcing shutdown. */ const END_CALL_REPLY_TIMEOUT = 5000; -/** - * `events.once` typed against {@link AgentSessionCallbacks}, resolving to the event payload (e.g. - * `CloseEvent`) — every session event is single-arg — instead of the raw `any[]` tuple. The cast - * `events.once` forces is confined here. - * - * If `signal` aborts before the event fires, `events.once` rejects with an AbortError. Since these - * waits are fire-and-forget (or race losers), an uncaught rejection would surface as an unhandled - * rejection — so we absorb the abort and resolve to `undefined`, letting callers treat "aborted" - * as a normal (typed) outcome. Other rejections (e.g. an emitter `error`) still propagate. - */ +/** Typed wrapper around `events.once`; abort resolves to `undefined`, other errors propagate. */ function onceEvent( // eslint-disable-next-line @typescript-eslint/no-explicit-any -- callbacks don't depend on UserData session: AgentSession, @@ -137,22 +128,24 @@ export function createEndCallTool({ ? AbortSignal.any([abortSignal, controller.signal]) : controller.signal; - void onceEvent(session, AgentSessionEventTypes.Close, { signal }).then((event) => { - if (!event) return; // signal aborted before close fired - controller.abort(); // stop the delayed-shutdown race + void onceEvent(session, AgentSessionEventTypes.Close, { signal }) + .then((event) => { + if (!event) return; // signal aborted before close fired + controller.abort(); // stop the delayed-shutdown race - const jobCtx = getJobContext(false); - if (!jobCtx) return; + const jobCtx = getJobContext(false); + if (!jobCtx) return; - if (deleteRoom) { - jobCtx.addShutdownCallback(async () => { - log().info('deleting the room because the user ended the call'); - await jobCtx.deleteRoom(); - }); - } + if (deleteRoom) { + jobCtx.addShutdownCallback(async () => { + log().info('deleting the room because the user ended the call'); + await jobCtx.deleteRoom(); + }); + } - jobCtx.shutdown(String(event.reason)); - }); + jobCtx.shutdown(String(event.reason)); + }) + .catch((error) => log().error({ error }, 'error during end call shutdown')); ctx.speechHandle.addDoneCallback(() => { if (!(llm instanceof RealtimeModel) || !llm.capabilities.autoToolReplyGeneration) { diff --git a/agents/src/llm/tool_context.test.ts b/agents/src/llm/tool_context.test.ts index a18946eb2..5f2f80cca 100644 --- a/agents/src/llm/tool_context.test.ts +++ b/agents/src/llm/tool_context.test.ts @@ -631,7 +631,7 @@ describe('Toolset', () => { it('lets subclasses override lifecycle hooks', async () => { const events: string[] = []; class Recording extends Toolset { - override async setup(): Promise { + override async setup(_ctx: ToolsetContext): Promise { events.push(`setup:${this.id}`); } override async aclose(): Promise { diff --git a/agents/src/llm/tool_context.ts b/agents/src/llm/tool_context.ts index 4cc087352..50c4220f2 100644 --- a/agents/src/llm/tool_context.ts +++ b/agents/src/llm/tool_context.ts @@ -608,34 +608,30 @@ export function tool(tool: any): any { // eslint-disable-next-line @typescript-eslint/no-explicit-any export function isTool(tool: any): tool is Tool { - return tool && tool[TOOL_SYMBOL] === true; + return !!tool && tool[TOOL_SYMBOL] === true; } // eslint-disable-next-line @typescript-eslint/no-explicit-any export function isFunctionTool(tool: any): tool is FunctionTool { - const isTool = tool && tool[TOOL_SYMBOL] === true; - const isFunctionTool = tool[FUNCTION_TOOL_SYMBOL] === true; - return isTool && isFunctionTool; + return isTool(tool) && (tool as FunctionTool)[FUNCTION_TOOL_SYMBOL] === true; } // eslint-disable-next-line @typescript-eslint/no-explicit-any export function isProviderTool(tool: any): tool is ProviderTool { - const isTool = tool && tool[TOOL_SYMBOL] === true; - const isProviderTool = tool[PROVIDER_TOOL_SYMBOL] === true; - return isTool && isProviderTool; + return isTool(tool) && (tool as ProviderTool)[PROVIDER_TOOL_SYMBOL] === true; } // eslint-disable-next-line @typescript-eslint/no-explicit-any export function isToolset(value: any): value is Toolset { - return value && value[TOOLSET_SYMBOL] === true; + return !!value && value[TOOLSET_SYMBOL] === true; } // eslint-disable-next-line @typescript-eslint/no-explicit-any export function isToolError(error: any): error is ToolError { - return error && error[TOOL_ERROR_SYMBOL] === true; + return !!error && error[TOOL_ERROR_SYMBOL] === true; } // eslint-disable-next-line @typescript-eslint/no-explicit-any export function isAgentHandoff(handoff: any): handoff is AgentHandoff { - return handoff && handoff[HANDOFF_SYMBOL] === true; + return !!handoff && handoff[HANDOFF_SYMBOL] === true; } From 0daf4ef5616b342ce99b08dfd9241d23b44b03ae Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Wed, 10 Jun 2026 16:48:19 +0100 Subject: [PATCH 06/15] fix(elevenlabs): end server vad turns (#1745) Co-authored-by: rosetta-livekit-bot[bot] <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> --- plugins/elevenlabs/src/stt.test.ts | 14 +++++++----- plugins/elevenlabs/src/stt.ts | 36 +++++++++++++----------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/plugins/elevenlabs/src/stt.test.ts b/plugins/elevenlabs/src/stt.test.ts index c0349641a..918c321dc 100644 --- a/plugins/elevenlabs/src/stt.test.ts +++ b/plugins/elevenlabs/src/stt.test.ts @@ -280,7 +280,7 @@ describe('ElevenLabs STT', () => { expect(url.pathname).toBe('/speech-to-text/realtime'); expect(url.searchParams.get('model_id')).toBe('scribe_v2_realtime'); expect(url.searchParams.get('audio_format')).toBe('pcm_16000'); - expect(url.searchParams.get('commit_strategy')).toBe('vad'); + expect(url.searchParams.get('commit_strategy')).toBe('manual'); expect(url.searchParams.get('include_language_detection')).toBe('true'); expect(receivedMessages[0]).toMatchObject({ message_type: 'input_audio_chunk', @@ -334,7 +334,6 @@ describe('ElevenLabs STT', () => { words: [{ text: 'kept', start: 0, end: 0.2 }], }), ); - ws.send(JSON.stringify({ message_type: 'committed_transcript_with_timestamps', text: '' })); }); }); @@ -364,6 +363,7 @@ describe('ElevenLabs STT', () => { const url = new URL(`ws://127.0.0.1${requestUrl}`); expect(url.searchParams.get('language_code')).toBe('en'); + expect(url.searchParams.get('commit_strategy')).toBe('vad'); expect(url.searchParams.get('include_language_detection')).toBeNull(); expect(url.searchParams.get('include_timestamps')).toBe('true'); expect(url.searchParams.get('vad_silence_threshold_secs')).toBe('0.5'); @@ -393,14 +393,18 @@ describe('ElevenLabs STT', () => { const stream = eleven.stream(); await waitUntil(() => urls.length === 1); - eleven.updateOptions({ serverVad: null }); + eleven.updateOptions({ serverVad: { vadSilenceThresholdSecs: 0.5 } }); await waitUntil(() => urls.length === 2, 2000); + eleven.updateOptions({ serverVad: null }); + await waitUntil(() => urls.length === 3, 2000); stream.close(); const first = new URL(`ws://127.0.0.1${urls[0]}`); const second = new URL(`ws://127.0.0.1${urls[1]}`); - expect(first.searchParams.get('commit_strategy')).toBe('vad'); - expect(second.searchParams.get('commit_strategy')).toBe('manual'); + const third = new URL(`ws://127.0.0.1${urls[2]}`); + expect(first.searchParams.get('commit_strategy')).toBe('manual'); + expect(second.searchParams.get('commit_strategy')).toBe('vad'); + expect(third.searchParams.get('commit_strategy')).toBe('manual'); } finally { await closeWebSocketServer(wss); } diff --git a/plugins/elevenlabs/src/stt.ts b/plugins/elevenlabs/src/stt.ts index a205a29b7..d179752de 100644 --- a/plugins/elevenlabs/src/stt.ts +++ b/plugins/elevenlabs/src/stt.ts @@ -650,7 +650,8 @@ export class SpeechStream extends stt.SpeechStream { } async #connectWs(): Promise { - const commitStrategy = this.#opts.serverVad === null ? 'manual' : 'vad'; + const serverVad = this.#opts.serverVad; + const commitStrategy = serverVad === undefined || serverVad === null ? 'manual' : 'vad'; const params = [ `model_id=${this.#opts.modelId}`, `audio_format=pcm_${this.#opts.sampleRate}`, @@ -661,30 +662,21 @@ export class SpeechStream extends stt.SpeechStream { params.push('include_language_detection=true'); } - if (this.#opts.serverVad) { + if (serverVad !== undefined && serverVad !== null) { if ( - this.#opts.serverVad.vadSilenceThresholdSecs !== undefined && - this.#opts.serverVad.vadSilenceThresholdSecs !== null + serverVad.vadSilenceThresholdSecs !== undefined && + serverVad.vadSilenceThresholdSecs !== null ) { - params.push(`vad_silence_threshold_secs=${this.#opts.serverVad.vadSilenceThresholdSecs}`); + params.push(`vad_silence_threshold_secs=${serverVad.vadSilenceThresholdSecs}`); } - if ( - this.#opts.serverVad.vadThreshold !== undefined && - this.#opts.serverVad.vadThreshold !== null - ) { - params.push(`vad_threshold=${this.#opts.serverVad.vadThreshold}`); + if (serverVad.vadThreshold !== undefined && serverVad.vadThreshold !== null) { + params.push(`vad_threshold=${serverVad.vadThreshold}`); } - if ( - this.#opts.serverVad.minSpeechDurationMs !== undefined && - this.#opts.serverVad.minSpeechDurationMs !== null - ) { - params.push(`min_speech_duration_ms=${this.#opts.serverVad.minSpeechDurationMs}`); + if (serverVad.minSpeechDurationMs !== undefined && serverVad.minSpeechDurationMs !== null) { + params.push(`min_speech_duration_ms=${serverVad.minSpeechDurationMs}`); } - if ( - this.#opts.serverVad.minSilenceDurationMs !== undefined && - this.#opts.serverVad.minSilenceDurationMs !== null - ) { - params.push(`min_silence_duration_ms=${this.#opts.serverVad.minSilenceDurationMs}`); + if (serverVad.minSilenceDurationMs !== undefined && serverVad.minSilenceDurationMs !== null) { + params.push(`min_silence_duration_ms=${serverVad.minSilenceDurationMs}`); } } @@ -804,6 +796,10 @@ export class SpeechStream extends stt.SpeechStream { type: stt.SpeechEventType.FINAL_TRANSCRIPT, alternatives: [speechData], }); + if (this.#opts.serverVad !== undefined && this.#opts.serverVad !== null) { + this.queue.put({ type: stt.SpeechEventType.END_OF_SPEECH }); + this.#speaking = false; + } } else if (this.#speaking) { this.queue.put({ type: stt.SpeechEventType.END_OF_SPEECH }); this.#speaking = false; From d423f3f89528662b1e5777f7e24989946784879d Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Wed, 10 Jun 2026 17:14:00 -0400 Subject: [PATCH 07/15] Add Inworld delivery mode inference TTS option (#1749) Co-authored-by: rosetta-livekit-bot[bot] <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> --- .changeset/inworld-delivery-mode.md | 5 +++++ agents/src/inference/tts.ts | 2 ++ 2 files changed, 7 insertions(+) create mode 100644 .changeset/inworld-delivery-mode.md diff --git a/.changeset/inworld-delivery-mode.md b/.changeset/inworld-delivery-mode.md new file mode 100644 index 000000000..c02a8c94e --- /dev/null +++ b/.changeset/inworld-delivery-mode.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Add Inworld `delivery_mode` to inference TTS model options. diff --git a/agents/src/inference/tts.ts b/agents/src/inference/tts.ts index 47a93bb04..78c0d2a26 100644 --- a/agents/src/inference/tts.ts +++ b/agents/src/inference/tts.ts @@ -127,6 +127,8 @@ export interface InworldOptions { speaking_rate?: number; /** Range 0-2. */ temperature?: number; + /** Controls output variation on inworld-tts-2 only. */ + delivery_mode?: 'DELIVERY_MODE_UNSPECIFIED' | 'STABLE' | 'BALANCED' | 'CREATIVE'; timestamp_type?: 'TIMESTAMP_TYPE_UNSPECIFIED' | 'WORD' | 'CHARACTER'; apply_text_normalization?: 'APPLY_TEXT_NORMALIZATION_UNSPECIFIED' | 'ON' | 'OFF'; /** @deprecated Backward-compatible alias. Use `apply_text_normalization`. */ From 9d86bc5c5f11018cc000841899e959542eafca28 Mon Sep 17 00:00:00 2001 From: Brian Yin Date: Thu, 11 Jun 2026 07:10:57 +0800 Subject: [PATCH 08/15] Don't retain recorded events when recording is disabled (#1750) --- .changeset/honest-swans-drum.md | 5 +++++ agents/src/voice/agent_session.ts | 11 +++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 .changeset/honest-swans-drum.md diff --git a/.changeset/honest-swans-drum.md b/.changeset/honest-swans-drum.md new file mode 100644 index 000000000..e4782071e --- /dev/null +++ b/.changeset/honest-swans-drum.md @@ -0,0 +1,5 @@ +--- +"@livekit/agents": patch +--- + +Don't retain recorded events when recording is disabled diff --git a/agents/src/voice/agent_session.ts b/agents/src/voice/agent_session.ts index 3cade1f2f..d308df854 100644 --- a/agents/src/voice/agent_session.ts +++ b/agents/src/voice/agent_session.ts @@ -480,8 +480,15 @@ export class AgentSession< event: K, ...args: Parameters ): boolean { - const eventData = args[0] as AgentEvent; - this._recordedEvents.push(eventData); + // Only retain events when recording is actually enabled. Otherwise this + // array grows unbounded for the entire (potentially hours-long) session, + // pinning every event's graph (SpeechHandle, OTel spans/contexts, streams) + // and leaking memory even though the events are never reported. The buffer + // is only consumed by makeSessionReport() when recording is enabled. + if (this._enableRecording) { + const eventData = args[0] as AgentEvent; + this._recordedEvents.push(eventData); + } return super.emit(event, ...args); } From 03429b82ac96c37272800aa54bb478a91744753f Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 17:43:52 +0400 Subject: [PATCH 09/15] ci: speed up type checking (#1742) Co-authored-by: rosetta-livekit-bot[bot] <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 8 ++++++++ .github/workflows/test.yml | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a527b4fe4..5681c1fca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,5 +48,13 @@ jobs: cache: pnpm - name: Install dependencies run: pnpm install --frozen-lockfile --ignore-scripts + - name: Cache turbo + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: .turbo + key: turbo-${{ runner.os }}-node20-${{ hashFiles('pnpm-lock.yaml') }}-${{ github.run_id }} + restore-keys: | + turbo-${{ runner.os }}-node20-${{ hashFiles('pnpm-lock.yaml') }}- + turbo-${{ runner.os }}-node20- - name: Build run: pnpm build diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index de5e3535e..d205e932d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,6 +29,14 @@ jobs: cache: pnpm - name: Install dependencies run: pnpm install --frozen-lockfile --ignore-scripts + - name: Cache turbo + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: .turbo + key: turbo-${{ runner.os }}-node20-${{ hashFiles('pnpm-lock.yaml') }}-${{ github.run_id }} + restore-keys: | + turbo-${{ runner.os }}-node20-${{ hashFiles('pnpm-lock.yaml') }}- + turbo-${{ runner.os }}-node20- - name: Build run: pnpm build - name: Check which tests to run From ec765eaaaef8af73a453793b1dd7ff2ff0836288 Mon Sep 17 00:00:00 2001 From: Chenghao Mou Date: Thu, 11 Jun 2026 17:15:57 +0100 Subject: [PATCH 10/15] fix(voice): scope forwardAudio playback-started listener to its own segment (#1760) Co-authored-by: Cursor --- .../scope-forward-audio-playback-started.md | 15 +++++ agents/src/voice/generation.ts | 17 +++++- .../src/voice/generation_tts_timeout.test.ts | 57 +++++++++++++++++++ 3 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 .changeset/scope-forward-audio-playback-started.md diff --git a/.changeset/scope-forward-audio-playback-started.md b/.changeset/scope-forward-audio-playback-started.md new file mode 100644 index 000000000..5c5cd277e --- /dev/null +++ b/.changeset/scope-forward-audio-playback-started.md @@ -0,0 +1,15 @@ +--- +'@livekit/agents': patch +--- + +fix(voice): scope forwardAudio's playback-started listener to its own segment + +When a speech is interrupted, the scheduling loop immediately authorizes the next +speech, so the new segment's `forwardAudio` registers its `playback_started` +listener on the shared audio output while the interrupted segment is still +emitting events during teardown. The stray event resolved the new segment's +`firstFrameFut` before its first frame was captured, which skipped resampler +creation and pushed an unresampled frame straight to the `AudioSource` +(`RtcError: sample_rate and num_channels don't match`) and corrupted playback +bookkeeping. The listener now only resolves `firstFrameFut` after the segment has +captured its own first frame. diff --git a/agents/src/voice/generation.ts b/agents/src/voice/generation.ts index 8cd1e84bd..59b2d663d 100644 --- a/agents/src/voice/generation.ts +++ b/agents/src/voice/generation.ts @@ -836,8 +836,19 @@ async function forwardAudio( const reader = ttsStream.getReader(); let resampler: AudioResampler | null = null; + // The audio output is shared across overlapping segments. When a speech is + // interrupted, the main loop immediately authorizes the next speech, so this + // forwarder can register its listener while the interrupted segment's teardown + // is still emitting PLAYBACK_STARTED on the same output. Only honor the event + // once this loop has captured its own first frame, so a stray event from + // another segment can't resolve our `firstFrameFut` prematurely. A premature + // resolution skips resampler creation (gated on `!firstFrameFut.done`) and + // pushes an unresampled frame to the AudioSource, raising + // `RtcError: sample_rate and num_channels don't match`. + let hasCapturedOwnFrame = false; + const onPlaybackStarted = (ev: { createdAt: number }) => { - if (!out.firstFrameFut.done) { + if (hasCapturedOwnFrame && !out.firstFrameFut.done) { out.firstFrameFut.resolve(ev.createdAt); } }; @@ -868,6 +879,10 @@ async function forwardAudio( resampler = new AudioResampler(frame.sampleRate, audioOutput.sampleRate, 1); } + // Mark before capturing so the PLAYBACK_STARTED emitted synchronously inside + // the first captureFrame is attributed to this segment. + hasCapturedOwnFrame = true; + if (resampler) { for (const f of resampler.push(frame)) { await audioOutput.captureFrame(f); diff --git a/agents/src/voice/generation_tts_timeout.test.ts b/agents/src/voice/generation_tts_timeout.test.ts index c7d2c0717..5cb3803f9 100644 --- a/agents/src/voice/generation_tts_timeout.test.ts +++ b/agents/src/voice/generation_tts_timeout.test.ts @@ -105,6 +105,63 @@ describe('TTS stream idle timeout', () => { expect(audioOut.firstFrameFut.done).toBe(true); }); + it('ignores PLAYBACK_STARTED from another segment before its own first frame', async () => { + // Stalled stream so the forwarder is still waiting on its first read when a + // stray event (from an interrupted overlapping segment) arrives; the idle + // timeout then ends the loop without this segment ever capturing a frame. + const stalledStream = new ReadableStream({ start() {} }); + + const audioOutput = new MockAudioOutput(); + const controller = new AbortController(); + const [task, audioOut] = performAudioForwarding(stalledStream, audioOutput, controller, 500); + + // Reject path is expected (no first frame ever captured). + audioOut.firstFrameFut.await.catch(() => {}); + + vi.useFakeTimers(); + + // Stray PLAYBACK_STARTED before this segment captures anything must be ignored. + audioOutput.onPlaybackStarted(Date.now()); + expect(audioOut.firstFrameFut.done).toBe(false); + + const taskPromise = task.result; + await vi.advanceTimersByTimeAsync(600); + await taskPromise; + + vi.useRealTimers(); + + expect(audioOutput.capturedFrames.length).toBe(0); + expect(audioOut.firstFrameFut.rejected).toBe(true); + }); + + it('resamples a rate-mismatched frame even after a stray PLAYBACK_STARTED', async () => { + // Output is 24kHz; frames are 16kHz and must be resampled regardless of any + // stray PLAYBACK_STARTED resolving firstFrameFut early. + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(createSilentFrame(16000)); + controller.enqueue(createSilentFrame(16000)); + controller.close(); + }, + }); + + const audioOutput = new MockAudioOutput(); + const controller = new AbortController(); + const [task, audioOut] = performAudioForwarding(stream, audioOutput, controller); + + // Stray event before the loop captures anything must not skip resampling. + audioOutput.onPlaybackStarted(Date.now()); + + await task.result; + + expect(audioOut.firstFrameFut.done).toBe(true); + // Every captured frame must match the output sample rate (i.e. was resampled). + expect(audioOutput.capturedFrames.length).toBeGreaterThan(0); + for (const f of audioOutput.capturedFrames) { + expect(f.sampleRate).toBe(24000); + } + }); + it('performTTSInference completes when TTS node returns stalled stream', async () => { const stalledTtsStream = new ReadableStream({ start(controller) { From 416871a5e381ff34e59e1507787b4a9c5a1679f8 Mon Sep 17 00:00:00 2001 From: Chenghao Mou Date: Thu, 11 Jun 2026 17:21:09 +0100 Subject: [PATCH 11/15] feat(barge-in): add default threshold support and drop http transport (#1698) Co-authored-by: Claude Opus 4.8 (1M context) --- .../bargein-default-threshold-drop-http.md | 5 + agents/src/inference/interruption/_mock_ws.ts | 58 +++ agents/src/inference/interruption/defaults.ts | 8 +- .../inference/interruption/http_transport.ts | 208 --------- .../interruption/interruption_detector.ts | 64 +-- .../interruption_failover.test.ts | 408 ++++++++++++++++++ .../interruption_session_create.test.ts | 127 ++++++ .../interruption/interruption_stream.ts | 53 ++- agents/src/inference/interruption/types.ts | 3 +- .../inference/interruption/ws_transport.ts | 231 +++++++--- agents/src/inference/utils.ts | 16 +- agents/src/voice/audio_recognition.ts | 3 + agents/src/worker.ts | 6 + turbo.json | 1 + 14 files changed, 835 insertions(+), 356 deletions(-) create mode 100644 .changeset/bargein-default-threshold-drop-http.md create mode 100644 agents/src/inference/interruption/_mock_ws.ts delete mode 100644 agents/src/inference/interruption/http_transport.ts create mode 100644 agents/src/inference/interruption/interruption_failover.test.ts create mode 100644 agents/src/inference/interruption/interruption_session_create.test.ts diff --git a/.changeset/bargein-default-threshold-drop-http.md b/.changeset/bargein-default-threshold-drop-http.md new file mode 100644 index 000000000..2d937eb5c --- /dev/null +++ b/.changeset/bargein-default-threshold-drop-http.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Adaptive interruption detection now omits the threshold from `session.create` unless the user explicitly overrides it, letting the gateway apply its fetched default (surfaced via `default_threshold` on `session.created`). The HTTP transport has been dropped — detection always connects over WebSocket and always requires LiveKit credentials, and its base URL now defaults from `LIVEKIT_INFERENCE_URL` instead of `LIVEKIT_REMOTE_EOT_URL`. Inference requests also send an `X-LiveKit-Worker-Token` header when `LIVEKIT_WORKER_TOKEN` is set (hosted agents); a token supplied via the `--worker-token` CLI flag is now re-exported into the environment so forked job subprocesses inherit it and include the header. The `X-LiveKit-Agent-Id` header is now only attached once the room is connected to avoid leaking an unset local-participant SID. The interruption WebSocket is now closed deterministically on stream teardown (including error and cancel paths) instead of only on graceful completion — previously an orphaned socket leaked per session/activity and accumulated for the worker's lifetime. Mid-session threshold/duration changes via `updateOptions` now reconnect the WebSocket in place rather than closing it and letting the next send error the stream — so option changes no longer consume a failover retry (previously enough updates in a session could exhaust the retry budget and stop interruption detection). diff --git a/agents/src/inference/interruption/_mock_ws.ts b/agents/src/inference/interruption/_mock_ws.ts new file mode 100644 index 000000000..978224a0b --- /dev/null +++ b/agents/src/inference/interruption/_mock_ws.ts @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { EventEmitter } from 'node:events'; + +/** + * Minimal stand-in for the `ws` WebSocket, used to drive the interruption transport in tests. + * + * Lives in its own module (rather than inline in each test) so the `vi.mock('ws')` factory can + * `await import()` it without a top-level await — the tsup build transpiles test files to CJS, + * which does not support top-level await. + */ +export class MockWebSocket extends EventEmitter { + static OPEN = 1; + static instances: MockWebSocket[] = []; + + readyState = 0; // CONNECTING + readonly sent: unknown[] = []; + terminated = false; + + constructor( + public url: string, + public opts: unknown, + ) { + super(); + MockWebSocket.instances.push(this); + } + + send(data: unknown): void { + this.sent.push(data); + } + + close(): void { + this.readyState = 3; // CLOSED + this.emit('close', 1000, Buffer.from('')); + } + + terminate(): void { + this.terminated = true; + this.readyState = 3; + } + + /** Simulate a successful upgrade. */ + simulateOpen(): void { + this.readyState = MockWebSocket.OPEN; + this.emit('open'); + } + + /** Simulate the server rejecting the upgrade with an HTTP status. */ + simulateUnexpectedResponse(statusCode: number): void { + this.emit('unexpected-response', {}, { statusCode }); + } + + /** Simulate a server message frame carrying a JSON payload. */ + simulateMessage(payload: unknown): void { + this.emit('message', Buffer.from(JSON.stringify(payload))); + } +} diff --git a/agents/src/inference/interruption/defaults.ts b/agents/src/inference/interruption/defaults.ts index df6134bc4..c01c127a1 100644 --- a/agents/src/inference/interruption/defaults.ts +++ b/agents/src/inference/interruption/defaults.ts @@ -5,9 +5,8 @@ import type { ApiConnectOptions } from './interruption_stream.js'; import type { InterruptionOptions } from './types.js'; export const MIN_INTERRUPTION_DURATION_IN_S = 0.025 * 2; // 25ms per frame, 2 consecutive frames -export const THRESHOLD = 0.5; export const MAX_AUDIO_DURATION_IN_S = 3.0; -export const AUDIO_PREFIX_DURATION_IN_S = 0.5; +export const AUDIO_PREFIX_DURATION_IN_S = 1.0; export const DETECTION_INTERVAL_IN_S = 0.1; export const REMOTE_INFERENCE_TIMEOUT_IN_S = 0.7; export const SAMPLE_RATE = 16000; @@ -36,12 +35,13 @@ export function intervalForRetry( } // env-derived fields are resolved in the constructor, not at module load. +// `threshold` is intentionally omitted: when the user does not override it, it stays undefined +// so the server applies its fetched default. export const interruptionOptionDefaults: Omit< InterruptionOptions, - 'baseUrl' | 'useProxy' | 'apiKey' | 'apiSecret' + 'baseUrl' | 'apiKey' | 'apiSecret' | 'threshold' > = { sampleRate: SAMPLE_RATE, - threshold: THRESHOLD, minFrames: Math.ceil(MIN_INTERRUPTION_DURATION_IN_S * FRAMES_PER_SECOND), maxAudioDurationInS: MAX_AUDIO_DURATION_IN_S, audioPrefixDurationInS: AUDIO_PREFIX_DURATION_IN_S, diff --git a/agents/src/inference/interruption/http_transport.ts b/agents/src/inference/interruption/http_transport.ts deleted file mode 100644 index bd7953688..000000000 --- a/agents/src/inference/interruption/http_transport.ts +++ /dev/null @@ -1,208 +0,0 @@ -// SPDX-FileCopyrightText: 2026 LiveKit, Inc. -// -// SPDX-License-Identifier: Apache-2.0 -import type { Throws } from '@livekit/throws-transformer/throws'; -import { FetchError, ofetch } from 'ofetch'; -import { TransformStream } from 'stream/web'; -import { z } from 'zod'; -import { APIConnectionError, APIError, APIStatusError, isAPIError } from '../../_exceptions.js'; -import { log } from '../../log.js'; -import { buildMetadataHeaders, createAccessToken } from '../utils.js'; -import { InterruptionCacheEntry } from './interruption_cache_entry.js'; -import type { OverlappingSpeechEvent } from './types.js'; -import type { BoundedCache } from './utils.js'; - -export interface PostOptions { - baseUrl: string; - token: string; - signal?: AbortSignal; - timeout?: number; - maxRetries?: number; -} - -export interface PredictOptions { - threshold: number; - minFrames: number; -} - -export const predictEndpointResponseSchema = z.object({ - created_at: z.number(), - is_bargein: z.boolean(), - probabilities: z.array(z.number()), -}); - -export type PredictEndpointResponse = z.infer; - -export interface PredictResponse { - createdAt: number; - isBargein: boolean; - probabilities: number[]; - predictionDurationInS: number; -} - -export async function predictHTTP( - data: Int16Array, - predictOptions: PredictOptions, - options: PostOptions, -): Promise> { - const createdAt = performance.now(); - const url = new URL(`/bargein`, options.baseUrl); - url.searchParams.append('threshold', predictOptions.threshold.toString()); - url.searchParams.append('min_frames', predictOptions.minFrames.toFixed()); - url.searchParams.append('created_at', createdAt.toFixed()); - - try { - const response = await ofetch(url.toString(), { - retry: 0, - headers: { - ...buildMetadataHeaders(), - 'Content-Type': 'application/octet-stream', - Authorization: `Bearer ${options.token}`, - }, - signal: options.signal, - timeout: options.timeout, - method: 'POST', - body: data, - }); - const { created_at, is_bargein, probabilities } = predictEndpointResponseSchema.parse(response); - - return { - createdAt: created_at, - isBargein: is_bargein, - probabilities, - predictionDurationInS: (performance.now() - createdAt) / 1000, - }; - } catch (err) { - if (isAPIError(err)) throw err; - if (err instanceof FetchError) { - if (err.statusCode) { - throw new APIStatusError({ - message: `error during interruption prediction: ${err.message}`, - options: { statusCode: err.statusCode, body: err.data }, - }); - } - if ( - err.cause instanceof Error && - (err.cause.name === 'TimeoutError' || err.cause.name === 'AbortError') - ) { - throw new APIStatusError({ - message: `interruption inference timeout: ${err.message}`, - options: { statusCode: 408, retryable: false }, - }); - } - throw new APIConnectionError({ - message: `interruption inference connection error: ${err.message}`, - }); - } - throw new APIError(`error during interruption prediction: ${err}`); - } -} - -export interface HttpTransportOptions { - baseUrl: string; - apiKey: string; - apiSecret: string; - threshold: number; - minFrames: number; - timeout: number; - maxRetries?: number; -} - -export interface HttpTransportState { - overlapSpeechStarted: boolean; - overlapSpeechStartedAt: number | undefined; - cache: BoundedCache; -} - -/** - * Creates an HTTP transport TransformStream for interruption detection. - * - * This transport receives Int16Array audio slices and outputs InterruptionEvents. - * Each audio slice triggers an HTTP POST request. - * - * @param options - Transport options object. This is read on each request, so mutations - * to threshold/minFrames will be picked up dynamically. - */ -export function createHttpTransport( - options: HttpTransportOptions, - getState: () => HttpTransportState, - setState: (partial: Partial) => void, - updateUserSpeakingSpan?: (entry: InterruptionCacheEntry) => void, - getAndResetNumRequests?: () => number, -): TransformStream { - const logger = log(); - - return new TransformStream( - { - async transform(chunk, controller) { - if (!(chunk instanceof Int16Array)) { - controller.enqueue(chunk); - return; - } - - const state = getState(); - const overlapSpeechStartedAt = state.overlapSpeechStartedAt; - if (overlapSpeechStartedAt === undefined || !state.overlapSpeechStarted) return; - - try { - const resp = await predictHTTP( - chunk, - { threshold: options.threshold, minFrames: options.minFrames }, - { - baseUrl: options.baseUrl, - timeout: options.timeout, - maxRetries: options.maxRetries, - token: await createAccessToken(options.apiKey, options.apiSecret), - }, - ); - - const { createdAt, isBargein, probabilities, predictionDurationInS } = resp; - const entry = state.cache.setOrUpdate( - createdAt, - () => new InterruptionCacheEntry({ createdAt }), - { - probabilities, - isInterruption: isBargein, - speechInput: chunk, - totalDurationInS: (performance.now() - createdAt) / 1000, - detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000, - predictionDurationInS, - }, - ); - - if (state.overlapSpeechStarted && entry.isInterruption) { - if (updateUserSpeakingSpan) { - updateUserSpeakingSpan(entry); - } - const event: OverlappingSpeechEvent = { - type: 'overlapping_speech', - detectedAt: Date.now(), - overlapStartedAt: overlapSpeechStartedAt, - isInterruption: entry.isInterruption, - speechInput: entry.speechInput, - probabilities: entry.probabilities, - totalDurationInS: entry.totalDurationInS, - predictionDurationInS: entry.predictionDurationInS, - detectionDelayInS: entry.detectionDelayInS, - probability: entry.probability, - numRequests: getAndResetNumRequests?.() ?? 0, - }; - logger.debug( - { - detectionDelayInS: entry.detectionDelayInS, - totalDurationInS: entry.totalDurationInS, - }, - 'interruption detected', - ); - setState({ overlapSpeechStarted: false }); - controller.enqueue(event); - } - } catch (err) { - controller.error(err); - } - }, - }, - { highWaterMark: 2 }, - { highWaterMark: 2 }, - ); -} diff --git a/agents/src/inference/interruption/interruption_detector.ts b/agents/src/inference/interruption/interruption_detector.ts index 40c36e17d..0f4c341a9 100644 --- a/agents/src/inference/interruption/interruption_detector.ts +++ b/agents/src/inference/interruption/interruption_detector.ts @@ -7,7 +7,7 @@ import EventEmitter from 'events'; import { log } from '../../log.js'; import type { InterruptionMetrics } from '../../metrics/base.js'; import { asError } from '../../utils.js'; -import { DEFAULT_INFERENCE_URL, STAGING_INFERENCE_URL, getDefaultInferenceUrl } from '../utils.js'; +import { getDefaultInferenceUrl } from '../utils.js'; import { FRAMES_PER_SECOND, SAMPLE_RATE, interruptionOptionDefaults } from './defaults.js'; import { InterruptionDetectionError } from './errors.js'; import { InterruptionStreamBase } from './interruption_stream.js'; @@ -19,7 +19,7 @@ type InterruptionCallbacks = { error: (error: InterruptionDetectionError) => void; }; -export type AdaptiveInterruptionDetectorOptions = Omit, 'useProxy'>; +export type AdaptiveInterruptionDetectorOptions = Partial; export class AdaptiveInterruptionDetector extends (EventEmitter as new () => TypedEventEmitter) { options: InterruptionOptions; @@ -46,46 +46,23 @@ export class AdaptiveInterruptionDetector extends (EventEmitter as new () => Typ throw new RangeError('maxAudioDurationInS must be less than or equal to 3.0 seconds'); } - const lkBaseUrl = baseUrl ?? process.env.LIVEKIT_REMOTE_EOT_URL ?? getDefaultInferenceUrl(); - let lkApiKey = apiKey || ''; - let lkApiSecret = apiSecret || ''; - let useProxy: boolean; - - // Use LiveKit credentials if using the inference service (production or staging) - const isInferenceUrl = - lkBaseUrl === DEFAULT_INFERENCE_URL || lkBaseUrl === STAGING_INFERENCE_URL; - if (isInferenceUrl) { - lkApiKey = - apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY || ''; - if (!lkApiKey) { - throw new TypeError( - 'apiKey is required, either as argument or set LIVEKIT_API_KEY environmental variable', - ); - } - - lkApiSecret = - apiSecret || - process.env.LIVEKIT_INFERENCE_API_SECRET || - process.env.LIVEKIT_API_SECRET || - ''; - if (!lkApiSecret) { - throw new TypeError( - 'apiSecret is required, either as argument or set LIVEKIT_API_SECRET environmental variable', - ); - } - useProxy = true; - } else { - useProxy = false; + const lkBaseUrl = baseUrl ?? getDefaultInferenceUrl(); + + const lkApiKey = + apiKey || process.env.LIVEKIT_INFERENCE_API_KEY || process.env.LIVEKIT_API_KEY || ''; + if (!lkApiKey) { + throw new TypeError( + 'apiKey is required, either as argument or set LIVEKIT_API_KEY environmental variable', + ); + } + + const lkApiSecret = + apiSecret || process.env.LIVEKIT_INFERENCE_API_SECRET || process.env.LIVEKIT_API_SECRET || ''; + if (!lkApiSecret) { + throw new TypeError( + 'apiSecret is required, either as argument or set LIVEKIT_API_SECRET environmental variable', + ); } - const transport = useProxy ? 'websocket' : 'http'; - this.logger.debug( - { - baseUrl: lkBaseUrl, - useProxy, - transport, - }, - '=== Resolved interruption detector transport configuration', - ); this.options = { sampleRate: SAMPLE_RATE, @@ -98,7 +75,6 @@ export class AdaptiveInterruptionDetector extends (EventEmitter as new () => Typ baseUrl: lkBaseUrl, apiKey: lkApiKey, apiSecret: lkApiSecret, - useProxy, minInterruptionDurationInS, }; @@ -111,10 +87,8 @@ export class AdaptiveInterruptionDetector extends (EventEmitter as new () => Typ audioPrefixDurationInS: this.options.audioPrefixDurationInS, maxAudioDurationInS: this.options.maxAudioDurationInS, minFrames: this.options.minFrames, - threshold: this.options.threshold, + threshold: this.options.threshold ?? null, inferenceTimeout: this.options.inferenceTimeout, - useProxy: this.options.useProxy, - transport, }, '=== Adaptive interruption detector initialized', ); diff --git a/agents/src/inference/interruption/interruption_failover.test.ts b/agents/src/inference/interruption/interruption_failover.test.ts new file mode 100644 index 000000000..b1cd77bed --- /dev/null +++ b/agents/src/inference/interruption/interruption_failover.test.ts @@ -0,0 +1,408 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Tests for interruption detection failover (transport error production + error-emission) +// behavior. Ported from the Python `test_interruption_failover.py` suite. +// +// Covers, for the WebSocket-only transport (HTTP transport was dropped): +// - connection timeout -> non-retryable APITimeoutError +// - connection 429 -> non-retryable APIStatusError +// - cache-based inference timeout -> non-retryable APIStatusError (408) +// and that a non-retryable transport error surfaces as exactly one unrecoverable +// InterruptionDetectionError (zero recoverable) through AudioRecognition's retry loop. +// +// Also covers in-place reconnect on updateOptions: it opens a fresh socket with the updated +// settings without erroring the stream (so it never consumes a failover retry), while a genuine +// reconnect failure still surfaces as a stream error. +import { AudioFrame } from '@livekit/rtc-node'; +import { ReadableStream } from 'node:stream/web'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { APIError } from '../../_exceptions.js'; +import { APIStatusError, APITimeoutError } from '../../_exceptions.js'; +import { ChatContext } from '../../llm/chat_context.js'; +import { initializeLogger } from '../../log.js'; +import { AudioRecognition, type RecognitionHooks } from '../../voice/audio_recognition.js'; +import { MockWebSocket } from './_mock_ws.js'; +import { apiConnectDefaults } from './defaults.js'; +import { AdaptiveInterruptionDetector } from './interruption_detector.js'; +import { InterruptionStreamBase, InterruptionStreamSentinel } from './interruption_stream.js'; + +// --------------------------------------------------------------------------- +// Mock `ws` so the WebSocket transport can be driven deterministically. +// --------------------------------------------------------------------------- + +vi.mock('ws', async () => { + const { MockWebSocket } = await import('./_mock_ws.js'); + return { default: MockWebSocket, WebSocket: MockWebSocket }; +}); + +type MockSocket = MockWebSocket; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +initializeLogger({ pretty: false, level: 'silent' }); + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function waitForInstance(timeoutMs = 2000): Promise { + const start = performance.now(); + while (MockWebSocket.instances.length === 0) { + if (performance.now() - start > timeoutMs) { + throw new Error('WebSocket instance was never constructed'); + } + await sleep(5); + } + return MockWebSocket.instances[MockWebSocket.instances.length - 1]!; +} + +/** Wait until at least `n` sockets have been constructed and return the nth (1-indexed). */ +async function waitForInstanceCount(n: number, timeoutMs = 2000): Promise { + const start = performance.now(); + while (MockWebSocket.instances.length < n) { + if (performance.now() - start > timeoutMs) { + throw new Error(`expected ${n} WebSocket instances, saw ${MockWebSocket.instances.length}`); + } + await sleep(5); + } + return MockWebSocket.instances[n - 1]!; +} + +async function waitFor(predicate: () => boolean, timeoutMs = 2000): Promise { + const start = performance.now(); + while (!predicate()) { + if (performance.now() - start > timeoutMs) { + throw new Error('condition not met within timeout'); + } + await sleep(5); + } +} + +/** Resolves to 'alive' if the stream has not errored/ended within `ms`. */ +function stillAlive(errPromise: Promise, ms = 30): Promise<'alive' | unknown> { + return Promise.race([errPromise, sleep(ms).then(() => 'alive' as const)]); +} + +function sessionCreateSettings(ws: MockSocket): Record { + return JSON.parse(String(ws.sent[0])).settings; +} + +function makeAudioFrame(numSamples = 1600, sampleRate = 16000): AudioFrame { + const data = new Int16Array(numSamples); + return new AudioFrame(data, sampleRate, 1, numSamples); +} + +function createDetector(opts: { inferenceTimeout?: number } = {}): AdaptiveInterruptionDetector { + return new AdaptiveInterruptionDetector({ + baseUrl: 'http://localhost:9999', + apiKey: 'test-key', + apiSecret: 'test-secret', + ...opts, + }); +} + +/** Drain a stream's event side and return the rejection (or undefined on clean end). */ +async function readError(stream: InterruptionStreamBase): Promise { + const reader = stream.stream().getReader(); + try { + for (;;) { + const { done } = await reader.read(); + if (done) return undefined; + } + } catch (e) { + return e; + } finally { + try { + reader.releaseLock(); + } catch { + // already released + } + } +} + +beforeEach(() => { + MockWebSocket.instances.length = 0; +}); + +afterEach(() => { + vi.clearAllMocks(); +}); + +// --------------------------------------------------------------------------- +// WebSocket transport error production +// --------------------------------------------------------------------------- + +describe('interruption WebSocket transport failover', () => { + it('surfaces a non-retryable APITimeoutError on connection timeout', async () => { + const detector = createDetector(); + // Short connect timeout so the test does not wait on the default. + const stream = new InterruptionStreamBase(detector, { timeout: 50 }); + + const err = await readError(stream); + + expect(err).toBeInstanceOf(APITimeoutError); + expect((err as APIError).retryable).toBe(false); + + await stream.close(); + }); + + it('surfaces a non-retryable APIStatusError on connection 429', async () => { + const detector = createDetector(); + const stream = new InterruptionStreamBase(detector, {}); + + const errPromise = readError(stream); + const ws = await waitForInstance(); + ws.simulateUnexpectedResponse(429); + + const err = await errPromise; + + expect(err).toBeInstanceOf(APIStatusError); + expect((err as APIStatusError).statusCode).toBe(429); + expect((err as APIError).retryable).toBe(false); + + await stream.close(); + }); + + it('surfaces a non-retryable 408 APIStatusError when inference responses time out', async () => { + const inferenceTimeout = 50; + const detector = createDetector({ inferenceTimeout }); + const stream = new InterruptionStreamBase(detector, {}); + + const errPromise = readError(stream); + const ws = await waitForInstance(); + ws.simulateOpen(); + // let ensureConnection() resolve and send session.create + await sleep(5); + + // Drive overlap audio so the transport sends a request and caches it, then never + // answers — the next slice must trip the cache-timeout guard. + await stream.pushFrame(InterruptionStreamSentinel.agentSpeechStarted()); + await stream.pushFrame(InterruptionStreamSentinel.overlapSpeechStarted(500, Date.now())); + await stream.pushFrame(makeAudioFrame()); + await sleep(inferenceTimeout + 40); + await stream.pushFrame(makeAudioFrame()); + + const err = await errPromise; + + expect(err).toBeInstanceOf(APIStatusError); + expect((err as APIStatusError).statusCode).toBe(408); + expect((err as APIError).retryable).toBe(false); + + await stream.close(); + }); + + it('errors when session.created reports no threshold and the user did not override', async () => { + const detector = createDetector(); + const stream = new InterruptionStreamBase(detector, {}); + + const errPromise = readError(stream); + const ws = await waitForInstance(); + ws.simulateOpen(); + await sleep(5); + ws.simulateMessage({ type: 'session.created' }); + + const err = await errPromise; + + expect(err).toBeInstanceOf(APIStatusError); + expect((err as APIStatusError).statusCode).toBe(500); + expect((err as APIError).retryable).toBe(false); + + await stream.close(); + }); + + it('does not error when session.created carries a default_threshold', async () => { + const detector = createDetector(); + const stream = new InterruptionStreamBase(detector, {}); + + const errPromise = readError(stream); + const ws = await waitForInstance(); + ws.simulateOpen(); + await sleep(5); + ws.simulateMessage({ type: 'session.created', default_threshold: 0.42 }); + await sleep(5); + + await stream.close(); + expect(await errPromise).toBeUndefined(); + }); + + it('closes the underlying WebSocket on teardown even after the transport errored', async () => { + // The transport's flush() only runs on graceful stream completion. When the stream is torn + // down via an error (here a 408 inference timeout) the socket is still open, so close() must + // tear it down directly — otherwise the WebSocket leaks. + const inferenceTimeout = 50; + const detector = createDetector({ inferenceTimeout }); + const stream = new InterruptionStreamBase(detector, {}); + + const errPromise = readError(stream); + const ws = await waitForInstance(); + ws.simulateOpen(); + await sleep(5); + + await stream.pushFrame(InterruptionStreamSentinel.agentSpeechStarted()); + await stream.pushFrame(InterruptionStreamSentinel.overlapSpeechStarted(500, Date.now())); + await stream.pushFrame(makeAudioFrame()); + await sleep(inferenceTimeout + 40); + await stream.pushFrame(makeAudioFrame()); + + await errPromise; + expect(ws.readyState).toBe(MockWebSocket.OPEN); // socket still open after the stream errored + + await stream.close(); + + expect(ws.readyState).toBe(3); // CLOSED — close() tore the socket down despite the error + }); +}); + +// --------------------------------------------------------------------------- +// In-place reconnect on updateOptions +// --------------------------------------------------------------------------- + +describe('interruption updateOptions reconnect', () => { + it('reconnects in place with the updated threshold without erroring the stream', async () => { + const detector = createDetector(); + const stream = new InterruptionStreamBase(detector, {}); + const errPromise = readError(stream); + + const ws1 = await waitForInstance(); + ws1.simulateOpen(); + await waitFor(() => ws1.sent.length > 0); // session.create #1 + // No user override → the first session.create omits threshold (server applies its default). + expect('threshold' in sessionCreateSettings(ws1)).toBe(false); + + await stream.updateOptions({ threshold: 0.7 }); + + // A fresh socket is opened in place and the old one is closed — no error on the stream. + const ws2 = await waitForInstanceCount(2); + expect(ws2).not.toBe(ws1); + expect(ws1.readyState).toBe(3); // old socket closed + ws2.simulateOpen(); + await waitFor(() => ws2.sent.length > 0); // session.create #2 + + expect(sessionCreateSettings(ws2).threshold).toBe(0.7); + await expect(stillAlive(errPromise)).resolves.toBe('alive'); + + await stream.close(); + }); + + it('does not error the stream across more reconnects than the failover budget', async () => { + // The original bug: each updateOptions errored the stream and burned one (never-reset) retry, + // so exceeding maxRetries killed detection. In-place reconnect must keep the stream live. + const detector = createDetector(); + const stream = new InterruptionStreamBase(detector, {}); + const errPromise = readError(stream); + + const ws1 = await waitForInstance(); + ws1.simulateOpen(); + await waitFor(() => ws1.sent.length > 0); + + const rounds = apiConnectDefaults.maxRetries + 2; // deliberately exceed the retry budget + for (let i = 0; i < rounds; i++) { + await stream.updateOptions({ threshold: 0.5 + (i + 1) * 0.01 }); + const ws = await waitForInstanceCount(i + 2); + ws.simulateOpen(); + await waitFor(() => ws.sent.length > 0); + } + + expect(MockWebSocket.instances.length).toBe(rounds + 1); + await expect(stillAlive(errPromise)).resolves.toBe('alive'); + + await stream.close(); + }); + + it('errors the stream when the post-updateOptions reconnect genuinely fails', async () => { + // A real reconnect failure (here connection 429) must still surface as a stream error so the + // failover path runs. + const detector = createDetector(); + const stream = new InterruptionStreamBase(detector, {}); + const errPromise = readError(stream); + + const ws1 = await waitForInstance(); + ws1.simulateOpen(); + await waitFor(() => ws1.sent.length > 0); + + await stream.updateOptions({ threshold: 0.7 }); + const ws2 = await waitForInstanceCount(2); + ws2.simulateUnexpectedResponse(429); // the reconnect's connect attempt is rejected + + const err = await errPromise; + expect(err).toBeInstanceOf(APIStatusError); + expect((err as APIStatusError).statusCode).toBe(429); + + await stream.close(); + }); +}); + +// --------------------------------------------------------------------------- +// AudioRecognition error-emission classification +// --------------------------------------------------------------------------- + +function createHooks(): RecognitionHooks { + return { + onInterruption: vi.fn(), + onStartOfSpeech: vi.fn(), + onVADInferenceDone: vi.fn(), + onEndOfSpeech: vi.fn(), + onInterimTranscript: vi.fn(), + onFinalTranscript: vi.fn(), + onPreemptiveGeneration: vi.fn(), + retrieveChatCtx: () => ChatContext.empty(), + onEndOfTurn: vi.fn(async () => true), + }; +} + +describe('interruption failover error emission', () => { + it('emits exactly one unrecoverable error for a non-retryable transport failure', async () => { + // Mirrors how ws_transport constructs the connection-rejected error (retryable forced off). + const transportError = new APIStatusError({ + message: 'WebSocket connection rejected with status 429', + options: { statusCode: 429, retryable: false }, + }); + expect(transportError.retryable).toBe(false); + + const errors: Array<{ recoverable: boolean }> = []; + const erroringStream = { + stream: () => + new ReadableStream({ + start(controller) { + controller.error(transportError); + }, + }), + pushFrame: async () => {}, + close: async () => {}, + }; + const mockDetector = { + label: 'mock-detector', + createStream: () => erroringStream, + emitError: (e: { recoverable: boolean }) => errors.push(e), + }; + + const recognition = new AudioRecognition({ + recognitionHooks: createHooks(), + interruptionDetection: mockDetector as unknown as AdaptiveInterruptionDetector, + }); + + const ac = new AbortController(); + const task = ( + recognition as unknown as { + createInterruptionTask: ( + d: AdaptiveInterruptionDetector, + signal: AbortSignal, + ) => Promise; + } + ).createInterruptionTask(mockDetector as unknown as AdaptiveInterruptionDetector, ac.signal); + + // The non-retryable path emits the unrecoverable error then blocks in `finally` + // awaiting the (idle) input-forwarding task; abort to let it wind down. + const start = performance.now(); + while (errors.length === 0 && performance.now() - start < 2000) { + await sleep(5); + } + ac.abort(); + await task; + + expect(errors.filter((e) => e.recoverable)).toHaveLength(0); + expect(errors.filter((e) => !e.recoverable)).toHaveLength(1); + }); +}); diff --git a/agents/src/inference/interruption/interruption_session_create.test.ts b/agents/src/inference/interruption/interruption_session_create.test.ts new file mode 100644 index 000000000..7db426df9 --- /dev/null +++ b/agents/src/inference/interruption/interruption_session_create.test.ts @@ -0,0 +1,127 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Tests for the adaptive-interruption threshold negotiation contract. Ported from the Python +// `test_interruption_session_create.py` suite. +// +// The feature is server-driven: the SDK only sends `threshold` in `session.create` when the user +// explicitly overrode it, and otherwise omits the field so the server applies its fetched default. +// These tests lock that serialization contract plus the parsing of the server's `default_threshold` +// off `session.created` and the observability-only effective-threshold resolution. +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { initializeLogger } from '../../log.js'; +import { MockWebSocket } from './_mock_ws.js'; +import { AdaptiveInterruptionDetector } from './interruption_detector.js'; +import { InterruptionStreamBase } from './interruption_stream.js'; +import { resolveEffectiveThreshold, wsMessageSchema } from './ws_transport.js'; + +// --------------------------------------------------------------------------- +// Mock `ws` so the WebSocket transport can be driven deterministically. +// --------------------------------------------------------------------------- + +vi.mock('ws', async () => { + const { MockWebSocket } = await import('./_mock_ws.js'); + return { default: MockWebSocket, WebSocket: MockWebSocket }; +}); + +type MockSocket = MockWebSocket; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +initializeLogger({ pretty: false, level: 'silent' }); + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function waitFor(predicate: () => boolean, timeoutMs = 2000): Promise { + const start = performance.now(); + while (!predicate()) { + if (performance.now() - start > timeoutMs) { + throw new Error('condition not met within timeout'); + } + await sleep(5); + } +} + +function createDetector(opts: { threshold?: number } = {}): AdaptiveInterruptionDetector { + return new AdaptiveInterruptionDetector({ + baseUrl: 'http://localhost:9999', + apiKey: 'test-key', + apiSecret: 'test-secret', + ...opts, + }); +} + +/** Run the real connect path and return the parsed session.create payload the transport sent. */ +async function captureSessionCreate( + detector: AdaptiveInterruptionDetector, +): Promise<{ settings: Record }> { + const stream = new InterruptionStreamBase(detector, {}); + try { + await waitFor(() => MockWebSocket.instances.length > 0); + const ws = MockWebSocket.instances[MockWebSocket.instances.length - 1] as MockSocket; + ws.simulateOpen(); + await waitFor(() => ws.sent.length > 0); + return JSON.parse(String(ws.sent[0])); + } finally { + await stream.close(); + } +} + +beforeEach(() => { + MockWebSocket.instances.length = 0; +}); + +// --------------------------------------------------------------------------- +// session.create threshold serialization contract +// --------------------------------------------------------------------------- + +describe('session.create threshold', () => { + it('omits threshold when not given', async () => { + const payload = await captureSessionCreate(createDetector()); + expect('threshold' in payload.settings).toBe(false); + }); + + it('includes threshold when overridden', async () => { + const payload = await captureSessionCreate(createDetector({ threshold: 0.7 })); + expect(payload.settings.threshold).toBe(0.7); + }); +}); + +// --------------------------------------------------------------------------- +// session.created default_threshold parsing +// --------------------------------------------------------------------------- + +describe('session.created default_threshold parsing', () => { + it('parses default_threshold', () => { + const msg = wsMessageSchema.parse({ type: 'session.created', default_threshold: 0.42 }); + expect(msg).toMatchObject({ type: 'session.created', default_threshold: 0.42 }); + }); + + it('treats default_threshold as optional', () => { + const msg = wsMessageSchema.parse({ type: 'session.created' }) as { + default_threshold?: number | null; + }; + expect(msg.default_threshold ?? null).toBeNull(); + }); +}); + +// --------------------------------------------------------------------------- +// effective-threshold resolution (observability only) +// --------------------------------------------------------------------------- + +describe('resolveEffectiveThreshold', () => { + it('prefers the user override', () => { + expect(resolveEffectiveThreshold(0.7, 0.3)).toBe(0.7); + }); + + it('falls back to the server default', () => { + expect(resolveEffectiveThreshold(undefined, 0.3)).toBe(0.3); + }); + + it('returns null when neither the user nor the server provides a value', () => { + expect(resolveEffectiveThreshold(undefined, null)).toBeNull(); + }); +}); diff --git a/agents/src/inference/interruption/interruption_stream.ts b/agents/src/inference/interruption/interruption_stream.ts index 37bd2f201..486a95eb3 100644 --- a/agents/src/inference/interruption/interruption_stream.ts +++ b/agents/src/inference/interruption/interruption_stream.ts @@ -10,7 +10,6 @@ import { type StreamChannel, createStreamChannel } from '../../stream/stream_cha import { traceTypes } from '../../telemetry/index.js'; import { FRAMES_PER_SECOND, apiConnectDefaults } from './defaults.js'; import type { InterruptionDetectionError } from './errors.js'; -import { createHttpTransport } from './http_transport.js'; import { InterruptionCacheEntry } from './interruption_cache_entry.js'; import type { AdaptiveInterruptionDetector } from './interruption_detector.js'; import { @@ -99,13 +98,15 @@ export class InterruptionStreamBase { // Store reconnect function for WebSocket transport private wsReconnect?: () => Promise; + private wsClose?: () => void; + // Mutable transport options that can be updated via updateOptions() private transportOptions: { baseUrl: string; apiKey: string; apiSecret: string; sampleRate: number; - threshold: number; + threshold?: number; minFrames: number; timeout: number; connectTimeout: number; @@ -154,8 +155,8 @@ export class InterruptionStreamBase { this.options.minFrames = Math.ceil(options.minInterruptionDurationInS * FRAMES_PER_SECOND); this.transportOptions.minFrames = this.options.minFrames; } - // Trigger WebSocket reconnection if using proxy (WebSocket transport) - if (this.options.useProxy && this.wsReconnect) { + // Trigger WebSocket reconnection to apply updated settings. + if (this.wsReconnect) { await this.wsReconnect(); } } @@ -309,30 +310,20 @@ export class InterruptionStreamBase { { highWaterMark: 32 }, ); - // Second transform: transport layer (HTTP or WebSocket based on useProxy) + // Second transform: WebSocket transport layer. const transportOptions = this.transportOptions; - let transport: TransformStream; - if (this.options.useProxy) { - const wsResult = createWsTransport( - transportOptions, - getState, - setState, - handleSpanUpdate, - onRequestSent, - getAndResetNumRequests, - ); - transport = wsResult.transport; - this.wsReconnect = wsResult.reconnect; - } else { - transport = createHttpTransport( - transportOptions, - getState, - setState, - handleSpanUpdate, - getAndResetNumRequests, - ); - } + const wsResult = createWsTransport( + transportOptions, + getState, + setState, + handleSpanUpdate, + onRequestSent, + getAndResetNumRequests, + ); + const transport = wsResult.transport; + this.wsReconnect = wsResult.reconnect; + this.wsClose = wsResult.close; const eventEmitter = new TransformStream({ transform: (chunk, controller) => { @@ -415,9 +406,13 @@ export class InterruptionStreamBase { } async close(): Promise { - if (!this.inputStream.closed) await this.inputStream.close(); - this.resampler?.close(); - this.model.removeStream(this); + try { + if (!this.inputStream.closed) await this.inputStream.close(); + } finally { + this.wsClose?.(); + this.resampler?.close(); + this.model.removeStream(this); + } } } diff --git a/agents/src/inference/interruption/types.ts b/agents/src/inference/interruption/types.ts index d3aae7b95..8306684a7 100644 --- a/agents/src/inference/interruption/types.ts +++ b/agents/src/inference/interruption/types.ts @@ -22,7 +22,7 @@ export interface OverlappingSpeechEvent { */ export interface InterruptionOptions { sampleRate: number; - threshold: number; + threshold?: number; minFrames: number; maxAudioDurationInS: number; audioPrefixDurationInS: number; @@ -32,7 +32,6 @@ export interface InterruptionOptions { baseUrl: string; apiKey: string; apiSecret: string; - useProxy: boolean; } /** diff --git a/agents/src/inference/interruption/ws_transport.ts b/agents/src/inference/interruption/ws_transport.ts index 89f97e5fc..f0bd91b9f 100644 --- a/agents/src/inference/interruption/ws_transport.ts +++ b/agents/src/inference/interruption/ws_transport.ts @@ -7,6 +7,7 @@ import WebSocket from 'ws'; import { z } from 'zod'; import { APIConnectionError, APIStatusError, APITimeoutError } from '../../_exceptions.js'; import { log } from '../../log.js'; +import { Event } from '../../utils.js'; import { buildMetadataHeaders, createAccessToken } from '../utils.js'; import { InterruptionCacheEntry } from './interruption_cache_entry.js'; import type { OverlappingSpeechEvent } from './types.js'; @@ -26,7 +27,7 @@ export interface WsTransportOptions { apiKey: string; apiSecret: string; sampleRate: number; - threshold: number; + threshold?: number; minFrames: number; timeout: number; connectTimeout: number; @@ -39,9 +40,10 @@ export interface WsTransportState { cache: BoundedCache; } -const wsMessageSchema = z.discriminatedUnion('type', [ +export const wsMessageSchema = z.discriminatedUnion('type', [ z.object({ type: z.literal(MSG_SESSION_CREATED), + default_threshold: z.number().nullish(), }), z.object({ type: z.literal(MSG_SESSION_CLOSED), @@ -69,6 +71,23 @@ const wsMessageSchema = z.discriminatedUnion('type', [ type WsMessage = z.infer; +/** + * Resolve the effective interruption threshold for observability only — the server makes the + * actual decision. Precedence: user override, then server default; null when neither is known. + */ +export function resolveEffectiveThreshold( + threshold: number | undefined, + defaultThreshold: number | null | undefined, +): number | null { + if (threshold !== undefined) { + return threshold; + } + if (defaultThreshold != null) { + return defaultThreshold; + } + return null; +} + /** * Creates a WebSocket connection and waits for it to open. */ @@ -83,39 +102,46 @@ async function connectWebSocket( headers: { ...buildMetadataHeaders(), Authorization: `Bearer ${token}` }, }); - await new ThrowsPromise( - (resolve, reject) => { - const timeout = setTimeout(() => { - ws.terminate(); - reject( - new APITimeoutError({ - message: 'WebSocket connection timeout', - options: { retryable: false }, - }), - ); - }, options.connectTimeout); - ws.once('open', () => { - clearTimeout(timeout); - resolve(); - }); - ws.once('unexpected-response', (_req, res) => { - clearTimeout(timeout); - ws.terminate(); - const statusCode = res.statusCode ?? -1; - reject( - new APIStatusError({ - message: `WebSocket connection rejected with status ${statusCode}`, - options: { statusCode, retryable: false }, - }), - ); - }); - ws.once('error', (err: Error) => { - clearTimeout(timeout); - ws.terminate(); - reject(new APIConnectionError({ message: `WebSocket connection error: ${err.message}` })); - }); - }, - ); + try { + await new ThrowsPromise( + (resolve, reject) => { + const timeout = setTimeout(() => { + ws.terminate(); + reject( + new APITimeoutError({ + message: 'WebSocket connection timeout', + options: { retryable: false }, + }), + ); + }, options.connectTimeout); + ws.once('open', () => { + clearTimeout(timeout); + resolve(); + }); + ws.once('unexpected-response', (_req, res) => { + clearTimeout(timeout); + ws.terminate(); + const statusCode = res.statusCode ?? -1; + reject( + new APIStatusError({ + message: `WebSocket connection rejected with status ${statusCode}`, + options: { statusCode, retryable: false }, + }), + ); + }); + ws.once('error', (err: Error) => { + clearTimeout(timeout); + ws.terminate(); + reject(new APIConnectionError({ message: `WebSocket connection error: ${err.message}` })); + }); + }, + ); + } finally { + // Drop the connection-phase once() listeners so a later socket error can't fire the stale + // once('error') alongside the operational on('error'). Safe to remove all: the message handler + // is attached after this returns. + ws.removeAllListeners(); + } return ws; } @@ -123,6 +149,7 @@ async function connectWebSocket( export interface WsTransportResult { transport: TransformStream; reconnect: () => Promise; + close: () => void; } /** @@ -141,9 +168,15 @@ export function createWsTransport( getAndResetNumRequests?: () => number, ): WsTransportResult { const logger = log(); - let ws: WebSocket | null = null; + let activeWs: WebSocket | null = null; let outputController: TransformStreamDefaultController | null = null; + // `reconnecting` is the in-flight reconnect; transform() awaits it so it never sends on a socket + // being torn down. `closed` lets the background watcher exit its loop. + const reconnectEvent = new Event(); + let reconnecting: Promise | null = null; + let closed = false; + function setupMessageHandler(socket: WebSocket): void { socket.on('message', (data: WebSocket.Data) => { let message: WsMessage; @@ -182,31 +215,56 @@ export function createWsTransport( async function ensureConnection(): Promise< Throws > { - if (ws && ws.readyState === WebSocket.OPEN) return; - - ws = await connectWebSocket(options); - setupMessageHandler(ws); - + if (activeWs && activeWs.readyState === WebSocket.OPEN) return; + + activeWs = await connectWebSocket(options); + setupMessageHandler(activeWs); + + const settings: Record = { + sample_rate: options.sampleRate, + num_channels: 1, + min_frames: options.minFrames, + encoding: 's16le', + }; + if (options.threshold !== undefined) { + settings.threshold = options.threshold; + } const sessionCreateMsg = JSON.stringify({ type: MSG_SESSION_CREATE, - settings: { - sample_rate: options.sampleRate, - num_channels: 1, - threshold: options.threshold, - min_frames: options.minFrames, - encoding: 's16le', - }, + settings, }); - ws.send(sessionCreateMsg); + activeWs.send(sessionCreateMsg); } function handleMessage(message: WsMessage): void { const state = getState(); switch (message.type) { - case MSG_SESSION_CREATED: - logger.debug('WebSocket session created'); + case MSG_SESSION_CREATED: { + if (options.threshold === undefined && message.default_threshold == null) { + outputController?.error( + new APIStatusError({ + message: + 'adaptive interruption session created without a threshold: no user override and the server did not report a default_threshold', + options: { statusCode: 500, retryable: false }, + }), + ); + break; + } + // Observability only — the server makes the actual decision. + logger.debug( + { + defaultThreshold: message.default_threshold, + effectiveThreshold: resolveEffectiveThreshold( + options.threshold, + message.default_threshold, + ), + userOverride: options.threshold !== undefined, + }, + 'adaptive interruption session created', + ); break; + } case MSG_INTERRUPTION_DETECTED: { const createdAt = message.created_at; @@ -325,7 +383,9 @@ export function createWsTransport( } function sendAudioData(audioSlice: Int16Array): void { - if (!ws || ws.readyState !== WebSocket.OPEN) { + // Backstop for a genuine unexpected drop: throws a retryable error so the stream fails over. An + // intentional reconnect is awaited in transform() before we get here, so it won't fire then. + if (!activeWs || activeWs.readyState !== WebSocket.OPEN) { throw new APIConnectionError({ message: 'WebSocket not connected' }); } @@ -355,29 +415,69 @@ export function createWsTransport( combined.set(new Uint8Array(header), 0); combined.set(audioBytes, 8); - ws.send(combined); + activeWs.send(combined); onRequestSent?.(); } - function close(): void { - if (ws?.readyState === WebSocket.OPEN) { + // Close the current socket without ending the transport (used by both close() and reconnect). + function closeSocket(): void { + if (activeWs?.readyState === WebSocket.OPEN) { const closeMsg = JSON.stringify({ type: MSG_SESSION_CLOSE }); try { - ws.send(closeMsg); + activeWs.send(closeMsg); } catch (e: unknown) { logger.error(e, 'failed to send close message'); } } - ws?.close(1000); // signal normal websocket closure - ws = null; + // The abandoned socket can still emit 'error' during its close handshake; that handler closes + // over the shared outputController, so a late error would tear down the replacement stream. + activeWs?.removeAllListeners(); + activeWs?.close(1000); // signal normal websocket closure + activeWs = null; + } + + function close(): void { + closed = true; + reconnectEvent.set(); // wake the watcher so it exits its loop + closeSocket(); } /** - * Reconnect the WebSocket with updated options. - * This is called when options are updated via updateOptions(). + * Request an in-place reconnect to apply updated options (threshold / min frames): it does not + * error the stream and does not consume a failover retry. The work happens in reconnectWatcher(). */ async function reconnect(): Promise { - close(); + if (closed) return; + reconnectEvent.set(); + } + + // Background loop that reconnects in place when reconnect() fires, so applying new options keeps + // the stream alive and off the failover retry path. + async function reconnectWatcher(): Promise { + while (!closed) { + await reconnectEvent.wait(); + if (closed) break; + reconnectEvent.clear(); + + // `.catch` keeps `reconnecting` non-rejecting (transform awaits it); a genuine reconnect + // failure is routed to the stream as an error — a legitimate retry. + const done = (async () => { + closeSocket(); + getState().cache.clear(); // abandon the old socket's unanswered in-flight requests + await ensureConnection(); + // close() may have raced in during the await; its closeSocket() saw activeWs === null and + // was a no-op, so tear down the socket we just opened — else it leaks with live handlers. + if (closed) closeSocket(); + })().catch((e: unknown) => { + outputController?.error(e); + }); + reconnecting = done; + try { + await done; + } finally { + if (reconnecting === done) reconnecting = null; + } + } } const transport = new TransformStream< @@ -390,14 +490,19 @@ export function createWsTransport( await ensureConnection().catch((e) => { controller.error(e); }); + void reconnectWatcher(); }, - transform(chunk, controller) { + async transform(chunk, controller) { if (!(chunk instanceof Int16Array)) { controller.enqueue(chunk); return; } + // Wait out any in-flight reconnect so we don't send on a socket being torn down. It never + // rejects — a failed reconnect has already errored the stream via outputController. + if (reconnecting) await reconnecting; + // Only forwards buffered audio while overlap speech is actively on. const state = getState(); if (!state.overlapSpeechStartedAt || !state.overlapSpeechStarted) return; @@ -434,5 +539,5 @@ export function createWsTransport( { highWaterMark: 2 }, ); - return { transport, reconnect }; + return { transport, reconnect, close }; } diff --git a/agents/src/inference/utils.ts b/agents/src/inference/utils.ts index 8a9b8319e..2849ed9ab 100644 --- a/agents/src/inference/utils.ts +++ b/agents/src/inference/utils.ts @@ -56,6 +56,7 @@ export async function createAccessToken( /** * Build metadata headers for inference requests. * Includes SDK version/platform, and optionally room/job/agent IDs from the current job context. + * Includes X-LiveKit-Worker-Token when LIVEKIT_WORKER_TOKEN is set (hosted agents). */ export function buildMetadataHeaders(): Record { const headers: Record = { @@ -71,11 +72,16 @@ export function buildMetadataHeaders(): Record { if (ctx.job.id) { headers['X-LiveKit-Job-Id'] = ctx.job.id; } - if (ctx.room.isConnected) { - const agentSid = ctx.agent?.sid; - if (agentSid) { - headers['X-LiveKit-Agent-Id'] = agentSid; - } + // for hosted agents where job context is always present + const workerToken = process.env.LIVEKIT_WORKER_TOKEN; + if (workerToken) { + headers['X-LiveKit-Worker-Token'] = workerToken; + } + // Only emit the agent SID once the room is connected: before connection the + // local participant SID is unset/placeholder and would leak into requests. + const agentSid = ctx.agent?.sid; + if (ctx.room.isConnected && agentSid) { + headers['X-LiveKit-Agent-Id'] = agentSid; } } diff --git a/agents/src/voice/audio_recognition.ts b/agents/src/voice/audio_recognition.ts index c85d926bd..a46841dd3 100644 --- a/agents/src/voice/audio_recognition.ts +++ b/agents/src/voice/audio_recognition.ts @@ -1506,6 +1506,9 @@ export class AudioRecognition { if (!res) break; const { done, value: ev } = res; if (done) break; + // A healthy stream delivering events recovers the failover budget, so a later transient + // failure isn't charged against earlier ones. + numRetries = 0; this.onOverlapSpeechEvent(ev); } break; diff --git a/agents/src/worker.ts b/agents/src/worker.ts index 3ecc45dc3..860b57005 100644 --- a/agents/src/worker.ts +++ b/agents/src/worker.ts @@ -307,6 +307,12 @@ export class AgentServer { ); if (opts.workerToken) { + // Re-export into the environment so forked subprocesses inherit it (fork() + // copies process.env by default). The inference-header code in the child reads + // process.env.LIVEKIT_WORKER_TOKEN — see inference/utils.ts buildMetadataHeaders(). + // Mirrors Python worker.py, which sets os.environ before spawning job procs. + process.env.LIVEKIT_WORKER_TOKEN = opts.workerToken; + if (opts.loadFunc !== defaultCpuLoad) { this.#logger.warn( 'custom loadFunc is not supported when deploying to Cloud, using defaults', diff --git a/turbo.json b/turbo.json index 325cafca6..3ccab37dd 100644 --- a/turbo.json +++ b/turbo.json @@ -42,6 +42,7 @@ "LLAMA_API_KEY", "LIVEKIT_AGENT_ID", "LIVEKIT_AGENT_NAME", + "LIVEKIT_WORKER_TOKEN", "LOG_LEVEL", "OCTOAI_TOKEN", "OPENAI_API_KEY", From eae60747e308f063dd15e1d19b55fe7cefa00b26 Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:44:33 -0700 Subject: [PATCH 12/15] feat(voice): add agent instruction updates (#1783) Co-authored-by: rosetta-livekit-bot[bot] <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Co-authored-by: Toubat --- .changeset/agent-update-instructions.md | 6 +++ agents/src/voice/agent.ts | 9 +++++ agents/src/voice/agent_activity.ts | 18 +++++++++ .../src/realtime/realtime_model.test.ts | 38 +++++++++++++++++++ plugins/openai/src/realtime/realtime_model.ts | 13 ++++++- 5 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 .changeset/agent-update-instructions.md diff --git a/.changeset/agent-update-instructions.md b/.changeset/agent-update-instructions.md new file mode 100644 index 000000000..69ecaebb4 --- /dev/null +++ b/.changeset/agent-update-instructions.md @@ -0,0 +1,6 @@ +--- +'@livekit/agents': patch +'@livekit/agents-plugin-openai': patch +--- + +Add `Agent.updateInstructions()` to update an agent's instructions mid-session (parity with Python). The change propagates the new instructions through the active `AgentActivity`, records an `AgentConfigUpdate` in the chat and session history, and syncs the realtime/stateless contexts. For OpenAI realtime, per-response instructions now preserve the session-level instructions instead of replacing them. diff --git a/agents/src/voice/agent.ts b/agents/src/voice/agent.ts index f65d45425..7139d3588 100644 --- a/agents/src/voice/agent.ts +++ b/agents/src/voice/agent.ts @@ -366,6 +366,15 @@ export class Agent { this._agentActivity.updateChatCtx(chatCtx); } + async updateInstructions(instructions: string | Instructions): Promise { + if (!this._agentActivity) { + this._instructions = instructions; + return; + } + + await this._agentActivity.updateInstructions(instructions); + } + // TODO(parity): Add when AgentConfigUpdate is ported to ChatContext. async updateTools(tools: readonly ToolContextEntry[]): Promise { if (!this._agentActivity) { diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 66c55a747..a04a21c35 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -781,6 +781,24 @@ export class AgentActivity implements RecognitionHooks { } } + async updateInstructions(instructions: string | Instructions): Promise { + this.agent._instructions = instructions; + + const configUpdate = new AgentConfigUpdate({ instructions }); + this.agent._chatCtx.insert(configUpdate); + this.agentSession.history.insert(configUpdate); + + if (this.realtimeSession) { + await this.realtimeSession.updateInstructions(renderInstructions(instructions)); + } else { + updateInstructions({ + chatCtx: this.agent._chatCtx, + instructions, + addIfMissing: true, + }); + } + } + async updateTools(tools: readonly ToolContextEntry[]): Promise { const oldToolCtx = this.agent._toolCtx; const oldToolNames = new Set(Object.keys(oldToolCtx.functionTools)); diff --git a/plugins/openai/src/realtime/realtime_model.test.ts b/plugins/openai/src/realtime/realtime_model.test.ts index 6080cdbd6..d52d1090a 100644 --- a/plugins/openai/src/realtime/realtime_model.test.ts +++ b/plugins/openai/src/realtime/realtime_model.test.ts @@ -13,9 +13,15 @@ import { type RealtimeSessionInternals = { generateReply: RealtimeSession['generateReply']; + updateInstructions: RealtimeSession['updateInstructions']; responseCreatedFutures: Record; sendEvent: ReturnType; textModeRecoveryRetries: number; + instructions?: string; + _options: { + isAzure?: boolean; + apiVersion?: string; + }; }; type ResponseDoneSessionInternals = { @@ -36,6 +42,7 @@ function createSessionForTest(): RealtimeSessionInternals { session.responseCreatedFutures = {}; session.sendEvent = vi.fn(); session.textModeRecoveryRetries = 0; + session._options = {}; return session; } @@ -49,6 +56,37 @@ function stubTaskRuntime(): void { } describe('RealtimeSession.generateReply', () => { + it('preserves session instructions when generating with per-response instructions', async () => { + const session = createSessionForTest(); + await session.updateInstructions('Your name is Kelly. Always respond in English.'); + + const abortController = new AbortController(); + const promise = session.generateReply('Tell the user what your name is.', { + signal: abortController.signal, + }); + abortController.abort(); + + await expect(promise).rejects.toThrow('generateReply aborted'); + expect(session.instructions).toBe('Your name is Kelly. Always respond in English.'); + expect(session.sendEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'session.update', + session: expect.objectContaining({ + instructions: 'Your name is Kelly. Always respond in English.', + }), + }), + ); + expect(session.sendEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'response.create', + response: expect.objectContaining({ + instructions: + 'Your name is Kelly. Always respond in English.\nTell the user what your name is.', + }), + }), + ); + }); + it('cancels an in-flight response when aborted before response.created', async () => { const session = createSessionForTest(); const abortController = new AbortController(); diff --git a/plugins/openai/src/realtime/realtime_model.ts b/plugins/openai/src/realtime/realtime_model.ts index 1f7de8ef7..e50ea5d3a 100644 --- a/plugins/openai/src/realtime/realtime_model.ts +++ b/plugins/openai/src/realtime/realtime_model.ts @@ -842,7 +842,18 @@ export class RealtimeSession extends llm.RealtimeSession { instructions?: string, options: { signal?: AbortSignal } = {}, ): Promise { - const handle = this.createResponse({ instructions, userInitiated: true }); + // In OpenAI realtime, the session-level instructions are completely replaced by the + // per-response instructions for this response. Prepend the session instructions so they + // are preserved (parity with the Python implementation). + let responseInstructions = instructions; + if (instructions && this.instructions) { + responseInstructions = `${this.instructions}\n${instructions}`; + } + + const handle = this.createResponse({ + instructions: responseInstructions, + userInitiated: true, + }); this.textModeRecoveryRetries = 0; const onAbort = () => { From b57921663cc85c8a823e26ec63346e73852fe561 Mon Sep 17 00:00:00 2001 From: Toubat Date: Fri, 12 Jun 2026 18:14:47 -0700 Subject: [PATCH 13/15] test: use fake timers in manual-abort Task test to deflake CI The test asserted an exact tick count ([0,1,2]) against real timers with a 5ms margin, which flakes on loaded CI runners. Co-Authored-By: Claude Fable 5 --- agents/src/utils.test.ts | 48 ++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/agents/src/utils.test.ts b/agents/src/utils.test.ts index 824305313..421697fb9 100644 --- a/agents/src/utils.test.ts +++ b/agents/src/utils.test.ts @@ -239,29 +239,39 @@ describe('utils', () => { }); it('should handle task that checks abort signal manually', async () => { - const arr: number[] = []; - const task = Task.from(async (controller) => { - for (let i = 0; i < 10; i++) { - if (controller.signal.aborted) { - throw new Error('Task was aborted'); + // fake timers: with real timers the exact tick count is scheduling- + // dependent and flakes on loaded CI runners + vi.useFakeTimers(); + try { + const arr: number[] = []; + const task = Task.from(async (controller) => { + for (let i = 0; i < 10; i++) { + if (controller.signal.aborted) { + throw new Error('Task was aborted'); + } + await delay(10); + arr.push(i); } - await delay(10); - arr.push(i); - } - return 'completed'; - }); + return 'completed'; + }); - await delay(35); - task.cancel(); + await vi.advanceTimersByTimeAsync(35); + task.cancel(); - expect(arr).toEqual([0, 1, 2]); - try { - await task.result; - } catch (error: unknown) { - expect((error as Error).message).toBe('Task was aborted'); - } + expect(arr).toEqual([0, 1, 2]); + // the pending (signal-less) delay must elapse for the loop to reach + // its manual abort checkpoint + await vi.advanceTimersByTimeAsync(10); + try { + await task.result; + } catch (error: unknown) { + expect((error as Error).message).toBe('Task was aborted'); + } - expect(task.done).toBe(true); + expect(task.done).toBe(true); + } finally { + vi.useRealTimers(); + } }); it('should handle cleanup in finally block', async () => { From 2cd85daed2fb44d6183c97d162f888e090b1c0eb Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2026 10:41:47 +0100 Subject: [PATCH 14/15] fix(elevenlabs): stop recognize from mutating instance language (#1789) Co-authored-by: rosetta-livekit-bot[bot] <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> --- .changeset/odd-languages-speak.md | 5 +++++ plugins/elevenlabs/src/stt.ts | 11 +++++------ 2 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 .changeset/odd-languages-speak.md diff --git a/.changeset/odd-languages-speak.md b/.changeset/odd-languages-speak.md new file mode 100644 index 000000000..b70a32109 --- /dev/null +++ b/.changeset/odd-languages-speak.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-elevenlabs': patch +--- + +Avoid persisting per-call STT language overrides on ElevenLabs STT instances. diff --git a/plugins/elevenlabs/src/stt.ts b/plugins/elevenlabs/src/stt.ts index d179752de..5ff10488b 100644 --- a/plugins/elevenlabs/src/stt.ts +++ b/plugins/elevenlabs/src/stt.ts @@ -330,17 +330,16 @@ export class STT extends stt.STT { language?: string, abortSignal?: AbortSignal, ): Promise { - if (language !== undefined) { - this.#opts.languageCode = normalizeLanguage(language); - } + const languageCode = + language !== undefined ? normalizeLanguage(language) : this.#opts.languageCode; const wavBytes = createWav(mergeFrames(buffer)); const form = new FormData(); form.append('file', new Blob([new Uint8Array(wavBytes)], { type: 'audio/x-wav' }), 'audio.wav'); form.append('model_id', this.#opts.modelId); form.append('tag_audio_events', String(this.#opts.tagAudioEvents)); - if (this.#opts.languageCode) { - form.append('language_code', this.#opts.languageCode); + if (languageCode) { + form.append('language_code', languageCode); } if (this.#opts.keyterms !== undefined) { for (const keyterm of this.#opts.keyterms) { @@ -369,7 +368,7 @@ export class STT extends stt.STT { const startTime = words.length > 0 ? Math.min(...words.map((word) => word.start ?? 0)) : 0; const endTime = words.length > 0 ? Math.max(...words.map((word) => word.end ?? 0)) : 0; const normalizedLanguage = normalizeLanguage( - responseJson.language_code ?? this.#opts.languageCode ?? '', + responseJson.language_code ?? languageCode ?? '', ); return this.#transcriptionToSpeechEvent( From 01d44f4d19c7dc4958c47751cbac621f7e61e08c Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2026 09:58:48 +0000 Subject: [PATCH 15/15] fix(avatar): preserve audio wrappers across hot-swaps --- .changeset/polite-avatars-swap.md | 5 + .../src/voice/avatar/avatar_session.test.ts | 27 ---- agents/src/voice/avatar/avatar_session.ts | 12 -- agents/src/voice/io.ts | 126 +++++++++++++++++- plugins/anam/src/avatar.ts | 12 +- plugins/bey/src/avatar.ts | 12 +- plugins/did/src/avatar.ts | 14 +- plugins/lemonslice/src/avatar.ts | 16 ++- plugins/liveavatar/src/avatar.ts | 2 +- plugins/runway/src/avatar.ts | 14 +- plugins/tavus/src/avatar.ts | 14 +- plugins/trugen/src/avatar.ts | 12 +- 12 files changed, 179 insertions(+), 87 deletions(-) create mode 100644 .changeset/polite-avatars-swap.md diff --git a/.changeset/polite-avatars-swap.md b/.changeset/polite-avatars-swap.md new file mode 100644 index 000000000..4ed64af3f --- /dev/null +++ b/.changeset/polite-avatars-swap.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +fix(avatar): preserve audio wrappers across avatar hot-swaps diff --git a/agents/src/voice/avatar/avatar_session.test.ts b/agents/src/voice/avatar/avatar_session.test.ts index e1c798c07..a4d3f6811 100644 --- a/agents/src/voice/avatar/avatar_session.test.ts +++ b/agents/src/voice/avatar/avatar_session.test.ts @@ -3,7 +3,6 @@ // SPDX-License-Identifier: Apache-2.0 import { afterEach, describe, expect, it, vi } from 'vitest'; import * as jobModule from '../../job.js'; -import * as logModule from '../../log.js'; import { AvatarSession } from './avatar_session.js'; describe('AvatarSession base', () => { @@ -46,30 +45,4 @@ describe('AvatarSession base', () => { await shutdownCallback?.(); expect(acloseSpy).toHaveBeenCalledTimes(1); }); - - it('warns when avatar is started after AgentSession.start()', async () => { - const warn = vi.fn(); - vi.spyOn(logModule, 'log').mockReturnValue({ - warn, - debug: vi.fn(), - } as unknown as ReturnType); - vi.spyOn(jobModule, 'getJobContext').mockReturnValue(undefined); - - const session = new AvatarSession(); - - await session.start( - { - ...mockAgentSession({ - _started: true, - output: { audio: { constructor: { name: 'MockAudioOutput' } } }, - }), - } as any, - mockRoom() as any, - ); - - expect(warn).toHaveBeenCalledWith( - { audioOutput: 'MockAudioOutput' }, - expect.stringContaining('AvatarSession.start() was called after AgentSession.start()'), - ); - }); }); diff --git a/agents/src/voice/avatar/avatar_session.ts b/agents/src/voice/avatar/avatar_session.ts index a2ad4bef1..4bd7bfee8 100644 --- a/agents/src/voice/avatar/avatar_session.ts +++ b/agents/src/voice/avatar/avatar_session.ts @@ -28,8 +28,6 @@ export type AvatarSessionCallbacks = { * first in their own `start()` method. The base: * - Registers {@link AvatarSession.aclose} as a job shutdown callback, so avatar resources * are released when the job shuts down. - * - Warns when the avatar session is started after {@link AgentSession.start} — in that - * case the existing audio output will be replaced by the avatar's. */ export class AvatarSession extends (EventEmitter as new () => TypedEmitter) { #logger = log(); @@ -64,16 +62,6 @@ export class AvatarSession extends (EventEmitter as new () => TypedEmitter - this.onPlaybackStarted(ev.createdAt), - ); - this.nextInChain.on(AudioOutput.EVENT_PLAYBACK_FINISHED, (ev: PlaybackFinishedEvent) => - this.onPlaybackFinished(ev), - ); + this.nextInChain.on(AudioOutput.EVENT_PLAYBACK_STARTED, this.forwardNextPlaybackStarted); + this.nextInChain.on(AudioOutput.EVENT_PLAYBACK_FINISHED, this.forwardNextPlaybackFinished); } } + get nextInChain(): AudioOutput | undefined { + return this._nextInChain; + } + + protected forwardNextPlaybackStarted = (ev: PlaybackStartedEvent) => { + this.onPlaybackStarted(ev.createdAt); + }; + + protected forwardNextPlaybackFinished = (ev: PlaybackFinishedEvent) => { + this.onPlaybackFinished(ev); + }; + /** * Whether this output and all outputs in the chain support pause/resume. */ @@ -242,6 +261,87 @@ export abstract class AudioOutput extends EventEmitter { } } +class AudioSinkProxy extends AudioOutput { + private attached: boolean = false; + private capturing: boolean = false; + private pushedDuration: number = 0; + + constructor(nextInChain: AudioOutput) { + super(undefined, undefined, { pause: true }); + this.setNextInChain(nextInChain); + } + + override get nextInChain(): AudioOutput { + if (this._nextInChain === undefined) { + throw new Error('AudioSinkProxy has no downstream sink'); + } + return this._nextInChain; + } + + override get canPause(): boolean { + return this.nextInChain.canPause; + } + + override onAttached(): void { + this.attached = true; + super.onAttached(); + } + + override onDetached(): void { + this.attached = false; + super.onDetached(); + } + + setNextInChain(sink: AudioOutput): void { + if (sink === this._nextInChain) return; + + const oldSink = this._nextInChain; + if (oldSink !== undefined) { + oldSink.off(AudioOutput.EVENT_PLAYBACK_STARTED, this.forwardNextPlaybackStarted); + oldSink.off(AudioOutput.EVENT_PLAYBACK_FINISHED, this.forwardNextPlaybackFinished); + if (this.pendingPlayoutSegments > 0) { + oldSink.clearBuffer(); + } + if (this.attached) { + oldSink.onDetached(); + } + } + + this._nextInChain = sink; + this.sampleRate = sink.sampleRate; + sink.on(AudioOutput.EVENT_PLAYBACK_STARTED, this.forwardNextPlaybackStarted); + sink.on(AudioOutput.EVENT_PLAYBACK_FINISHED, this.forwardNextPlaybackFinished); + if (this.attached) { + sink.onAttached(); + } + + if (oldSink !== undefined && this.pendingPlayoutSegments > 0 && !this.capturing) { + this.onPlaybackFinished({ playbackPosition: this.pushedDuration, interrupted: true }); + } + } + + override async captureFrame(frame: AudioFrame): Promise { + if (!this.capturing) { + this.capturing = true; + this.pushedDuration = 0; + } + + await super.captureFrame(frame); + await this.nextInChain.captureFrame(frame); + this.pushedDuration += frame.samplesPerChannel / frame.sampleRate; + } + + override flush(): void { + super.flush(); + this.nextInChain.flush(); + this.capturing = false; + } + + override clearBuffer(): void { + this.nextInChain.clearBuffer(); + } +} + export interface PlaybackFinishedEvent { /** How much of the audio was played back, in seconds */ playbackPosition: number; @@ -397,6 +497,18 @@ export class AgentOutput { } } + replaceAudioTail(sink: AudioOutput): void { + let current = this._audioSink; + while (current !== null && current !== undefined) { + if (current instanceof AudioSinkProxy) { + current.setNextInChain(sink); + return; + } + current = current.nextInChain ?? null; + } + this.audio = sink; + } + get transcription(): TextOutput | null { return this._transcriptionSink; } diff --git a/plugins/anam/src/avatar.ts b/plugins/anam/src/avatar.ts index 9e2f40e62..05c8543d6 100644 --- a/plugins/anam/src/avatar.ts +++ b/plugins/anam/src/avatar.ts @@ -122,10 +122,12 @@ export class AvatarSession extends voice.AvatarSession { const started = await anam.startEngineSession({ sessionToken }); this.sessionId = started.sessionId; - agentSession.output.audio = new voice.DataStreamAudioOutput({ - room, - destinationIdentity: this.avatarIdentity, - waitRemoteTrack: TrackKind.KIND_VIDEO, - }); + agentSession.output.replaceAudioTail( + new voice.DataStreamAudioOutput({ + room, + destinationIdentity: this.avatarIdentity, + waitRemoteTrack: TrackKind.KIND_VIDEO, + }), + ); } } diff --git a/plugins/bey/src/avatar.ts b/plugins/bey/src/avatar.ts index ce23bb31e..53d091216 100644 --- a/plugins/bey/src/avatar.ts +++ b/plugins/bey/src/avatar.ts @@ -204,11 +204,13 @@ export class AvatarSession extends voice.AvatarSession { this.#logger.debug('starting avatar session'); await this.startAgent(livekitUrl, livekitToken); - agentSession.output.audio = new voice.DataStreamAudioOutput({ - room, - destinationIdentity: this.avatarIdentity, - waitRemoteTrack: TrackKind.KIND_VIDEO, - }); + agentSession.output.replaceAudioTail( + new voice.DataStreamAudioOutput({ + room, + destinationIdentity: this.avatarIdentity, + waitRemoteTrack: TrackKind.KIND_VIDEO, + }), + ); } private async startAgent(livekitUrl: string, livekitToken: string): Promise { diff --git a/plugins/did/src/avatar.ts b/plugins/did/src/avatar.ts index 7906cc47f..d1610a1e5 100644 --- a/plugins/did/src/avatar.ts +++ b/plugins/did/src/avatar.ts @@ -159,11 +159,13 @@ export class AvatarSession extends voice.AvatarSession { audioConfig: { sample_rate: sampleRate }, }); - agentSession.output.audio = new voice.DataStreamAudioOutput({ - room, - destinationIdentity: this.avatarIdentity, - sampleRate, - waitRemoteTrack: TrackKind.KIND_VIDEO, - }); + agentSession.output.replaceAudioTail( + new voice.DataStreamAudioOutput({ + room, + destinationIdentity: this.avatarIdentity, + sampleRate, + waitRemoteTrack: TrackKind.KIND_VIDEO, + }), + ); } } diff --git a/plugins/lemonslice/src/avatar.ts b/plugins/lemonslice/src/avatar.ts index e4d6a970b..9861d0b4d 100644 --- a/plugins/lemonslice/src/avatar.ts +++ b/plugins/lemonslice/src/avatar.ts @@ -252,13 +252,15 @@ export class AvatarSession extends voice.AvatarSession { this.#logger.debug('starting avatar session'); const sessionId = await this.startAgent(livekitUrl, livekitToken); - agentSession.output.audio = new voice.DataStreamAudioOutput({ - room, - destinationIdentity: this.avatarIdentity, - sampleRate: SAMPLE_RATE, - waitRemoteTrack: TrackKind.KIND_VIDEO, - waitPlaybackStart: true, - }); + agentSession.output.replaceAudioTail( + new voice.DataStreamAudioOutput({ + room, + destinationIdentity: this.avatarIdentity, + sampleRate: SAMPLE_RATE, + waitRemoteTrack: TrackKind.KIND_VIDEO, + waitPlaybackStart: true, + }), + ); return sessionId; } diff --git a/plugins/liveavatar/src/avatar.ts b/plugins/liveavatar/src/avatar.ts index ce0575d3b..ff88af749 100644 --- a/plugins/liveavatar/src/avatar.ts +++ b/plugins/liveavatar/src/avatar.ts @@ -238,7 +238,7 @@ export class AvatarSession extends voice.AvatarSession { this.audioBuffer.on('clear_buffer', (ev: voice.QueueAudioOutputClearEvent) => this.onClearBuffer(ev), ); - agentSession.output.audio = this.audioBuffer; + agentSession.output.replaceAudioTail(this.audioBuffer); // Spawn the main task with an attached error handler so a websocket open or // protocol failure does not surface as an unhandled rejection. The main task diff --git a/plugins/runway/src/avatar.ts b/plugins/runway/src/avatar.ts index c1a23c228..5361a49df 100644 --- a/plugins/runway/src/avatar.ts +++ b/plugins/runway/src/avatar.ts @@ -155,12 +155,14 @@ export class AvatarSession extends voice.AvatarSession { void this.ensureEndSessionPromise(); }); - agentSession.output.audio = new voice.DataStreamAudioOutput({ - room, - destinationIdentity: this.avatarIdentity, - waitRemoteTrack: TrackKind.KIND_VIDEO, - sampleRate: SAMPLE_RATE, - }); + agentSession.output.replaceAudioTail( + new voice.DataStreamAudioOutput({ + room, + destinationIdentity: this.avatarIdentity, + waitRemoteTrack: TrackKind.KIND_VIDEO, + sampleRate: SAMPLE_RATE, + }), + ); } private async createSession( diff --git a/plugins/tavus/src/avatar.ts b/plugins/tavus/src/avatar.ts index 1dd19ec03..28f47f9fc 100644 --- a/plugins/tavus/src/avatar.ts +++ b/plugins/tavus/src/avatar.ts @@ -149,11 +149,13 @@ export class AvatarSession extends voice.AvatarSession { properties: { livekit_ws_url: livekitUrl, livekit_room_token: livekitToken }, }); - agentSession.output.audio = new voice.DataStreamAudioOutput({ - room, - destinationIdentity: this.avatarIdentity, - sampleRate: SAMPLE_RATE, - waitRemoteTrack: TrackKind.KIND_VIDEO, - }); + agentSession.output.replaceAudioTail( + new voice.DataStreamAudioOutput({ + room, + destinationIdentity: this.avatarIdentity, + sampleRate: SAMPLE_RATE, + waitRemoteTrack: TrackKind.KIND_VIDEO, + }), + ); } } diff --git a/plugins/trugen/src/avatar.ts b/plugins/trugen/src/avatar.ts index 71276e012..11620a75b 100644 --- a/plugins/trugen/src/avatar.ts +++ b/plugins/trugen/src/avatar.ts @@ -167,11 +167,13 @@ export class AvatarSession extends voice.AvatarSession { this.#logger.debug('starting avatar session'); await this.startAgent(livekitUrl, livekitToken); - agentSession.output.audio = new voice.DataStreamAudioOutput({ - room, - destinationIdentity: this.avatarIdentity, - waitRemoteTrack: TrackKind.KIND_VIDEO, - }); + agentSession.output.replaceAudioTail( + new voice.DataStreamAudioOutput({ + room, + destinationIdentity: this.avatarIdentity, + waitRemoteTrack: TrackKind.KIND_VIDEO, + }), + ); } private async startAgent(livekitUrl: string, livekitToken: string): Promise {