diff --git a/packages/agent/README.md b/packages/agent/README.md index 5371d01..bc8205b 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -99,6 +99,68 @@ Both classes mirror pi constructor shapes and behavior, with minimal additions: - `extraTools` to add your own pi tools alongside the built-in browser tools - `playwright: true` to let the model run Playwright/TypeScript against the live browser session +- `toolResultImageReplayLimit` to control how many recent tool-result images + are included in each model request +- `responseThreading` to choose whether OpenAI and Tzafon continue from + provider-stored response state or receive the full request context each time + +### Context management + +Both constructors accept the same request-context options: + +```ts +const agent = new CuaAgent({ + // ...browser, client, and initialState + toolResultImageReplayLimit: 4, + responseThreading: true, +}); +``` + +`toolResultImageReplayLimit` defaults to `4`. A non-negative integer keeps that +many of the newest image blocks from tool results in each model request. `0` +removes all tool-result images from the request, while `false` disables this +filter and includes every tool-result image. Text, metadata, and message order +are preserved; a changed tool result gets one +`[stale tool-result images omitted]` marker where its first removed image was. +The limit applies across built-in tools, custom tools, and multi-image results. + +This is a request-time view of the transcript, not a history mutation. +`CuaAgent.state.messages`, harness session entries, and +`session.buildContext()` retain the original images. + +#### How this uses pi context primitives + +pi provides several context-management layers with different lifetimes: + +- `Agent.transformContext` changes the messages passed to one model request. + `CuaAgent` uses this hook for the image limit. If you provide your own + `transformContext`, it runs first and the image limit is applied to its + result. +- `AgentHarness` exposes the same request-time stage through its `context` + hook. pi selects their final result first, then `CuaAgentHarness` applies + the image limit at the `Models` boundary before the provider call. + User handlers therefore compose with the built-in limit instead of replacing + it. Set `toolResultImageReplayLimit: false` if a handler should own image + filtering completely. +- A harness `Session` can use `entryTransforms` and `entryProjectors` while + building model context from stored session entries. These run before the + request-time `context` hooks; the image limit runs afterward. None of these + projections rewrite stored entries. +- `harness.compact()` is the persistent, model-generated option for reducing a + long transcript. It writes a summary entry that future session contexts use. + It is broader than the image limit and is not called automatically by this + package. + +`responseThreading` defaults to `true`. For OpenAI and Tzafon Responses APIs, +that means each request can continue from `previous_response_id`; the provider +keeps earlier response state and CUA sends only messages added since that +response. Because the provider retains that state, removing an older image from +the current request view does not remove it from an existing provider thread. + +Set `responseThreading: false` when every request should be built entirely from +the current pi context instead. In that mode no `previous_response_id` is sent, +and `toolResultImageReplayLimit` determines which tool-result images are +included in the full request. Other providers do not use this option. If auth callbacks are omitted, both classes default to CUA env var conventions: - OpenAI: `OPENAI_API_KEY` diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index cde7de4..6b18861 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -2,6 +2,7 @@ import { Agent, AgentHarness, type AgentHarnessOptions, + type AgentMessage, type AgentOptions, type AgentState, type AgentTool, @@ -11,6 +12,7 @@ import { } from "@earendil-works/pi-agent-core"; import { type Api, + type Context, CUA_NAVIGATION_TOOL_NAME, CUA_PLAYWRIGHT_TOOL_NAME, cuaModels, @@ -42,6 +44,64 @@ import { InternalComputerTranslator, type KernelBrowser } from "./translator/tra */ type CuaRuntimeInput = CuaModelRef | Model; +const DEFAULT_TOOL_RESULT_IMAGE_REPLAY_LIMIT = 4; +const OMITTED_TOOL_RESULT_IMAGES = "[stale tool-result images omitted]"; + +/** + * Maximum number of tool-result images included in the request-time message + * projection, or `false` to leave image blocks unchanged. + */ +export type ToolResultImageReplayLimit = number | false; + +function resolveToolResultImageReplayLimit(limit: ToolResultImageReplayLimit | undefined): ToolResultImageReplayLimit { + if (limit === undefined) return DEFAULT_TOOL_RESULT_IMAGE_REPLAY_LIMIT; + if (limit !== false && (!Number.isFinite(limit) || !Number.isInteger(limit) || limit < 0)) { + throw new TypeError("toolResultImageReplayLimit must be a finite non-negative integer or false"); + } + return limit; +} + +function projectToolResultImages( + messages: TMessage[], + limit: ToolResultImageReplayLimit, +): TMessage[] { + if (limit === false) return messages; + + let imageCount = 0; + for (const message of messages) { + if (message.role === "toolResult") { + imageCount += message.content.filter((block) => block.type === "image").length; + } + } + if (imageCount <= limit) return messages; + + const firstRetainedImage = Math.max(0, imageCount - limit); + let imageOrdinal = 0; + return messages.map((message) => { + if (message.role !== "toolResult") return message; + + let changed = false; + let markerInserted = false; + const content: typeof message.content = []; + for (const block of message.content) { + if (block.type !== "image") { + content.push(block); + continue; + } + if (imageOrdinal++ >= firstRetainedImage) { + content.push(block); + continue; + } + changed = true; + if (!markerInserted) { + content.push({ type: "text", text: OMITTED_TOOL_RESULT_IMAGES }); + markerInserted = true; + } + } + return changed ? ({ ...message, content } as TMessage) : message; + }); +} + /** * Agent state exposed by {@link CuaAgent}. * @@ -84,6 +144,10 @@ export type CuaAgentOptions = Omit & { nativeTool?: CuaNativeToolSpec; /** Expose a tool that runs Playwright code against the browser session. */ playwright?: boolean; + /** Maximum tool-result images included from message history per provider request. Defaults to 4; false disables projection. */ + toolResultImageReplayLimit?: ToolResultImageReplayLimit; + /** Chain OpenAI and Tzafon requests through provider-stored response state. Defaults to true. */ + responseThreading?: boolean; /** Optional CUA-level retries around each provider request. Disabled by default. */ retry?: CuaRetryOptions; }; @@ -124,6 +188,10 @@ export type CuaAgentHarnessOptions< playwright?: boolean; /** Optional payload hook composed after the provider-specific CUA payload hook. */ onPayload?: SimpleStreamOptions["onPayload"]; + /** Maximum tool-result images included from message history per provider request. Defaults to 4; false disables projection. */ + toolResultImageReplayLimit?: ToolResultImageReplayLimit; + /** Chain OpenAI and Tzafon requests through provider-stored response state. Defaults to true. */ + responseThreading?: boolean; /** Optional CUA-level retries around each provider request. Disabled by default. */ retry?: CuaRetryOptions; }; @@ -279,6 +347,70 @@ class CuaRuntimeController { /** Default stream path: the shared CUA `Models` collection. */ const defaultCuaStream: StreamFn = (model, context, options) => cuaModels().streamSimple(model, context, options); +function resolveResponseThreading(responseThreading: boolean | undefined): boolean { + if (responseThreading !== undefined && typeof responseThreading !== "boolean") { + throw new TypeError("responseThreading must be a boolean"); + } + return responseThreading ?? true; +} + +function withResponseThreading( + options: TOptions, + enabled: boolean, +): TOptions { + return enabled ? options : { ...options, disableResponseThreading: true }; +} + +function projectModelContext(context: Context, imageReplayLimit: ToolResultImageReplayLimit): Context { + const messages = projectToolResultImages(context.messages, imageReplayLimit); + return messages === context.messages ? context : { ...context, messages }; +} + +function withContextManagement( + models: Models, + imageReplayLimit: ToolResultImageReplayLimit, + responseThreading: boolean, +): Models { + if (imageReplayLimit === false && responseThreading) return models; + + const stream: Models["stream"] = (model, context, options) => + models.stream( + model, + projectModelContext(context, imageReplayLimit), + withResponseThreading(options, responseThreading), + ); + const complete: Models["complete"] = (model, context, options) => + models.complete( + model, + projectModelContext(context, imageReplayLimit), + withResponseThreading(options, responseThreading), + ); + const streamSimple: Models["streamSimple"] = (model, context, options) => + models.streamSimple( + model, + projectModelContext(context, imageReplayLimit), + withResponseThreading(options, responseThreading), + ); + const completeSimple: Models["completeSimple"] = (model, context, options) => + models.completeSimple( + model, + projectModelContext(context, imageReplayLimit), + withResponseThreading(options, responseThreading), + ); + return { + getProviders: () => models.getProviders(), + getProvider: (id) => models.getProvider(id), + getModels: (provider) => models.getModels(provider), + getModel: (provider, id) => models.getModel(provider, id), + refresh: (provider) => models.refresh(provider), + getAuth: (model) => models.getAuth(model), + stream, + complete, + streamSimple, + completeSimple, + }; +} + /** * Pi `Agent` configured for Kernel browser computer use. * @@ -302,13 +434,18 @@ export class CuaAgent extends Agent { onPayload, streamFn, prepareNextTurn, + transformContext, extraTools, mode, nativeTool, playwright, + toolResultImageReplayLimit, + responseThreading, retry, ...agentOptions } = options; + const imageReplayLimit = resolveToolResultImageReplayLimit(toolResultImageReplayLimit); + const useResponseThreading = resolveResponseThreading(responseThreading); const runtime = new CuaRuntimeController({ browser, client, @@ -328,6 +465,7 @@ export class CuaAgent extends Agent { ...streamOptions, onPayload: runtime.onPayload(), keepToolNames: runtime.keepToolNames(), + disableResponseThreading: !useResponseThreading, }; return retryingStream(model, context, optionsWithCuaRuntime); }; @@ -336,6 +474,11 @@ export class CuaAgent extends Agent { ...agentOptions, getApiKey: agentOptions.getApiKey ?? getCuaEnvApiKey, streamFn: wrappedStreamFn, + transformContext: async (messages, signal) => + projectToolResultImages( + transformContext ? await transformContext(messages, signal) : messages, + imageReplayLimit, + ), initialState: { ...initialState, model: runtime.model, @@ -461,9 +604,13 @@ export class CuaAgentHarness< systemPrompt, onPayload, activeToolNames, + toolResultImageReplayLimit, + responseThreading, retry, ...harnessOptions } = options; + const imageReplayLimit = resolveToolResultImageReplayLimit(toolResultImageReplayLimit); + const useResponseThreading = resolveResponseThreading(responseThreading); const runtime = new CuaRuntimeController({ browser, client, @@ -479,11 +626,12 @@ export class CuaAgentHarness< models ?? cuaModels(), resolveProviderRetryPolicy(retry), ); + const contextModels = withContextManagement(retryingModels, imageReplayLimit, useResponseThreading); super({ ...harnessOptions, model: runtime.model, - models: retryingModels, + models: contextModels, tools: resolvedTools, systemPrompt: systemPrompt ?? (() => runtime.systemPrompt), activeToolNames: activeToolNames ?? resolvedTools.map((tool) => tool.name), diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index a27747f..197b490 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -16,5 +16,10 @@ export type { PlaywrightDetails, } from "./tools"; export { CuaAgent, CuaAgentHarness } from "./agent"; -export type { CuaAgentHarnessOptions, CuaAgentOptions, CuaAgentState } from "./agent"; +export type { + CuaAgentHarnessOptions, + CuaAgentOptions, + CuaAgentState, + ToolResultImageReplayLimit, +} from "./agent"; export type { CuaRetryOptions } from "./provider-retry"; diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index b699f7c..cc02565 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it, vi } from "vitest"; import { createAssistantMessageEventStream, type AssistantMessage } from "@earendil-works/pi-ai"; -import { resolveCuaRuntimeSpec } from "@onkernel/cua-ai"; +import { + type Context, + createCuaModels, + type CuaSimpleStreamOptions, + resolveCuaRuntimeSpec, +} from "@onkernel/cua-ai"; import type Kernel from "@onkernel/sdk"; import { Agent, @@ -9,6 +14,7 @@ import { CuaAgentHarness, InMemorySessionRepo, NodeExecutionEnv, + type AgentMessage, type AgentTool, type KernelBrowser, type StreamFn, @@ -454,9 +460,644 @@ describe("CuaAgent", () => { expect(fedBack, "second provider request should carry the tool result").toBeDefined(); expect(fedBack!.content.some((block) => block.type === "image" && block.mimeType === "image/png")).toBe(true); }); + + it("applies screenshot projection after a caller context transform", async () => { + const history: AgentMessage[] = []; + for (let index = 1; index <= 5; index += 1) { + const assistant = createAssistantMessage({ api: "anthropic-messages", provider: "anthropic", id: "test" }); + assistant.content = [{ type: "toolCall", id: `call-${index}`, name: "custom_image", arguments: {} }]; + assistant.stopReason = "toolUse"; + const images = index < 5 ? [index] : [5, 6, 7, 8, 9]; + history.push(assistant, { + role: "toolResult", + toolCallId: `call-${index}`, + toolName: "custom_image", + content: [ + { type: "text", text: `result-${index}` }, + ...images.map((image) => ({ + type: "image" as const, + data: Buffer.from(`image-${image}`).toString("base64"), + mimeType: "image/png", + })), + ], + details: { index }, + isError: false, + timestamp: index, + }); + } + + let providerContext: Parameters[1] | undefined; + const streamFn: StreamFn = (model, context) => { + providerContext = context; + const stream = createAssistantMessageEventStream(); + const message = createAssistantMessage(model); + message.content = [{ type: "text", text: "done" }]; + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: "stop", message }); + stream.end(message); + return stream; + }; + const agent = new CuaAgent({ + browser, + client, + streamFn, + transformContext: async (messages) => + messages.map((message) => + message.role === "user" && Array.isArray(message.content) + ? { + ...message, + content: message.content.map((block) => + block.type === "text" ? { ...block, text: `caller:${block.text}` } : block, + ), + } + : message, + ), + initialState: { model: "anthropic:claude-haiku-4-5", messages: history }, + }); + + await agent.prompt("next"); + + const projected = providerContext!.messages; + const projectedImages = projected.flatMap((message) => message.content).filter((block) => block.type === "image"); + expect(projectedImages.map((block) => Buffer.from(block.data, "base64").toString())).toEqual([ + "image-6", + "image-7", + "image-8", + "image-9", + ]); + const finalResult = projected.find( + (message) => message.role === "toolResult" && message.toolCallId === "call-5", + ); + expect(finalResult?.content).toContainEqual({ type: "text", text: "result-5" }); + expect( + finalResult?.content.filter( + (block) => block.type === "text" && block.text === "[stale tool-result images omitted]", + ), + ).toHaveLength(1); + expect( + projected.flatMap((message) => message.content).some((block) => block.type === "text" && block.text === "caller:next"), + ).toBe(true); + expect( + projected.filter((message) => message.role === "toolResult").map(({ toolCallId, toolName, details, isError, timestamp }) => ({ + toolCallId, + toolName, + details, + isError, + timestamp, + })), + ).toEqual( + Array.from({ length: 5 }, (_, index) => ({ + toolCallId: `call-${index + 1}`, + toolName: "custom_image", + details: { index: index + 1 }, + isError: false, + timestamp: index + 1, + })), + ); + expect(agent.state.messages.flatMap((message) => message.content).filter((block) => block.type === "image")).toHaveLength(9); + }); + + it.each([ + { limit: 2 as const, expected: ["image-4", "image-5"] }, + { limit: 0 as const, expected: [] }, + { limit: false as const, expected: ["image-1", "image-2", "image-3", "image-4", "image-5"] }, + ])("supports toolResultImageReplayLimit=$limit", async ({ limit, expected }) => { + const history: AgentMessage[] = Array.from({ length: 5 }, (_, index) => ({ + role: "toolResult" as const, + toolCallId: `call-${index + 1}`, + toolName: "custom_image", + content: [{ + type: "image" as const, + data: Buffer.from(`image-${index + 1}`).toString("base64"), + mimeType: "image/png", + }], + details: {}, + isError: false, + timestamp: index + 1, + })); + let providerContext: Parameters[1] | undefined; + const streamFn: StreamFn = (model, context) => { + providerContext = context; + const stream = createAssistantMessageEventStream(); + const message = createAssistantMessage(model); + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: "stop", message }); + stream.end(message); + return stream; + }; + const agent = new CuaAgent({ + browser, + client, + streamFn, + toolResultImageReplayLimit: limit, + initialState: { model: "anthropic:claude-haiku-4-5", messages: history }, + }); + + await agent.prompt("next"); + + expect( + providerContext!.messages + .flatMap((message) => message.content) + .filter((block) => block.type === "image") + .map((block) => Buffer.from(block.data, "base64").toString()), + ).toEqual(expected); + expect(agent.state.messages.flatMap((message) => message.content).filter((block) => block.type === "image")).toHaveLength(5); + }); + + it.each([true, false])("passes responseThreading=$enabled to provider streams", async (enabled) => { + let streamOptions: CuaSimpleStreamOptions | undefined; + const streamFn: StreamFn = (model, _context, options) => { + streamOptions = options as CuaSimpleStreamOptions; + const stream = createAssistantMessageEventStream(); + const message = createAssistantMessage(model); + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: "stop", message }); + stream.end(message); + return stream; + }; + const agent = new CuaAgent({ + browser, + client, + streamFn, + responseThreading: enabled, + initialState: { model: "openai:gpt-5.5" }, + }); + + await agent.prompt("next"); + + expect(streamOptions?.disableResponseThreading).toBe(!enabled); + }); + + it("rejects invalid context-management options", async () => { + expect(() => new CuaAgent({ + browser, + client, + responseThreading: "false" as never, + initialState: { model: "openai:gpt-5.5" }, + })).toThrow(new TypeError("responseThreading must be a boolean")); + + const services = await createHarnessServices(); + expect(() => new CuaAgentHarness({ + ...services, + browser, + client, + model: "openai:gpt-5.5", + responseThreading: "false" as never, + })).toThrow(new TypeError("responseThreading must be a boolean")); + + for (const limit of [-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY]) { + expect(() => new CuaAgent({ + browser, + client, + toolResultImageReplayLimit: limit, + initialState: { model: "openai:gpt-5.5" }, + })).toThrow(new TypeError("toolResultImageReplayLimit must be a finite non-negative integer or false")); + + const services = await createHarnessServices(); + expect(() => new CuaAgentHarness({ + ...services, + browser, + client, + model: "openai:gpt-5.5", + toolResultImageReplayLimit: limit, + })).toThrow(new TypeError("toolResultImageReplayLimit must be a finite non-negative integer or false")); + } + }); }); describe("CuaAgentHarness", () => { + it.each([ + { limit: 2 as const, expected: ["image-4", "image-5"] }, + { limit: 0 as const, expected: [] }, + { limit: false as const, expected: ["image-1", "image-2", "image-3", "image-4", "image-5"] }, + ])("supports toolResultImageReplayLimit=$limit", async ({ limit, expected }) => { + const services = await createHarnessServices(); + for (let index = 1; index <= 5; index += 1) { + await services.session.appendMessage({ + role: "toolResult", + toolCallId: `call-${index}`, + toolName: "custom_image", + content: [{ type: "image", data: Buffer.from(`image-${index}`).toString("base64"), mimeType: "image/png" }], + details: {}, + isError: false, + timestamp: index, + }); + } + let providerContext: Parameters[1] | undefined; + const streamFn: StreamFn = (model, context) => { + providerContext = context; + const stream = createAssistantMessageEventStream(); + const message = createAssistantMessage(model); + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: "stop", message }); + stream.end(message); + return stream; + }; + const models = createCuaModels(); + models.setProvider({ + id: "anthropic", + name: "stateless test provider", + auth: { apiKey: { name: "test key", resolve: async () => ({ auth: { apiKey: "test-key" } }) } }, + getModels: () => [], + stream: streamFn, + streamSimple: streamFn, + }); + const harness = new CuaAgentHarness({ + ...services, + browser, + client, + model: "anthropic:claude-haiku-4-5", + models, + toolResultImageReplayLimit: limit, + }); + + await harness.prompt("next"); + + expect( + providerContext!.messages + .flatMap((message) => message.content) + .filter((block) => block.type === "image") + .map((block) => Buffer.from(block.data, "base64").toString()), + ).toEqual(expected); + const stored = await services.session.buildContext(); + expect(stored.messages.flatMap((message) => message.content).filter((block) => block.type === "image")).toHaveLength(5); + }); + + it.each([true, false])("applies context management through all harness model methods", async (enabled) => { + const services = await createHarnessServices(); + let streamOptions: CuaSimpleStreamOptions | undefined; + let providerImageCount = -1; + const streamFn: StreamFn = (model, context, options) => { + streamOptions = options as CuaSimpleStreamOptions; + providerImageCount = context.messages.flatMap((message) => message.content).filter((block) => block.type === "image").length; + const stream = createAssistantMessageEventStream(); + const message = createAssistantMessage(model); + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: "stop", message }); + stream.end(message); + return stream; + }; + const models = createCuaModels(); + models.setProvider({ + id: "openai", + name: "threading test provider", + auth: { apiKey: { name: "test key", resolve: async () => ({ auth: { apiKey: "test-key" } }) } }, + getModels: () => [], + stream: streamFn, + streamSimple: streamFn, + }); + const harness = new CuaAgentHarness({ + ...services, + browser, + client, + model: "openai:gpt-5.5", + models, + responseThreading: enabled, + retry: { enabled: true, maxRetries: 1, baseDelayMs: 0 }, + }); + + const expected = enabled ? undefined : true; + const context: Context = { + messages: Array.from({ length: 5 }, (_, index) => ({ + role: "toolResult" as const, + toolCallId: `call-${index}`, + toolName: "screenshot", + content: [{ type: "image" as const, data: `${index}`, mimeType: "image/png" }], + isError: false, + timestamp: index, + })), + }; + const model = harness.getModel(); + + await harness.models.stream(model, context).result(); + expect(streamOptions?.disableResponseThreading).toBe(expected); + expect(providerImageCount).toBe(4); + streamOptions = undefined; + + await harness.models.complete(model, context); + expect(streamOptions?.disableResponseThreading).toBe(expected); + expect(providerImageCount).toBe(4); + streamOptions = undefined; + + await harness.models.streamSimple(model, context).result(); + expect(streamOptions?.disableResponseThreading).toBe(expected); + expect(providerImageCount).toBe(4); + streamOptions = undefined; + + await harness.models.completeSimple(model, context); + expect(streamOptions?.disableResponseThreading).toBe(expected); + expect(providerImageCount).toBe(4); + }); + + it("applies image projection after user context hooks", async () => { + const run = async (replace: boolean) => { + const services = await createHarnessServices(); + for (let index = 1; index <= 5; index += 1) { + await services.session.appendMessage({ + role: "toolResult", + toolCallId: `call-${index}`, + toolName: "custom_image", + content: [{ type: "image", data: `${index}`, mimeType: "image/png" }], + details: {}, + isError: false, + timestamp: index, + }); + } + let providerImageCount = -1; + let providerOmissionCount = -1; + let observedImageCount = -1; + const streamFn: StreamFn = (model, context) => { + providerImageCount = context.messages.flatMap((message) => message.content).filter((block) => block.type === "image").length; + providerOmissionCount = context.messages + .flatMap((message) => message.content) + .filter((block) => block.type === "text" && block.text === "[stale tool-result images omitted]").length; + const stream = createAssistantMessageEventStream(); + const message = createAssistantMessage(model); + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: "stop", message }); + stream.end(message); + return stream; + }; + const models = createCuaModels(); + models.setProvider({ + id: "anthropic", + name: "stateless test provider", + auth: { apiKey: { name: "test key", resolve: async () => ({ auth: { apiKey: "test-key" } }) } }, + getModels: () => [], + stream: streamFn, + streamSimple: streamFn, + }); + const harness = new CuaAgentHarness({ ...services, browser, client, model: "anthropic:claude-haiku-4-5", models }); + harness.on("context", ({ messages }) => { + observedImageCount = messages.flatMap((message) => message.content).filter((block) => block.type === "image").length; + return replace ? { messages } : undefined; + }); + + await harness.prompt("next"); + return { observedImageCount, providerImageCount, providerOmissionCount }; + }; + + await expect(run(false)).resolves.toEqual({ observedImageCount: 5, providerImageCount: 4, providerOmissionCount: 1 }); + await expect(run(true)).resolves.toEqual({ observedImageCount: 5, providerImageCount: 4, providerOmissionCount: 1 }); + }); + + it("bounds screenshots retained across long stateless CUA trajectories", async () => { + const contexts: Parameters[1][] = []; + let providerCalls = 0; + const models = createCuaModels(); + const streamFn: StreamFn = (model, context) => { + contexts.push({ ...context, messages: structuredClone(context.messages) }); + const stream = createAssistantMessageEventStream(); + const message = createAssistantMessage(model); + if (providerCalls++ % 2 === 0) { + message.content = [{ type: "toolCall", id: `screenshot-${providerCalls}`, name: "screenshot", arguments: {} }]; + message.stopReason = "toolUse"; + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: "toolUse", message }); + } else { + message.content = [{ type: "text", text: "continue" }]; + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: "stop", message }); + } + stream.end(message); + return stream; + }; + models.setProvider({ + id: "anthropic", + name: "stateless test provider", + auth: { apiKey: { name: "test key", resolve: async () => ({ auth: { apiKey: "test-key" } }) } }, + getModels: () => [], + stream: streamFn, + streamSimple: streamFn, + }); + + let screenshots = 0; + const screenshotClient = { + browsers: { + computer: { captureScreenshot: async () => new Response(Buffer.from(`screenshot-${++screenshots}`)) }, + }, + } as unknown as Kernel; + const services = await createHarnessServices(); + const harness = new CuaAgentHarness({ + ...services, + browser, + client: screenshotClient, + model: "anthropic:claude-haiku-4-5", + models, + }); + + for (let turn = 0; turn < 9; turn += 1) await harness.prompt(`turn ${turn}`); + + const providerImages = contexts.map((context) => + context.messages + .flatMap((message) => message.content) + .filter((block) => block.type === "image") + .map((block) => Buffer.from(block.data, "base64").toString()), + ); + expect(providerImages).toEqual([ + [], + ["screenshot-1"], + ["screenshot-1"], + ["screenshot-1", "screenshot-2"], + ["screenshot-1", "screenshot-2"], + ["screenshot-1", "screenshot-2", "screenshot-3"], + ["screenshot-1", "screenshot-2", "screenshot-3"], + ["screenshot-1", "screenshot-2", "screenshot-3", "screenshot-4"], + ["screenshot-1", "screenshot-2", "screenshot-3", "screenshot-4"], + ["screenshot-2", "screenshot-3", "screenshot-4", "screenshot-5"], + ["screenshot-2", "screenshot-3", "screenshot-4", "screenshot-5"], + ["screenshot-3", "screenshot-4", "screenshot-5", "screenshot-6"], + ["screenshot-3", "screenshot-4", "screenshot-5", "screenshot-6"], + ["screenshot-4", "screenshot-5", "screenshot-6", "screenshot-7"], + ["screenshot-4", "screenshot-5", "screenshot-6", "screenshot-7"], + ["screenshot-5", "screenshot-6", "screenshot-7", "screenshot-8"], + ["screenshot-5", "screenshot-6", "screenshot-7", "screenshot-8"], + ["screenshot-6", "screenshot-7", "screenshot-8", "screenshot-9"], + ]); + expect(Math.max(...providerImages.map((images) => images.length))).toBeLessThanOrEqual(4); + + const omissionCounts = contexts.map((context) => + context.messages + .flatMap((message) => message.content) + .filter((block) => block.type === "text" && block.text === "[stale tool-result images omitted]").length, + ); + expect(omissionCounts[9]).toBe(1); + expect(omissionCounts[17]).toBe(5); + + const stored = await services.session.buildContext(); + expect( + stored.messages.flatMap((message) => message.content).filter((block) => block.type === "image"), + ).toHaveLength(9); + }); + + it("projects stale images returned by extra tools without removing their result shell", async () => { + const contexts: Parameters[1][] = []; + const tools = ["image_extra", "screenshot", "screenshot", "screenshot", "screenshot"]; + let providerCalls = 0; + const models = createCuaModels(); + const streamFn: StreamFn = (model, context) => { + contexts.push({ ...context, messages: structuredClone(context.messages) }); + const stream = createAssistantMessageEventStream(); + const message = createAssistantMessage(model); + if (providerCalls % 2 === 0) { + const turn = providerCalls / 2; + message.content = [{ type: "toolCall", id: `call-${turn}`, name: tools[turn]!, arguments: {} }]; + message.stopReason = "toolUse"; + } else { + message.content = [{ type: "text", text: "continue" }]; + } + providerCalls += 1; + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: message.stopReason, message }); + stream.end(message); + return stream; + }; + models.setProvider({ + id: "anthropic", + name: "stateless test provider", + auth: { apiKey: { name: "test key", resolve: async () => ({ auth: { apiKey: "test-key" } }) } }, + getModels: () => [], + stream: streamFn, + streamSimple: streamFn, + }); + let screenshots = 0; + const services = await createHarnessServices(); + const harness = new CuaAgentHarness({ + ...services, + browser, + client: { + browsers: { computer: { captureScreenshot: async () => new Response(Buffer.from(`builtin-${++screenshots}`)) } }, + } as unknown as Kernel, + model: "anthropic:claude-haiku-4-5", + models, + extraTools: [{ + ...createCustomTool("image_extra"), + async execute() { + return { + content: [ + { type: "text" as const, text: "extra text" }, + { type: "image" as const, data: Buffer.from("extra-image").toString("base64"), mimeType: "image/png" }, + ], + details: { source: "extra" }, + }; + }, + }], + }); + + for (let turn = 0; turn < tools.length; turn += 1) await harness.prompt(`turn ${turn}`); + + const finalContext = contexts.at(-1)!; + const extraResult = finalContext.messages.find( + (message) => message.role === "toolResult" && message.toolCallId === "call-0", + ); + expect(extraResult).toMatchObject({ + role: "toolResult", + toolCallId: "call-0", + toolName: "image_extra", + details: { source: "extra" }, + isError: false, + }); + expect(extraResult?.content).toEqual([ + { type: "text", text: "extra text" }, + { type: "text", text: "[stale tool-result images omitted]" }, + ]); + expect(finalContext.messages.some((message) => + message.role === "assistant" && message.content.some((block) => block.type === "toolCall" && block.id === "call-0"), + )).toBe(true); + expect( + finalContext.messages.flatMap((message) => message.content).filter((block) => block.type === "image").map((block) => Buffer.from(block.data, "base64").toString()), + ).toEqual(["builtin-1", "builtin-2", "builtin-3", "builtin-4"]); + const stored = await services.session.buildContext(); + expect(stored.messages.flatMap((message) => message.content).filter((block) => block.type === "image")).toHaveLength(5); + }); + + it("caps a multi-screenshot computer_batch result by image ordinal", async () => { + const contexts: Parameters[1][] = []; + let providerCalls = 0; + const models = createCuaModels(); + const streamFn: StreamFn = (model, context) => { + contexts.push({ ...context, messages: structuredClone(context.messages) }); + const stream = createAssistantMessageEventStream(); + const message = createAssistantMessage(model); + if (providerCalls++ === 0) { + message.content = [{ + type: "toolCall", + id: "batch-1", + name: ANTHROPIC_BATCH_TOOL_NAME, + arguments: { actions: [ + { type: "screenshot" }, { type: "screenshot" }, { type: "screenshot" }, + { type: "screenshot" }, { type: "cursor_position" }, + { type: "screenshot" }, { type: "screenshot" }, + ] }, + }]; + message.stopReason = "toolUse"; + } else { + message.content = [{ type: "text", text: "done" }]; + } + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: message.stopReason, message }); + stream.end(message); + return stream; + }; + models.setProvider({ + id: "anthropic", + name: "stateless test provider", + auth: { apiKey: { name: "test key", resolve: async () => ({ auth: { apiKey: "test-key" } }) } }, + getModels: () => [], + stream: streamFn, + streamSimple: streamFn, + }); + let screenshots = 0; + const services = await createHarnessServices(); + const harness = new CuaAgentHarness({ + ...services, + browser, + client: { browsers: { computer: { + captureScreenshot: async () => new Response(Buffer.from(`batch-${++screenshots}`)), + getMousePosition: async () => ({ x: 17, y: 23 }), + } } } as unknown as Kernel, + model: "anthropic:claude-haiku-4-5", + models, + }); + + await harness.prompt("inspect several frames"); + + const followUp = contexts[1]!; + const result = followUp.messages.find( + (message) => message.role === "toolResult" && message.toolCallId === "batch-1", + ); + expect(result).toMatchObject({ + role: "toolResult", + toolCallId: "batch-1", + toolName: ANTHROPIC_BATCH_TOOL_NAME, + isError: false, + details: { + statusText: "Actions executed successfully.", + readResults: [ + { type: "screenshot", bytes: 7 }, { type: "screenshot", bytes: 7 }, + { type: "screenshot", bytes: 7 }, { type: "screenshot", bytes: 7 }, + { type: "cursor_position", x: 17, y: 23 }, + { type: "screenshot", bytes: 7 }, { type: "screenshot", bytes: 7 }, + ], + }, + }); + expect(result?.content).toEqual([ + { type: "text", text: "[stale tool-result images omitted]" }, + { type: "image", data: Buffer.from("batch-3").toString("base64"), mimeType: "image/png" }, + { type: "image", data: Buffer.from("batch-4").toString("base64"), mimeType: "image/png" }, + { type: "text", text: "cursor_position(): 17,23" }, + { type: "image", data: Buffer.from("batch-5").toString("base64"), mimeType: "image/png" }, + { type: "image", data: Buffer.from("batch-6").toString("base64"), mimeType: "image/png" }, + ]); + expect(result?.content.filter((block) => block.type === "text" && block.text === "[stale tool-result images omitted]")).toHaveLength(1); + expect(followUp.messages.some((message) => + message.role === "assistant" && message.content.some((block) => block.type === "toolCall" && block.id === "batch-1"), + )).toBe(true); + expect(followUp.messages.flatMap((message) => message.content).filter((block) => block.type === "image")).toHaveLength(4); + const stored = await services.session.buildContext(); + expect(stored.messages.flatMap((message) => message.content).filter((block) => block.type === "image")).toHaveLength(6); + }); + it("extends pi AgentHarness and resolves model refs", async () => { const harness = new CuaAgentHarness({ ...(await createHarnessServices()), diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index 3b66770..49e89d4 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -260,33 +260,28 @@ export interface CuaPayloadContext { export type CuaPayloadHook = (payload: unknown, model: Model, context?: CuaPayloadContext) => unknown | Promise; /** - * pi-ai `SimpleStreamOptions` plus the CUA extension consumed by the - * Yutori/Tzafon stream adapters. Pass `keepToolNames` for caller tools that - * must survive provider-native tool-set substitution. + * pi-ai `SimpleStreamOptions` plus CUA request controls consumed by provider + * adapters. Pass `keepToolNames` for caller tools that must survive native + * tool-set substitution; use `disableResponseThreading` for a full-context + * OpenAI or Tzafon request. */ -export interface CuaSimpleStreamOptions extends SimpleStreamOptions { +export interface CuaSimpleStreamOptions extends SimpleStreamOptions, ResponseThreadingOptions { keepToolNames?: readonly string[]; } -/** Environment variable that disables server-side `previous_response_id` threading when truthy. */ -export const CUA_DISABLE_RESPONSE_THREADING_ENV_VAR = "CUA_DISABLE_RESPONSE_THREADING"; - /** Per-call control over `previous_response_id` threading for Responses API providers. */ export interface ResponseThreadingOptions { - /** Force full-history replay for this request, overriding the environment default. */ + /** Force this request to send its full message context instead of chaining to a stored provider response. */ disableResponseThreading?: boolean; } /** * Whether a Responses API provider should thread requests with - * `previous_response_id` + delta input instead of replaying the full message - * history. Threading is on by default and disabled by an explicit option or a - * truthy {@link CUA_DISABLE_RESPONSE_THREADING_ENV_VAR}. + * `previous_response_id` + delta input instead of sending the full message + * context. Threading is on by default and disabled per request. */ export function responseThreadingEnabled(options?: ResponseThreadingOptions): boolean { - if (options?.disableResponseThreading) return false; - const flag = process.env[CUA_DISABLE_RESPONSE_THREADING_ENV_VAR]; - return !(flag && flag !== "0" && flag.toLowerCase() !== "false"); + return options?.disableResponseThreading !== true; } /** Result of {@link responseThreadingDelta}: the chaining id and the messages to send this turn. */ diff --git a/packages/ai/src/providers/openai/provider.ts b/packages/ai/src/providers/openai/provider.ts index bca616a..b9fbfc8 100644 --- a/packages/ai/src/providers/openai/provider.ts +++ b/packages/ai/src/providers/openai/provider.ts @@ -1,7 +1,6 @@ import { type Context, type OpenAIResponsesOptions as PiOpenAIResponsesOptions, - type SimpleStreamOptions, type StreamFunction, type StreamOptions, } from "@earendil-works/pi-ai"; @@ -9,7 +8,12 @@ import { stream as piStreamOpenAIResponses, streamSimple as piStreamSimpleOpenAIResponses, } from "@earendil-works/pi-ai/api/openai-responses"; -import { responseThreadingDelta, responseThreadingEnabled, type ResponseThreadingOptions } from "../common"; +import { + type CuaSimpleStreamOptions, + responseThreadingDelta, + responseThreadingEnabled, + type ResponseThreadingOptions, +} from "../common"; export const OPENAI_CUA_RESPONSES_API = "openai-cua-responses"; @@ -49,7 +53,7 @@ export const streamOpenAIResponses: StreamFunction = (model, context, options) => { +export const streamSimpleOpenAIResponses: StreamFunction = (model, context, options) => { const threaded = threadRequest(context, options); return piStreamSimpleOpenAIResponses(model as never, threaded.context, { ...options, onPayload: threaded.onPayload }); }; diff --git a/packages/ai/src/providers/tzafon/provider.ts b/packages/ai/src/providers/tzafon/provider.ts index 95f31de..1337af5 100644 --- a/packages/ai/src/providers/tzafon/provider.ts +++ b/packages/ai/src/providers/tzafon/provider.ts @@ -6,7 +6,6 @@ import { type ImageContent, type Message, type Model, - type SimpleStreamOptions, type StreamFunction, type StreamOptions, type TextContent, @@ -17,6 +16,7 @@ import Lightcone from "@tzafon/lightcone"; import { canonicalToolCallArguments, canonicalToolCallName, + type CuaSimpleStreamOptions, CUA_ACTION_TYPES, responseThreadingDelta, responseThreadingEnabled, @@ -81,7 +81,7 @@ export function buildTzafonRequestInput(model: Model, context: Context, opt return { ...body, input: convertMessages(deltaMessages), previous_response_id: previousResponseId, store: true }; } -export const streamSimpleTzafonResponses: StreamFunction = (model, context, options) => { +export const streamSimpleTzafonResponses: StreamFunction = (model, context, options) => { return streamTzafonResponses(model, context, options); }; diff --git a/packages/ai/test/openai-threading.test.ts b/packages/ai/test/openai-threading.test.ts index d4821d0..d41cbd0 100644 --- a/packages/ai/test/openai-threading.test.ts +++ b/packages/ai/test/openai-threading.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, it } from "vitest"; +import { describe, expect, it } from "vitest"; import type { Context, Message, Model } from "@earendil-works/pi-ai"; import { OPENAI_CUA_RESPONSES_API, threadRequest } from "../src/providers/openai/provider"; @@ -33,10 +33,6 @@ function multiTurnContext(): Context { } describe("openai threadRequest", () => { - afterEach(() => { - delete process.env.CUA_DISABLE_RESPONSE_THREADING; - }); - it("prunes to the delta and injects store + previous_response_id when threading (default)", async () => { const { context, onPayload } = threadRequest(multiTurnContext(), undefined); // Only the latest tool result (after the last assistant turn) is sent; the rest lives server-side. @@ -45,21 +41,13 @@ describe("openai threadRequest", () => { expect(await onPayload({ input: [] }, model)).toEqual({ input: [], store: true, previous_response_id: `resp_${TURNS - 1}` }); }); - it("replays full history with store but no previous_response_id when disabled by option", async () => { + it("sends full history with store but no previous_response_id when disabled by option", async () => { const ctx = multiTurnContext(); const { context, onPayload } = threadRequest(ctx, { disableResponseThreading: true }); expect(context).toBe(ctx); expect(await onPayload({}, model)).toEqual({ store: true }); }); - it("replays full history when CUA_DISABLE_RESPONSE_THREADING is set", async () => { - process.env.CUA_DISABLE_RESPONSE_THREADING = "1"; - const ctx = multiTurnContext(); - const { context, onPayload } = threadRequest(ctx, undefined); - expect(context).toBe(ctx); - expect(((await onPayload({}, model)) as Record).previous_response_id).toBeUndefined(); - }); - it("falls back to full history when the latest assistant turn lacks a responseId", async () => { const ctx = multiTurnContext(); ctx.messages.push({ diff --git a/packages/ai/test/tzafon-threading.test.ts b/packages/ai/test/tzafon-threading.test.ts index 959b55c..8287402 100644 --- a/packages/ai/test/tzafon-threading.test.ts +++ b/packages/ai/test/tzafon-threading.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, it } from "vitest"; +import { describe, expect, it } from "vitest"; import type { Context, Message, Model } from "@earendil-works/pi-ai"; import { tzafon } from "../src/index"; @@ -48,12 +48,8 @@ function screenshotImageUrls(input: Array>): string[] { } describe("buildTzafonRequestInput response threading", () => { - afterEach(() => { - delete process.env.CUA_DISABLE_RESPONSE_THREADING; - }); - // Threading ON is the fix: chain via previous_response_id and send only the - // latest screenshot. OFF replays every screenshot — the per-turn growth that + // latest screenshot. OFF sends every screenshot — the per-turn growth that // overflows Tzafon's real 64K window after a few turns. it("threads the latest delta when enabled (default)", () => { const body = tzafon.buildTzafonRequestInput(model, multiTurnContext()); @@ -65,7 +61,7 @@ describe("buildTzafonRequestInput response threading", () => { expect(body.store).toBe(true); }); - it("replays the full screenshot history when threading is disabled by option (locks the failure mode)", () => { + it("sends the full screenshot history when threading is disabled by option (locks the failure mode)", () => { const body = tzafon.buildTzafonRequestInput(model, multiTurnContext(), { disableResponseThreading: true }); const screenshots = screenshotImageUrls(body.input); @@ -75,14 +71,6 @@ describe("buildTzafonRequestInput response threading", () => { expect(body.store).toBeUndefined(); }); - it("replays the full screenshot history when CUA_DISABLE_RESPONSE_THREADING is set", () => { - process.env.CUA_DISABLE_RESPONSE_THREADING = "1"; - const body = tzafon.buildTzafonRequestInput(model, multiTurnContext()); - - expect(screenshotImageUrls(body.input)).toHaveLength(TURNS); - expect(body.previous_response_id).toBeUndefined(); - }); - it("falls back to full history when no prior turn carries a responseId", () => { const context = multiTurnContext(); for (const message of context.messages) { diff --git a/packages/cli/src/harness.ts b/packages/cli/src/harness.ts index 3b1f83f..2ffdf30 100644 --- a/packages/cli/src/harness.ts +++ b/packages/cli/src/harness.ts @@ -43,6 +43,10 @@ export interface BuildCuaHarnessOptions { extraTools?: CuaAgentHarnessOptions["extraTools"]; /** Override the pi `Models` collection requests stream through (mainly for tests). */ models?: Models; + /** Maximum tool-result images included from message history per provider request. */ + toolResultImageReplayLimit?: CuaAgentHarnessOptions["toolResultImageReplayLimit"]; + /** Chain OpenAI and Tzafon requests through provider-stored response state. Defaults to true. */ + responseThreading?: CuaAgentHarnessOptions["responseThreading"]; /** Optional CUA-level retries around each provider request. Disabled by default. */ retry?: CuaAgentHarnessOptions["retry"]; /** Override the catalog `baseUrl` on the resolved model (e.g. from `_BASE_URL`). */ @@ -86,6 +90,8 @@ export function buildCuaHarness(opts: BuildCuaHarnessOptions): CuaAgentHarness { return composeSystemPrompt(runtime.defaultSystemPrompt, resources.skills ?? [], contextFiles); }, models: opts.models, + toolResultImageReplayLimit: opts.toolResultImageReplayLimit, + responseThreading: opts.responseThreading, retry: opts.retry, }); return harness; diff --git a/packages/cli/test/fixtures/scripted-provider.ts b/packages/cli/test/fixtures/scripted-provider.ts index 1fa8da4..601b2d7 100644 --- a/packages/cli/test/fixtures/scripted-provider.ts +++ b/packages/cli/test/fixtures/scripted-provider.ts @@ -4,6 +4,7 @@ import { type Context, createAssistantMessageEventStream, createCuaModels, + type CuaSimpleStreamOptions, type Model, type MutableModels, } from "@onkernel/cua-ai"; @@ -33,6 +34,8 @@ export interface ScriptedProviderHandle { callCount(): number; /** Latest context the provider was called with (assistant-side mock). */ lastContext(): Context | undefined; + /** Latest stream options the provider was called with. */ + lastStreamOptions(): CuaSimpleStreamOptions | undefined; } /** @@ -45,12 +48,14 @@ export function createScriptedCuaModels(providerId: string, turns: ScriptedTurn[ const state = { index: 0, lastContext: undefined as Context | undefined, + lastStreamOptions: undefined as CuaSimpleStreamOptions | undefined, }; - const dispatch = (model: Model, context: Context, signal?: AbortSignal) => { + const dispatch = (model: Model, context: Context, options?: CuaSimpleStreamOptions) => { state.lastContext = context; + state.lastStreamOptions = options; const turn = turns[state.index]; state.index += 1; - return buildStream(model, turn, signal); + return buildStream(model, turn, options?.signal); }; const models = createCuaModels(); models.setProvider({ @@ -58,13 +63,15 @@ export function createScriptedCuaModels(providerId: string, turns: ScriptedTurn[ name: `Scripted ${providerId}`, auth: { apiKey: { name: "scripted test key", resolve: async () => ({ auth: { apiKey: "test-key" } }) } }, getModels: () => [], - stream: (model, context, options) => dispatch(model, context, options?.signal), - streamSimple: (model, context, options) => dispatch(model, context, options?.signal), + stream: (model, context, options) => dispatch(model, context, options), + streamSimple: (model, context, options) => dispatch(model, context, options), }); return { models, reset(): void { state.index = 0; + state.lastContext = undefined; + state.lastStreamOptions = undefined; }, callCount(): number { return state.index; @@ -72,6 +79,9 @@ export function createScriptedCuaModels(providerId: string, turns: ScriptedTurn[ lastContext(): Context | undefined { return state.lastContext; }, + lastStreamOptions(): CuaSimpleStreamOptions | undefined { + return state.lastStreamOptions; + }, }; } diff --git a/packages/cli/test/harness-assembly.test.ts b/packages/cli/test/harness-assembly.test.ts index a1da9ce..386de87 100644 --- a/packages/cli/test/harness-assembly.test.ts +++ b/packages/cli/test/harness-assembly.test.ts @@ -94,6 +94,29 @@ describe("buildCuaHarness", () => { expect(capturedSystemPrompt).toContain(join(cwd, "AGENTS.md")); }); + it("forwards response-threading configuration to the provider", async () => { + const provider = createScriptedCuaModels("openai", [ + { steps: [{ type: "text", text: "ok" }] }, + ]); + const cwd = mkdtempSync(join(tmpdir(), "cua-cli-harness-")); + const kernel = createFakeKernelEnvironment(); + const session = await new InMemorySessionRepo().create(); + const harness = buildCuaHarness({ + cwd, + client: kernel.client, + browser: kernel.browser, + session, + model: "openai:gpt-5.5", + extraTools: [], + models: provider.models, + responseThreading: false, + }); + + await harness.prompt("hi"); + + expect(provider.lastStreamOptions()?.disableResponseThreading).toBe(true); + }); + it("delivers the first prompt with an image attached via harness.prompt({ images })", async () => { const provider = createScriptedCuaModels("openai", [ { steps: [{ type: "text", text: "done" }] },