From fb600e6115f4e6d28e4deb67e1782b9bd376d1af Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:35:53 +0000 Subject: [PATCH 01/15] Add Meta Muse Spark provider --- .agents/skills/update-models/SKILL.md | 17 +++- .../skills/update-models/reference/README.md | 3 +- .../reference/audit-official-examples.ts | 11 ++- .../reference/discover-models.ts | 79 +++++++++++++++++-- .../reference/provider-doc-drift.ts | 33 +++++++- .../update-models/reference/report-schema.md | 1 + .github/workflows/ci.yml | 3 + README.md | 8 +- packages/agent/README.md | 7 +- packages/agent/src/agent.ts | 4 +- packages/agent/test/e2e.live.test.ts | 15 ++++ packages/ai/README.md | 13 ++- packages/ai/docs/supported-models.md | 15 ++++ packages/ai/examples/quickstart.ts | 1 + packages/ai/src/api-keys.ts | 1 + packages/ai/src/index.ts | 1 + packages/ai/src/models.ts | 38 ++++++--- packages/ai/src/providers.ts | 16 +++- packages/ai/src/providers/meta/index.ts | 47 +++++++++++ packages/ai/src/providers/meta/provider.ts | 70 ++++++++++++++++ packages/ai/src/runtime-spec.ts | 2 + packages/ai/test/api-keys.test.ts | 9 +++ .../ai/test/computer-tool.integration.test.ts | 73 ++++++++++++++++- packages/ai/test/meta-threading.test.ts | 78 ++++++++++++++++++ packages/ai/test/models.test.ts | 17 +++- packages/ai/test/provider-module.test.ts | 9 ++- packages/ai/test/providers.test.ts | 5 +- packages/ai/test/runtime-spec.test.ts | 11 ++- packages/cli/README.md | 8 +- packages/cli/src/cli-harness.ts | 2 +- packages/cli/src/cli.ts | 4 + packages/cli/src/harness.ts | 2 +- packages/cli/test/cli-executor.test.ts | 2 + packages/cli/test/harness-models.test.ts | 6 ++ 34 files changed, 561 insertions(+), 50 deletions(-) create mode 100644 packages/ai/src/providers/meta/index.ts create mode 100644 packages/ai/src/providers/meta/provider.ts create mode 100644 packages/ai/test/meta-threading.test.ts diff --git a/.agents/skills/update-models/SKILL.md b/.agents/skills/update-models/SKILL.md index 9ffb5628..6e7d6f5a 100644 --- a/.agents/skills/update-models/SKILL.md +++ b/.agents/skills/update-models/SKILL.md @@ -1,6 +1,6 @@ --- name: update-models -description: Discover latest OpenAI, Anthropic, Google/Gemini, Tzafon, and Yutori models and verify computer-use support. Use when updating CUA model defaults, checking new model releases, auditing provider-native computer tool actions, or comparing provider metadata, official examples, and smoke-test results. +description: Discover latest OpenAI, Anthropic, Google/Gemini, Meta, Tzafon, and Yutori models and verify computer-use support. Use when updating CUA model defaults, checking new model releases, auditing provider-native computer tool actions, or comparing provider metadata, official examples, and smoke-test results. --- # Update Models @@ -9,14 +9,14 @@ Use this workflow to keep CUA current with provider model releases and computer- ## Quick Start -1. Verify credentials are available: `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY` or `GEMINI_API_KEY`, `TZAFON_API_KEY`, and `YUTORI_API_KEY`. +1. Verify credentials are available: `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY` or `GEMINI_API_KEY`, `MODEL_API_KEY` or `META_MODEL_API_KEY`, `TZAFON_API_KEY`, and `YUTORI_API_KEY`. 2. If credentials live in `~/AGENTS.md`, load them into the current shell without printing them: ```bash eval "$(python3 - <<'PY' import pathlib, re, shlex text = pathlib.Path('~/AGENTS.md').expanduser().read_text() -for key in ['OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'GOOGLE_API_KEY', 'TZAFON_API_KEY', 'YUTORI_API_KEY']: +for key in ['OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'GOOGLE_API_KEY', 'MODEL_API_KEY', 'TZAFON_API_KEY', 'YUTORI_API_KEY']: m = re.search(r'export\s+' + re.escape(key) + r'=(?:"([^"]+)"|([^\s\n]+))', text) if m: print(f'export {key}={shlex.quote(m.group(1) or m.group(2))}') @@ -68,6 +68,15 @@ When live discovery finds a new model with passing smoke tests, update `packages ## Provider Checks +Meta: + +- Discover with the OpenAI SDK against `https://api.meta.ai/v1` using `MODEL_API_KEY` (or `META_MODEL_API_KEY`). +- Smoke-test the Responses API with screenshot input and explicit function tools matching CUA's canonical actions. +- Pass condition: response output contains a `function_call` for one of the supplied browser actions. +- Use `store: true` plus `previous_response_id` for CUA tool loops. Meta rejects `include: ["reasoning.encrypted_content"]` on requests that set `previous_response_id`. +- Set `parallel_tool_calls: false` because browser actions mutate shared state. +- Treat Meta computer use as custom-function-tool support, not a provider-native `{ type: "computer" }` tool. + OpenAI: - Discover with `OpenAI().models.list()` and optionally `models.retrieve(modelId)`. @@ -116,6 +125,7 @@ Yutori: Run action probes when updating adapters or when docs/examples show drift: ```bash +npx tsx .agents/skills/update-models/reference/discover-models.ts --provider meta --models muse-spark-1.1 npx tsx .agents/skills/update-models/reference/native-action-probe.ts --provider openai --model gpt-5.5 npx tsx .agents/skills/update-models/reference/native-action-probe.ts --provider anthropic --model claude-opus-4-7 npx tsx .agents/skills/update-models/reference/native-action-probe.ts --provider gemini --model gemini-3-flash-preview @@ -160,6 +170,7 @@ All CUA model and adapter support lives in `packages/ai` (`@onkernel/cua-ai`). W - Update the snapshot in `packages/ai/docs/supported-models.md` to match. - New provider-native action, response field, or tool version: + - Meta: update `packages/ai/src/providers/meta/index.ts` and `provider.ts`, including Responses threading and reasoning compatibility. - OpenAI: update `packages/ai/src/providers/openai/index.ts` and its action vocabulary, plus the shared canonical types in `packages/ai/src/providers/common.ts` if the action set changes. - Anthropic: update the `ANTHROPIC_CUA_ACTION_TYPES` set in `packages/ai/src/providers/anthropic/actions.ts` and `index.ts`. The computer tool version and `computer-use-*` beta header are selected by `pi-ai` per model, so a new dated tool version usually means bumping `@earendil-works/pi-ai`, not editing this package. - Gemini: update `packages/ai/src/providers/gemini/index.ts`, including coordinate handling if needed. diff --git a/.agents/skills/update-models/reference/README.md b/.agents/skills/update-models/reference/README.md index 4569bda7..af11c833 100644 --- a/.agents/skills/update-models/reference/README.md +++ b/.agents/skills/update-models/reference/README.md @@ -11,10 +11,11 @@ These scripts support the `update-models` skill. Run them from the repository ro - `OPENAI_API_KEY` - `ANTHROPIC_API_KEY` - `GOOGLE_API_KEY` or `GEMINI_API_KEY` + - `MODEL_API_KEY` or `META_MODEL_API_KEY` - `TZAFON_API_KEY` - `YUTORI_API_KEY` -The scripts never print API keys. Smoke tests are non-destructive: they ask each model to emit a provider-native computer-use tool call, then inspect the response without executing the action. +The scripts never print API keys. Smoke tests are non-destructive: they ask each model to emit a computer-use tool call, then inspect the response without executing the action. Meta uses supplied function tools; other providers may use provider-native computer tools. ## Common Commands diff --git a/.agents/skills/update-models/reference/audit-official-examples.ts b/.agents/skills/update-models/reference/audit-official-examples.ts index 3940e017..247d295a 100644 --- a/.agents/skills/update-models/reference/audit-official-examples.ts +++ b/.agents/skills/update-models/reference/audit-official-examples.ts @@ -5,7 +5,7 @@ import { basename, join, resolve } from "node:path"; import { spawnSync } from "node:child_process"; import process from "node:process"; -type Provider = "openai" | "anthropic" | "gemini" | "yutori"; +type Provider = "openai" | "anthropic" | "gemini" | "meta" | "yutori"; interface ExampleRepo { provider: Provider; @@ -51,6 +51,14 @@ const EXAMPLES: ExampleRepo[] = [ confidence: "provider-owned", patterns: ["computer_use", "ComputerUse", "function_call", "functionCall", "FunctionResponse", "safety_decision"], }, + { + provider: "meta", + name: "meta-model-cookbook", + repo: "https://github.com/meta-models/meta-model-cookbook.git", + confidence: "provider-owned", + pathHint: "03_use_cases/13_macos_cua", + patterns: ["muse-spark-1.1", "function_call", "function_call_output", "previous_response_id", "reasoning.encrypted_content", "parallel_tool_calls"], + }, { provider: "yutori", name: "kernel-cli-yutori-template", @@ -65,6 +73,7 @@ const ACTION_REGEXES: Record = { openai: [/\b(click|double_click|scroll|type|wait|keypress|drag|move|screenshot)\b/g], anthropic: [/\b(screenshot|left_click|right_click|middle_click|double_click|triple_click|left_click_drag|mouse_move|key|type|scroll|hold_key|wait|left_mouse_down|left_mouse_up|cursor_position|zoom)\b/g], gemini: [/\b(open_web_browser|open_web|wait_5_seconds|go_back|go_forward|search|navigate|click_at|hover_at|type_text_at|key_combination|scroll_document|scroll_at|drag_and_drop)\b/g], + meta: [/\b(screenshot|left_click|right_click|middle_click|double_click|triple_click|left_click_drag|mouse_move|key|type|scroll|hold_key|wait|left_mouse_down|left_mouse_up)\b/g], yutori: [/\b(left_click|double_click|triple_click|right_click|scroll|type|key_press|hover|drag|wait|refresh|go_back|goto_url|mouse_move|middle_click|mouse_down|mouse_up|go_forward|hold_key|extract_elements|find|set_element_value|execute_js)\b/g], }; diff --git a/.agents/skills/update-models/reference/discover-models.ts b/.agents/skills/update-models/reference/discover-models.ts index c8bb4db5..d1c853c8 100644 --- a/.agents/skills/update-models/reference/discover-models.ts +++ b/.agents/skills/update-models/reference/discover-models.ts @@ -3,9 +3,8 @@ import { readFile, writeFile } from "node:fs/promises"; import { homedir } from "node:os"; import { join } from "node:path"; import process from "node:process"; -import { parse as parseToml } from "smol-toml"; -type Provider = "openai" | "anthropic" | "gemini" | "tzafon" | "yutori"; +type Provider = "openai" | "anthropic" | "gemini" | "meta" | "tzafon" | "yutori"; interface Args { provider: Provider | "all"; @@ -38,7 +37,7 @@ interface ModelResult { cua?: Record; } -const PROVIDERS: Provider[] = ["openai", "anthropic", "gemini", "tzafon", "yutori"]; +const PROVIDERS: Provider[] = ["openai", "anthropic", "gemini", "meta", "tzafon", "yutori"]; const TZAFON_KNOWN_MODELS = [ "tzafon.northstar-cua-fast", ]; @@ -109,7 +108,7 @@ function usage(): never { npx tsx .agents/skills/update-models/reference/discover-models.ts --provider openai --models gpt-5.5,gpt-5.4 Options: - --provider + --provider --models Smoke-test explicit models instead of inferred candidates. --candidate-limit Max inferred candidates per provider. Default: 20. --no-smoke Only list metadata. @@ -137,6 +136,7 @@ async function runProvider(provider: Provider, args: Args): Promise> { return { provider: "openai", metadata_source: "client.models.list()", models, candidates }; } +async function discoverMeta(args: Args): Promise> { + const OpenAI = await importDefault("openai", "OpenAI"); + const apiKey = process.env.MODEL_API_KEY ?? process.env.META_MODEL_API_KEY; + const client = new OpenAI({ apiKey, baseURL: "https://api.meta.ai/v1" }); + const rawModels = await collectAsync(client.models.list()); + const models: ModelResult[] = rawModels.map((m) => ({ + id: String(m.id), + display_name: String(m.id), + created_at: typeof m.created === "number" && m.created > 0 ? new Date(m.created * 1000).toISOString() : null, + raw: m, + supports_generation: true, + })); + const candidates = explicitOrCandidates(args, models.map((model) => model.id)); + if (args.smoke) { + await Promise.all(candidates.map(async (id) => { + const model = models.find((candidate) => candidate.id === id) ?? { id, display_name: id, supports_generation: true }; + model.computer_use = await smokeMeta(client, id); + if (!models.find((candidate) => candidate.id === id)) models.unshift(model); + })); + } + await annotateCuaSupport("meta", models); + return { provider: "meta", metadata_source: "Meta Model API models.list()", models, candidates }; +} + +async function smokeMeta(client: any, model: string): Promise { + try { + const screenshot = await readFile(join(process.cwd(), "packages", "ai", "examples", "screenshot.png")); + const response = await client.responses.create({ + model, + store: false, + parallel_tool_calls: false, + max_output_tokens: 512, + reasoning: { effort: "low" }, + input: [{ + role: "user", + content: [ + { type: "input_text", text: "Call the click tool for the sign in link. Do not answer only in text." }, + { type: "input_image", image_url: `data:image/png;base64,${screenshot.toString("base64")}` }, + ], + }], + tools: [{ + type: "function", + name: "click", + description: "Click at normalized 0-1000 screen coordinates.", + parameters: { + type: "object", + properties: { x: { type: "number" }, y: { type: "number" } }, + required: ["x", "y"], + additionalProperties: false, + }, + }], + }); + const output: any[] = response.output ?? []; + const calls = output.filter((item) => item?.type === "function_call"); + return { + status: calls.length > 0 ? "pass" : "inconclusive", + tool_name: "function_tools", + tool_version: null, + beta_header: null, + observed_actions: unique(calls.map((call) => call?.name).filter(Boolean)), + response_item_types: unique(output.map((item) => item?.type).filter(Boolean)), + error: null, + }; + } catch (err) { + return smokeError(err, { tool_name: "function_tools" }); + } +} + function likelyOpenAIGenerationModel(id: string): boolean { const lower = id.toLowerCase(); if (OPENAI_EXCLUDE.some((needle) => lower.includes(needle))) return false; @@ -478,7 +546,8 @@ async function yutoriApiKey(): Promise { if (env && env.trim()) return env; const cfgPath = join(process.env.XDG_CONFIG_HOME ?? join(homedir(), ".config"), "cua", "config.toml"); try { - const raw = parseToml(await readFile(cfgPath, "utf8")) as any; + const { parse } = await import("smol-toml"); + const raw = parse(await readFile(cfgPath, "utf8")) as any; const profile = typeof raw?.default_profile === "string" ? raw.default_profile : undefined; const key = profile ? raw?.profiles?.[profile]?.yutori_api_key : undefined; if (typeof key === "string" && key.trim()) return key; diff --git a/.agents/skills/update-models/reference/provider-doc-drift.ts b/.agents/skills/update-models/reference/provider-doc-drift.ts index c60fbd42..7b7d10c8 100644 --- a/.agents/skills/update-models/reference/provider-doc-drift.ts +++ b/.agents/skills/update-models/reference/provider-doc-drift.ts @@ -2,7 +2,7 @@ import { readFile, writeFile } from "node:fs/promises"; import process from "node:process"; -type Provider = "openai" | "anthropic" | "gemini" | "yutori"; +type Provider = "openai" | "anthropic" | "gemini" | "meta" | "yutori"; interface Args { examples: string; @@ -30,6 +30,11 @@ const DOCS: Record = { "https://ai.google.dev/gemini-api/docs/computer-use", "https://ai.google.dev/api/models", ], + meta: [ + "https://dev.meta.ai/docs/getting-started/cookbook/computer-use-macos", + "https://dev.meta.ai/docs/features/responses", + "https://dev.meta.ai/docs/features/tool-calling", + ], yutori: [ "https://docs.yutori.com/reference/navigator", "https://docs.yutori.com/reference/n1", @@ -42,6 +47,7 @@ const LOCAL_FILES: Record = { openai: "packages/ai/src/providers/openai/index.ts", anthropic: "packages/ai/src/providers/anthropic/actions.ts", gemini: "packages/ai/src/providers/gemini/index.ts", + meta: "packages/ai/src/providers/meta/index.ts", yutori: "packages/ai/src/providers/yutori/actions.ts", }; @@ -49,6 +55,7 @@ const ACTION_REGEXES: Record = { openai: /\b(click|double_click|scroll|type|wait|keypress|drag|move|screenshot)\b/g, anthropic: /\b(screenshot|left_click|right_click|middle_click|double_click|triple_click|left_click_drag|mouse_move|key|type|scroll|hold_key|wait|left_mouse_down|left_mouse_up|cursor_position|zoom)\b/g, gemini: /\b(open_web_browser|open_web|wait_5_seconds|go_back|go_forward|search|navigate|click_at|hover_at|type_text_at|key_combination|scroll_document|scroll_at|drag_and_drop)\b/g, + meta: /\b(click|double_click|mouse_down|mouse_up|scroll|type|wait|keypress|drag|move|screenshot|goto|back|forward|url|cursor_position|left_click|right_click|middle_click|triple_click|left_click_drag|mouse_move|key|hold_key|left_mouse_down|left_mouse_up)\b/g, yutori: /\b(left_click|double_click|triple_click|right_click|scroll|type|key_press|hover|drag|wait|refresh|go_back|go_forward|goto_url|mouse_move|middle_click|mouse_down|mouse_up|hold_key|extract_elements|find|set_element_value|execute_js)\b/g, }; @@ -123,8 +130,8 @@ async function checkProvider(provider: Provider, examples: any): Promise normalizeProviderAction(provider, action))), new Set(localActions))), + unknown_example_actions: sorted(difference(new Set(exampleActions.map((action) => normalizeProviderAction(provider, action))), new Set(localActions))), response_fields_from_examples: sorted(example.response_fields ?? []), notes: notesFor(provider, documentedToolVersions, exampleToolVersions, localToolVersions), }; @@ -140,6 +147,23 @@ async function fetchText(url: string): Promise { } } +function normalizeProviderAction(provider: Provider, action: string): string { + if (provider !== "meta") return action; + const aliases: Record = { + left_click: "click", + right_click: "click", + middle_click: "click", + triple_click: "click", + left_click_drag: "drag", + mouse_move: "move", + key: "keypress", + hold_key: "keypress", + left_mouse_down: "mouse_down", + left_mouse_up: "mouse_up", + }; + return aliases[action] ?? action; +} + function notesFor(provider: Provider, documentedToolVersions: string[], exampleToolVersions: string[], localToolVersions: string[]): string[] { const notes: string[] = []; if (provider === "openai") { @@ -154,6 +178,9 @@ function notesFor(provider: Provider, documentedToolVersions: string[], exampleT if (provider === "gemini") { notes.push("Gemini official computer use emits predefined function-call names; keep this separate from CUA custom function declarations."); } + if (provider === "meta") { + notes.push("Meta computer use uses developer-defined function tools over the Responses API; compare payload threading and coordinate conventions as well as action names."); + } if (provider === "yutori") { notes.push("Yutori Navigator emits OpenAI-compatible tool_calls for built-in browser actions; local AgentTools should execute those names but outbound payloads should not duplicate the built-in browser schemas."); notes.push("Track n1 vs n1.5 separately because n1.5 can add tool_set/disable_tools and expanded browser actions."); diff --git a/.agents/skills/update-models/reference/report-schema.md b/.agents/skills/update-models/reference/report-schema.md index 7247f449..52e9c93e 100644 --- a/.agents/skills/update-models/reference/report-schema.md +++ b/.agents/skills/update-models/reference/report-schema.md @@ -12,6 +12,7 @@ Use this shape for JSON reports and the same fields when writing a Markdown summ "openai": {}, "anthropic": {}, "gemini": {}, + "meta": {}, "tzafon": {}, "yutori": {} }, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1aac9651..fbe507dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,6 +132,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + MODEL_API_KEY: ${{ secrets.MODEL_API_KEY }} TZAFON_API_KEY: ${{ secrets.TZAFON_API_KEY }} YUTORI_API_KEY: ${{ secrets.YUTORI_API_KEY }} run: npm run test:integration --workspace @onkernel/cua-ai @@ -154,10 +155,12 @@ jobs: - name: Agent live smoke tests (all providers) env: CUA_E2E_LIVE: "1" + CUA_E2E_META: "1" KERNEL_API_KEY: ${{ secrets.KERNEL_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + MODEL_API_KEY: ${{ secrets.MODEL_API_KEY }} TZAFON_API_KEY: ${{ secrets.TZAFON_API_KEY }} YUTORI_API_KEY: ${{ secrets.YUTORI_API_KEY }} run: npm test --workspace @onkernel/cua-agent -- test/e2e.live.test.ts diff --git a/README.md b/README.md index dbbec9bf..e049d6eb 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ cua "go to news.ycombinator.com and tell me the top 3 story titles" ## Why this exists -Every frontier model now ships its own first-party "computer use" tool: +Frontier models expose computer use through different tool protocols: - **OpenAI gpt-5.5**: a built-in `computer` tool that emits actions like `{type:"click", x, y}`, `{type:"scroll", x, y, scroll_x, scroll_y}`, @@ -23,6 +23,8 @@ Every frontier model now ships its own first-party "computer use" tool: - **Google gemini-2.5-pro / gemini-3.x**: a set of predefined computer-use functions (`click_at`, `type_text_at`, `scroll_at`, `navigate`, `go_back`, …) with 0-1000 normalized coordinates. +- **Meta Muse Spark 1.1**: OpenAI-compatible Responses API function calls + paired with screenshot inputs and 0-1000 normalized coordinates. - **Yutori Navigator n1 / n1.5**: OpenAI-compatible `chat.completions` responses with built-in browser action `tool_calls` like `left_click`, `goto_url`, `type`, and `scroll` in 0-1000 normalized coordinates. @@ -100,6 +102,7 @@ npx tsx packages/cli/src/cli.ts --help export OPENAI_API_KEY=sk-... # for gpt-5.5 export ANTHROPIC_API_KEY=sk-ant-... # for claude-opus-4-7 export GOOGLE_API_KEY=... # for gemini-3-flash-preview +export MODEL_API_KEY=... # for muse-spark-1.1 export YUTORI_API_KEY=yt_... # for n1.5-latest export KERNEL_API_KEY=sk_... # always required @@ -115,6 +118,9 @@ cua -p --model claude-opus-4-7 "Same prompt" # Gemini 3 Flash (built-in computer use) cua -p --model gemini-3-flash-preview "Same prompt" +# Meta Muse Spark +cua -p --model meta:muse-spark-1.1 "Same prompt" + # Yutori Navigator cua -p --model n1.5-latest "Same prompt" diff --git a/packages/agent/README.md b/packages/agent/README.md index 7b7404cd..c88cc5a7 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -127,7 +127,7 @@ Both classes mirror pi constructor shapes and behavior, with minimal additions: live browser session - `toolResultImageReplayLimit` to control how many recent tool-result images are included in each model request -- `responseThreading` to choose whether OpenAI and Tzafon continue from +- `responseThreading` to choose whether OpenAI, Meta, and Tzafon continue from provider-stored response state or receive the full request context each time ### Context management @@ -177,7 +177,7 @@ pi provides several context-management layers with different lifetimes: It is broader than the image limit and is not called automatically by this package. -`responseThreading` defaults to `true`. For OpenAI and Tzafon Responses APIs, +`responseThreading` defaults to `true`. For OpenAI, Meta, and Tzafon Responses APIs, that means each request can continue from `previous_response_id`; the provider keeps earlier response state and CUA sends only messages added since that response. Because the provider retains that state, removing an older image from @@ -192,6 +192,7 @@ If auth callbacks are omitted, both classes default to CUA env var conventions: - OpenAI: `OPENAI_API_KEY` - Anthropic: `ANTHROPIC_OAUTH_TOKEN` or `ANTHROPIC_API_KEY` - Gemini: `GOOGLE_API_KEY` or `GEMINI_API_KEY` +- Meta: `MODEL_API_KEY` or `META_MODEL_API_KEY` - Tzafon: `TZAFON_API_KEY` - Yutori: `YUTORI_API_KEY` @@ -223,7 +224,7 @@ browser session does carry over. No screenshot is returned automatically; request one on a follow-up turn when the model needs to see the page. Playwright-level failures come back as tool content (so the model can adapt) rather than thrown errors. Verified e2e -against Anthropic, Tzafon, and Yutori CUA models; OpenAI and Google are +against Anthropic, Meta, Tzafon, and Yutori CUA models; OpenAI and Google are unit-tested. ### Model Switching diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index ce62089f..0380ac2d 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -156,7 +156,7 @@ export type CuaAgentOptions = Omit & { emptyResponseRecovery?: CuaEmptyResponseRecoveryOptions; /** Maximum tool-result images included from message history per provider request. Defaults to 4; false disables projection. */ toolResultImageReplayLimit?: ToolResultImageReplayLimit; - /** Chain OpenAI and Tzafon requests through provider-stored response state. Defaults to true. */ + /** Chain OpenAI, Meta, and Tzafon requests through provider-stored response state. Defaults to true. */ responseThreading?: boolean; /** Optional CUA-level retries around each provider request. Disabled by default. */ retry?: CuaRetryOptions; @@ -202,7 +202,7 @@ export type CuaAgentHarnessOptions< emptyResponseRecovery?: CuaEmptyResponseRecoveryOptions; /** Maximum tool-result images included from message history per provider request. Defaults to 4; false disables projection. */ toolResultImageReplayLimit?: ToolResultImageReplayLimit; - /** Chain OpenAI and Tzafon requests through provider-stored response state. Defaults to true. */ + /** Chain OpenAI, Meta, and Tzafon requests through provider-stored response state. Defaults to true. */ responseThreading?: boolean; /** Optional CUA-level retries around each provider request. Disabled by default. */ retry?: CuaRetryOptions; diff --git a/packages/agent/test/e2e.live.test.ts b/packages/agent/test/e2e.live.test.ts index 41e3436a..3e932c8e 100644 --- a/packages/agent/test/e2e.live.test.ts +++ b/packages/agent/test/e2e.live.test.ts @@ -20,6 +20,7 @@ type ProviderCase = { | "openai:gpt-5.5" | "anthropic:claude-opus-4-7" | "google:gemini-3-flash-preview" + | "meta:muse-spark-1.1" | "tzafon:tzafon.northstar-cua-fast" | "yutori:n1.5-latest"; prompt: string; @@ -79,6 +80,20 @@ const cases: ProviderCase[] = [ timeoutMs: 300_000, ciOptInEnvVar: "CUA_E2E_GEMINI", }, + { + name: "meta", + apiKeyEnvVar: "MODEL_API_KEY", + modelRef: "meta:muse-spark-1.1", + prompt: [ + "Use the tool named `screenshot` exactly once to inspect the browser.", + "Pass empty arguments (`{}`).", + "Do not call any other tools.", + "After the tool result, provide a one-sentence summary.", + ].join("\n"), + expectToolCalls: true, + timeoutMs: 180_000, + ciOptInEnvVar: "CUA_E2E_META", + }, { name: "tzafon", apiKeyEnvVar: "TZAFON_API_KEY", diff --git a/packages/ai/README.md b/packages/ai/README.md index 97ab4b14..15a0422f 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -20,6 +20,7 @@ check these environment variables, in order: | `openai` | `OPENAI_API_KEY` | | `anthropic` | `ANTHROPIC_OAUTH_TOKEN`, `ANTHROPIC_API_KEY` | | `google` | `GOOGLE_API_KEY`, `GEMINI_API_KEY` | +| `meta` | `MODEL_API_KEY`, `META_MODEL_API_KEY` | | `tzafon` | `TZAFON_API_KEY` | | `yutori` | `YUTORI_API_KEY` | @@ -210,6 +211,7 @@ provider/tool metadata. getCuaModel("openai:gpt-5.5"); getCuaModel("anthropic:claude-opus-4-7"); getCuaModel("google:gemini-3-flash-preview"); +getCuaModel("meta:muse-spark-1.1"); getCuaModel("tzafon:tzafon.northstar-cua-fast"); getCuaModel("yutori:n1.5-latest"); ``` @@ -226,7 +228,7 @@ list of CUA-supporting models per provider. `CuaProvider` is the string union of provider IDs this package targets: ```ts -type CuaProvider = "openai" | "anthropic" | "google" | "tzafon" | "yutori"; +type CuaProvider = "openai" | "anthropic" | "google" | "meta" | "tzafon" | "yutori"; ``` The IDs match pi-ai's `Model.provider` values exactly. `providerForModel(model)` @@ -332,7 +334,7 @@ definitions and executors; it is forwarded to the provider module's with pi's builtin providers plus CUA's adjustments: OpenAI models route through cua's `openai-cua-responses` stream (threads `previous_response_id`), Google resolves `GOOGLE_API_KEY` or - `GEMINI_API_KEY`, and the Tzafon/Yutori providers are registered. Each call + `GEMINI_API_KEY`, and the Meta/Tzafon/Yutori providers are registered. Each call returns an independent collection. - `cuaModels(): MutableModels` — the shared default collection, created on first use. `CuaAgent` and `CuaAgentHarness` stream through it unless given @@ -370,6 +372,7 @@ Per-provider canonical action subsets (each namespace exports its list as | `openai` | all 16 | | `anthropic` | 13 — everything except `back`, `forward`, `url`; adds `computer_batch` by default | | `gemini` | all 16 | +| `meta` | all 16 | | `tzafon` | all 16 (replaced on the wire by Tzafon's native `computer_use` tool) | | `yutori` | 13 — everything except `screenshot`, `url`, `cursor_position` (local mirrors only) | @@ -404,6 +407,7 @@ Current coordinate contracts: - `openai`: pixel coordinates - `anthropic`: pixel coordinates, matching Anthropic's computer-use quickstart - `gemini`: normalized coordinates in the 0-999 range ([source](https://ai.google.dev/gemini-api/docs/computer-use)) +- `meta`: normalized coordinates in the 0-1000 range ([source](https://dev.meta.ai/docs/getting-started/cookbook/computer-use-macos)) - `yutori`: normalized coordinates in the 0-1000 range ([source](https://docs.yutori.com/reference/navigator), [SDK helper](https://github.com/yutori-ai/yutori-sdk-python/blob/main/yutori/navigator/coordinates.py)) - `tzafon`: normalized coordinates in the 0-999 range ([source](https://docs.lightcone.ai/guides/coordinates/), [model card](https://huggingface.co/Tzafon/Northstar-CUA-Fast)) @@ -480,7 +484,7 @@ type CuaNavigationInput = { ## Provider Namespaces -Every provider namespace (`openai`, `anthropic`, `gemini`, `tzafon`, +Every provider namespace (`openai`, `anthropic`, `gemini`, `meta`, `tzafon`, `yutori`) follows one convention: - `computerTools(options?)` and `computerToolExecutors(options?)` @@ -504,6 +508,9 @@ Provider-specific extras: `computer_use_extra` navigation aliases `OPENAI_EXTRA_TOOL_NAME`, `OPENAI_EXTRA_TOOL_DESCRIPTION`, `OpenAIExtraSchema`, `OpenAIExtraInput` - `anthropic`: `ANTHROPIC_BATCH_TOOL_NAME` (`"computer_batch"`) +- `meta`: the `meta-responses` stream adapter (`META_RESPONSES_API`, + `streamMetaResponses`, `streamSimpleMetaResponses`, `MetaResponsesOptions`), + with `previous_response_id` threading and Meta-compatible reasoning payloads - `tzafon`: the `tzafon-responses` stream adapter (`TZAFON_RESPONSES_API`, `streamTzafonResponses`, `streamSimpleTzafonResponses`, `TzafonResponsesOptions` with `keepToolNames`), `tzafonComputerUseOnPayload` diff --git a/packages/ai/docs/supported-models.md b/packages/ai/docs/supported-models.md index 1a51d8e1..b7df3d3e 100644 --- a/packages/ai/docs/supported-models.md +++ b/packages/ai/docs/supported-models.md @@ -58,6 +58,21 @@ Google's native `tools.computer_use` request wrapper instead. Source: [Gemini computer use docs](https://ai.google.dev/gemini-api/docs/computer-use). +## `meta` + +API: `meta-responses` · coordinates: normalized 0–1000 + +Exact IDs: + +- `muse-spark-1.1` + +Muse Spark uses Meta's OpenAI-compatible Responses API with ordinary function +tools. CUA stores each response and continues tool loops with +`previous_response_id`; threaded requests omit encrypted reasoning replay, +which Meta does not allow alongside a previous response ID. + +Source: [Meta computer-use cookbook](https://dev.meta.ai/docs/getting-started/cookbook/computer-use-macos). + ## `tzafon` API: `tzafon-responses` · coordinates: normalized 0–999 diff --git a/packages/ai/examples/quickstart.ts b/packages/ai/examples/quickstart.ts index cf9256e0..b9bb4f77 100644 --- a/packages/ai/examples/quickstart.ts +++ b/packages/ai/examples/quickstart.ts @@ -10,6 +10,7 @@ import { // Switch providers by setting CUA_MODEL (and the matching API key env var): // anthropic:claude-opus-4-7 ANTHROPIC_API_KEY // google:gemini-3-flash-preview GOOGLE_API_KEY +// meta:muse-spark-1.1 MODEL_API_KEY // tzafon:tzafon.northstar-cua-fast TZAFON_API_KEY // yutori:n1.5-latest YUTORI_API_KEY const modelRef = (process.env.CUA_MODEL ?? "openai:gpt-5.5") as CuaModelRef; diff --git a/packages/ai/src/api-keys.ts b/packages/ai/src/api-keys.ts index cfbbed74..c92b9590 100644 --- a/packages/ai/src/api-keys.ts +++ b/packages/ai/src/api-keys.ts @@ -12,6 +12,7 @@ const CUA_PROVIDER_API_KEY_ENV_VARS: Record = { openai: ["OPENAI_API_KEY"], anthropic: ["ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY"], google: ["GOOGLE_API_KEY", "GEMINI_API_KEY"], + meta: ["MODEL_API_KEY", "META_MODEL_API_KEY"], tzafon: ["TZAFON_API_KEY"], yutori: ["YUTORI_API_KEY"], }; diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index e71e3e61..54e05158 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -7,6 +7,7 @@ export * from "./runtime-spec"; export * from "./providers/common"; export * as anthropic from "./providers/anthropic/index"; export * as gemini from "./providers/gemini/index"; +export * as meta from "./providers/meta/index"; export * as openai from "./providers/openai/index"; export * as tzafon from "./providers/tzafon/index"; export * as yutori from "./providers/yutori/index"; diff --git a/packages/ai/src/models.ts b/packages/ai/src/models.ts index 5653e901..5827f872 100644 --- a/packages/ai/src/models.ts +++ b/packages/ai/src/models.ts @@ -1,9 +1,10 @@ import type { Api, Model } from "@earendil-works/pi-ai"; import { getBuiltinModel, getBuiltinModels } from "@earendil-works/pi-ai/providers/all"; +import { META_RESPONSES_API } from "./providers/meta/provider"; import { OPENAI_CUA_RESPONSES_API } from "./providers/openai/provider"; /** Providers with curated computer-use model support. */ -export type CuaProvider = "openai" | "anthropic" | "google" | "tzafon" | "yutori"; +export type CuaProvider = "openai" | "anthropic" | "google" | "meta" | "tzafon" | "yutori"; /** Provider-qualified model reference, e.g. `"openai:gpt-5.5"` or `"google:gemini-3-flash-preview"`. */ export type CuaModelRef = `${CuaProvider}:${string}`; @@ -20,7 +21,7 @@ export interface CuaModelInfo { } /** All providers this package curates computer-use models for. */ -export const CUA_PROVIDERS: readonly CuaProvider[] = ["openai", "anthropic", "google", "tzafon", "yutori"]; +export const CUA_PROVIDERS: readonly CuaProvider[] = ["openai", "anthropic", "google", "meta", "tzafon", "yutori"]; /** * How a {@link CuaModelAnnotation} matches model ids. @@ -78,6 +79,9 @@ export const CUA_MODEL_ANNOTATIONS: Record[]> = { openai: [], anthropic: [], google: [], + meta: [cuaModel("meta", "muse-spark-1.1", "Muse Spark 1.1")], tzafon: [ cuaModel("tzafon", "tzafon.northstar-cua-fast", "Tzafon Northstar CUA Fast"), cuaModel("tzafon", "tzafon.northstar-cua-fast-1.6", "Tzafon Northstar CUA Fast 1.6"), @@ -193,13 +198,17 @@ export function getCuaModel(ref: CuaModelRef): Model { throw new Error(`CUA model "${ref}" is supported but not registered. Add it to pi-ai (models.dev) or CUA_MODEL_OVERRIDES.`); } -// Route OpenAI CUA models to cua's own openai-cua-responses stream provider, -// which threads previous_response_id. Registry-resolved models (gpt-5.4, -// gpt-5.4-mini, gpt-5.5) otherwise carry pi-ai's builtin "openai-responses" api. +// Route OpenAI-compatible CUA models to provider-specific Responses transports +// that thread previous_response_id. Registry-resolved models otherwise carry +// pi-ai's builtin "openai-responses" api. export function routeCuaApi(model: Model): Model { - return model.provider === "openai" && model.api !== OPENAI_CUA_RESPONSES_API - ? { ...model, api: OPENAI_CUA_RESPONSES_API } - : model; + if (model.provider === "openai" && model.api !== OPENAI_CUA_RESPONSES_API) { + return { ...model, api: OPENAI_CUA_RESPONSES_API }; + } + if (model.provider === "meta" && model.api !== META_RESPONSES_API) { + return { ...model, api: META_RESPONSES_API }; + } + return model; } /** Return the {@link CuaProvider} for a concrete model, or throw when it is not a CUA provider. */ @@ -251,7 +260,7 @@ function cuaModel(provider: CuaProvider, id: string, name: string): Model { id, name, provider, - reasoning: provider === "openai" || provider === "anthropic" || provider === "google", + reasoning: provider === "openai" || provider === "anthropic" || provider === "google" || provider === "meta", input: ["text", "image"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, } satisfies Partial>; @@ -263,6 +272,17 @@ function cuaModel(provider: CuaProvider, id: string, name: string): Model { return { ...base, api: "anthropic-messages", baseUrl: "https://api.anthropic.com", contextWindow: 200_000, maxTokens: 64_000 } as Model; case "google": return { ...base, api: "google-generative-ai", baseUrl: "https://generativelanguage.googleapis.com/v1beta", contextWindow: 1_048_576, maxTokens: 65_536 } as Model; + case "meta": + return { + ...base, + api: META_RESPONSES_API, + baseUrl: "https://api.meta.ai/v1", + thinkingLevelMap: { off: null, xhigh: "xhigh" }, + cost: { input: 1.25, output: 4.25, cacheRead: 0.15, cacheWrite: 0 }, + contextWindow: 1_048_576, + maxTokens: 128_000, + compat: { supportsDeveloperRole: true, sendSessionIdHeader: false, supportsLongCacheRetention: true }, + } as Model; case "tzafon": return { ...base, api: "tzafon-responses", baseUrl: "https://api.lightcone.ai", contextWindow: 128_000, maxTokens: 4_096 } as Model; case "yutori": diff --git a/packages/ai/src/providers.ts b/packages/ai/src/providers.ts index 9a5d1c18..cdd8b225 100644 --- a/packages/ai/src/providers.ts +++ b/packages/ai/src/providers.ts @@ -14,6 +14,7 @@ import { builtinModels } from "@earendil-works/pi-ai/providers/all"; import { cuaApiKeyEnvVarsForProvider } from "./api-keys"; import { cuaOverrideModels } from "./models"; import { ANTHROPIC_NATIVE_API_BETA_HEADERS, withAnthropicBetaHeader } from "./providers/anthropic/native"; +import { META_RESPONSES_API, streamMetaResponses, streamSimpleMetaResponses } from "./providers/meta/provider"; import { OPENAI_CUA_RESPONSES_API, streamOpenAIResponses, streamSimpleOpenAIResponses } from "./providers/openai/provider"; import { streamSimpleTzafonResponses, streamTzafonResponses, TZAFON_RESPONSES_API } from "./providers/tzafon/provider"; import { streamSimpleYutori, streamYutori, YUTORI_CHAT_COMPLETIONS_API } from "./providers/yutori/provider"; @@ -32,7 +33,7 @@ import { streamSimpleYutori, streamYutori, YUTORI_CHAT_COMPLETIONS_API } from ". * `anthropic-beta` header merged in. * - `google` resolves its API key from `GOOGLE_API_KEY` or `GEMINI_API_KEY` * (pi's builtin only reads `GEMINI_API_KEY`). - * - `tzafon` and `yutori` are CUA-only providers pi does not ship. + * - `meta`, `tzafon`, and `yutori` are CUA-only providers pi does not ship. * * Each call returns an independent collection; register additional providers * or credentials on it freely. Use {@link cuaModels} for the shared default. @@ -45,6 +46,7 @@ export function createCuaModels(options?: CreateModelsOptions): MutableModels { if (anthropic) models.setProvider(withAnthropicNativeTools(anthropic)); const google = models.getProvider("google"); if (google) models.setProvider(withGoogleEnvKeys(google)); + models.setProvider(metaProvider()); models.setProvider(tzafonProvider()); models.setProvider(yutoriProvider()); return models; @@ -106,6 +108,17 @@ function withGoogleEnvKeys(base: Provider): Provider { return { ...base, auth: { ...base.auth, apiKey: envApiKeyAuth("Google API key", cuaApiKeyEnvVarsForProvider("google")) } }; } +function metaProvider(): Provider { + return createProvider({ + id: "meta", + name: "Meta", + baseUrl: "https://api.meta.ai/v1", + auth: { apiKey: envApiKeyAuth("Meta Model API key", cuaApiKeyEnvVarsForProvider("meta")) }, + models: cuaOverrideModels("meta"), + api: { stream: streamMetaResponses, streamSimple: streamSimpleMetaResponses }, + }); +} + function tzafonProvider(): Provider { return createProvider({ id: "tzafon", @@ -128,6 +141,7 @@ function yutoriProvider(): Provider { }); } +export { META_RESPONSES_API, streamMetaResponses, streamSimpleMetaResponses }; export { OPENAI_CUA_RESPONSES_API, streamOpenAIResponses, streamSimpleOpenAIResponses }; export { TZAFON_RESPONSES_API, streamSimpleTzafonResponses, streamTzafonResponses }; export { YUTORI_CHAT_COMPLETIONS_API, streamSimpleYutori, streamYutori }; diff --git a/packages/ai/src/providers/meta/index.ts b/packages/ai/src/providers/meta/index.ts new file mode 100644 index 00000000..5d08b047 --- /dev/null +++ b/packages/ai/src/providers/meta/index.ts @@ -0,0 +1,47 @@ +import { assertComputerModeOnly, computerToolExecutors, computerTools } from "../common"; +import type { ComputerToolCoordinateSystem, ComputerToolsOptions, CuaProviderModule } from "../common"; + +export { + CUA_ACTION_TYPES as META_CUA_ACTION_TYPES, + computerToolExecutors, + computerTools, + createCuaActionSchema as createActionSchema, +} from "../common"; +export type { + CuaAction as MetaAction, + ComputerToolsOptions, +} from "../common"; +export { + META_RESPONSES_API, + streamMetaResponses, + streamSimpleMetaResponses, + threadMetaRequest, +} from "./provider"; +export type { MetaResponsesOptions } from "./provider"; + +// CUA supplies Muse Spark with its canonical function actions: click, +// double_click, mouse_down, mouse_up, type, keypress, scroll, move, drag, +// wait, screenshot, goto, back, forward, url, and cursor_position. +// Meta's computer-use reference agent uses normalized 0-1000 coordinates. +export function coordinateSystem(): ComputerToolCoordinateSystem { + return { type: "normalized", range: [0, 1000] }; +} + +export const META_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Coordinates are normalized from 0 to 1000 relative to the most recent screenshot, with (0, 0) at the top left and (1000, 1000) at the bottom right. Base each action on the latest observed state and request a screenshot when you need a fresh view.`; + +export function buildMetaSystemPrompt(opts: { suffix?: string } = {}): string { + return [META_COMPUTER_INSTRUCTIONS, opts.suffix].filter(Boolean).join("\n\n"); +} + +export const providerModule = { + toolDefinitions: (options?: ComputerToolsOptions) => { + assertComputerModeOnly("meta", options); + return computerTools(options); + }, + toolExecutors: (options?: ComputerToolsOptions) => { + assertComputerModeOnly("meta", options); + return computerToolExecutors(options); + }, + coordinateSystem, + buildSystemPrompt: buildMetaSystemPrompt, +} satisfies CuaProviderModule; diff --git a/packages/ai/src/providers/meta/provider.ts b/packages/ai/src/providers/meta/provider.ts new file mode 100644 index 00000000..f8fa59d9 --- /dev/null +++ b/packages/ai/src/providers/meta/provider.ts @@ -0,0 +1,70 @@ +import { + type Context, + type OpenAIResponsesOptions as PiOpenAIResponsesOptions, + type StreamFunction, + type StreamOptions, +} from "@earendil-works/pi-ai"; +import { + stream as piStreamOpenAIResponses, + streamSimple as piStreamSimpleOpenAIResponses, +} from "@earendil-works/pi-ai/api/openai-responses"; +import { + type CuaSimpleStreamOptions, + responseThreadingDelta, + responseThreadingEnabled, + type ResponseThreadingOptions, +} from "../common"; + +export const META_RESPONSES_API = "meta-responses"; + +/** Stream options for Meta's OpenAI-compatible Responses API. */ +export interface MetaResponsesOptions extends PiOpenAIResponsesOptions, ResponseThreadingOptions {} + +type OnPayload = NonNullable; + +/** + * Prepare a Meta Responses request with server-managed conversation state. + * Meta rejects reasoning.encrypted_content when previous_response_id is set, + * so threaded turns remove that include while retaining reasoning server-side. + */ +export function threadMetaRequest( + context: Context, + options: (ResponseThreadingOptions & { onPayload?: OnPayload }) | undefined, +): { context: Context; onPayload: OnPayload } { + const delta = responseThreadingEnabled(options) ? responseThreadingDelta(context.messages, META_RESPONSES_API) : undefined; + const previousResponseId = delta?.previousResponseId; + const messages = previousResponseId && delta ? delta.deltaMessages : context.messages; + const onPayload: OnPayload = async (payload, model) => { + const threaded = applyMetaPayloadDefaults(payload, previousResponseId); + const transformed = options?.onPayload ? ((await options.onPayload(threaded, model)) ?? threaded) : threaded; + return applyMetaPayloadDefaults(transformed, previousResponseId); + }; + return { context: messages === context.messages ? context : { ...context, messages }, onPayload }; +} + +function applyMetaPayloadDefaults(payload: unknown, previousResponseId?: string): Record { + const next: Record = { + ...(payload as Record), + store: true, + parallel_tool_calls: false, + ...(previousResponseId ? { previous_response_id: previousResponseId } : {}), + }; + if (!previousResponseId || !Array.isArray(next.include)) return next; + + const include = next.include.filter((value: unknown) => value !== "reasoning.encrypted_content"); + if (include.length > 0) next.include = include; + else delete next.include; + return next; +} + +// Meta implements the OpenAI Responses wire protocol. The model carries a +// distinct api id so response ids can never be threaded across providers. +export const streamMetaResponses: StreamFunction = (model, context, options) => { + const threaded = threadMetaRequest(context, options); + return piStreamOpenAIResponses(model as never, threaded.context, { ...options, onPayload: threaded.onPayload }); +}; + +export const streamSimpleMetaResponses: StreamFunction = (model, context, options) => { + const threaded = threadMetaRequest(context, options); + return piStreamSimpleOpenAIResponses(model as never, threaded.context, { ...options, onPayload: threaded.onPayload }); +}; diff --git a/packages/ai/src/runtime-spec.ts b/packages/ai/src/runtime-spec.ts index e8fbee66..867d4df7 100644 --- a/packages/ai/src/runtime-spec.ts +++ b/packages/ai/src/runtime-spec.ts @@ -5,6 +5,7 @@ import { modeForNativeTool, resolveNativeTool, type CuaNativeToolSpec } from "./ import { providerModule as anthropic } from "./providers/anthropic/index"; import { createNativeToolOnPayload, nativeApiForToolType, nativeToolExecutors } from "./providers/anthropic/native"; import { providerModule as gemini } from "./providers/gemini/index"; +import { providerModule as meta } from "./providers/meta/index"; import { providerModule as openai } from "./providers/openai/index"; import { providerModule as tzafon } from "./providers/tzafon/index"; import { providerModule as yutori } from "./providers/yutori/index"; @@ -20,6 +21,7 @@ const PROVIDERS = { openai, anthropic, google: gemini, + meta, tzafon, yutori, } satisfies Record; diff --git a/packages/ai/test/api-keys.test.ts b/packages/ai/test/api-keys.test.ts index 501b0106..b4b553a2 100644 --- a/packages/ai/test/api-keys.test.ts +++ b/packages/ai/test/api-keys.test.ts @@ -12,6 +12,8 @@ const ENV_KEYS = [ "ANTHROPIC_API_KEY", "GOOGLE_API_KEY", "GEMINI_API_KEY", + "MODEL_API_KEY", + "META_MODEL_API_KEY", "TZAFON_API_KEY", "YUTORI_API_KEY", ] as const; @@ -31,10 +33,12 @@ describe("cua api key helpers", () => { expect(cuaApiKeyEnvVarsForProvider("openai")).toEqual(["OPENAI_API_KEY"]); expect(cuaApiKeyEnvVarsForProvider("google")).toEqual(["GOOGLE_API_KEY", "GEMINI_API_KEY"]); expect(cuaApiKeyEnvVarsForProvider("gemini")).toEqual(["GOOGLE_API_KEY", "GEMINI_API_KEY"]); + expect(cuaApiKeyEnvVarsForProvider("meta")).toEqual(["MODEL_API_KEY", "META_MODEL_API_KEY"]); expect(cuaApiKeyEnvVarsForProvider("unknown")).toEqual([]); }); it("resolves provider api keys with fallback order", () => { + delete process.env.GOOGLE_API_KEY; process.env.GEMINI_API_KEY = "gemini"; expect(getCuaEnvApiKey("google")).toBe("gemini"); process.env.GOOGLE_API_KEY = "google"; @@ -44,6 +48,11 @@ describe("cua api key helpers", () => { it("resolves keys from model refs", () => { process.env.OPENAI_API_KEY = "openai"; expect(getCuaEnvApiKeyForModel("openai:gpt-5.5")).toBe("openai"); + delete process.env.MODEL_API_KEY; + process.env.META_MODEL_API_KEY = "meta-fallback"; + expect(getCuaEnvApiKeyForModel("meta:muse-spark-1.1")).toBe("meta-fallback"); + process.env.MODEL_API_KEY = "meta"; + expect(getCuaEnvApiKeyForModel("meta:muse-spark-1.1")).toBe("meta"); }); it("throws readable errors when missing", () => { diff --git a/packages/ai/test/computer-tool.integration.test.ts b/packages/ai/test/computer-tool.integration.test.ts index 3aa06717..e85e2d77 100644 --- a/packages/ai/test/computer-tool.integration.test.ts +++ b/packages/ai/test/computer-tool.integration.test.ts @@ -1,6 +1,5 @@ import { readFile } from "node:fs/promises"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; +import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { CUA_ACTION_TYPES, @@ -11,13 +10,13 @@ import { cuaModels, gemini, getCuaModel, + meta, openai, tzafon, yutori, } from "../src/index"; -const here = dirname(fileURLToPath(import.meta.url)); -const screenshotPath = join(here, "..", "examples", "screenshot.png"); +const screenshotPath = join(process.cwd(), "examples", "screenshot.png"); interface ProviderCase { provider: CuaProvider; @@ -56,6 +55,14 @@ const cases: ProviderCase[] = [ coordinateRange: [0, 999], requireToolCalls: true, }, + { + provider: "meta", + envVar: "MODEL_API_KEY", + modelRef: "meta:muse-spark-1.1", + tools: () => meta.computerTools({ actions: ["click"] }), + coordinateRange: [0, 1000], + requireToolCalls: true, + }, { provider: "tzafon", envVar: "TZAFON_API_KEY", @@ -142,6 +149,64 @@ describe("individual computer action integration", () => { }, 60_000); } + const metaHasKey = !!process.env.MODEL_API_KEY; + (metaHasKey ? it : it.skip)( + "meta continues a screenshot tool loop with previous_response_id", + async () => { + const screenshot = await readFile(screenshotPath); + const model = getCuaModel("meta:muse-spark-1.1"); + const tools = meta.computerTools({ actions: ["click"] }); + const context: Context = { + systemPrompt: "Use normalized 0-1000 coordinates. Call click exactly once, then answer in text after its result.", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Click the sign in / up link. After the tool result, report that the click completed without calling another tool." }, + { type: "image", data: screenshot.toString("base64"), mimeType: "image/png" }, + ], + timestamp: Date.now(), + }, + ], + tools, + }; + const first = await cuaModels().complete(model, context, { + apiKey: process.env.MODEL_API_KEY, + maxTokens: 1024, + }); + const click = first.content.find((part) => part.type === "toolCall" && part.name === "click"); + expect(click).toBeDefined(); + expect(first.responseId).toBeTruthy(); + context.messages.push(first, { + role: "toolResult", + toolCallId: click!.id, + toolName: click!.name, + content: [ + { type: "text", text: "Click executed successfully." }, + { type: "image", data: screenshot.toString("base64"), mimeType: "image/png" }, + ], + isError: false, + timestamp: Date.now(), + }); + + let payload: Record | undefined; + const second = await cuaModels().complete(model, context, { + apiKey: process.env.MODEL_API_KEY, + maxTokens: 1024, + onPayload: (value) => { + payload = value as Record; + }, + }); + expect(second.stopReason).not.toBe("error"); + expect(second.responseId).toBeTruthy(); + expect(payload?.previous_response_id).toBe(first.responseId); + expect(payload?.store).toBe(true); + expect(payload?.parallel_tool_calls).toBe(false); + expect(payload?.include).toBeUndefined(); + }, + 90_000, + ); + const yutoriHasKey = !!process.env.YUTORI_API_KEY; (yutoriHasKey ? it : it.skip)( "yutori translates native tool calls into canonical individual actions", diff --git a/packages/ai/test/meta-threading.test.ts b/packages/ai/test/meta-threading.test.ts new file mode 100644 index 00000000..c37a2e16 --- /dev/null +++ b/packages/ai/test/meta-threading.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; +import type { Context, Message, Model } from "@earendil-works/pi-ai"; +import { META_RESPONSES_API, threadMetaRequest } from "../src/providers/meta/provider"; + +const model = {} as Model; + +function multiTurnContext(): Context { + const messages: Message[] = [ + { role: "user", content: "inspect the browser", timestamp: 0 }, + { + role: "assistant", + content: [{ type: "toolCall", id: "call_1|fc_1", name: "screenshot", arguments: {} }], + api: META_RESPONSES_API, + provider: "meta", + model: "muse-spark-1.1", + responseId: "resp_meta_1", + usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + stopReason: "toolUse", + timestamp: 0, + }, + { + role: "toolResult", + toolCallId: "call_1|fc_1", + toolName: "screenshot", + content: [{ type: "image", mimeType: "image/png", data: "screenshot" }], + isError: false, + timestamp: 0, + }, + ]; + return { messages, tools: [], systemPrompt: "control the browser" }; +} + +describe("Meta Responses threading", () => { + it("threads the latest response and applies serial computer-use defaults", async () => { + const { context, onPayload } = threadMetaRequest(multiTurnContext(), undefined); + expect(context.messages).toHaveLength(1); + expect(await onPayload({ input: [], include: ["reasoning.encrypted_content"] }, model)).toEqual({ + input: [], + store: true, + parallel_tool_calls: false, + previous_response_id: "resp_meta_1", + }); + }); + + it("keeps encrypted reasoning on the initial unthreaded request", async () => { + const context: Context = { messages: [{ role: "user", content: "hello", timestamp: 0 }] }; + const { onPayload } = threadMetaRequest(context, undefined); + expect(await onPayload({ input: [], include: ["reasoning.encrypted_content"] }, model)).toEqual({ + input: [], + include: ["reasoning.encrypted_content"], + store: true, + parallel_tool_calls: false, + }); + }); + + it("sends full history without previous_response_id when threading is disabled", async () => { + const context = multiTurnContext(); + const prepared = threadMetaRequest(context, { disableResponseThreading: true }); + expect(prepared.context).toBe(context); + expect(await prepared.onPayload({ input: [] }, model)).toEqual({ + input: [], + store: true, + parallel_tool_calls: false, + }); + }); + + it("reapplies protocol invariants after a caller payload hook", async () => { + const { onPayload } = threadMetaRequest(multiTurnContext(), { + onPayload: (payload) => ({ ...(payload as object), store: false, parallel_tool_calls: true, include: ["reasoning.encrypted_content", "other"] }), + }); + expect(await onPayload({}, model)).toEqual({ + store: true, + parallel_tool_calls: false, + previous_response_id: "resp_meta_1", + include: ["other"], + }); + }); +}); diff --git a/packages/ai/test/models.test.ts b/packages/ai/test/models.test.ts index f383547a..622d73df 100644 --- a/packages/ai/test/models.test.ts +++ b/packages/ai/test/models.test.ts @@ -6,6 +6,7 @@ import { formatCuaModelRef, getCuaModel, listCuaModels, + meta, openai, parseCuaModelRef, } from "../src/index"; @@ -24,7 +25,7 @@ describe("CUA model refs", () => { it("names the valid providers in the unsupported-provider error", () => { expect(() => parseCuaModelRef("bogus:model")).toThrow( - 'unsupported CUA provider "bogus" (expected one of: openai, anthropic, google, tzafon, yutori)', + 'unsupported CUA provider "bogus" (expected one of: openai, anthropic, google, meta, tzafon, yutori)', ); }); @@ -50,6 +51,14 @@ describe("CUA model refs", () => { const model = getCuaModel("yutori:n1.5-latest"); expect(model.provider).toBe("yutori"); expect(model.api).toBe("yutori-chat-completions"); + + const muse = getCuaModel("meta:muse-spark-1.1"); + expect(muse.provider).toBe("meta"); + expect(muse.api).toBe(meta.META_RESPONSES_API); + expect(muse.baseUrl).toBe("https://api.meta.ai/v1"); + expect(muse.contextWindow).toBe(1_048_576); + expect(muse.maxTokens).toBe(128_000); + expect(muse.thinkingLevelMap?.off).toBeNull(); }); it("loads supported custom provider models without explicit registration", () => { @@ -57,11 +66,12 @@ describe("CUA model refs", () => { expect(getCuaModel("yutori:n1.5-latest").api).toBe("yutori-chat-completions"); }); - it("routes OpenAI CUA models to the threading-aware openai-cua-responses api", () => { + it("routes Responses models to provider-specific threading APIs", () => { // Registry models carry pi-ai's builtin "openai-responses" api and must - // be routed to cua's own previous_response_id-threading provider. + // be routed to provider-specific previous_response_id transports. expect(getCuaModel("openai:gpt-5.5").api).toBe(openai.OPENAI_CUA_RESPONSES_API); expect(getCuaModel("openai:gpt-5.4-mini").api).toBe(openai.OPENAI_CUA_RESPONSES_API); + expect(getCuaModel("meta:muse-spark-1.1").api).toBe(meta.META_RESPONSES_API); }); it("rejects supported model IDs that are not in pi-ai or overrides", () => { @@ -115,6 +125,7 @@ describe("CUA support annotations", () => { it("matches exact-id annotations", () => { expect(findCuaAnnotation("google", "gemini-3-flash-preview")).toBeDefined(); + expect(findCuaAnnotation("meta", "muse-spark-1.1")).toBeDefined(); expect(findCuaAnnotation("google", "gemini-3.1-flash-lite")).toBeDefined(); expect(findCuaAnnotation("yutori", "n1.5-latest")).toBeDefined(); expect(findCuaAnnotation("tzafon", "tzafon.northstar-cua-fast")).toBeDefined(); diff --git a/packages/ai/test/provider-module.test.ts b/packages/ai/test/provider-module.test.ts index 88e81af8..b3f48106 100644 --- a/packages/ai/test/provider-module.test.ts +++ b/packages/ai/test/provider-module.test.ts @@ -1,11 +1,12 @@ import { describe, expect, it } from "vitest"; -import { anthropic, CUA_COMPUTER_ACTION_TYPES, CUA_PROVIDERS, type CuaProvider, gemini, openai, tzafon, yutori } from "../src/index"; +import { anthropic, CUA_COMPUTER_ACTION_TYPES, CUA_PROVIDERS, type CuaProvider, gemini, meta, openai, tzafon, yutori } from "../src/index"; import type { CuaProviderModule } from "../src/providers/common"; const MODULES: Record = { openai, anthropic, google: gemini, + meta, tzafon, yutori, }; @@ -14,6 +15,7 @@ const NAMESPACES: Record; pref openai: { namespace: openai, prefix: "OPENAI" }, anthropic: { namespace: anthropic, prefix: "ANTHROPIC" }, google: { namespace: gemini, prefix: "GEMINI" }, + meta: { namespace: meta, prefix: "META" }, tzafon: { namespace: tzafon, prefix: "TZAFON" }, yutori: { namespace: yutori, prefix: "YUTORI" }, }; @@ -51,6 +53,11 @@ describe("provider modules satisfy the uniform contract", () => { it("yutori sends no model-facing tool definitions", () => { expect(yutori.providerModule.toolDefinitions()).toEqual([]); }); + + it("meta currently restricts its normalized coordinate contract to computer mode", () => { + expect(() => meta.providerModule.toolDefinitions({ mode: "browser" })).toThrow(/computer only/); + expect(() => meta.providerModule.toolDefinitions({ mode: "hybrid" })).toThrow(/computer only/); + }); }); describe("provider namespaces export a uniform surface", () => { diff --git a/packages/ai/test/providers.test.ts b/packages/ai/test/providers.test.ts index c2b89737..8eb90faa 100644 --- a/packages/ai/test/providers.test.ts +++ b/packages/ai/test/providers.test.ts @@ -8,7 +8,7 @@ const YUTORI_CHAT_COMPLETIONS_API = yutori.YUTORI_CHAT_COMPLETIONS_API; describe("createCuaModels", () => { it("registers the CUA-only providers alongside pi's builtins", () => { const models = createCuaModels(); - for (const id of ["openai", "anthropic", "google", "tzafon", "yutori"]) { + for (const id of ["openai", "anthropic", "google", "meta", "tzafon", "yutori"]) { const provider = models.getProvider(id); expect(provider, id).toBeDefined(); expect(provider?.stream).toBeTypeOf("function"); @@ -16,8 +16,9 @@ describe("createCuaModels", () => { } }); - it("lists the tzafon and yutori catalogs on their providers", () => { + it("lists the CUA-only provider catalogs", () => { const models = createCuaModels(); + expect(models.getModel("meta", "muse-spark-1.1")?.api).toBe("meta-responses"); const tzafonIds = models.getModels("tzafon").map((m) => m.id); expect(tzafonIds).toContain("tzafon.northstar-cua-fast"); const yutoriIds = models.getModels("yutori").map((m) => m.id); diff --git a/packages/ai/test/runtime-spec.test.ts b/packages/ai/test/runtime-spec.test.ts index bd7ee938..4233339a 100644 --- a/packages/ai/test/runtime-spec.test.ts +++ b/packages/ai/test/runtime-spec.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { CUA_NAVIGATION_TOOL_NAME, CUA_PROVIDERS, getCuaModel, listCuaModels, openai, resolveCuaRuntimeSpec } from "../src/index"; +import { CUA_NAVIGATION_TOOL_NAME, CUA_PROVIDERS, getCuaModel, listCuaModels, meta, openai, resolveCuaRuntimeSpec } from "../src/index"; describe("resolveCuaRuntimeSpec", () => { it("resolves a runtime spec for every CUA provider", () => { @@ -45,9 +45,12 @@ describe("resolveCuaRuntimeSpec", () => { expect(anthropicSpec.onPayload).toBeTypeOf("function"); }); - it("routes a concrete OpenAI Model input (not just a string ref) to openai-cua-responses", () => { - const raw = { ...getCuaModel("openai:gpt-5.5"), api: "openai-responses" } as Parameters[0]; - expect(resolveCuaRuntimeSpec(raw).model.api).toBe(openai.OPENAI_CUA_RESPONSES_API); + it("routes concrete Responses models to provider-specific APIs", () => { + const openaiModel = { ...getCuaModel("openai:gpt-5.5"), api: "openai-responses" } as Parameters[0]; + expect(resolveCuaRuntimeSpec(openaiModel).model.api).toBe(openai.OPENAI_CUA_RESPONSES_API); + + const metaModel = { ...getCuaModel("meta:muse-spark-1.1"), api: "openai-responses" } as Parameters[0]; + expect(resolveCuaRuntimeSpec(metaModel).model.api).toBe(meta.META_RESPONSES_API); }); it("threads tool options through to the provider module", () => { diff --git a/packages/cli/README.md b/packages/cli/README.md index 2e122292..ae70ccf7 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -45,6 +45,7 @@ cua models -p openai cua --print --model openai:gpt-5.5 "..." cua --print --model anthropic:claude-opus-4-7 "..." cua --print --model google:gemini-3-flash-preview "..." +cua --print --model meta:muse-spark-1.1 "..." cua --print --model yutori:n1.5-latest "..." # Named sessions (browser stays alive across calls): @@ -67,8 +68,8 @@ cua --session abc12345 # by id prefix Run `cua models` to list every supported `-m` / `--model` value and the provider it routes to. Filter by provider with `cua models -p openai`, -`cua models -p anthropic`, `cua models -p google` (alias: `gemini`), or -`cua models -p yutori`. +`cua models -p anthropic`, `cua models -p google` (alias: `gemini`), +`cua models -p meta`, or `cua models -p yutori`. `-m` / `--model` accepts a provider-qualified `provider:model` ref (e.g. `openai:gpt-5.5`) or a bare model id when it matches exactly one catalog @@ -85,12 +86,15 @@ Configuration is by environment variable. There is no config file. | `ANTHROPIC_API_KEY` | Anthropic API key (required when `-m anthropic:…`) | | `GOOGLE_API_KEY` | Google API key (required when `-m google:…`) | | `GEMINI_API_KEY` | alias of `GOOGLE_API_KEY` | +| `MODEL_API_KEY` | Meta Model API key (required when `-m meta:…`) | +| `META_MODEL_API_KEY` | alias of `MODEL_API_KEY` | | `TZAFON_API_KEY` | Tzafon API key (required when `-m tzafon:…`) | | `YUTORI_API_KEY` | Yutori API key (required when `-m yutori:…`) | | `KERNEL_BASE_URL` | override Kernel base URL | | `OPENAI_BASE_URL` | override OpenAI base URL | | `ANTHROPIC_BASE_URL` | override Anthropic base URL | | `GOOGLE_BASE_URL` | override Google base URL | +| `META_BASE_URL` | override Meta Model API base URL | | `TZAFON_BASE_URL` | override Tzafon base URL | | `YUTORI_BASE_URL` | override Yutori base URL | | `XDG_DATA_HOME` | sessions dir base (defaults to `~/.local/share`) | diff --git a/packages/cli/src/cli-harness.ts b/packages/cli/src/cli-harness.ts index 2de6d209..64d2d2eb 100644 --- a/packages/cli/src/cli-harness.ts +++ b/packages/cli/src/cli-harness.ts @@ -60,7 +60,7 @@ Usage: cua models --json Options: - -p, --provider Filter by provider: openai | anthropic | google | gemini | tzafon | yutori + -p, --provider Filter by provider: openai | anthropic | google | gemini | meta | tzafon | yutori --json Output JSON -h, --help Show this help `; diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index e474b565..569338f2 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -53,6 +53,7 @@ Options: openai: openai:gpt-5.5 anthropic: anthropic:claude-opus-4-7 google: google:gemini-3-flash-preview + meta: meta:muse-spark-1.1 tzafon: tzafon:tzafon.northstar-cua-fast yutori: yutori:n1.5-latest --thinking Thinking level: off | minimal | low | medium | high | xhigh @@ -101,12 +102,15 @@ Environment: ANTHROPIC_API_KEY Anthropic API key (required when -m anthropic:…) GOOGLE_API_KEY Google API key (required when -m google:…) GEMINI_API_KEY Alias for GOOGLE_API_KEY + MODEL_API_KEY Meta Model API key (required when -m meta:…) + META_MODEL_API_KEY Alias for MODEL_API_KEY TZAFON_API_KEY Tzafon API key (required when -m tzafon:…) YUTORI_API_KEY Yutori API key (required when -m yutori:…) KERNEL_BASE_URL Override Kernel base URL OPENAI_BASE_URL Override OpenAI base URL ANTHROPIC_BASE_URL Override Anthropic base URL GOOGLE_BASE_URL Override Google base URL + META_BASE_URL Override Meta Model API base URL TZAFON_BASE_URL Override Tzafon base URL YUTORI_BASE_URL Override Yutori base URL XDG_DATA_HOME Sessions are stored under \$XDG_DATA_HOME/cua/sessions diff --git a/packages/cli/src/harness.ts b/packages/cli/src/harness.ts index 2ffdf30f..41cda4b3 100644 --- a/packages/cli/src/harness.ts +++ b/packages/cli/src/harness.ts @@ -45,7 +45,7 @@ export interface BuildCuaHarnessOptions { models?: Models; /** Maximum tool-result images included from message history per provider request. */ toolResultImageReplayLimit?: CuaAgentHarnessOptions["toolResultImageReplayLimit"]; - /** Chain OpenAI and Tzafon requests through provider-stored response state. Defaults to true. */ + /** Chain OpenAI, Meta, and Tzafon requests through provider-stored response state. Defaults to true. */ responseThreading?: CuaAgentHarnessOptions["responseThreading"]; /** Optional CUA-level retries around each provider request. Disabled by default. */ retry?: CuaAgentHarnessOptions["retry"]; diff --git a/packages/cli/test/cli-executor.test.ts b/packages/cli/test/cli-executor.test.ts index 6cfb791c..af9fbbae 100644 --- a/packages/cli/test/cli-executor.test.ts +++ b/packages/cli/test/cli-executor.test.ts @@ -22,6 +22,8 @@ const PROVIDER_ENV_KEYS = [ "ANTHROPIC_API_KEY", "GOOGLE_API_KEY", "GEMINI_API_KEY", + "MODEL_API_KEY", + "META_MODEL_API_KEY", "TZAFON_API_KEY", "YUTORI_API_KEY", ]; diff --git a/packages/cli/test/harness-models.test.ts b/packages/cli/test/harness-models.test.ts index 9354f364..d58e852e 100644 --- a/packages/cli/test/harness-models.test.ts +++ b/packages/cli/test/harness-models.test.ts @@ -9,16 +9,22 @@ describe("resolveCuaModelRef", () => { it("passes provider-qualified refs through", () => { expect(resolveCuaModelRef("openai:gpt-5.5")).toBe("openai:gpt-5.5"); + expect(resolveCuaModelRef("meta:muse-spark-1.1")).toBe("meta:muse-spark-1.1"); }); it("accepts bare ids when they match exactly one catalog entry", () => { expect(resolveCuaModelRef("gpt-5.5")).toBe("openai:gpt-5.5"); + expect(resolveCuaModelRef("muse-spark-1.1")).toBe("meta:muse-spark-1.1"); }); it("throws on unknown bare ids", () => { expect(() => resolveCuaModelRef("does-not-exist")).toThrow(/unknown model/); }); + it("filters the Meta catalog", () => { + expect(listSupportedModels("meta").map((model) => model.ref)).toEqual(["meta:muse-spark-1.1"]); + }); + it("treats 'gemini' as an alias for google when filtering", () => { const fromGemini = listSupportedModels("gemini"); const fromGoogle = listSupportedModels("google"); From a6ce06fc0a6ded457356eed09c2cbd67360ee4a9 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:21:06 +0000 Subject: [PATCH 02/15] Prefer scoped Meta API key name --- .agents/skills/update-models/SKILL.md | 6 +++--- .agents/skills/update-models/reference/README.md | 2 +- .../skills/update-models/reference/discover-models.ts | 2 +- .github/workflows/ci.yml | 4 ++-- README.md | 2 +- packages/agent/README.md | 2 +- packages/agent/test/e2e.live.test.ts | 2 +- packages/ai/README.md | 2 +- packages/ai/examples/quickstart.ts | 2 +- packages/ai/src/api-keys.ts | 2 +- packages/ai/test/api-keys.test.ts | 10 +++++----- packages/ai/test/computer-tool.integration.test.ts | 8 ++++---- packages/cli/README.md | 4 ++-- packages/cli/src/cli.ts | 4 ++-- 14 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.agents/skills/update-models/SKILL.md b/.agents/skills/update-models/SKILL.md index 6e7d6f5a..7cb292a1 100644 --- a/.agents/skills/update-models/SKILL.md +++ b/.agents/skills/update-models/SKILL.md @@ -9,14 +9,14 @@ Use this workflow to keep CUA current with provider model releases and computer- ## Quick Start -1. Verify credentials are available: `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY` or `GEMINI_API_KEY`, `MODEL_API_KEY` or `META_MODEL_API_KEY`, `TZAFON_API_KEY`, and `YUTORI_API_KEY`. +1. Verify credentials are available: `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY` or `GEMINI_API_KEY`, `META_MODEL_API_KEY` or `MODEL_API_KEY`, `TZAFON_API_KEY`, and `YUTORI_API_KEY`. 2. If credentials live in `~/AGENTS.md`, load them into the current shell without printing them: ```bash eval "$(python3 - <<'PY' import pathlib, re, shlex text = pathlib.Path('~/AGENTS.md').expanduser().read_text() -for key in ['OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'GOOGLE_API_KEY', 'MODEL_API_KEY', 'TZAFON_API_KEY', 'YUTORI_API_KEY']: +for key in ['OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'GOOGLE_API_KEY', 'META_MODEL_API_KEY', 'TZAFON_API_KEY', 'YUTORI_API_KEY']: m = re.search(r'export\s+' + re.escape(key) + r'=(?:"([^"]+)"|([^\s\n]+))', text) if m: print(f'export {key}={shlex.quote(m.group(1) or m.group(2))}') @@ -70,7 +70,7 @@ When live discovery finds a new model with passing smoke tests, update `packages Meta: -- Discover with the OpenAI SDK against `https://api.meta.ai/v1` using `MODEL_API_KEY` (or `META_MODEL_API_KEY`). +- Discover with the OpenAI SDK against `https://api.meta.ai/v1` using `META_MODEL_API_KEY` (or Meta's documented `MODEL_API_KEY` alias). - Smoke-test the Responses API with screenshot input and explicit function tools matching CUA's canonical actions. - Pass condition: response output contains a `function_call` for one of the supplied browser actions. - Use `store: true` plus `previous_response_id` for CUA tool loops. Meta rejects `include: ["reasoning.encrypted_content"]` on requests that set `previous_response_id`. diff --git a/.agents/skills/update-models/reference/README.md b/.agents/skills/update-models/reference/README.md index af11c833..b7af4cec 100644 --- a/.agents/skills/update-models/reference/README.md +++ b/.agents/skills/update-models/reference/README.md @@ -11,7 +11,7 @@ These scripts support the `update-models` skill. Run them from the repository ro - `OPENAI_API_KEY` - `ANTHROPIC_API_KEY` - `GOOGLE_API_KEY` or `GEMINI_API_KEY` - - `MODEL_API_KEY` or `META_MODEL_API_KEY` + - `META_MODEL_API_KEY` or `MODEL_API_KEY` - `TZAFON_API_KEY` - `YUTORI_API_KEY` diff --git a/.agents/skills/update-models/reference/discover-models.ts b/.agents/skills/update-models/reference/discover-models.ts index d1c853c8..2b186ce1 100644 --- a/.agents/skills/update-models/reference/discover-models.ts +++ b/.agents/skills/update-models/reference/discover-models.ts @@ -177,7 +177,7 @@ async function discoverOpenAI(args: Args): Promise> { async function discoverMeta(args: Args): Promise> { const OpenAI = await importDefault("openai", "OpenAI"); - const apiKey = process.env.MODEL_API_KEY ?? process.env.META_MODEL_API_KEY; + const apiKey = process.env.META_MODEL_API_KEY ?? process.env.MODEL_API_KEY; const client = new OpenAI({ apiKey, baseURL: "https://api.meta.ai/v1" }); const rawModels = await collectAsync(client.models.list()); const models: ModelResult[] = rawModels.map((m) => ({ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbe507dd..602749a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,7 +132,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - MODEL_API_KEY: ${{ secrets.MODEL_API_KEY }} + META_MODEL_API_KEY: ${{ secrets.META_MODEL_API_KEY }} TZAFON_API_KEY: ${{ secrets.TZAFON_API_KEY }} YUTORI_API_KEY: ${{ secrets.YUTORI_API_KEY }} run: npm run test:integration --workspace @onkernel/cua-ai @@ -160,7 +160,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - MODEL_API_KEY: ${{ secrets.MODEL_API_KEY }} + META_MODEL_API_KEY: ${{ secrets.META_MODEL_API_KEY }} TZAFON_API_KEY: ${{ secrets.TZAFON_API_KEY }} YUTORI_API_KEY: ${{ secrets.YUTORI_API_KEY }} run: npm test --workspace @onkernel/cua-agent -- test/e2e.live.test.ts diff --git a/README.md b/README.md index e049d6eb..8e3be1fa 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ npx tsx packages/cli/src/cli.ts --help export OPENAI_API_KEY=sk-... # for gpt-5.5 export ANTHROPIC_API_KEY=sk-ant-... # for claude-opus-4-7 export GOOGLE_API_KEY=... # for gemini-3-flash-preview -export MODEL_API_KEY=... # for muse-spark-1.1 +export META_MODEL_API_KEY=... # for muse-spark-1.1 export YUTORI_API_KEY=yt_... # for n1.5-latest export KERNEL_API_KEY=sk_... # always required diff --git a/packages/agent/README.md b/packages/agent/README.md index c88cc5a7..fddc97f3 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -192,7 +192,7 @@ If auth callbacks are omitted, both classes default to CUA env var conventions: - OpenAI: `OPENAI_API_KEY` - Anthropic: `ANTHROPIC_OAUTH_TOKEN` or `ANTHROPIC_API_KEY` - Gemini: `GOOGLE_API_KEY` or `GEMINI_API_KEY` -- Meta: `MODEL_API_KEY` or `META_MODEL_API_KEY` +- Meta: `META_MODEL_API_KEY` or `MODEL_API_KEY` - Tzafon: `TZAFON_API_KEY` - Yutori: `YUTORI_API_KEY` diff --git a/packages/agent/test/e2e.live.test.ts b/packages/agent/test/e2e.live.test.ts index 3e932c8e..7fb47e6e 100644 --- a/packages/agent/test/e2e.live.test.ts +++ b/packages/agent/test/e2e.live.test.ts @@ -82,7 +82,7 @@ const cases: ProviderCase[] = [ }, { name: "meta", - apiKeyEnvVar: "MODEL_API_KEY", + apiKeyEnvVar: "META_MODEL_API_KEY", modelRef: "meta:muse-spark-1.1", prompt: [ "Use the tool named `screenshot` exactly once to inspect the browser.", diff --git a/packages/ai/README.md b/packages/ai/README.md index 15a0422f..a07c0858 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -20,7 +20,7 @@ check these environment variables, in order: | `openai` | `OPENAI_API_KEY` | | `anthropic` | `ANTHROPIC_OAUTH_TOKEN`, `ANTHROPIC_API_KEY` | | `google` | `GOOGLE_API_KEY`, `GEMINI_API_KEY` | -| `meta` | `MODEL_API_KEY`, `META_MODEL_API_KEY` | +| `meta` | `META_MODEL_API_KEY`, `MODEL_API_KEY` | | `tzafon` | `TZAFON_API_KEY` | | `yutori` | `YUTORI_API_KEY` | diff --git a/packages/ai/examples/quickstart.ts b/packages/ai/examples/quickstart.ts index b9bb4f77..431a998e 100644 --- a/packages/ai/examples/quickstart.ts +++ b/packages/ai/examples/quickstart.ts @@ -10,7 +10,7 @@ import { // Switch providers by setting CUA_MODEL (and the matching API key env var): // anthropic:claude-opus-4-7 ANTHROPIC_API_KEY // google:gemini-3-flash-preview GOOGLE_API_KEY -// meta:muse-spark-1.1 MODEL_API_KEY +// meta:muse-spark-1.1 META_MODEL_API_KEY // tzafon:tzafon.northstar-cua-fast TZAFON_API_KEY // yutori:n1.5-latest YUTORI_API_KEY const modelRef = (process.env.CUA_MODEL ?? "openai:gpt-5.5") as CuaModelRef; diff --git a/packages/ai/src/api-keys.ts b/packages/ai/src/api-keys.ts index c92b9590..98481574 100644 --- a/packages/ai/src/api-keys.ts +++ b/packages/ai/src/api-keys.ts @@ -12,7 +12,7 @@ const CUA_PROVIDER_API_KEY_ENV_VARS: Record = { openai: ["OPENAI_API_KEY"], anthropic: ["ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY"], google: ["GOOGLE_API_KEY", "GEMINI_API_KEY"], - meta: ["MODEL_API_KEY", "META_MODEL_API_KEY"], + meta: ["META_MODEL_API_KEY", "MODEL_API_KEY"], tzafon: ["TZAFON_API_KEY"], yutori: ["YUTORI_API_KEY"], }; diff --git a/packages/ai/test/api-keys.test.ts b/packages/ai/test/api-keys.test.ts index b4b553a2..c17f7c4c 100644 --- a/packages/ai/test/api-keys.test.ts +++ b/packages/ai/test/api-keys.test.ts @@ -33,7 +33,7 @@ describe("cua api key helpers", () => { expect(cuaApiKeyEnvVarsForProvider("openai")).toEqual(["OPENAI_API_KEY"]); expect(cuaApiKeyEnvVarsForProvider("google")).toEqual(["GOOGLE_API_KEY", "GEMINI_API_KEY"]); expect(cuaApiKeyEnvVarsForProvider("gemini")).toEqual(["GOOGLE_API_KEY", "GEMINI_API_KEY"]); - expect(cuaApiKeyEnvVarsForProvider("meta")).toEqual(["MODEL_API_KEY", "META_MODEL_API_KEY"]); + expect(cuaApiKeyEnvVarsForProvider("meta")).toEqual(["META_MODEL_API_KEY", "MODEL_API_KEY"]); expect(cuaApiKeyEnvVarsForProvider("unknown")).toEqual([]); }); @@ -48,10 +48,10 @@ describe("cua api key helpers", () => { it("resolves keys from model refs", () => { process.env.OPENAI_API_KEY = "openai"; expect(getCuaEnvApiKeyForModel("openai:gpt-5.5")).toBe("openai"); - delete process.env.MODEL_API_KEY; - process.env.META_MODEL_API_KEY = "meta-fallback"; - expect(getCuaEnvApiKeyForModel("meta:muse-spark-1.1")).toBe("meta-fallback"); - process.env.MODEL_API_KEY = "meta"; + delete process.env.META_MODEL_API_KEY; + process.env.MODEL_API_KEY = "meta-official-alias"; + expect(getCuaEnvApiKeyForModel("meta:muse-spark-1.1")).toBe("meta-official-alias"); + process.env.META_MODEL_API_KEY = "meta"; expect(getCuaEnvApiKeyForModel("meta:muse-spark-1.1")).toBe("meta"); }); diff --git a/packages/ai/test/computer-tool.integration.test.ts b/packages/ai/test/computer-tool.integration.test.ts index e85e2d77..be77619c 100644 --- a/packages/ai/test/computer-tool.integration.test.ts +++ b/packages/ai/test/computer-tool.integration.test.ts @@ -57,7 +57,7 @@ const cases: ProviderCase[] = [ }, { provider: "meta", - envVar: "MODEL_API_KEY", + envVar: "META_MODEL_API_KEY", modelRef: "meta:muse-spark-1.1", tools: () => meta.computerTools({ actions: ["click"] }), coordinateRange: [0, 1000], @@ -149,7 +149,7 @@ describe("individual computer action integration", () => { }, 60_000); } - const metaHasKey = !!process.env.MODEL_API_KEY; + const metaHasKey = !!process.env.META_MODEL_API_KEY; (metaHasKey ? it : it.skip)( "meta continues a screenshot tool loop with previous_response_id", async () => { @@ -171,7 +171,7 @@ describe("individual computer action integration", () => { tools, }; const first = await cuaModels().complete(model, context, { - apiKey: process.env.MODEL_API_KEY, + apiKey: process.env.META_MODEL_API_KEY, maxTokens: 1024, }); const click = first.content.find((part) => part.type === "toolCall" && part.name === "click"); @@ -191,7 +191,7 @@ describe("individual computer action integration", () => { let payload: Record | undefined; const second = await cuaModels().complete(model, context, { - apiKey: process.env.MODEL_API_KEY, + apiKey: process.env.META_MODEL_API_KEY, maxTokens: 1024, onPayload: (value) => { payload = value as Record; diff --git a/packages/cli/README.md b/packages/cli/README.md index ae70ccf7..da0df71b 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -86,8 +86,8 @@ Configuration is by environment variable. There is no config file. | `ANTHROPIC_API_KEY` | Anthropic API key (required when `-m anthropic:…`) | | `GOOGLE_API_KEY` | Google API key (required when `-m google:…`) | | `GEMINI_API_KEY` | alias of `GOOGLE_API_KEY` | -| `MODEL_API_KEY` | Meta Model API key (required when `-m meta:…`) | -| `META_MODEL_API_KEY` | alias of `MODEL_API_KEY` | +| `META_MODEL_API_KEY` | Meta Model API key (required when `-m meta:…`) | +| `MODEL_API_KEY` | alias accepted for Meta's official examples | | `TZAFON_API_KEY` | Tzafon API key (required when `-m tzafon:…`) | | `YUTORI_API_KEY` | Yutori API key (required when `-m yutori:…`) | | `KERNEL_BASE_URL` | override Kernel base URL | diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 569338f2..5754d071 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -102,8 +102,8 @@ Environment: ANTHROPIC_API_KEY Anthropic API key (required when -m anthropic:…) GOOGLE_API_KEY Google API key (required when -m google:…) GEMINI_API_KEY Alias for GOOGLE_API_KEY - MODEL_API_KEY Meta Model API key (required when -m meta:…) - META_MODEL_API_KEY Alias for MODEL_API_KEY + META_MODEL_API_KEY Meta Model API key (required when -m meta:…) + MODEL_API_KEY Alias accepted for Meta's official examples TZAFON_API_KEY Tzafon API key (required when -m tzafon:…) YUTORI_API_KEY Yutori API key (required when -m yutori:…) KERNEL_BASE_URL Override Kernel base URL From a48624c43544aa74500b3f882e8c25771c6428f5 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:27:54 +0000 Subject: [PATCH 03/15] Harden Meta live test setup --- packages/agent/test/e2e.live.test.ts | 20 +++++++++------ .../ai/test/computer-tool.integration.test.ts | 25 +++++++++++++------ 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/packages/agent/test/e2e.live.test.ts b/packages/agent/test/e2e.live.test.ts index 7fb47e6e..59ca46b6 100644 --- a/packages/agent/test/e2e.live.test.ts +++ b/packages/agent/test/e2e.live.test.ts @@ -140,11 +140,17 @@ type RunStats = { assistantErrors: string[]; }; +function apiKeyForCase(c: ProviderCase): string | undefined { + return c.name === "meta" + ? process.env.META_MODEL_API_KEY ?? process.env.MODEL_API_KEY + : process.env[c.apiKeyEnvVar]; +} + function shouldRunCase(c: ProviderCase): boolean { if (!LIVE) return false; if (!KERNEL_API_KEY) return false; if (c.ciOptInEnvVar && process.env.CI && process.env[c.ciOptInEnvVar] !== "1") return false; - return Boolean(process.env[c.apiKeyEnvVar]); + return Boolean(apiKeyForCase(c)); } function shouldRunSwitchCase(c: ModelSwitchCase): boolean { @@ -244,7 +250,7 @@ describe("Cua live e2e", () => { const agent = new CuaAgent({ browser, client, - getApiKey: () => process.env[c.apiKeyEnvVar], + getApiKey: () => apiKeyForCase(c), afterToolCall: async () => ({ terminate: true }), initialState: { model: c.modelRef, @@ -272,7 +278,7 @@ describe("Cua live e2e", () => { client, model: c.modelRef, getApiKeyAndHeaders: async () => { - const apiKey = process.env[c.apiKeyEnvVar]; + const apiKey = apiKeyForCase(c); return apiKey ? { apiKey } : undefined; }, }); @@ -302,8 +308,8 @@ describe("Cua live e2e", () => { browser, client, getApiKey: (provider) => { - if (provider === c.from.modelRef.split(":")[0]) return process.env[c.from.apiKeyEnvVar]; - if (provider === c.to.modelRef.split(":")[0]) return process.env[c.to.apiKeyEnvVar]; + if (provider === c.from.modelRef.split(":")[0]) return apiKeyForCase(c.from); + if (provider === c.to.modelRef.split(":")[0]) return apiKeyForCase(c.to); return undefined; }, initialState: { @@ -338,11 +344,11 @@ describe("Cua live e2e", () => { model: c.from.modelRef, getApiKeyAndHeaders: async (model) => { if (model.provider === c.from.modelRef.split(":")[0]) { - const apiKey = process.env[c.from.apiKeyEnvVar]; + const apiKey = apiKeyForCase(c.from); return apiKey ? { apiKey } : undefined; } if (model.provider === c.to.modelRef.split(":")[0]) { - const apiKey = process.env[c.to.apiKeyEnvVar]; + const apiKey = apiKeyForCase(c.to); return apiKey ? { apiKey } : undefined; } return undefined; diff --git a/packages/ai/test/computer-tool.integration.test.ts b/packages/ai/test/computer-tool.integration.test.ts index be77619c..1f879080 100644 --- a/packages/ai/test/computer-tool.integration.test.ts +++ b/packages/ai/test/computer-tool.integration.test.ts @@ -1,3 +1,4 @@ +import { existsSync } from "node:fs"; import { readFile } from "node:fs/promises"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -9,6 +10,7 @@ import { anthropic, cuaModels, gemini, + getCuaEnvApiKey, getCuaModel, meta, openai, @@ -16,7 +18,12 @@ import { yutori, } from "../src/index"; -const screenshotPath = join(process.cwd(), "examples", "screenshot.png"); +const screenshotPath = [ + join(process.cwd(), "examples", "screenshot.png"), + join(process.cwd(), "packages", "ai", "examples", "screenshot.png"), +].find(existsSync); + +if (!screenshotPath) throw new Error("could not find packages/ai/examples/screenshot.png"); interface ProviderCase { provider: CuaProvider; @@ -112,9 +119,13 @@ async function buildYutoriContext(): Promise { }; } +function apiKeyForCase(c: ProviderCase): string | undefined { + return c.provider === "meta" ? getCuaEnvApiKey("meta") : process.env[c.envVar]; +} + describe("individual computer action integration", () => { for (const c of cases) { - const hasKey = !!process.env[c.envVar]; + const hasKey = !!apiKeyForCase(c); const ciEnabled = !c.ciOptInEnvVar || !process.env.CI || process.env[c.ciOptInEnvVar] === "1"; const test = hasKey ? it : it.skip; @@ -122,7 +133,7 @@ describe("individual computer action integration", () => { const model = getCuaModel(c.modelRef as never); const context = await buildContext(c.tools); const response = await cuaModels().complete(model, context, { - apiKey: process.env[c.envVar], + apiKey: apiKeyForCase(c), maxTokens: 1024, ...c.extraOptions, }); @@ -149,8 +160,8 @@ describe("individual computer action integration", () => { }, 60_000); } - const metaHasKey = !!process.env.META_MODEL_API_KEY; - (metaHasKey ? it : it.skip)( + const metaApiKey = getCuaEnvApiKey("meta"); + (metaApiKey ? it : it.skip)( "meta continues a screenshot tool loop with previous_response_id", async () => { const screenshot = await readFile(screenshotPath); @@ -171,7 +182,7 @@ describe("individual computer action integration", () => { tools, }; const first = await cuaModels().complete(model, context, { - apiKey: process.env.META_MODEL_API_KEY, + apiKey: metaApiKey, maxTokens: 1024, }); const click = first.content.find((part) => part.type === "toolCall" && part.name === "click"); @@ -191,7 +202,7 @@ describe("individual computer action integration", () => { let payload: Record | undefined; const second = await cuaModels().complete(model, context, { - apiKey: process.env.META_MODEL_API_KEY, + apiKey: metaApiKey, maxTokens: 1024, onPayload: (value) => { payload = value as Record; From 63646c970bbf272b36a5e7cef0a4b61cd8fcb7d3 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:40:45 +0000 Subject: [PATCH 04/15] Address Meta provider review findings --- .agents/skills/update-models/SKILL.md | 2 +- packages/ai/src/providers/meta/index.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.agents/skills/update-models/SKILL.md b/.agents/skills/update-models/SKILL.md index 7cb292a1..47612dde 100644 --- a/.agents/skills/update-models/SKILL.md +++ b/.agents/skills/update-models/SKILL.md @@ -16,7 +16,7 @@ Use this workflow to keep CUA current with provider model releases and computer- eval "$(python3 - <<'PY' import pathlib, re, shlex text = pathlib.Path('~/AGENTS.md').expanduser().read_text() -for key in ['OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'GOOGLE_API_KEY', 'META_MODEL_API_KEY', 'TZAFON_API_KEY', 'YUTORI_API_KEY']: +for key in ['OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'GOOGLE_API_KEY', 'META_MODEL_API_KEY', 'MODEL_API_KEY', 'TZAFON_API_KEY', 'YUTORI_API_KEY']: m = re.search(r'export\s+' + re.escape(key) + r'=(?:"([^"]+)"|([^\s\n]+))', text) if m: print(f'export {key}={shlex.quote(m.group(1) or m.group(2))}') diff --git a/packages/ai/src/providers/meta/index.ts b/packages/ai/src/providers/meta/index.ts index 5d08b047..b54df032 100644 --- a/packages/ai/src/providers/meta/index.ts +++ b/packages/ai/src/providers/meta/index.ts @@ -29,6 +29,7 @@ export function coordinateSystem(): ComputerToolCoordinateSystem { export const META_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Coordinates are normalized from 0 to 1000 relative to the most recent screenshot, with (0, 0) at the top left and (1000, 1000) at the bottom right. Base each action on the latest observed state and request a screenshot when you need a fresh view.`; +/** Build the default system prompt used with Meta CUA models. */ export function buildMetaSystemPrompt(opts: { suffix?: string } = {}): string { return [META_COMPUTER_INSTRUCTIONS, opts.suffix].filter(Boolean).join("\n\n"); } From 350314272b55b6807a86b269b178d7ccfb090b46 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:47:10 +0000 Subject: [PATCH 05/15] Resolve Meta smoke fixture across workspaces --- .agents/skills/update-models/reference/discover-models.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.agents/skills/update-models/reference/discover-models.ts b/.agents/skills/update-models/reference/discover-models.ts index 2b186ce1..42da9c70 100644 --- a/.agents/skills/update-models/reference/discover-models.ts +++ b/.agents/skills/update-models/reference/discover-models.ts @@ -1,4 +1,5 @@ #!/usr/bin/env tsx +import { existsSync } from "node:fs"; import { readFile, writeFile } from "node:fs/promises"; import { homedir } from "node:os"; import { join } from "node:path"; @@ -201,7 +202,12 @@ async function discoverMeta(args: Args): Promise> { async function smokeMeta(client: any, model: string): Promise { try { - const screenshot = await readFile(join(process.cwd(), "packages", "ai", "examples", "screenshot.png")); + const screenshotPath = [ + join(process.cwd(), "examples", "screenshot.png"), + join(process.cwd(), "packages", "ai", "examples", "screenshot.png"), + ].find(existsSync); + if (!screenshotPath) throw new Error("could not find packages/ai/examples/screenshot.png"); + const screenshot = await readFile(screenshotPath); const response = await client.responses.create({ model, store: false, From 3758e3f392d3e50b45eda1545cda658804754bb8 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 12 Jul 2026 18:35:28 +0000 Subject: [PATCH 06/15] Simplify Meta provider integration --- .agents/skills/update-models/SKILL.md | 6 ++-- .../skills/update-models/reference/README.md | 2 +- .../reference/discover-models.ts | 6 ++-- .../reference/provider-doc-drift.ts | 36 ++++++++++--------- .github/workflows/ci.yml | 4 +-- README.md | 2 +- package-lock.json | 14 ++++++++ package.json | 1 + packages/agent/README.md | 2 +- packages/agent/test/e2e.live.test.ts | 6 ++-- packages/ai/README.md | 2 +- packages/ai/docs/supported-models.md | 6 ++-- packages/ai/examples/quickstart.ts | 2 +- packages/ai/src/api-keys.ts | 2 +- packages/ai/src/models.ts | 3 ++ packages/ai/src/providers/meta/index.ts | 2 +- packages/ai/src/providers/meta/provider.ts | 33 ++++++----------- packages/ai/test/api-keys.test.ts | 10 ++---- .../ai/test/computer-tool.integration.test.ts | 2 +- packages/ai/test/meta-threading.test.ts | 18 +++++----- packages/cli/README.md | 3 +- packages/cli/src/cli.ts | 3 +- packages/cli/test/cli-executor.test.ts | 3 +- 23 files changed, 82 insertions(+), 86 deletions(-) diff --git a/.agents/skills/update-models/SKILL.md b/.agents/skills/update-models/SKILL.md index 47612dde..fe6ff729 100644 --- a/.agents/skills/update-models/SKILL.md +++ b/.agents/skills/update-models/SKILL.md @@ -9,14 +9,14 @@ Use this workflow to keep CUA current with provider model releases and computer- ## Quick Start -1. Verify credentials are available: `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY` or `GEMINI_API_KEY`, `META_MODEL_API_KEY` or `MODEL_API_KEY`, `TZAFON_API_KEY`, and `YUTORI_API_KEY`. +1. Verify credentials are available: `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY` or `GEMINI_API_KEY`, `META_API_KEY`, `TZAFON_API_KEY`, and `YUTORI_API_KEY`. 2. If credentials live in `~/AGENTS.md`, load them into the current shell without printing them: ```bash eval "$(python3 - <<'PY' import pathlib, re, shlex text = pathlib.Path('~/AGENTS.md').expanduser().read_text() -for key in ['OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'GOOGLE_API_KEY', 'META_MODEL_API_KEY', 'MODEL_API_KEY', 'TZAFON_API_KEY', 'YUTORI_API_KEY']: +for key in ['OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'GOOGLE_API_KEY', 'META_API_KEY', 'TZAFON_API_KEY', 'YUTORI_API_KEY']: m = re.search(r'export\s+' + re.escape(key) + r'=(?:"([^"]+)"|([^\s\n]+))', text) if m: print(f'export {key}={shlex.quote(m.group(1) or m.group(2))}') @@ -70,7 +70,7 @@ When live discovery finds a new model with passing smoke tests, update `packages Meta: -- Discover with the OpenAI SDK against `https://api.meta.ai/v1` using `META_MODEL_API_KEY` (or Meta's documented `MODEL_API_KEY` alias). +- Discover with the OpenAI SDK against `https://api.meta.ai/v1` using `META_API_KEY`. - Smoke-test the Responses API with screenshot input and explicit function tools matching CUA's canonical actions. - Pass condition: response output contains a `function_call` for one of the supplied browser actions. - Use `store: true` plus `previous_response_id` for CUA tool loops. Meta rejects `include: ["reasoning.encrypted_content"]` on requests that set `previous_response_id`. diff --git a/.agents/skills/update-models/reference/README.md b/.agents/skills/update-models/reference/README.md index b7af4cec..686fa65f 100644 --- a/.agents/skills/update-models/reference/README.md +++ b/.agents/skills/update-models/reference/README.md @@ -11,7 +11,7 @@ These scripts support the `update-models` skill. Run them from the repository ro - `OPENAI_API_KEY` - `ANTHROPIC_API_KEY` - `GOOGLE_API_KEY` or `GEMINI_API_KEY` - - `META_MODEL_API_KEY` or `MODEL_API_KEY` + - `META_API_KEY` - `TZAFON_API_KEY` - `YUTORI_API_KEY` diff --git a/.agents/skills/update-models/reference/discover-models.ts b/.agents/skills/update-models/reference/discover-models.ts index 42da9c70..8984a138 100644 --- a/.agents/skills/update-models/reference/discover-models.ts +++ b/.agents/skills/update-models/reference/discover-models.ts @@ -4,6 +4,7 @@ import { readFile, writeFile } from "node:fs/promises"; import { homedir } from "node:os"; import { join } from "node:path"; import process from "node:process"; +import { parse as parseToml } from "smol-toml"; type Provider = "openai" | "anthropic" | "gemini" | "meta" | "tzafon" | "yutori"; @@ -178,7 +179,7 @@ async function discoverOpenAI(args: Args): Promise> { async function discoverMeta(args: Args): Promise> { const OpenAI = await importDefault("openai", "OpenAI"); - const apiKey = process.env.META_MODEL_API_KEY ?? process.env.MODEL_API_KEY; + const apiKey = process.env.META_API_KEY; const client = new OpenAI({ apiKey, baseURL: "https://api.meta.ai/v1" }); const rawModels = await collectAsync(client.models.list()); const models: ModelResult[] = rawModels.map((m) => ({ @@ -552,8 +553,7 @@ async function yutoriApiKey(): Promise { if (env && env.trim()) return env; const cfgPath = join(process.env.XDG_CONFIG_HOME ?? join(homedir(), ".config"), "cua", "config.toml"); try { - const { parse } = await import("smol-toml"); - const raw = parse(await readFile(cfgPath, "utf8")) as any; + const raw = parseToml(await readFile(cfgPath, "utf8")) as any; const profile = typeof raw?.default_profile === "string" ? raw.default_profile : undefined; const key = profile ? raw?.profiles?.[profile]?.yutori_api_key : undefined; if (typeof key === "string" && key.trim()) return key; diff --git a/.agents/skills/update-models/reference/provider-doc-drift.ts b/.agents/skills/update-models/reference/provider-doc-drift.ts index 7b7d10c8..b6b39e0c 100644 --- a/.agents/skills/update-models/reference/provider-doc-drift.ts +++ b/.agents/skills/update-models/reference/provider-doc-drift.ts @@ -51,6 +51,19 @@ const LOCAL_FILES: Record = { yutori: "packages/ai/src/providers/yutori/actions.ts", }; +const META_ACTION_ALIASES: Record = { + left_click: "click", + right_click: "click", + middle_click: "click", + triple_click: "click", + left_click_drag: "drag", + mouse_move: "move", + key: "keypress", + hold_key: "keypress", + left_mouse_down: "mouse_down", + left_mouse_up: "mouse_up", +}; + const ACTION_REGEXES: Record = { openai: /\b(click|double_click|scroll|type|wait|keypress|drag|move|screenshot)\b/g, anthropic: /\b(screenshot|left_click|right_click|middle_click|double_click|triple_click|left_click_drag|mouse_move|key|type|scroll|hold_key|wait|left_mouse_down|left_mouse_up|cursor_position|zoom)\b/g, @@ -115,6 +128,8 @@ async function checkProvider(provider: Provider, examples: any): Promise normalizeProviderAction(provider, action))), new Set(localActions))), - unknown_example_actions: sorted(difference(new Set(exampleActions.map((action) => normalizeProviderAction(provider, action))), new Set(localActions))), + unknown_documented_actions: sorted(difference(new Set(documentedActionsForComparison), new Set(localActions))), + unknown_example_actions: sorted(difference(new Set(exampleActionsForComparison), new Set(localActions))), response_fields_from_examples: sorted(example.response_fields ?? []), notes: notesFor(provider, documentedToolVersions, exampleToolVersions, localToolVersions), }; @@ -147,21 +162,8 @@ async function fetchText(url: string): Promise { } } -function normalizeProviderAction(provider: Provider, action: string): string { - if (provider !== "meta") return action; - const aliases: Record = { - left_click: "click", - right_click: "click", - middle_click: "click", - triple_click: "click", - left_click_drag: "drag", - mouse_move: "move", - key: "keypress", - hold_key: "keypress", - left_mouse_down: "mouse_down", - left_mouse_up: "mouse_up", - }; - return aliases[action] ?? action; +function normalizeMetaAction(action: string): string { + return META_ACTION_ALIASES[action] ?? action; } function notesFor(provider: Provider, documentedToolVersions: string[], exampleToolVersions: string[], localToolVersions: string[]): string[] { diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 602749a8..65517a57 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,7 +132,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - META_MODEL_API_KEY: ${{ secrets.META_MODEL_API_KEY }} + META_API_KEY: ${{ secrets.META_API_KEY }} TZAFON_API_KEY: ${{ secrets.TZAFON_API_KEY }} YUTORI_API_KEY: ${{ secrets.YUTORI_API_KEY }} run: npm run test:integration --workspace @onkernel/cua-ai @@ -160,7 +160,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - META_MODEL_API_KEY: ${{ secrets.META_MODEL_API_KEY }} + META_API_KEY: ${{ secrets.META_API_KEY }} TZAFON_API_KEY: ${{ secrets.TZAFON_API_KEY }} YUTORI_API_KEY: ${{ secrets.YUTORI_API_KEY }} run: npm test --workspace @onkernel/cua-agent -- test/e2e.live.test.ts diff --git a/README.md b/README.md index 8e3be1fa..97da85fd 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ npx tsx packages/cli/src/cli.ts --help export OPENAI_API_KEY=sk-... # for gpt-5.5 export ANTHROPIC_API_KEY=sk-ant-... # for claude-opus-4-7 export GOOGLE_API_KEY=... # for gemini-3-flash-preview -export META_MODEL_API_KEY=... # for muse-spark-1.1 +export META_API_KEY=... # for muse-spark-1.1 export YUTORI_API_KEY=yt_... # for n1.5-latest export KERNEL_API_KEY=sk_... # always required diff --git a/package-lock.json b/package-lock.json index 663eb65b..fdcf3859 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ ], "devDependencies": { "@types/node": "22.18.4", + "smol-toml": "^1.7.0", "tsx": "^4.21.0", "typescript": "5.9.3" }, @@ -5569,6 +5570,19 @@ "dev": true, "license": "ISC" }, + "node_modules/smol-toml": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.7.0.tgz", + "integrity": "sha512-aqVvWoyO21L23mb+drl4RmMXbf6N7FdHjAhTRA9ZBL7apWBgfWC16KjrASI+1p9GAroljyMHj6fK67i0UiTNvQ==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">= 18" + }, + "funding": { + "url": "https://github.com/sponsors/cyyynthia" + } + }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", diff --git a/package.json b/package.json index 1f5397e0..0e3e7a67 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ }, "devDependencies": { "@types/node": "22.18.4", + "smol-toml": "^1.7.0", "tsx": "^4.21.0", "typescript": "5.9.3" } diff --git a/packages/agent/README.md b/packages/agent/README.md index fddc97f3..77d3dfed 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -192,7 +192,7 @@ If auth callbacks are omitted, both classes default to CUA env var conventions: - OpenAI: `OPENAI_API_KEY` - Anthropic: `ANTHROPIC_OAUTH_TOKEN` or `ANTHROPIC_API_KEY` - Gemini: `GOOGLE_API_KEY` or `GEMINI_API_KEY` -- Meta: `META_MODEL_API_KEY` or `MODEL_API_KEY` +- Meta: `META_API_KEY` - Tzafon: `TZAFON_API_KEY` - Yutori: `YUTORI_API_KEY` diff --git a/packages/agent/test/e2e.live.test.ts b/packages/agent/test/e2e.live.test.ts index 59ca46b6..202a9498 100644 --- a/packages/agent/test/e2e.live.test.ts +++ b/packages/agent/test/e2e.live.test.ts @@ -82,7 +82,7 @@ const cases: ProviderCase[] = [ }, { name: "meta", - apiKeyEnvVar: "META_MODEL_API_KEY", + apiKeyEnvVar: "META_API_KEY", modelRef: "meta:muse-spark-1.1", prompt: [ "Use the tool named `screenshot` exactly once to inspect the browser.", @@ -141,9 +141,7 @@ type RunStats = { }; function apiKeyForCase(c: ProviderCase): string | undefined { - return c.name === "meta" - ? process.env.META_MODEL_API_KEY ?? process.env.MODEL_API_KEY - : process.env[c.apiKeyEnvVar]; + return process.env[c.apiKeyEnvVar]; } function shouldRunCase(c: ProviderCase): boolean { diff --git a/packages/ai/README.md b/packages/ai/README.md index a07c0858..f8c9739c 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -20,7 +20,7 @@ check these environment variables, in order: | `openai` | `OPENAI_API_KEY` | | `anthropic` | `ANTHROPIC_OAUTH_TOKEN`, `ANTHROPIC_API_KEY` | | `google` | `GOOGLE_API_KEY`, `GEMINI_API_KEY` | -| `meta` | `META_MODEL_API_KEY`, `MODEL_API_KEY` | +| `meta` | `META_API_KEY` | | `tzafon` | `TZAFON_API_KEY` | | `yutori` | `YUTORI_API_KEY` | diff --git a/packages/ai/docs/supported-models.md b/packages/ai/docs/supported-models.md index b7df3d3e..63fbe4cb 100644 --- a/packages/ai/docs/supported-models.md +++ b/packages/ai/docs/supported-models.md @@ -60,16 +60,14 @@ Source: [Gemini computer use docs](https://ai.google.dev/gemini-api/docs/compute ## `meta` -API: `meta-responses` · coordinates: normalized 0–1000 +Coordinates: normalized 0–1000 Exact IDs: - `muse-spark-1.1` Muse Spark uses Meta's OpenAI-compatible Responses API with ordinary function -tools. CUA stores each response and continues tool loops with -`previous_response_id`; threaded requests omit encrypted reasoning replay, -which Meta does not allow alongside a previous response ID. +tools. CUA continues tool loops through `previous_response_id`. Source: [Meta computer-use cookbook](https://dev.meta.ai/docs/getting-started/cookbook/computer-use-macos). diff --git a/packages/ai/examples/quickstart.ts b/packages/ai/examples/quickstart.ts index 431a998e..fed43c6f 100644 --- a/packages/ai/examples/quickstart.ts +++ b/packages/ai/examples/quickstart.ts @@ -10,7 +10,7 @@ import { // Switch providers by setting CUA_MODEL (and the matching API key env var): // anthropic:claude-opus-4-7 ANTHROPIC_API_KEY // google:gemini-3-flash-preview GOOGLE_API_KEY -// meta:muse-spark-1.1 META_MODEL_API_KEY +// meta:muse-spark-1.1 META_API_KEY // tzafon:tzafon.northstar-cua-fast TZAFON_API_KEY // yutori:n1.5-latest YUTORI_API_KEY const modelRef = (process.env.CUA_MODEL ?? "openai:gpt-5.5") as CuaModelRef; diff --git a/packages/ai/src/api-keys.ts b/packages/ai/src/api-keys.ts index 98481574..769599f3 100644 --- a/packages/ai/src/api-keys.ts +++ b/packages/ai/src/api-keys.ts @@ -12,7 +12,7 @@ const CUA_PROVIDER_API_KEY_ENV_VARS: Record = { openai: ["OPENAI_API_KEY"], anthropic: ["ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY"], google: ["GOOGLE_API_KEY", "GEMINI_API_KEY"], - meta: ["META_MODEL_API_KEY", "MODEL_API_KEY"], + meta: ["META_API_KEY"], tzafon: ["TZAFON_API_KEY"], yutori: ["YUTORI_API_KEY"], }; diff --git a/packages/ai/src/models.ts b/packages/ai/src/models.ts index 5827f872..df574b44 100644 --- a/packages/ai/src/models.ts +++ b/packages/ai/src/models.ts @@ -104,6 +104,7 @@ const CUA_MODEL_OVERRIDES: Record[]> = { openai: [], anthropic: [], google: [], + // pi-ai 0.80.3 predates Meta's models.dev catalog entry. meta: [cuaModel("meta", "muse-spark-1.1", "Muse Spark 1.1")], tzafon: [ cuaModel("tzafon", "tzafon.northstar-cua-fast", "Tzafon Northstar CUA Fast"), @@ -273,6 +274,8 @@ function cuaModel(provider: CuaProvider, id: string, name: string): Model { case "google": return { ...base, api: "google-generative-ai", baseUrl: "https://generativelanguage.googleapis.com/v1beta", contextWindow: 1_048_576, maxTokens: 65_536 } as Model; case "meta": + // Meta documents the 1,048,576-token context window, and its + // computer-use cookbook configures 128,000 maximum output tokens. return { ...base, api: META_RESPONSES_API, diff --git a/packages/ai/src/providers/meta/index.ts b/packages/ai/src/providers/meta/index.ts index b54df032..fd17d385 100644 --- a/packages/ai/src/providers/meta/index.ts +++ b/packages/ai/src/providers/meta/index.ts @@ -27,7 +27,7 @@ export function coordinateSystem(): ComputerToolCoordinateSystem { return { type: "normalized", range: [0, 1000] }; } -export const META_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Coordinates are normalized from 0 to 1000 relative to the most recent screenshot, with (0, 0) at the top left and (1000, 1000) at the bottom right. Base each action on the latest observed state and request a screenshot when you need a fresh view.`; +export const META_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Base each action on the latest observed state and request a screenshot when you need a fresh view.`; /** Build the default system prompt used with Meta CUA models. */ export function buildMetaSystemPrompt(opts: { suffix?: string } = {}): string { diff --git a/packages/ai/src/providers/meta/provider.ts b/packages/ai/src/providers/meta/provider.ts index f8fa59d9..b976ac84 100644 --- a/packages/ai/src/providers/meta/provider.ts +++ b/packages/ai/src/providers/meta/provider.ts @@ -22,11 +22,7 @@ export interface MetaResponsesOptions extends PiOpenAIResponsesOptions, Response type OnPayload = NonNullable; -/** - * Prepare a Meta Responses request with server-managed conversation state. - * Meta rejects reasoning.encrypted_content when previous_response_id is set, - * so threaded turns remove that include while retaining reasoning server-side. - */ +/** Prepare a Meta Responses request that continues from the latest stored response. */ export function threadMetaRequest( context: Context, options: (ResponseThreadingOptions & { onPayload?: OnPayload }) | undefined, @@ -35,28 +31,19 @@ export function threadMetaRequest( const previousResponseId = delta?.previousResponseId; const messages = previousResponseId && delta ? delta.deltaMessages : context.messages; const onPayload: OnPayload = async (payload, model) => { - const threaded = applyMetaPayloadDefaults(payload, previousResponseId); - const transformed = options?.onPayload ? ((await options.onPayload(threaded, model)) ?? threaded) : threaded; - return applyMetaPayloadDefaults(transformed, previousResponseId); + const next: Record = { + ...(payload as Record), + store: true, + parallel_tool_calls: false, + ...(previousResponseId ? { previous_response_id: previousResponseId } : {}), + }; + // CUA uses stored response state instead of stateless encrypted reasoning replay. + delete next.include; + return options?.onPayload ? ((await options.onPayload(next, model)) ?? next) : next; }; return { context: messages === context.messages ? context : { ...context, messages }, onPayload }; } -function applyMetaPayloadDefaults(payload: unknown, previousResponseId?: string): Record { - const next: Record = { - ...(payload as Record), - store: true, - parallel_tool_calls: false, - ...(previousResponseId ? { previous_response_id: previousResponseId } : {}), - }; - if (!previousResponseId || !Array.isArray(next.include)) return next; - - const include = next.include.filter((value: unknown) => value !== "reasoning.encrypted_content"); - if (include.length > 0) next.include = include; - else delete next.include; - return next; -} - // Meta implements the OpenAI Responses wire protocol. The model carries a // distinct api id so response ids can never be threaded across providers. export const streamMetaResponses: StreamFunction = (model, context, options) => { diff --git a/packages/ai/test/api-keys.test.ts b/packages/ai/test/api-keys.test.ts index c17f7c4c..3ab7c86f 100644 --- a/packages/ai/test/api-keys.test.ts +++ b/packages/ai/test/api-keys.test.ts @@ -12,8 +12,7 @@ const ENV_KEYS = [ "ANTHROPIC_API_KEY", "GOOGLE_API_KEY", "GEMINI_API_KEY", - "MODEL_API_KEY", - "META_MODEL_API_KEY", + "META_API_KEY", "TZAFON_API_KEY", "YUTORI_API_KEY", ] as const; @@ -33,7 +32,7 @@ describe("cua api key helpers", () => { expect(cuaApiKeyEnvVarsForProvider("openai")).toEqual(["OPENAI_API_KEY"]); expect(cuaApiKeyEnvVarsForProvider("google")).toEqual(["GOOGLE_API_KEY", "GEMINI_API_KEY"]); expect(cuaApiKeyEnvVarsForProvider("gemini")).toEqual(["GOOGLE_API_KEY", "GEMINI_API_KEY"]); - expect(cuaApiKeyEnvVarsForProvider("meta")).toEqual(["META_MODEL_API_KEY", "MODEL_API_KEY"]); + expect(cuaApiKeyEnvVarsForProvider("meta")).toEqual(["META_API_KEY"]); expect(cuaApiKeyEnvVarsForProvider("unknown")).toEqual([]); }); @@ -48,10 +47,7 @@ describe("cua api key helpers", () => { it("resolves keys from model refs", () => { process.env.OPENAI_API_KEY = "openai"; expect(getCuaEnvApiKeyForModel("openai:gpt-5.5")).toBe("openai"); - delete process.env.META_MODEL_API_KEY; - process.env.MODEL_API_KEY = "meta-official-alias"; - expect(getCuaEnvApiKeyForModel("meta:muse-spark-1.1")).toBe("meta-official-alias"); - process.env.META_MODEL_API_KEY = "meta"; + process.env.META_API_KEY = "meta"; expect(getCuaEnvApiKeyForModel("meta:muse-spark-1.1")).toBe("meta"); }); diff --git a/packages/ai/test/computer-tool.integration.test.ts b/packages/ai/test/computer-tool.integration.test.ts index 1f879080..9c243beb 100644 --- a/packages/ai/test/computer-tool.integration.test.ts +++ b/packages/ai/test/computer-tool.integration.test.ts @@ -64,7 +64,7 @@ const cases: ProviderCase[] = [ }, { provider: "meta", - envVar: "META_MODEL_API_KEY", + envVar: "META_API_KEY", modelRef: "meta:muse-spark-1.1", tools: () => meta.computerTools({ actions: ["click"] }), coordinateRange: [0, 1000], diff --git a/packages/ai/test/meta-threading.test.ts b/packages/ai/test/meta-threading.test.ts index c37a2e16..f01ef766 100644 --- a/packages/ai/test/meta-threading.test.ts +++ b/packages/ai/test/meta-threading.test.ts @@ -34,7 +34,7 @@ describe("Meta Responses threading", () => { it("threads the latest response and applies serial computer-use defaults", async () => { const { context, onPayload } = threadMetaRequest(multiTurnContext(), undefined); expect(context.messages).toHaveLength(1); - expect(await onPayload({ input: [], include: ["reasoning.encrypted_content"] }, model)).toEqual({ + expect(await onPayload({ input: [] }, model)).toEqual({ input: [], store: true, parallel_tool_calls: false, @@ -42,12 +42,11 @@ describe("Meta Responses threading", () => { }); }); - it("keeps encrypted reasoning on the initial unthreaded request", async () => { + it("omits encrypted reasoning replay on the initial request", async () => { const context: Context = { messages: [{ role: "user", content: "hello", timestamp: 0 }] }; const { onPayload } = threadMetaRequest(context, undefined); expect(await onPayload({ input: [], include: ["reasoning.encrypted_content"] }, model)).toEqual({ input: [], - include: ["reasoning.encrypted_content"], store: true, parallel_tool_calls: false, }); @@ -64,15 +63,16 @@ describe("Meta Responses threading", () => { }); }); - it("reapplies protocol invariants after a caller payload hook", async () => { + it("passes the prepared request to a caller payload hook", async () => { const { onPayload } = threadMetaRequest(multiTurnContext(), { - onPayload: (payload) => ({ ...(payload as object), store: false, parallel_tool_calls: true, include: ["reasoning.encrypted_content", "other"] }), + onPayload: (payload) => ({ wrapped: payload }), }); expect(await onPayload({}, model)).toEqual({ - store: true, - parallel_tool_calls: false, - previous_response_id: "resp_meta_1", - include: ["other"], + wrapped: { + store: true, + parallel_tool_calls: false, + previous_response_id: "resp_meta_1", + }, }); }); }); diff --git a/packages/cli/README.md b/packages/cli/README.md index da0df71b..7debfac9 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -86,8 +86,7 @@ Configuration is by environment variable. There is no config file. | `ANTHROPIC_API_KEY` | Anthropic API key (required when `-m anthropic:…`) | | `GOOGLE_API_KEY` | Google API key (required when `-m google:…`) | | `GEMINI_API_KEY` | alias of `GOOGLE_API_KEY` | -| `META_MODEL_API_KEY` | Meta Model API key (required when `-m meta:…`) | -| `MODEL_API_KEY` | alias accepted for Meta's official examples | +| `META_API_KEY` | Meta Model API key (required when `-m meta:…`) | | `TZAFON_API_KEY` | Tzafon API key (required when `-m tzafon:…`) | | `YUTORI_API_KEY` | Yutori API key (required when `-m yutori:…`) | | `KERNEL_BASE_URL` | override Kernel base URL | diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 5754d071..5ed45148 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -102,8 +102,7 @@ Environment: ANTHROPIC_API_KEY Anthropic API key (required when -m anthropic:…) GOOGLE_API_KEY Google API key (required when -m google:…) GEMINI_API_KEY Alias for GOOGLE_API_KEY - META_MODEL_API_KEY Meta Model API key (required when -m meta:…) - MODEL_API_KEY Alias accepted for Meta's official examples + META_API_KEY Meta Model API key (required when -m meta:…) TZAFON_API_KEY Tzafon API key (required when -m tzafon:…) YUTORI_API_KEY Yutori API key (required when -m yutori:…) KERNEL_BASE_URL Override Kernel base URL diff --git a/packages/cli/test/cli-executor.test.ts b/packages/cli/test/cli-executor.test.ts index af9fbbae..1063b6a8 100644 --- a/packages/cli/test/cli-executor.test.ts +++ b/packages/cli/test/cli-executor.test.ts @@ -22,8 +22,7 @@ const PROVIDER_ENV_KEYS = [ "ANTHROPIC_API_KEY", "GOOGLE_API_KEY", "GEMINI_API_KEY", - "MODEL_API_KEY", - "META_MODEL_API_KEY", + "META_API_KEY", "TZAFON_API_KEY", "YUTORI_API_KEY", ]; From 839d4b08f563d48fce179185cce912f58d37ce66 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 12 Jul 2026 18:49:59 +0000 Subject: [PATCH 07/15] Drop Meta action alias normalization --- .../reference/provider-doc-drift.ts | 25 +++---------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/.agents/skills/update-models/reference/provider-doc-drift.ts b/.agents/skills/update-models/reference/provider-doc-drift.ts index b6b39e0c..bc652685 100644 --- a/.agents/skills/update-models/reference/provider-doc-drift.ts +++ b/.agents/skills/update-models/reference/provider-doc-drift.ts @@ -51,19 +51,6 @@ const LOCAL_FILES: Record = { yutori: "packages/ai/src/providers/yutori/actions.ts", }; -const META_ACTION_ALIASES: Record = { - left_click: "click", - right_click: "click", - middle_click: "click", - triple_click: "click", - left_click_drag: "drag", - mouse_move: "move", - key: "keypress", - hold_key: "keypress", - left_mouse_down: "mouse_down", - left_mouse_up: "mouse_up", -}; - const ACTION_REGEXES: Record = { openai: /\b(click|double_click|scroll|type|wait|keypress|drag|move|screenshot)\b/g, anthropic: /\b(screenshot|left_click|right_click|middle_click|double_click|triple_click|left_click_drag|mouse_move|key|type|scroll|hold_key|wait|left_mouse_down|left_mouse_up|cursor_position|zoom)\b/g, @@ -128,8 +115,6 @@ async function checkProvider(provider: Provider, examples: any): Promise { } } -function normalizeMetaAction(action: string): string { - return META_ACTION_ALIASES[action] ?? action; -} - function notesFor(provider: Provider, documentedToolVersions: string[], exampleToolVersions: string[], localToolVersions: string[]): string[] { const notes: string[] = []; if (provider === "openai") { @@ -181,7 +162,7 @@ function notesFor(provider: Provider, documentedToolVersions: string[], exampleT notes.push("Gemini official computer use emits predefined function-call names; keep this separate from CUA custom function declarations."); } if (provider === "meta") { - notes.push("Meta computer use uses developer-defined function tools over the Responses API; compare payload threading and coordinate conventions as well as action names."); + notes.push("Meta uses developer-defined function tools, so provider action-name drift does not apply; compare payload threading and coordinate conventions instead."); } if (provider === "yutori") { notes.push("Yutori Navigator emits OpenAI-compatible tool_calls for built-in browser actions; local AgentTools should execute those names but outbound payloads should not duplicate the built-in browser schemas."); From 9fdb09b20394e2a86f8251f8ab33e718521c9d30 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 12 Jul 2026 18:55:28 +0000 Subject: [PATCH 08/15] Sanitize Meta payload after hooks --- packages/ai/src/providers/meta/provider.ts | 5 ++++- packages/ai/test/meta-threading.test.ts | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/ai/src/providers/meta/provider.ts b/packages/ai/src/providers/meta/provider.ts index b976ac84..d1ca4b9e 100644 --- a/packages/ai/src/providers/meta/provider.ts +++ b/packages/ai/src/providers/meta/provider.ts @@ -39,7 +39,10 @@ export function threadMetaRequest( }; // CUA uses stored response state instead of stateless encrypted reasoning replay. delete next.include; - return options?.onPayload ? ((await options.onPayload(next, model)) ?? next) : next; + const prepared = options?.onPayload ? ((await options.onPayload(next, model)) ?? next) : next; + const sanitized = { ...(prepared as Record) }; + delete sanitized.include; + return sanitized; }; return { context: messages === context.messages ? context : { ...context, messages }, onPayload }; } diff --git a/packages/ai/test/meta-threading.test.ts b/packages/ai/test/meta-threading.test.ts index f01ef766..023b43c0 100644 --- a/packages/ai/test/meta-threading.test.ts +++ b/packages/ai/test/meta-threading.test.ts @@ -75,4 +75,15 @@ describe("Meta Responses threading", () => { }, }); }); + + it("removes include fields added by caller payload hooks", async () => { + const { onPayload } = threadMetaRequest(multiTurnContext(), { + onPayload: (payload) => ({ ...(payload as object), include: ["reasoning.encrypted_content"] }), + }); + expect(await onPayload({}, model)).toEqual({ + store: true, + parallel_tool_calls: false, + previous_response_id: "resp_meta_1", + }); + }); }); From bad7ac48db2120882603213b42e54a1bde706766 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 12 Jul 2026 18:58:28 +0000 Subject: [PATCH 09/15] Stabilize Meta integration coverage --- packages/ai/test/computer-tool.integration.test.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/ai/test/computer-tool.integration.test.ts b/packages/ai/test/computer-tool.integration.test.ts index 9c243beb..dcb59097 100644 --- a/packages/ai/test/computer-tool.integration.test.ts +++ b/packages/ai/test/computer-tool.integration.test.ts @@ -62,14 +62,6 @@ const cases: ProviderCase[] = [ coordinateRange: [0, 999], requireToolCalls: true, }, - { - provider: "meta", - envVar: "META_API_KEY", - modelRef: "meta:muse-spark-1.1", - tools: () => meta.computerTools({ actions: ["click"] }), - coordinateRange: [0, 1000], - requireToolCalls: true, - }, { provider: "tzafon", envVar: "TZAFON_API_KEY", @@ -187,6 +179,12 @@ describe("individual computer action integration", () => { }); const click = first.content.find((part) => part.type === "toolCall" && part.name === "click"); expect(click).toBeDefined(); + expect(typeof click!.arguments.x).toBe("number"); + expect(typeof click!.arguments.y).toBe("number"); + expect(click!.arguments.x as number).toBeGreaterThanOrEqual(0); + expect(click!.arguments.x as number).toBeLessThanOrEqual(1000); + expect(click!.arguments.y as number).toBeGreaterThanOrEqual(0); + expect(click!.arguments.y as number).toBeLessThanOrEqual(1000); expect(first.responseId).toBeTruthy(); context.messages.push(first, { role: "toolResult", From ac37c6bf0ea58e0b003ca8d2f96ec5cf3abca20c Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 12 Jul 2026 19:12:00 +0000 Subject: [PATCH 10/15] Remove internal API names from model docs --- packages/ai/docs/supported-models.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/ai/docs/supported-models.md b/packages/ai/docs/supported-models.md index 63fbe4cb..0812cfff 100644 --- a/packages/ai/docs/supported-models.md +++ b/packages/ai/docs/supported-models.md @@ -17,7 +17,7 @@ with CUA-only entries that pi-ai does not ship yet. ## `openai` -API: `openai-responses` · coordinates: pixel +Coordinates: pixel Family matches (root + numeric revision/dated-snapshot suffixes): @@ -27,7 +27,7 @@ Family matches (root + numeric revision/dated-snapshot suffixes): ## `anthropic` -API: `anthropic-messages` · coordinates: pixel +Coordinates: pixel Family matches (root + numeric revision/dated-snapshot suffixes): @@ -42,7 +42,7 @@ Source: [Anthropic computer use docs](https://docs.anthropic.com/en/docs/build-w ## `google` -API: `google-generative-ai` · coordinates: normalized 0–999 +Coordinates: normalized 0–999 Model refs use the `google:` prefix; `gemini:` is accepted as an alias. @@ -73,7 +73,7 @@ Source: [Meta computer-use cookbook](https://dev.meta.ai/docs/getting-started/co ## `tzafon` -API: `tzafon-responses` · coordinates: normalized 0–999 +Coordinates: normalized 0–999 Exact IDs: @@ -83,7 +83,7 @@ Exact IDs: ## `yutori` -API: `yutori-chat-completions` · coordinates: normalized 0–1000 +Coordinates: normalized 0–1000 Exact IDs: From 7592cc52835031b1e56f8858a5b99b4609ca0545 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 12 Jul 2026 19:29:11 +0000 Subject: [PATCH 11/15] Share Responses threading and computer prompt --- packages/ai/src/providers/common.ts | 39 +++++++++++++++++ packages/ai/src/providers/meta/index.ts | 19 +++------ packages/ai/src/providers/meta/provider.ts | 44 ++++++++------------ packages/ai/src/providers/openai/provider.ts | 36 +++------------- 4 files changed, 69 insertions(+), 69 deletions(-) diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index 49e89d48..cd4d951c 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -2,10 +2,12 @@ import { Type, type Api, type AssistantMessage, + type Context, type Message, type Model, type SimpleStreamOptions, type Static, + type StreamOptions, type TSchema, type Tool, } from "@earendil-works/pi-ai"; @@ -25,6 +27,13 @@ export * from "../native-tools"; */ export const CUA_ACTION_TYPES = CUA_DEFAULT_COMPUTER_ACTION_TYPES; +export const DEFAULT_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Base each action on the latest observed state and request a screenshot when you need a fresh view.`; + +/** Build the provider-neutral system prompt for canonical computer tools. */ +export function buildDefaultComputerSystemPrompt(opts: { suffix?: string } = {}): string { + return [DEFAULT_COMPUTER_INSTRUCTIONS, opts.suffix].filter(Boolean).join("\n\n"); +} + type ObjectSchemaWithProperties = TSchema & { properties: Record }; function createCuaActionArgumentSchema(action: CuaActionType, mode: CuaMode): TSchema { @@ -275,6 +284,36 @@ export interface ResponseThreadingOptions { disableResponseThreading?: boolean; } +type ResponsesOnPayload = NonNullable; + +/** Options shared by OpenAI-compatible Responses transports that support threading. */ +export interface ResponsesThreadingOptions extends ResponseThreadingOptions { + onPayload?: ResponsesOnPayload; +} + +/** + * Prepare an OpenAI-compatible Responses request using stored response state. + * When a prior response exists, only messages after it are sent. + */ +export function threadResponsesRequest( + context: Context, + api: Api, + options: ResponsesThreadingOptions | undefined, +): { context: Context; onPayload: ResponsesOnPayload } { + const delta = responseThreadingEnabled(options) ? responseThreadingDelta(context.messages, api) : undefined; + const previousResponseId = delta?.previousResponseId; + const messages = previousResponseId && delta ? delta.deltaMessages : context.messages; + const onPayload: ResponsesOnPayload = async (payload, model) => { + const threaded = { + ...(payload as Record), + store: true, + ...(previousResponseId ? { previous_response_id: previousResponseId } : {}), + }; + return options?.onPayload ? ((await options.onPayload(threaded, model)) ?? threaded) : threaded; + }; + return { context: messages === context.messages ? context : { ...context, messages }, onPayload }; +} + /** * Whether a Responses API provider should thread requests with * `previous_response_id` + delta input instead of sending the full message diff --git a/packages/ai/src/providers/meta/index.ts b/packages/ai/src/providers/meta/index.ts index fd17d385..26f7822d 100644 --- a/packages/ai/src/providers/meta/index.ts +++ b/packages/ai/src/providers/meta/index.ts @@ -1,8 +1,10 @@ -import { assertComputerModeOnly, computerToolExecutors, computerTools } from "../common"; +import { assertComputerModeOnly, buildDefaultComputerSystemPrompt, computerToolExecutors, computerTools } from "../common"; import type { ComputerToolCoordinateSystem, ComputerToolsOptions, CuaProviderModule } from "../common"; export { + buildDefaultComputerSystemPrompt as buildMetaSystemPrompt, CUA_ACTION_TYPES as META_CUA_ACTION_TYPES, + DEFAULT_COMPUTER_INSTRUCTIONS as META_COMPUTER_INSTRUCTIONS, computerToolExecutors, computerTools, createCuaActionSchema as createActionSchema, @@ -19,21 +21,12 @@ export { } from "./provider"; export type { MetaResponsesOptions } from "./provider"; -// CUA supplies Muse Spark with its canonical function actions: click, -// double_click, mouse_down, mouse_up, type, keypress, scroll, move, drag, -// wait, screenshot, goto, back, forward, url, and cursor_position. -// Meta's computer-use reference agent uses normalized 0-1000 coordinates. +// Muse Spark reports screenshot positions on a normalized 0-1000 grid. +// Source: https://dev.meta.ai/docs/features/image-understanding#coordinate-system export function coordinateSystem(): ComputerToolCoordinateSystem { return { type: "normalized", range: [0, 1000] }; } -export const META_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Base each action on the latest observed state and request a screenshot when you need a fresh view.`; - -/** Build the default system prompt used with Meta CUA models. */ -export function buildMetaSystemPrompt(opts: { suffix?: string } = {}): string { - return [META_COMPUTER_INSTRUCTIONS, opts.suffix].filter(Boolean).join("\n\n"); -} - export const providerModule = { toolDefinitions: (options?: ComputerToolsOptions) => { assertComputerModeOnly("meta", options); @@ -44,5 +37,5 @@ export const providerModule = { return computerToolExecutors(options); }, coordinateSystem, - buildSystemPrompt: buildMetaSystemPrompt, + buildSystemPrompt: buildDefaultComputerSystemPrompt, } satisfies CuaProviderModule; diff --git a/packages/ai/src/providers/meta/provider.ts b/packages/ai/src/providers/meta/provider.ts index d1ca4b9e..691860ae 100644 --- a/packages/ai/src/providers/meta/provider.ts +++ b/packages/ai/src/providers/meta/provider.ts @@ -2,7 +2,6 @@ import { type Context, type OpenAIResponsesOptions as PiOpenAIResponsesOptions, type StreamFunction, - type StreamOptions, } from "@earendil-works/pi-ai"; import { stream as piStreamOpenAIResponses, @@ -10,41 +9,34 @@ import { } from "@earendil-works/pi-ai/api/openai-responses"; import { type CuaSimpleStreamOptions, - responseThreadingDelta, - responseThreadingEnabled, - type ResponseThreadingOptions, + type ResponsesThreadingOptions, + threadResponsesRequest, } from "../common"; export const META_RESPONSES_API = "meta-responses"; /** Stream options for Meta's OpenAI-compatible Responses API. */ -export interface MetaResponsesOptions extends PiOpenAIResponsesOptions, ResponseThreadingOptions {} +export interface MetaResponsesOptions extends PiOpenAIResponsesOptions, ResponsesThreadingOptions {} -type OnPayload = NonNullable; - -/** Prepare a Meta Responses request that continues from the latest stored response. */ -export function threadMetaRequest( - context: Context, - options: (ResponseThreadingOptions & { onPayload?: OnPayload }) | undefined, -): { context: Context; onPayload: OnPayload } { - const delta = responseThreadingEnabled(options) ? responseThreadingDelta(context.messages, META_RESPONSES_API) : undefined; - const previousResponseId = delta?.previousResponseId; - const messages = previousResponseId && delta ? delta.deltaMessages : context.messages; - const onPayload: OnPayload = async (payload, model) => { - const next: Record = { - ...(payload as Record), - store: true, - parallel_tool_calls: false, - ...(previousResponseId ? { previous_response_id: previousResponseId } : {}), - }; - // CUA uses stored response state instead of stateless encrypted reasoning replay. - delete next.include; - const prepared = options?.onPayload ? ((await options.onPayload(next, model)) ?? next) : next; +/** Apply Meta's computer-use payload constraints on top of shared Responses threading. */ +export function threadMetaRequest(context: Context, options: ResponsesThreadingOptions | undefined) { + const callerOnPayload = options?.onPayload; + const threaded = threadResponsesRequest(context, META_RESPONSES_API, { + ...options, + onPayload: async (payload, model) => { + const constrained: Record = { ...(payload as Record), parallel_tool_calls: false }; + delete constrained.include; + return callerOnPayload ? ((await callerOnPayload(constrained, model)) ?? constrained) : constrained; + }, + }); + const onPayload: typeof threaded.onPayload = async (payload, model) => { + const prepared = await threaded.onPayload(payload, model); const sanitized = { ...(prepared as Record) }; + // CUA uses stored response state instead of stateless encrypted reasoning replay. delete sanitized.include; return sanitized; }; - return { context: messages === context.messages ? context : { ...context, messages }, onPayload }; + return { context: threaded.context, onPayload }; } // Meta implements the OpenAI Responses wire protocol. The model carries a diff --git a/packages/ai/src/providers/openai/provider.ts b/packages/ai/src/providers/openai/provider.ts index b9fbfc86..87abf0e9 100644 --- a/packages/ai/src/providers/openai/provider.ts +++ b/packages/ai/src/providers/openai/provider.ts @@ -2,7 +2,6 @@ import { type Context, type OpenAIResponsesOptions as PiOpenAIResponsesOptions, type StreamFunction, - type StreamOptions, } from "@earendil-works/pi-ai"; import { stream as piStreamOpenAIResponses, @@ -10,41 +9,18 @@ import { } from "@earendil-works/pi-ai/api/openai-responses"; import { type CuaSimpleStreamOptions, - responseThreadingDelta, - responseThreadingEnabled, - type ResponseThreadingOptions, + type ResponsesThreadingOptions, + threadResponsesRequest, } from "../common"; export const OPENAI_CUA_RESPONSES_API = "openai-cua-responses"; /** Stream options for the cua OpenAI Responses provider: pi-ai's options plus threading control. */ -export interface OpenAIResponsesOptions extends PiOpenAIResponsesOptions, ResponseThreadingOptions {} +export interface OpenAIResponsesOptions extends PiOpenAIResponsesOptions, ResponsesThreadingOptions {} -type OnPayload = NonNullable; - -/** - * Prepare a request for pi-ai's builtin OpenAI Responses stream so it threads - * `previous_response_id`. The public Responses API requires `store: true` to - * chain, so the payload always stores; when a prior assistant `responseId` - * exists, only the delta messages are sent with `previous_response_id` set. - * Any caller `onPayload` runs on top of the threaded payload. - */ -export function threadRequest( - context: Context, - options: (ResponseThreadingOptions & { onPayload?: OnPayload }) | undefined, -): { context: Context; onPayload: OnPayload } { - const delta = responseThreadingEnabled(options) ? responseThreadingDelta(context.messages, OPENAI_CUA_RESPONSES_API) : undefined; - const previousResponseId = delta?.previousResponseId; - const messages = previousResponseId && delta ? delta.deltaMessages : context.messages; - const onPayload: OnPayload = async (payload, model) => { - const threaded = { - ...(payload as Record), - store: true, - ...(previousResponseId ? { previous_response_id: previousResponseId } : {}), - }; - return options?.onPayload ? ((await options.onPayload(threaded, model)) ?? threaded) : threaded; - }; - return { context: messages === context.messages ? context : { ...context, messages }, onPayload }; +/** Prepare a request for the OpenAI Responses transport with stored response state. */ +export function threadRequest(context: Context, options: ResponsesThreadingOptions | undefined) { + return threadResponsesRequest(context, OPENAI_CUA_RESPONSES_API, options); } // pi-ai's builtin stream fns are typed to the "openai-responses" api; we reuse them under our routed api, hence `as never` on the model. From 1ba2bd25d9e35c4f23148195457e735453ff8476 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 12 Jul 2026 19:39:09 +0000 Subject: [PATCH 12/15] Preserve Meta payload constraints after shared threading --- packages/ai/src/providers/common.ts | 4 +-- packages/ai/src/providers/meta/provider.ts | 8 +++++- packages/ai/src/providers/openai/provider.ts | 3 ++- packages/ai/test/meta-threading.test.ts | 26 +++++++++++++------- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index cd4d951c..a33430ab 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -299,7 +299,7 @@ export function threadResponsesRequest( context: Context, api: Api, options: ResponsesThreadingOptions | undefined, -): { context: Context; onPayload: ResponsesOnPayload } { +): { context: Context; onPayload: ResponsesOnPayload; previousResponseId?: string } { const delta = responseThreadingEnabled(options) ? responseThreadingDelta(context.messages, api) : undefined; const previousResponseId = delta?.previousResponseId; const messages = previousResponseId && delta ? delta.deltaMessages : context.messages; @@ -311,7 +311,7 @@ export function threadResponsesRequest( }; return options?.onPayload ? ((await options.onPayload(threaded, model)) ?? threaded) : threaded; }; - return { context: messages === context.messages ? context : { ...context, messages }, onPayload }; + return { context: messages === context.messages ? context : { ...context, messages }, onPayload, previousResponseId }; } /** diff --git a/packages/ai/src/providers/meta/provider.ts b/packages/ai/src/providers/meta/provider.ts index 691860ae..ce8b894a 100644 --- a/packages/ai/src/providers/meta/provider.ts +++ b/packages/ai/src/providers/meta/provider.ts @@ -31,7 +31,13 @@ export function threadMetaRequest(context: Context, options: ResponsesThreadingO }); const onPayload: typeof threaded.onPayload = async (payload, model) => { const prepared = await threaded.onPayload(payload, model); - const sanitized = { ...(prepared as Record) }; + const sanitized: Record = { + ...(prepared as Record), + store: true, + parallel_tool_calls: false, + }; + delete sanitized.previous_response_id; + if (threaded.previousResponseId) sanitized.previous_response_id = threaded.previousResponseId; // CUA uses stored response state instead of stateless encrypted reasoning replay. delete sanitized.include; return sanitized; diff --git a/packages/ai/src/providers/openai/provider.ts b/packages/ai/src/providers/openai/provider.ts index 87abf0e9..a2f84da6 100644 --- a/packages/ai/src/providers/openai/provider.ts +++ b/packages/ai/src/providers/openai/provider.ts @@ -20,7 +20,8 @@ export interface OpenAIResponsesOptions extends PiOpenAIResponsesOptions, Respon /** Prepare a request for the OpenAI Responses transport with stored response state. */ export function threadRequest(context: Context, options: ResponsesThreadingOptions | undefined) { - return threadResponsesRequest(context, OPENAI_CUA_RESPONSES_API, options); + const { context: threadedContext, onPayload } = threadResponsesRequest(context, OPENAI_CUA_RESPONSES_API, options); + return { context: threadedContext, onPayload }; } // pi-ai's builtin stream fns are typed to the "openai-responses" api; we reuse them under our routed api, hence `as never` on the model. diff --git a/packages/ai/test/meta-threading.test.ts b/packages/ai/test/meta-threading.test.ts index 023b43c0..43f18a16 100644 --- a/packages/ai/test/meta-threading.test.ts +++ b/packages/ai/test/meta-threading.test.ts @@ -54,7 +54,10 @@ describe("Meta Responses threading", () => { it("sends full history without previous_response_id when threading is disabled", async () => { const context = multiTurnContext(); - const prepared = threadMetaRequest(context, { disableResponseThreading: true }); + const prepared = threadMetaRequest(context, { + disableResponseThreading: true, + onPayload: (payload) => ({ ...(payload as object), previous_response_id: "foreign-response" }), + }); expect(prepared.context).toBe(context); expect(await prepared.onPayload({ input: [] }, model)).toEqual({ input: [], @@ -65,20 +68,25 @@ describe("Meta Responses threading", () => { it("passes the prepared request to a caller payload hook", async () => { const { onPayload } = threadMetaRequest(multiTurnContext(), { - onPayload: (payload) => ({ wrapped: payload }), + onPayload: (payload) => ({ ...(payload as object), caller_field: true }), }); expect(await onPayload({}, model)).toEqual({ - wrapped: { - store: true, - parallel_tool_calls: false, - previous_response_id: "resp_meta_1", - }, + caller_field: true, + store: true, + parallel_tool_calls: false, + previous_response_id: "resp_meta_1", }); }); - it("removes include fields added by caller payload hooks", async () => { + it("reapplies Meta constraints after caller payload hooks", async () => { const { onPayload } = threadMetaRequest(multiTurnContext(), { - onPayload: (payload) => ({ ...(payload as object), include: ["reasoning.encrypted_content"] }), + onPayload: (payload) => ({ + ...(payload as object), + store: false, + parallel_tool_calls: true, + previous_response_id: "wrong-response", + include: ["reasoning.encrypted_content"], + }), }); expect(await onPayload({}, model)).toEqual({ store: true, From c4e92c56be87ed80c3626e4894b721d6cdfd88bb Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 10:57:23 +0000 Subject: [PATCH 13/15] Remove Meta-specific e2e gate --- .github/workflows/ci.yml | 1 - packages/agent/test/e2e.live.test.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65517a57..4f78ed44 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,7 +155,6 @@ jobs: - name: Agent live smoke tests (all providers) env: CUA_E2E_LIVE: "1" - CUA_E2E_META: "1" KERNEL_API_KEY: ${{ secrets.KERNEL_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} diff --git a/packages/agent/test/e2e.live.test.ts b/packages/agent/test/e2e.live.test.ts index 202a9498..55fbc1d6 100644 --- a/packages/agent/test/e2e.live.test.ts +++ b/packages/agent/test/e2e.live.test.ts @@ -92,7 +92,6 @@ const cases: ProviderCase[] = [ ].join("\n"), expectToolCalls: true, timeoutMs: 180_000, - ciOptInEnvVar: "CUA_E2E_META", }, { name: "tzafon", From b0905106dc273492d294f05fdbd496c3a4160a33 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 11:04:28 +0000 Subject: [PATCH 14/15] Document Meta response threading --- packages/ai/src/providers/common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index a33430ab..f4f764e1 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -272,7 +272,7 @@ export type CuaPayloadHook = (payload: unknown, model: Model, context?: Cua * pi-ai `SimpleStreamOptions` plus CUA request controls consumed by provider * adapters. Pass `keepToolNames` for caller tools that must survive native * tool-set substitution; use `disableResponseThreading` for a full-context - * OpenAI or Tzafon request. + * OpenAI, Meta, or Tzafon request. */ export interface CuaSimpleStreamOptions extends SimpleStreamOptions, ResponseThreadingOptions { keepToolNames?: readonly string[]; From 6bcb1c559df3c6303aa85fb37243163ebdc742c2 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 13 Jul 2026 13:10:47 +0000 Subject: [PATCH 15/15] Format exported API TSDoc --- packages/agent/src/agent.ts | 8 ++++++-- packages/ai/src/providers/common.ts | 8 ++++++-- packages/ai/src/providers/meta/provider.ts | 8 ++++++-- packages/ai/src/providers/openai/provider.ts | 4 +++- packages/cli/src/harness.ts | 4 +++- 5 files changed, 24 insertions(+), 8 deletions(-) diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index 0380ac2d..9d57eaca 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -156,7 +156,9 @@ export type CuaAgentOptions = Omit & { emptyResponseRecovery?: CuaEmptyResponseRecoveryOptions; /** Maximum tool-result images included from message history per provider request. Defaults to 4; false disables projection. */ toolResultImageReplayLimit?: ToolResultImageReplayLimit; - /** Chain OpenAI, Meta, and Tzafon requests through provider-stored response state. Defaults to true. */ + /** + * Chain OpenAI, Meta, and Tzafon requests through provider-stored response state. Defaults to true. + */ responseThreading?: boolean; /** Optional CUA-level retries around each provider request. Disabled by default. */ retry?: CuaRetryOptions; @@ -202,7 +204,9 @@ export type CuaAgentHarnessOptions< emptyResponseRecovery?: CuaEmptyResponseRecoveryOptions; /** Maximum tool-result images included from message history per provider request. Defaults to 4; false disables projection. */ toolResultImageReplayLimit?: ToolResultImageReplayLimit; - /** Chain OpenAI, Meta, and Tzafon requests through provider-stored response state. Defaults to true. */ + /** + * Chain OpenAI, Meta, and Tzafon requests through provider-stored response state. Defaults to true. + */ responseThreading?: boolean; /** Optional CUA-level retries around each provider request. Disabled by default. */ retry?: CuaRetryOptions; diff --git a/packages/ai/src/providers/common.ts b/packages/ai/src/providers/common.ts index f4f764e1..dc54a620 100644 --- a/packages/ai/src/providers/common.ts +++ b/packages/ai/src/providers/common.ts @@ -29,7 +29,9 @@ export const CUA_ACTION_TYPES = CUA_DEFAULT_COMPUTER_ACTION_TYPES; export const DEFAULT_COMPUTER_INSTRUCTIONS = `You control a Kernel cloud browser through individual browser tools. Base each action on the latest observed state and request a screenshot when you need a fresh view.`; -/** Build the provider-neutral system prompt for canonical computer tools. */ +/** + * Build the provider-neutral system prompt for canonical computer tools. + */ export function buildDefaultComputerSystemPrompt(opts: { suffix?: string } = {}): string { return [DEFAULT_COMPUTER_INSTRUCTIONS, opts.suffix].filter(Boolean).join("\n\n"); } @@ -286,7 +288,9 @@ export interface ResponseThreadingOptions { type ResponsesOnPayload = NonNullable; -/** Options shared by OpenAI-compatible Responses transports that support threading. */ +/** + * Options shared by OpenAI-compatible Responses transports that support threading. + */ export interface ResponsesThreadingOptions extends ResponseThreadingOptions { onPayload?: ResponsesOnPayload; } diff --git a/packages/ai/src/providers/meta/provider.ts b/packages/ai/src/providers/meta/provider.ts index ce8b894a..dffa9e15 100644 --- a/packages/ai/src/providers/meta/provider.ts +++ b/packages/ai/src/providers/meta/provider.ts @@ -15,10 +15,14 @@ import { export const META_RESPONSES_API = "meta-responses"; -/** Stream options for Meta's OpenAI-compatible Responses API. */ +/** + * Stream options for Meta's OpenAI-compatible Responses API. + */ export interface MetaResponsesOptions extends PiOpenAIResponsesOptions, ResponsesThreadingOptions {} -/** Apply Meta's computer-use payload constraints on top of shared Responses threading. */ +/** + * Apply Meta's computer-use payload constraints on top of shared Responses threading. + */ export function threadMetaRequest(context: Context, options: ResponsesThreadingOptions | undefined) { const callerOnPayload = options?.onPayload; const threaded = threadResponsesRequest(context, META_RESPONSES_API, { diff --git a/packages/ai/src/providers/openai/provider.ts b/packages/ai/src/providers/openai/provider.ts index a2f84da6..36febae6 100644 --- a/packages/ai/src/providers/openai/provider.ts +++ b/packages/ai/src/providers/openai/provider.ts @@ -18,7 +18,9 @@ export const OPENAI_CUA_RESPONSES_API = "openai-cua-responses"; /** Stream options for the cua OpenAI Responses provider: pi-ai's options plus threading control. */ export interface OpenAIResponsesOptions extends PiOpenAIResponsesOptions, ResponsesThreadingOptions {} -/** Prepare a request for the OpenAI Responses transport with stored response state. */ +/** + * Prepare a request for the OpenAI Responses transport with stored response state. + */ export function threadRequest(context: Context, options: ResponsesThreadingOptions | undefined) { const { context: threadedContext, onPayload } = threadResponsesRequest(context, OPENAI_CUA_RESPONSES_API, options); return { context: threadedContext, onPayload }; diff --git a/packages/cli/src/harness.ts b/packages/cli/src/harness.ts index 41cda4b3..ac1dbb6a 100644 --- a/packages/cli/src/harness.ts +++ b/packages/cli/src/harness.ts @@ -45,7 +45,9 @@ export interface BuildCuaHarnessOptions { models?: Models; /** Maximum tool-result images included from message history per provider request. */ toolResultImageReplayLimit?: CuaAgentHarnessOptions["toolResultImageReplayLimit"]; - /** Chain OpenAI, Meta, and Tzafon requests through provider-stored response state. Defaults to true. */ + /** + * Chain OpenAI, Meta, and Tzafon requests through provider-stored response state. Defaults to true. + */ responseThreading?: CuaAgentHarnessOptions["responseThreading"]; /** Optional CUA-level retries around each provider request. Disabled by default. */ retry?: CuaAgentHarnessOptions["retry"];